From f07596a5fdf9cefb9e49212477b08ebc1d68444d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 12:21:18 -0700 Subject: [PATCH 001/124] fix(helm): stop litellm-helm from injecting placeholder models into every install (#47056) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/README.md | 13 +++++++--- helm/litellm-helm/tests/deployment_tests.yaml | 26 +++++++++++++++++++ helm/litellm-helm/values.yaml | 12 +-------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index ff432c451e5..e059d45e037 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -70,11 +70,18 @@ proxyConfigMap: proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY + model_list: [] +``` + +The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true` + +``` +proxy_config: model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-5.6 litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy + model: openai/gpt-5.6 + api_key: os.environ/OPENAI_API_KEY ``` #### Example using existing `proxyConfigMap` instead of creating it: diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml index ee946038202..292eeadfd13 100644 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ b/helm/litellm-helm/tests/deployment_tests.yaml @@ -270,6 +270,32 @@ tests: - matchRegex: path: data["config.yaml"] pattern: '(?m)^\s*health_check_details:\s*false$' + - it: should not render any placeholder models when the user sets proxy_config without model_list + template: configmap-litellm.yaml + set: + proxy_config.general_settings.store_model_in_db: true + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: 'model_name:' + - matchRegex: + path: data["config.yaml"] + pattern: 'store_model_in_db: true' + - matchRegex: + path: data["config.yaml"] + pattern: 'master_key: os.environ/PROXY_MASTER_KEY' + - it: should render a user-supplied model_list as the only models + template: configmap-litellm.yaml + set: + proxy_config.model_list: + - model_name: my-model + litellm_params: + model: openai/my-model + api_key: os.environ/OPENAI_API_KEY + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$' - it: should allow overriding liveness, readiness, and startup probes template: deployment.yaml set: diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 199c4595149..8450c5f2840 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -180,17 +180,7 @@ proxyConfigMap: # Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml # Reference: https://docs.litellm.ai/docs/proxy/configs proxy_config: - model_list: - # At least one model must exist for the proxy to start. - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + model_list: [] general_settings: master_key: os.environ/PROXY_MASTER_KEY From 2e427c6247903b133bd8e3db9076bf5d70d3f9f0 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 12:37:47 -0700 Subject: [PATCH 002/124] chore(deps): require anyio>=4.14.0 to rule out the lock-waiter deadlock (#47378) anyio below 4.14.0 can leave a cancelled waiter queued on an asyncio Lock or Semaphore, so the next task to acquire waits forever on a lock nobody holds (agronholm/anyio#1145, fixed in 4.14.0). httpcore takes these locks for every httpx request, including the prisma query-engine client litellm imports anyio directly but never declared it, so pip and uv kept an already installed anyio 4.13.0 when installing litellm and accepted an explicit anyio==4.13.0 next to it. uv.lock already resolves 4.14.2, so Docker and CI installs do not change. Both the litellm and litellm-core manifests get the floor, and the lock only records the new direct dependency Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 1 + pyproject.toml | 1 + tests/unit/test_package_metadata.py | 17 +++++++++++++++++ uv.lock | 2 ++ 4 files changed, 21 insertions(+) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index cf87cc6b936..2add6868bd9 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/pyproject.toml b/pyproject.toml index f6697a1b57a..6c71140e6b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/tests/unit/test_package_metadata.py b/tests/unit/test_package_metadata.py index fb6069291d8..dc6e3dca64e 100644 --- a/tests/unit/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -5,6 +5,7 @@ from typing import Final from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import Version if sys.version_info >= (3, 11): import tomllib @@ -18,6 +19,8 @@ OPTIONAL_DEPENDENCIES: Final = PYPROJECT["project"]["optional-dependencies"] AIOHTTP_POOL_POISONING_RANGE: Final = SpecifierSet(">=3.14.0,<3.14.2") AIOHTTP_POOL_POISONING_RELEASES: Final = ("3.14.0", "3.14.1") AIOHTTP_POOL_POISONING_SOURCE: Final = "aio-libs/aiohttp#12953, fixed in aiohttp 3.14.2" +ANYIO_STRANDED_LOCK_WAITER_FIX: Final = Version("4.14.0") +ANYIO_STRANDED_LOCK_WAITER_SOURCE: Final = "agronholm/anyio#1145, fixed in anyio 4.14.0 released 2026-06-15" CLI_RUNTIME_DEPENDENCIES: Final = frozenset({"rich", "pyyaml", "requests"}) PROXY_SERVER_ONLY_DEPENDENCIES: Final = frozenset( { @@ -92,3 +95,17 @@ def test_locked_aiohttp_version_is_not_pool_poisoning() -> None: locked: Final = _locked_package("aiohttp")["version"] assert isinstance(locked, str) assert not AIOHTTP_POOL_POISONING_RANGE.contains(locked), AIOHTTP_POOL_POISONING_SOURCE + + +def test_declared_anyio_floor_is_at_or_above_stranded_lock_waiter_fix() -> None: + declared: Final = tuple( + Requirement(requirement) + for requirement in PYPROJECT["project"]["dependencies"] + if canonicalize_name(Requirement(requirement).name) == "anyio" + ) + assert len(declared) == 1, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert declared[0].marker is None, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert any( + specifier.operator == ">=" and Version(specifier.version) >= ANYIO_STRANDED_LOCK_WAITER_FIX + for specifier in declared[0].specifier + ), ANYIO_STRANDED_LOCK_WAITER_SOURCE diff --git a/uv.lock b/uv.lock index beb66fe42b2..69d9f777687 100644 --- a/uv.lock +++ b/uv.lock @@ -4508,6 +4508,7 @@ version = "1.107.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "anyio" }, { name = "async-timeout", marker = "python_full_version < '3.11'" }, { name = "boto3" }, { name = "click" }, @@ -4761,6 +4762,7 @@ requires-dist = [ { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=1.1.0,<2.0" }, { name = "aiohttp", specifier = ">=3.14.2,<4.0" }, { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" }, + { name = "anyio", specifier = ">=4.14.0,<5.0" }, { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "async-timeout", marker = "python_full_version < '3.11'", specifier = ">=4.0.3,<6.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, From 3e18a4b55d5dcab3a5514c554a890ed0f9441392 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:18:20 -0700 Subject: [PATCH 003/124] fix(router): record cooled deployments on the all-cooled failure row (#47120) * fix(router): record cooled deployments on the all-cooled failure row A RouterRateLimitError now carries the cooled deployments of the route the request resolves to for the caller's team (model group, alias, wildcard or team route), each with its masked api_base. get_error_information copies them into error_information.error_cooldown_deployments so the SpendLogs failure row and every logging callback can tell which deployments were cooling * fix(router): name cooled deployments from the router's own resolved route Use the deployments the router resolved for the request, before health and cooldown filtering, instead of re-deriving the route. Alias chains, the provider-model fallback, wildcards and team filtering now match routing by construction. A routing-strategy refusal records null. * fix(router): name only the cooled deployments tag and region routing allow The refusal's cooled-deployment list now leaves out deployments the request's tags or the end user's allowed region keep it from, so a tenant's refusal row no longer shows another tenant's api_base * fix(router): keep failure logging working for refusal subclasses that skip init A RouterRateLimitError subclass that never calls its parent init now logs a null cooled-deployment list instead of breaking the failure payload * test(router): call the refusal routing helpers directly The router coverage check needs each router function called by name from a unit test --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 3 + litellm/router.py | 104 +++++ litellm/router_utils/cooldown_handlers.py | 24 +- litellm/router_utils/handle_error.py | 12 +- litellm/types/router.py | 4 + litellm/types/utils.py | 6 + .../spend/test_router_cooldown_failure_row.py | 358 +++++++++++++++++ .../test_litellm_logging.py | 44 +++ .../router_utils/test_cooldown_handlers.py | 361 +++++++++++++++++- 9 files changed, 913 insertions(+), 3 deletions(-) create mode 100644 tests/integration/spend/test_router_cooldown_failure_row.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0ac216c2029..3a1dafa331c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -151,6 +151,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse +from litellm.types.router import RouterRateLimitError from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6692,6 +6693,7 @@ class StandardLoggingPayloadSetup: rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None)) rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None)) budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None + router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None return StandardLoggingPayloadErrorInformation( @@ -6707,6 +6709,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + error_cooldown_deployments=router_error.cooldown_deployments if router_error else None, normalized_error=normalize_error(original_exception, error_status, error_message), ) diff --git a/litellm/router.py b/litellm/router.py index 113565fecaa..af79fa763c4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -222,6 +222,7 @@ from litellm.router_utils.cooldown_handlers import ( _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matching the other cooldown_handlers imports on this line async_get_cooldown_deployments, async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, get_cooldown_deployments, is_advisor_orchestration_failure, is_background_response_poll_not_found, @@ -12598,6 +12599,80 @@ class Router: if isinstance(model_info, Mapping) and model_info.get("id") is not None ) + def _region_routable_deployments( + self, + deployments: Sequence[DeploymentTypedDict], + request_kwargs: Mapping[str, object] | None, + has_request_input: bool, + ) -> tuple[Mapping[str, Any], ...]: + """ + `deployments` without the ones `_pre_call_checks` keeps this request away from for being outside its allowed + region, typed for the tag routing filter + """ + allowed_model_region: Final = None if request_kwargs is None else request_kwargs.get("allowed_model_region") + if allowed_model_region is None or not self.enable_pre_call_checks or not has_request_input: + return tuple(deployments) + if not isinstance(allowed_model_region, str): + return () + return tuple( + deployment + for deployment in deployments + if is_region_allowed( + litellm_params=LiteLLM_Params.model_validate(deployment["litellm_params"]), + allowed_model_region=allowed_model_region, + ) + ) + + async def _async_routable_cooled_deployment_ids( + self, + model: str, + request_deployments: Sequence[DeploymentTypedDict], + cooldown_deployments: Iterable[str], + request_kwargs: Mapping[str, object], + has_request_input: bool, + ) -> frozenset[str]: + """ + Ids of the cooling deployments in `request_deployments` that region and tag routing would send this request + to, each tag routing pick counted along with the picks it falls back to once that one is unavailable, judged + without recording a tag match on the request + """ + cooled_ids: Final = frozenset(cooldown_deployments) + metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_kwargs) + metadata: Final = request_kwargs.get(metadata_variable_name) + if metadata_variable_name in request_kwargs and not isinstance(metadata, Mapping): + return frozenset() + tag_request: Final[Mapping[str, Any]] = ( + {**request_kwargs, metadata_variable_name: {**metadata}} + if isinstance(metadata, Mapping) + else request_kwargs + ) + routable_ids = frozenset[str]() # rebind-ok: gains each pass's tag routing picks + remaining = tuple( # rebind-ok: each pass drops what tag routing just picked + deployment + for deployment in self._region_routable_deployments(request_deployments, request_kwargs, has_request_input) + if self._deployment_ids((deployment,)) <= cooled_ids + ) + while remaining: + try: + picked = await get_deployments_for_tag( + llm_router_instance=self, + model=model, + healthy_deployments=remaining, + request_kwargs=tag_request, + metadata_variable_name=metadata_variable_name, + ) + except ValueError: # tag routing sends this request to none of the rest + break + picked_identities = frozenset(id(deployment) for deployment in picked) + rest = tuple(deployment for deployment in remaining if id(deployment) not in picked_identities) + if len(rest) == len(remaining): + break + routable_ids |= self._deployment_ids( + tuple(deployment for deployment in remaining if id(deployment) in picked_identities) + ) + remaining = rest + return routable_ids + def has_model_id(self, candidate_id: str) -> bool: """ O(1) membership check for a deployment ID without allocating large lists. @@ -14457,6 +14532,13 @@ class Router: litellm_router_instance=self, model=model, parent_otel_span=parent_otel_span, + route_model_ids=await self._async_routable_cooled_deployment_ids( + model=model, + request_deployments=request_deployments, + cooldown_deployments=cooldown_deployments, + request_kwargs=cast(Mapping[str, object], request_kwargs), # cast-ok: str-keyed request kwargs + has_request_input=messages is not None or input is not None, + ), ) raise exception @@ -15656,6 +15738,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) if strategy == "simple-shuffle": @@ -15830,6 +15923,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=pass_through_model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) # 6. Apply load balancing strategy diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bbd895320bd..7ed4864ebc3 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -8,7 +8,7 @@ Router cooldown handlers import asyncio import math -from collections.abc import Mapping +from collections.abc import Collection, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -28,12 +28,14 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.types.router import Deployment from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, + CooldownDeploymentInfo, InternalCallOrigin, ) @@ -644,6 +646,26 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel _get_cooldown_deployments = get_cooldown_deployments +def _deployment_api_base(litellm_router_instance: LitellmRouter, model_id: str) -> str | None: + deployment: Final = litellm_router_instance.get_deployment(model_id=model_id) + api_base: Final = deployment.litellm_params.api_base if deployment else None + return None if api_base is None else mask_api_base_credentials(api_base) + + +def get_cooldown_deployment_info( + litellm_router_instance: LitellmRouter, route_model_ids: Collection[str], cooldown_list: Sequence[str] +) -> tuple[CooldownDeploymentInfo, ...]: + """ + The deployments the router resolved for this request (`route_model_ids`) that are in `cooldown_list`, each + with its masked api_base + """ + return tuple( + CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(litellm_router_instance, model_id)) + for model_id in cooldown_list + if model_id in route_model_ids + ) + + def should_cooldown_based_on_allowed_fails_policy( litellm_router_instance: LitellmRouter, deployment: str, diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 14ed6ed2adc..38d9ef05706 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,9 +1,11 @@ +from collections.abc import Collection from typing import TYPE_CHECKING, Any, Final from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, ) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.router import RouterRateLimitError @@ -69,7 +71,10 @@ async def send_llm_exception_alert( async def async_raise_no_deployment_exception( - litellm_router_instance: LitellmRouter, model: str, parent_otel_span: Span | None + litellm_router_instance: LitellmRouter, + model: str, + parent_otel_span: Span | None, + route_model_ids: Collection[str] | None = None, ): """ Raises a RouterRateLimitError if no deployment is found for the given model. @@ -94,4 +99,9 @@ async def async_raise_no_deployment_exception( enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, cooldown_list=cooldown_list_ids, model_ids=model_ids, + cooldown_deployments=( + None + if route_model_ids is None + else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids) + ), ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 68dbc701c9e..46617e1b753 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -43,6 +43,7 @@ from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider from .utils import ( + CooldownDeploymentInfo, CustomPricingLiteLLMParams, MirroredPricingParams, ModelResponse, @@ -1106,6 +1107,7 @@ class RouterErrorTypes(str, enum.Enum): class RouterRateLimitError(ValueError): status_code: Final = 429 + cooldown_deployments: tuple[CooldownDeploymentInfo, ...] | None = None def __init__( self, @@ -1114,11 +1116,13 @@ class RouterRateLimitError(ValueError): enable_pre_call_checks: bool, cooldown_list: list, model_ids: Sequence[str] = (), + cooldown_deployments: Sequence[CooldownDeploymentInfo] | None = None, ) -> None: self.model = model self.cooldown_time = cooldown_time self.enable_pre_call_checks = enable_pre_call_checks self.cooldown_list = cooldown_list + self.cooldown_deployments = None if cooldown_deployments is None else tuple(cooldown_deployments) self.all_deployments_in_cooldown = bool(model_ids) and frozenset(model_ids) <= frozenset(cooldown_list) self.type = ( RouterErrorTypes.all_deployments_in_cooldown.value diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fd1d58adafb..2e05107f82d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3386,6 +3386,11 @@ class StandardLoggingZeroCostDiagnostic(TypedDict): missing_pricing_keys: ReadOnly[tuple[str, ...]] +class CooldownDeploymentInfo(TypedDict): + model_id: ReadOnly[str] + api_base: ReadOnly[str | None] + + class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_code: str | None error_class: str | None @@ -3414,6 +3419,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + error_cooldown_deployments: ReadOnly[tuple[CooldownDeploymentInfo, ...] | None] normalized_error: ReadOnly[str | None] diff --git a/tests/integration/spend/test_router_cooldown_failure_row.py b/tests/integration/spend/test_router_cooldown_failure_row.py new file mode 100644 index 00000000000..04e36785823 --- /dev/null +++ b/tests/integration/spend/test_router_cooldown_failure_row.py @@ -0,0 +1,358 @@ +import asyncio +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from integration._support.client import GATEWAY_LIMITS, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.redis_process import OwnedRedis, owned_redis +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue +from redis import Redis + +pytestmark: Final = pytest.mark.timeout(300) + +COOLDOWN_SECONDS: Final = 120 +NO_DEPLOYMENTS: Final = "No deployments available" +ROW_SECONDS: Final = 70 +CHAT: Final = "/v1/chat/completions" +MESSAGES: Final = "/v1/messages" +RESPONSES: Final = "/v1/responses" +FAILURE_ROW: Final = ( + "SELECT status, model_group, model_id, api_base, metadata::jsonb -> 'error_information' AS error " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) +ENDPOINT_CELLS: Final = tuple((path, stream) for path in (CHAT, MESSAGES, RESPONSES) for stream in (False, True)) +ENDPOINT_GROUPS: Final = tuple( + f"{path.rsplit('/', 1)[-1]}-{'stream' if stream else 'json'}" for path, stream in ENDPOINT_CELLS +) +SDK_CELLS: Final = ("openai-sync", "openai-async-stream", "anthropic-sync") +TAGGED: Final = "tagged" +REGIONAL: Final = "regional" +GROUPS: Final = (*ENDPOINT_GROUPS, *SDK_CELLS, "neighbour", "scoped", "burst", TAGGED, REGIONAL) +LANES: Final = ("a", "b") +LANE_PARAMS: Final[Mapping[tuple[str, str], Mapping[str, JsonValue]]] = { + (TAGGED, "a"): {"tags": ["tenant-a"]}, + (TAGGED, "b"): {"tags": ["default"]}, + (REGIONAL, "a"): {"region_name": "eu"}, + (REGIONAL, "b"): {"region_name": "us"}, +} +TENANT_A: Final[Mapping[str, JsonValue]] = {"metadata": {"tags": ["tenant-a"]}} +UNTAGGED: Final[Mapping[str, JsonValue]] = {} + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + cache: OwnedRedis + prefix: str + wire: Wire + + def group(self, name: str) -> str: + return f"{self.prefix}-{name}" + + def deployments(self, name: str) -> Mapping[str, str]: + return {f"{self.group(name)}-{lane}": f"{self.wire.url}/{self.group(name)}/{lane}/v1" for lane in LANES} + + +@dataclass(frozen=True, slots=True) +class Outcome: + status: int + detail: str + call_id: str + + +def is_model_probe(request: Request) -> bool: + return (request.method, request.target) == ("GET", "/v1/models") + + +def rate_limited(request: Request) -> Reply: + if is_model_probe(request): + return Reply(body=b'{"object": "list", "data": []}') + body: Final = {"error": {"message": "scripted 429", "type": "rate_limit_error", "code": "rate_limit_exceeded"}} + return Reply(status=429, body=json.dumps(body).encode()) + + +def cooldown_config(directory: Path, cache: OwnedRedis, prefix: str, wire: Wire) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + assert isinstance(base, dict) + model_list: Final = [ + { + "model_name": f"{prefix}-{name}", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": f"{wire.url}/{prefix}-{name}/{lane}/v1", + "api_key": "synthetic-openai-key", + **LANE_PARAMS.get((name, lane), {}), + }, + "model_info": { + "id": f"{prefix}-{name}-{lane}", + **({"enable_tag_filtering": True} if name == TAGGED else {}), + }, + } + for name, lane in product(GROUPS, LANES) + ] + config: Final = { + **base, + "model_list": model_list, + "router_settings": { + "num_retries": 0, + "allowed_fails": 0, + "cooldown_time": COOLDOWN_SECONDS, + "enable_pre_call_checks": True, + "redis_host": cache.host, + "redis_port": cache.port, + }, + } + path: Final = directory / f"cooldown-{prefix}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("router-cooldown-failure-row") + prefix: Final = f"cooldown-{uuid.uuid4().hex[:8]}" + with ExitStack() as stack: + upstream: Final = stack.enter_context(gateway_from_environment()) + cache: Final = stack.enter_context(owned_redis(directory)) + wire: Final = stack.enter_context(wire_server(rate_limited)) + config: Final = cooldown_config(directory, cache, prefix, wire) + owned: Final = stack.enter_context(owned_proxy_process(upstream, directory, {}, config=config, workers=2)) + yield Rig(owned.gateway, cache, prefix, wire) + + +def cooled(cache: OwnedRedis, deployment_id: str) -> bool: + with Redis(host=cache.host, port=cache.port) as client: + return int(client.exists(f"deployment:{deployment_id}:cooldown")) == 1 + + +def request_body(path: str, group: str, stream: bool) -> dict[str, JsonValue]: + content: Final = f"cooldown row {uuid.uuid4().hex}" + match path: + case "/v1/messages": + return { + "model": group, + "max_tokens": 8, + "messages": [{"role": "user", "content": content}], + "stream": stream, + } + case "/v1/responses": + return {"model": group, "input": content, "stream": stream} + case _: + return {"model": group, "messages": [{"role": "user", "content": content}], "stream": stream} + + +def send(gateway: Gateway, path: str, group: str, stream: bool = False) -> Outcome: + response: Final = gateway.request("POST", path, request_body(path, group, stream), headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def send_chat(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + body: Final = {**request_body(CHAT, rig.group(name), False), **extra} + response: Final = rig.gateway.request("POST", CHAT, body, headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def failure_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows(FAILURE_ROW, (call_id,)), lambda found: len(found) == 1, seconds=ROW_SECONDS + ) + assert rows[0]["status"] == "failure", rows + return rows[0] + + +def until_router_refusal(attempt: Callable[[], Outcome], tries: int = 10) -> tuple[Outcome, ...]: + def walk() -> Iterator[Outcome]: + for _ in range(tries): + outcome: Final = attempt() + yield outcome + if NO_DEPLOYMENTS in outcome.detail: + return + + history: Final = tuple(walk()) + assert NO_DEPLOYMENTS in history[-1].detail, [(o.status, o.detail[:200]) for o in history] + return history + + +def cool_group(rig: Rig, name: str) -> tuple[Outcome, ...]: + history: Final = until_router_refusal(lambda: send(rig.gateway, CHAT, rig.group(name))) + assert all(cooled(rig.cache, identity) for identity in rig.deployments(name)), history + return history + + +def expected_cooldown(deployments: Mapping[str, str]) -> list[JsonValue]: + return [{"model_id": identity, "api_base": deployments[identity]} for identity in sorted(deployments)] + + +def cooldown_entries(error: Mapping[str, JsonValue]) -> list[JsonValue] | None: + entries: Final = error.get("error_cooldown_deployments") + if not isinstance(entries, list): + return None + return sorted(entries, key=lambda entry: str(object_value(entry)["model_id"])) + + +def assert_refusal_row(call_id: str, deployments: Mapping[str, str]) -> Mapping[str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + assert (error["error_class"], error["error_code"]) == ("RouterRateLimitError", "429"), row + assert cooldown_entries(error) == expected_cooldown(deployments), error.get("error_message") + assert (row["model_id"], row["api_base"]) == ("", ""), row + return error + + +def provider_row(call_id: str) -> tuple[str, str, str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + return str(row["model_id"]), str(row["api_base"]), str(error["error_code"]), error.get("error_cooldown_deployments") + + +def assert_provider_rows(history: tuple[Outcome, ...], deployments: Mapping[str, str]) -> None: + provider: Final = history[:-1] + assert all(outcome.status == 429 and NO_DEPLOYMENTS not in outcome.detail for outcome in provider), [ + (o.status, o.detail[:200]) for o in provider + ] + rows: Final = tuple(provider_row(outcome.call_id) for outcome in provider) + assert {identity for identity, _, _, _ in rows} == set(deployments), rows + assert all(base.startswith(deployments[identity]) for identity, base, _, _ in rows), rows + assert all((code, cooldown) == ("429", None) for _, _, code, cooldown in rows), rows + + +@pytest.mark.parametrize(("path", "stream"), ENDPOINT_CELLS, ids=ENDPOINT_GROUPS) +def test_all_cooled_refusal_row_records_429_and_every_cooled_deployment(rig: Rig, path: str, stream: bool) -> None: + name: Final = ENDPOINT_GROUPS[ENDPOINT_CELLS.index((path, stream))] + history: Final = until_router_refusal(lambda: send(rig.gateway, path, rig.group(name), stream)) + refusal: Final = history[-1] + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + assert_refusal_row(refusal.call_id, rig.deployments(name)) + assert_provider_rows(history, rig.deployments(name)) + + +def openai_chat(rig: Rig, group: str) -> Outcome: + client: Final = openai.OpenAI(base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0) + with pytest.raises(openai.RateLimitError) as raised: + client.chat.completions.create(model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +def openai_async_stream(rig: Rig, group: str) -> Outcome: + async def call() -> Outcome: + client: Final = openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0 + ) + try: + with pytest.raises(openai.RateLimitError) as raised: + await client.chat.completions.create( + model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}], stream=True + ) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + finally: + await client.close() + + return asyncio.run(call()) + + +def anthropic_message(rig: Rig, group: str) -> Outcome: + client: Final = anthropic.Anthropic( + base_url=str(rig.gateway.client.base_url), api_key=rig.gateway.key, max_retries=0 + ) + with pytest.raises(anthropic.RateLimitError) as raised: + client.messages.create(model=group, max_tokens=8, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +SDK_CLIENTS: Final = { + "openai-sync": openai_chat, + "openai-async-stream": openai_async_stream, + "anthropic-sync": anthropic_message, +} + + +@pytest.mark.parametrize("client", SDK_CELLS) +def test_sdk_clients_get_429_and_the_row_names_the_cooled_deployments(rig: Rig, client: str) -> None: + history: Final = until_router_refusal(lambda: SDK_CLIENTS[client](rig, rig.group(client))) + assert_refusal_row(history[-1].call_id, rig.deployments(client)) + assert_provider_rows(history, rig.deployments(client)) + + +def test_the_row_lists_only_the_requested_groups_deployments_while_another_group_cools(rig: Rig) -> None: + cool_group(rig, "neighbour") + cool_group(rig, "scoped") + refusal: Final = eventually( + lambda: send(rig.gateway, CHAT, rig.group("scoped")), + lambda outcome: ( + NO_DEPLOYMENTS in outcome.detail + and all(identity in outcome.detail for identity in rig.deployments("neighbour")) + ), + ) + assert_refusal_row(refusal.call_id, rig.deployments("scoped")) + + +def test_a_concurrent_burst_on_a_cooled_group_writes_one_complete_row_per_refusal(rig: Rig) -> None: + cool_group(rig, "burst") + rig.wire.drain() + calls: Final = tuple(ENDPOINT_CELLS[index % len(ENDPOINT_CELLS)] for index in range(30)) + with ( + httpx.Client( + base_url=rig.gateway.client.base_url, timeout=60, trust_env=False, limits=GATEWAY_LIMITS + ) as client, + ThreadPoolExecutor(max_workers=30) as pool, + ): + patient: Final = Gateway(client, rig.gateway.key, rig.gateway.upstream_url) + outcomes: Final = tuple(pool.map(lambda call: send(patient, call[0], rig.group("burst"), call[1]), calls)) + assert all(outcome.status == 429 for outcome in outcomes), [(o.status, o.detail[:200]) for o in outcomes] + assert len({outcome.call_id for outcome in outcomes}) == len(outcomes), outcomes + refusals: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS in outcome.detail) + reached: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS not in outcome.detail) + assert refusals, [(o.status, o.detail[:200]) for o in outcomes] + assert len(tuple(request for request in rig.wire.drain() if not is_model_probe(request))) == len(reached) + for outcome in refusals: + assert_refusal_row(outcome.call_id, rig.deployments("burst")) + for outcome in reached: + identity, api_base, code, cooldown = provider_row(outcome.call_id) + assert api_base.startswith(rig.deployments("burst")[identity]), (identity, api_base) + assert (code, cooldown) == ("429", None), outcome + + +def cool_lane(rig: Rig, name: str, lane: str, extra: Mapping[str, JsonValue]) -> None: + identity: Final = f"{rig.group(name)}-{lane}" + outcomes: Final = eventually(lambda: send_chat(rig, name, extra), lambda _: cooled(rig.cache, identity)) + assert NO_DEPLOYMENTS not in outcomes.detail, outcomes.detail[:300] + + +def router_refusal(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + refusal: Final = eventually(lambda: send_chat(rig, name, extra), lambda outcome: NO_DEPLOYMENTS in outcome.detail) + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + return refusal + + +def test_a_tag_routed_refusal_row_names_only_the_cooled_deployments_its_tags_reach(rig: Rig) -> None: + deployments: Final = rig.deployments(TAGGED) + tenant: Final = f"{rig.group(TAGGED)}-a" + cool_lane(rig, TAGGED, "a", TENANT_A) + cool_lane(rig, TAGGED, "b", UNTAGGED) + assert_refusal_row(router_refusal(rig, TAGGED, TENANT_A).call_id, {tenant: deployments[tenant]}) + assert_refusal_row(router_refusal(rig, TAGGED, UNTAGGED).call_id, deployments) + + +def test_a_region_restricted_refusal_row_names_only_the_cooled_deployments_in_its_region(rig: Rig) -> None: + deployments: Final = rig.deployments(REGIONAL) + european: Final = f"{rig.group(REGIONAL)}-a" + customer: Final = f"{rig.prefix}-eu-customer" + with ExitStack() as cleanups: + rig.gateway.post("/customer/new", {"user_id": customer, "allowed_model_region": "eu"}) + cleanups.callback(rig.gateway.post, "/customer/delete", {"user_ids": [customer]}) + cool_group(rig, REGIONAL) + assert_refusal_row(router_refusal(rig, REGIONAL, {"user": customer}).call_id, {european: deployments[european]}) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index d1258e43154..37a985264b7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -4569,6 +4569,50 @@ def test_get_error_information_falls_back_to_str_when_no_message_attr(): assert result["error_class"] == "ValueError" +def test_get_error_information_records_the_cooled_deployments_of_a_router_refusal(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + from litellm.types.utils import CooldownDeploymentInfo + + cooled: Final = ( + CooldownDeploymentInfo(model_id="refused-1", api_base="https://first.example/v1"), + CooldownDeploymentInfo(model_id="refused-2", api_base=None), + ) + refusal: Final = RouterRateLimitError( + model="refused", + cooldown_time=60, + enable_pre_call_checks=False, + cooldown_list=["refused-1", "refused-2", "neighbour-1"], + model_ids=("refused-1", "refused-2"), + cooldown_deployments=cooled, + ) + + result: Final = StandardLoggingPayloadSetup.get_error_information(refusal) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "RouterRateLimitError", + "429", + cooled, + ), result + provider_429: Final = litellm.RateLimitError("scripted 429", "openai", "gpt-5.4") + assert StandardLoggingPayloadSetup.get_error_information(provider_429)["error_cooldown_deployments"] is None + + +def test_get_error_information_handles_a_router_refusal_subclass_that_skips_its_init(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + + class PluginRefusal(RouterRateLimitError): + def __init__(self, message: str) -> None: + ValueError.__init__(self, message) + + result: Final = StandardLoggingPayloadSetup.get_error_information(PluginRefusal("plugin refused")) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "PluginRefusal", + "429", + None, + ), result + + # ────────────────────────────────────────────────────────────────────── # Tests for _get_assembled_streaming_response non-streaming early return # ────────────────────────────────────────────────────────────────────── diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2376e0f4d0c..0c9d7ab9f85 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,5 +1,7 @@ import asyncio +import copy import importlib +import itertools import random import time from collections.abc import Mapping @@ -15,6 +17,7 @@ from litellm import Router from litellm._internal_context import current_service_target from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.router_utils.cooldown_handlers import ( @@ -44,8 +47,9 @@ from litellm.types.router import ( AllowedFailsPolicy, DeploymentTypedDict, LiteLLMParamsTypedDict, + RouterRateLimitError, ) -from litellm.types.utils import CredentialItem +from litellm.types.utils import CooldownDeploymentInfo, CredentialItem from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -3004,3 +3008,358 @@ def test_shared_mode_upstream_429_still_cools_down_the_deployment(single_deploym is True ) assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 429, exc) is True + + +REFUSAL_MESSAGES: Final = [{"role": "user", "content": "cooldown"}] +SECRET_API_BASE: Final = "https://first.example/v1?key=sk-scripted-secret" +FIRST_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-1", api_base=mask_api_base_credentials(SECRET_API_BASE)) +SECOND_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-2", api_base=None) +THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=None) +OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None) +FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1") +SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None) +ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1") +PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None) +EVERY_DEPLOYMENT: Final = ( + "refused-1", + "refused-2", + "refused-3", + "refused-team-b", + "neighbour-1", + "wildcard-1", + "wildcard-2", + "tier-1", + "friendly-1", +) + + +def _cooled_router(routing_strategy: str, cooled: tuple[str, ...]) -> Router: + router: Final = Router( + routing_strategy=routing_strategy, + model_group_alias={"entry": "tier", "tier": "refused"}, + model_list=[ + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": SECRET_API_BASE, "use_in_pass_through": True}, + "model_info": {"id": "refused-1"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True, "tpm": 1}, + "model_info": {"id": "refused-2"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "tpm": 1}, + "model_info": {"id": "refused-3"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True}, + "model_info": {"id": "refused-team-b", "team_id": "team-b"}, + }, + { + "model_name": "neighbour", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://neighbour.example/v1"}, + "model_info": {"id": "neighbour-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_base": "https://wildcard.example/v1"}, + "model_info": {"id": "wildcard-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "wildcard-2"}, + }, + { + "model_name": "tier", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://tier.example/v1"}, + "model_info": {"id": "tier-1"}, + }, + { + "model_name": "friendly", + "litellm_params": {"model": "anthropic/claude-friendly"}, + "model_info": {"id": "friendly-1"}, + }, + ], + ) + for model_id in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=model_id, + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +def _team_request(team_id: str | None) -> dict[str, object]: + return {"metadata": {"user_api_key_team_id": team_id} if team_id else {}} + + +def _select(router: Router, entrypoint: str, model: str, request_kwargs: dict[str, object]) -> object: + match entrypoint: + case "async": + return asyncio.run( + router.async_get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + ) + case "async-pass-through": + return asyncio.run( + router.async_get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + ) + case "pass-through": + return router.get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + case "pass-through-chat": + return router.get_available_deployment_for_pass_through( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + case _: + return router.get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + + +ALL_COOLED: Final = (FIRST_REFUSED, SECOND_REFUSED, THIRD_REFUSED) +STRATEGY_COOLED: Final = ("refused-1", "refused-team-b", "neighbour-1") +WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD) + + +@pytest.mark.parametrize( + ("routing_strategy", "cooled", "entrypoint", "model", "team_id", "expected"), + ( + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "pass-through", "refused", "team-a", (FIRST_REFUSED, SECOND_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async-pass-through", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-b", (*ALL_COOLED, OTHER_TEAMS_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED), + ("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ), + ids=( + "all-cooled-sync", + "all-cooled-async", + "all-cooled-pass-through", + "all-cooled-async-pass-through", + "all-cooled-owning-team", + "all-cooled-async-no-team", + "strategy-refused-sync", + "strategy-refused-async", + "strategy-refused-pass-through", + "wildcard-sync", + "wildcard-async", + "alias-chain-sync", + "alias-chain-async", + "provider-model-sync", + "provider-model-async", + ), +) +def test_no_deployment_error_names_the_cooled_deployments_the_request_could_route_to( + routing_strategy: str, + cooled: tuple[str, ...], + entrypoint: str, + model: str, + team_id: str | None, + expected: tuple[CooldownDeploymentInfo, ...] | None, +) -> None: + router: Final = _cooled_router(routing_strategy, cooled) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, model, _team_request(team_id)) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +@pytest.mark.parametrize("entrypoint", ("sync", "async")) +def test_no_deployment_error_names_only_the_cooled_part_of_a_route_emptied_by_other_filters(entrypoint: str) -> None: + router: Final = _cooled_router("simple-shuffle", STRATEGY_COOLED) + request_kwargs: Final = {**_team_request("team-a"), "_excluded_deployment_ids": ["refused-2", "refused-3"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "refused", request_kwargs) + assert refusal.value.cooldown_deployments == (FIRST_REFUSED,), refusal.value.cooldown_list + assert "sk-scripted-secret" not in repr(refusal.value.cooldown_deployments) + + +TENANT_A: Final = CooldownDeploymentInfo(model_id="tenant-a-1", api_base="https://tenant-a.example/v1") +TENANT_B: Final = CooldownDeploymentInfo(model_id="tenant-b-1", api_base="https://tenant-b.example/v1") +SHARED: Final = CooldownDeploymentInfo(model_id="shared-1", api_base="https://shared.example/v1") +EVERY_TENANT: Final = (TENANT_A, TENANT_B, SHARED) +TENANT_ROUTES: Final = ((TENANT_A, "tenant-a", "eu"), (TENANT_B, "tenant-b", "us"), (SHARED, "default", "eu")) + + +def _tenant_router(cooled: tuple[CooldownDeploymentInfo, ...], **router_settings: bool) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "tenants", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": deployment["api_base"], + "tags": [tag], + "region_name": region, + "use_in_pass_through": True, + }, + "model_info": {"id": deployment["model_id"]}, + } + for deployment, tag, region in TENANT_ROUTES + ], + **router_settings, + ) + for deployment in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=deployment["model_id"], + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +TAGS_ON: Final = {"enable_tag_filtering": True} +REGION_CHECKS_ON: Final = {"enable_pre_call_checks": True} +BOTH_ON: Final = {**TAGS_ON, **REGION_CHECKS_ON} + + +@pytest.mark.parametrize( + ("router_settings", "entrypoint", "request_kwargs", "expected"), + ( + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {}}, EVERY_TENANT), + (TAGS_ON, "async", {}, EVERY_TENANT), + (TAGS_ON, "async", {"metadata": "unparsed"}, ()), + (TAGS_ON, "async", {"litellm_metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a"]}}, (TENANT_B, SHARED)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-z"]}}, (SHARED,)), + (TAGS_ON, "async-pass-through", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}, "enable_tag_filtering": True}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}}, EVERY_TENANT), + (REGION_CHECKS_ON, "async", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through-chat", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + ({}, "sync", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": 1}, ()), + ( + {**TAGS_ON, **REGION_CHECKS_ON}, + "async", + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, + (), + ), + ), + ids=( + "tag-a", + "tag-b", + "untagged-falls-back-past-the-default", + "no-metadata", + "metadata-not-a-mapping", + "tag-litellm-metadata", + "tag-constraints", + "tag-exclusion-falls-back-in-tiers", + "unknown-tag-gets-default", + "tag-async-pass-through", + "tag-request-level-switch", + "tags-off", + "region-async", + "region-sync", + "region-pass-through", + "region-pass-through-without-input", + "region-without-pre-call-checks", + "region-not-a-string", + "region-and-tag-allow-none", + ), +) +def test_no_deployment_error_names_only_the_cooled_deployments_tag_and_region_routing_allow( + router_settings: dict[str, bool], + entrypoint: str, + request_kwargs: dict[str, object], + expected: tuple[CooldownDeploymentInfo, ...], +) -> None: + router: Final = _tenant_router(EVERY_TENANT, **router_settings) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "tenants", request_kwargs) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +def test_naming_tag_routed_deployments_leaves_the_request_metadata_untouched() -> None: + metadata: Final[dict[str, object]] = {"tags": ["tenant-a"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **TAGS_ON), "async", "tenants", {"metadata": metadata}) + assert refusal.value.cooldown_deployments == (TENANT_A,) + assert metadata == {"tags": ["tenant-a"]} + + +@pytest.mark.parametrize( + ("has_request_input", "expected"), ((True, (TENANT_A, SHARED)), (False, EVERY_TENANT)), ids=("chat", "no-input") +) +def test_region_routable_deployments_follow_the_pre_call_region_check( + has_request_input: bool, expected: tuple[CooldownDeploymentInfo, ...] +) -> None: + router: Final = _tenant_router((), **REGION_CHECKS_ON) + kept: Final = router._region_routable_deployments( + router.get_model_list(model_name="tenants") or [], {"allowed_model_region": "eu"}, has_request_input + ) + assert router._deployment_ids(kept) == frozenset(deployment["model_id"] for deployment in expected) + + +def test_routable_cooled_deployment_ids_skip_healthy_deployments_and_peel_tag_fallbacks() -> None: + router: Final = _tenant_router((TENANT_A, SHARED), **TAGS_ON) + routable: Final = asyncio.run( + router._async_routable_cooled_deployment_ids( + model="tenants", + request_deployments=router.get_model_list(model_name="tenants") or [], + cooldown_deployments=(TENANT_A["model_id"], SHARED["model_id"]), + request_kwargs={"metadata": {}}, + has_request_input=True, + ) + ) + assert routable == frozenset((TENANT_A["model_id"], SHARED["model_id"])) + + +REACHABILITY_REQUESTS: Final = ( + {"metadata": {"tags": ["tenant-a"]}}, + {"metadata": {}}, + {"metadata": {"tags": ["!tenant-a"]}}, + {"metadata": {"tags": ["tenant-z"]}}, + {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, + {"metadata": {}, "allowed_model_region": "eu"}, + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, +) +HEALTH_STATES: Final = tuple( + itertools.chain.from_iterable( + itertools.combinations(EVERY_TENANT, size) for size in range(1, len(EVERY_TENANT) + 1) + ) +) + + +def _routable_while_up(up: tuple[CooldownDeploymentInfo, ...], request_kwargs: dict[str, object]) -> set[str]: + router: Final = _tenant_router(tuple(deployment for deployment in EVERY_TENANT if deployment not in up), **BOTH_ON) + try: + pool: Final = asyncio.run( + router.async_get_healthy_deployments( + model="tenants", request_kwargs=copy.deepcopy(request_kwargs), messages=REFUSAL_MESSAGES + ) + ) + except ValueError: + return set() + return {str(deployment["model_info"]["id"]) for deployment in pool} + + +@pytest.mark.parametrize("request_kwargs", REACHABILITY_REQUESTS) +def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the_request_to( + request_kwargs: dict[str, object], +) -> None: + reachable: Final = set().union(*(_routable_while_up(up, request_kwargs) for up in HEALTH_STATES)) + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) + assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable From 89a194bbd0fd25c57df5456ccdc223978e4ac084 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:20:50 -0700 Subject: [PATCH 004/124] test(integration): hold the upstream until the disconnect is settled before reading its spend row (#47303) test_client_disconnect_before_any_content_sends_no_response_scan released the gated upstream right after the client closed, so the upstream's next chunk raced the disconnect into the proxy. When the chunk won, the proxy had " tail" on record for the client and the cut-short scan recorded a post_call success on the spend row, which the test asserts is absent (CircleCI integration-extensions job 2263560 on 68db022d5704: `AssertionError: (('success',),)`). The test now reads the spend row first, which is written once the proxy has settled the disconnect, and releases the upstream after it. The proxy's behavior is unchanged; only the test's ordering is. Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- tests/integration/observability/test_guardrail_effects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index b3481057b3b..086baac6ee5 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -2836,9 +2836,9 @@ def test_client_disconnect_before_any_content_sends_no_response_scan(gateway: Ga with _disconnect_rig(gateway, tmp_path, shape="empty") as rig: try: _stream_and_close(rig, "/v1/chat/completions", _chat_body(rig, 0), "data: ") + rows: Final = _post_call_statuses(rig, rig.model) finally: rig.gate.set() - rows: Final = _post_call_statuses(rig, rig.model) assert rig.scans.response_scans(rig.secret()) == (), rig.scans.seen assert len(rows) == 1 and "success" not in rows[0], rows From 6e148255d921de26381aed7fa199af0e68bd5bb6 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:30:25 -0700 Subject: [PATCH 005/124] chore(pricing): add meta sam-3.1 image segmentation price (#47395) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 +++++++++++++++++ model_prices_and_context_window.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6e573402427..9d5e0aa8021 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e573402427..9d5e0aa8021 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } From 9bafafc2df3c8871bb30277f47d9fb6600e707e4 Mon Sep 17 00:00:00 2001 From: litellm-stampbot Date: Mon, 19 Oct 2026 13:35:04 -0700 Subject: [PATCH 006/124] feat(search): add DeepSeek as a /v1/search provider (#47401) * feat(search): add DeepSeek as a /v1/search provider Adds DeepSeekSearchConfig for POST https://api.deepseek.com/v1/tools/search, registers it under search_provider=deepseek, prices deepseek/search at 0.00075 per query and marks search support in provider_endpoints_support.json * fix(search): reject DeepSeek queries over the 128 character limit before the call * test(search): escape regex metacharacters in DeepSeek query limit match --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/deepseek/search/__init__.py | 7 + .../llms/deepseek/search/transformation.py | 172 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 5 + litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 5 + provider_endpoints_support.json | 3 +- tests/unit/llms/deepseek/search/__init__.py | 0 .../test_deepseek_search_transformation.py | 143 +++++++++++++++ 9 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/deepseek/search/__init__.py create mode 100644 litellm/llms/deepseek/search/transformation.py create mode 100644 tests/unit/llms/deepseek/search/__init__.py create mode 100644 tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py diff --git a/litellm/llms/deepseek/search/__init__.py b/litellm/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..528dceb223f --- /dev/null +++ b/litellm/llms/deepseek/search/__init__.py @@ -0,0 +1,7 @@ +""" +DeepSeek Search API module. +""" + +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + +__all__ = ["DeepSeekSearchConfig"] diff --git a/litellm/llms/deepseek/search/transformation.py b/litellm/llms/deepseek/search/transformation.py new file mode 100644 index 00000000000..b6bb1c343d8 --- /dev/null +++ b/litellm/llms/deepseek/search/transformation.py @@ -0,0 +1,172 @@ +""" +DeepSeek Search API. +Endpoint: POST https://api.deepseek.com/v1/tools/search +Docs: https://api-docs.deepseek.com/guides/search/ +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_JSON_OBJECTS: Final = TypeAdapter(Sequence[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_STR_LIST: Final = TypeAdapter(list[str]) + +DEEPSEEK_SEARCH_MAX_COUNT: Final = 50 +DEEPSEEK_SEARCH_MAX_QUERY_CHARS: Final = 128 +_DEEPSEEK_SEARCH_PATH: Final = "/v1/tools/search" +# DeepSeek chat defaults DEEPSEEK_API_BASE to the /beta prefix; the search route lives at the root. +_CHAT_ONLY_BASE_SUFFIXES: Final = ("/beta", "/v1") + + +class DeepSeekSearchConfig(BaseSearchConfig): + DEEPSEEK_API_BASE = "https://api.deepseek.com" + + @staticmethod + def ui_friendly_name() -> str: + return "DeepSeek" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature + ) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers + resolved_key: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("DEEPSEEK_API_KEY",), + base_env_var="DEEPSEEK_API_BASE", + default_api_base=self.DEEPSEEK_API_BASE, + ) + if not resolved_key: + raise ValueError("DEEPSEEK_API_KEY is not set. Set `DEEPSEEK_API_KEY` environment variable.") + return {**headers, "Authorization": f"Bearer {resolved_key}", "Content-Type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature + data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature + ) -> str: + resolved_base: Final = api_base or get_secret_str("DEEPSEEK_API_BASE") or self.DEEPSEEK_API_BASE + clean_base: Final = resolved_base.rstrip("/") + if clean_base.endswith(_DEEPSEEK_SEARCH_PATH): + return clean_base + return f"{_strip_chat_suffix(clean_base)}{_DEEPSEEK_SEARCH_PATH}" + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature + optional_params: dict[str, object], # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature + ) -> dict[str, object]: # mutable-ok: the http handler sends this as the JSON body + """ + Unified params -> DeepSeek body: ``query`` (list joined with spaces, ``search_domain_filter`` + folded in as ``site:`` operators, the whole string capped at 128 characters by DeepSeek), + ``max_results`` -> ``count`` clamped to [1, 50]. + ``country`` and ``max_tokens_per_page`` have no DeepSeek equivalent and are dropped. + Everything outside the unified spec (``need_content``, ``ori_question``, ``localization``) + is forwarded as-is. + """ + joined_query: Final = " ".join(query) if isinstance(query, list) else query + domains: Final = _domain_filter(optional_params.get("search_domain_filter")) + count: Final = _count(optional_params.get("max_results")) + unified: Final = self.get_supported_perplexity_optional_params() + passthrough: Final = {k: v for k, v in optional_params.items() if k not in unified} + return { + **passthrough, + "query": _checked_query(_with_site_filters(joined_query, domains), domains), + **({"count": count} if count is not None else {}), + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature + ) -> SearchResponse: + response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) + raw_results: Final = _JSON_OBJECTS.validate_python(response_json.get("results", ())) + return SearchResponse( + results=[_to_search_result(item) for item in raw_results], + object="search", + ) + + +def _strip_chat_suffix(base: str) -> str: + return next( + (base.removesuffix(suffix) for suffix in _CHAT_ONLY_BASE_SUFFIXES if base.endswith(suffix)), + base, + ) + + +def _domain_filter(raw: object) -> tuple[str, ...]: + try: + return tuple(_STR_LIST.validate_python(raw)) + except ValidationError: + return () + + +def _with_site_filters(query: str, domains: tuple[str, ...]) -> str: + if not domains: + return query + site_clause: Final = " OR ".join(f"site:{domain}" for domain in domains) + return f"{query} {site_clause}" + + +def _checked_query(query: str, domains: tuple[str, ...]) -> str: + if len(query) <= DEEPSEEK_SEARCH_MAX_QUERY_CHARS: + return query + filter_note: Final = ( + f" ({len(domains)} search_domain_filter entries are sent as site: operators inside the query)" + if domains + else "" + ) + raise BadRequestError( + message=( + f"DeepSeek Search accepts a query of at most {DEEPSEEK_SEARCH_MAX_QUERY_CHARS} characters, " + f"got {len(query)}{filter_note}. Shorten the query or the domain filter." + ), + model="deepseek/search", + llm_provider="deepseek", + ) + + +def _count(raw: object) -> int | None: + if isinstance(raw, bool) or not isinstance(raw, int): + return None + return max(1, min(raw, DEEPSEEK_SEARCH_MAX_COUNT)) + + +def _to_search_result(item: Mapping[str, object]) -> SearchResult: + date: Final = item.get("date") + return SearchResult.model_validate( + { + **item, + "title": _as_str(item.get("title")), + "url": _as_str(item.get("url")), + "snippet": _as_str(item.get("snippet")), + "date": date if isinstance(date, str) else None, + "last_updated": None, + } + ) + + +def _as_str(value: object) -> str: + return value if isinstance(value, str) else "" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9d5e0aa8021..f5eec206e04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e05107f82d..41581d702f3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4469,6 +4469,7 @@ class SearchProviders(str, Enum): AGENTCORE = "agentcore" NIMBLE = "nimble" BING_GROUNDING = "bing_grounding" + DEEPSEEK = "deepseek" # Create a set of all search provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index ff2c9012ec8..cf3b8756e2f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10297,6 +10297,7 @@ class ProviderConfigManager: from litellm.llms.bedrock.search.transformation import AgentCoreSearchConfig from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig + from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig @@ -10336,6 +10337,7 @@ class ProviderConfigManager: SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, SearchProviders.BING_GROUNDING: BingGroundingSearchConfig, + SearchProviders.DEEPSEEK: DeepSeekSearchConfig, } config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None) if config_class is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9d5e0aa8021..f5eec206e04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c2cec2c5490..d65da3f41fd 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -923,7 +923,8 @@ "batches": false, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "search": true } }, "edenai": { diff --git a/tests/unit/llms/deepseek/search/__init__.py b/tests/unit/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py new file mode 100644 index 00000000000..7539876bd37 --- /dev/null +++ b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py @@ -0,0 +1,143 @@ +""" +Tests for the DeepSeek Search provider (POST https://api.deepseek.com/v1/tools/search). +""" + +from typing import Final +import re +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + + +@pytest.mark.asyncio +async def test_asearch_sends_deepseek_body_and_maps_results(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "results": [ + { + "rerank_score": 2.4, + "title": "DeepSeek V4 model released", + "url": "https://example.com/deepseek-v4", + "snippet": "short snippet", + "summary": "long summary for the model", + "content": "full page content", + "site_name": None, + "date": "2026-06-01T00:00:00+00:00", + "authority": 3, + }, + {"title": None, "url": "https://example.com/degraded", "snippet": None, "date": None}, + ] + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_post: + response: Final = await litellm.asearch( + query=["DeepSeek V4", "release"], + search_provider="deepseek", + max_results=5, + search_domain_filter=["github.com", "huggingface.co"], + need_content=True, + ) + + assert mock_post.call_count == 1 + call: Final = mock_post.call_args + assert call.kwargs["url"] == "https://api.deepseek.com/v1/tools/search" + assert call.kwargs["headers"]["Authorization"] == "Bearer test-deepseek-key" + assert call.kwargs["headers"]["Content-Type"] == "application/json" + assert call.kwargs["json"] == { + "query": "DeepSeek V4 release site:github.com OR site:huggingface.co", + "count": 5, + "need_content": True, + } + + assert response.object == "search" + assert len(response.results) == 2 + first: Final = response.results[0] + assert (first.title, first.url, first.snippet) == ( + "DeepSeek V4 model released", + "https://example.com/deepseek-v4", + "short snippet", + ) + assert first.date == "2026-06-01T00:00:00+00:00" + assert first.last_updated is None + assert first.model_extra["summary"] == "long summary for the model" + assert first.model_extra["content"] == "full page content" + assert first.model_extra["rerank_score"] == 2.4 + degraded: Final = response.results[1] + assert (degraded.title, degraded.snippet, degraded.date) == ("", "", None) + + +@pytest.mark.parametrize( + ("max_results", "expected_count"), + [(0, 1), (3, 3), (50, 50), (500, 50)], +) +def test_max_results_clamps_to_deepseek_count_range(max_results, expected_count): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", optional_params={"max_results": max_results} + ) + assert body == {"query": "litellm", "count": expected_count} + + +def test_query_over_deepseek_limit_fails_before_the_call(): + with pytest.raises( + litellm.BadRequestError, match=re.escape("at most 128 characters, got 144 (3 search_domain_filter") + ): + DeepSeekSearchConfig().transform_search_request( + query="litellm proxy search endpoint configuration guide for enterprise deployments with", + optional_params={"search_domain_filter": ["docs.litellm.ai", "github.com", "huggingface.co"]}, + ) + + +def test_unified_params_without_deepseek_equivalent_are_not_sent(): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", + optional_params={"country": "US", "max_tokens_per_page": 1024, "ori_question": "what is litellm"}, + ) + assert body == {"query": "litellm", "ori_question": "what is litellm"} + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.deepseek.com", + "https://api.deepseek.com/", + "https://api.deepseek.com/beta", + "https://api.deepseek.com/v1", + "https://api.deepseek.com/v1/tools/search", + ], +) +def test_complete_url_resolves_search_route_from_chat_style_api_base(api_base, monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + url: Final = DeepSeekSearchConfig().get_complete_url(api_base=api_base, optional_params={}) + assert url == "https://api.deepseek.com/v1/tools/search" + + +def test_missing_api_key_raises(monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + with pytest.raises(ValueError, match="DEEPSEEK_API_KEY"): + DeepSeekSearchConfig().validate_environment(headers={}) + + +def test_search_cost_comes_from_cost_map(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + priced: Final[float] = litellm.model_cost["deepseek/search"]["input_cost_per_query"] + assert priced > 0 + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"results": [{"title": "t", "url": "https://e.com", "snippet": "s"}]} + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=mock_response, + ): + response: Final = litellm.search(query="litellm", search_provider="deepseek") + assert response._hidden_params["response_cost"] == pytest.approx(priced) From c0d88b46c9b5a462405e557782341d3d07663aa9 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:35:27 +0000 Subject: [PATCH 007/124] fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop (#47348) * fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): await the now async azure health-check auth header helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(azure): await async azure_ad_token_provider results in the threaded wrapper * fix(azure): keep transport params when validating async Azure client params on pydantic 2.11 * test(azure): assert the token and validation thread hops by thread identity instead of sleeps * fix(azure): give the OIDC token exchange its own short-lived HTTP client The exchange now runs from worker threads, so it no longer shares litellm.module_level_client's connection pool with other threads * test(azure): check the OIDC exchange closes its client through an in-process transport * refactor(responses): merge caller extra_headers into a new dict instead of updating validated headers in place --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/azure/common_utils.py | 121 +++++++++++----- litellm/llms/azure/search/transformation.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 71 ++++++---- litellm/realtime_api/main.py | 23 ++- litellm/secret_managers/main.py | 9 +- .../test_health_check_helpers.py | 4 +- ...st_bing_grounding_search_transformation.py | 22 +++ .../llms/azure/test_azure_common_utils.py | 133 +++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 36 +++++ 9 files changed, 348 insertions(+), 76 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f9f636244f4..52e3cdf380e 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -3,13 +3,15 @@ import hashlib import json import os import re -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping +from contextlib import closing from functools import lru_cache from types import MappingProxyType from typing import Final, Literal, NamedTuple, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from pydantic import ConfigDict, TypeAdapter, with_config from typing_extensions import ReadOnly, TypedDict import litellm @@ -17,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RETRIES from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -38,6 +41,16 @@ class _AzureV1ClientParams(TypedDict, total=False, extra_items=object): base_url: ReadOnly[str] +@with_config(ConfigDict(extra="allow")) +class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object): + api_key: ReadOnly[str | None] + azure_ad_token: ReadOnly[str | None] + azure_ad_token_provider: ReadOnly[Callable[[], str | Awaitable[str]] | None] + + +_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams) + + class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): api_version: ReadOnly[str] base_url: ReadOnly[str] @@ -98,11 +111,42 @@ def _cached_entra_id_token_provider( return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope) +@lru_cache(maxsize=128) +def _cached_username_password_token_provider( + client_id: str, + azure_username: str, + azure_password: str, + scope: str, +) -> Callable[[], str]: + from azure.identity import UsernamePasswordCredential, get_bearer_token_provider + + credential: Final = UsernamePasswordCredential( + client_id=client_id, + username=azure_username, + password=azure_password, + ) + return get_bearer_token_provider(credential, scope) + + @lru_cache(maxsize=128) def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: return get_azure_ad_token_provider(azure_scope=scope) +def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: + return _cached_azure_ad_token_refresh_provider(scope) + + +def _async_azure_ad_token_provider( + provider: Callable[[], str | Awaitable[str]], +) -> Callable[[], Awaitable[str]]: + async def async_provider() -> str: + token: Final = await asyncio.to_thread(provider) + return token if isinstance(token, str) else await token + + return async_provider + + def get_azure_ad_token_from_entra_id( tenant_id: str, client_id: str, @@ -177,22 +221,19 @@ def get_azure_ad_token_from_username_password( Returns: callable that returns a bearer token. """ - from azure.identity import UsernamePasswordCredential, get_bearer_token_provider - verbose_logger.debug( "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]", client_id, azure_username is not None, azure_password is not None, ) - credential: Final = UsernamePasswordCredential( + token_provider: Final = _cached_username_password_token_provider( client_id=client_id, - username=azure_username, - password=azure_password, + azure_username=azure_username, + azure_password=azure_password, + scope=scope, ) - token_provider: Final = get_bearer_token_provider(credential, scope) - verbose_logger.debug("token_provider %s", token_provider) return token_provider @@ -248,18 +289,17 @@ def get_azure_ad_token_from_oidc( if azure_ad_token_access_token is not None: return azure_ad_token_access_token - client: Final = litellm.module_level_client - - req_token: Final = client.post( - f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", - data={ - "client_id": azure_client_id, - "grant_type": "client_credentials", - "scope": scope, - "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - "client_assertion": oidc_token, - }, - ) + with closing(HTTPHandler(timeout=litellm.request_timeout)) as client: + req_token: Final = client.post( + f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": scope, + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) if req_token.status_code != 200: raise AzureOpenAIError( @@ -560,22 +600,19 @@ class BaseAzureLLM(BaseOpenAILLM): # on every request (via `_refresh_api_key`), so passing # `azure_ad_token_provider` directly preserves Azure AD token refresh # behavior that the regular AzureOpenAI client provides. - v1_api_key: str | Callable[[], object] | None = ( - azure_client_params.get("api_key") - or azure_client_params.get("azure_ad_token_provider") - or azure_client_params.get("azure_ad_token") + v1_client_params: Final[_AsyncAzureClientParams] = _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python( + azure_client_params + ) + v1_token_provider: Final = ( + v1_client_params.get("api_key") + or v1_client_params.get("azure_ad_token_provider") + or v1_client_params.get("azure_ad_token") + ) + v1_api_key: Final[str | Callable[[], str | Awaitable[str]] | None] = ( + _async_azure_ad_token_provider(v1_token_provider) + if _is_async is True and callable(v1_token_provider) + else v1_token_provider ) - if _is_async is True and callable(v1_api_key): - # AsyncOpenAI expects an async provider; wrap the sync provider - # returned by azure-identity. Offload to a thread so a token - # refresh (blocking HTTP call to AAD on cache miss) does not - # stall the event loop. - _sync_provider: Final = v1_api_key - - async def _async_v1_api_key() -> str: - return await asyncio.to_thread(_sync_provider) - - v1_api_key = _async_v1_api_key v1_params: Final[_AzureV1ClientParams] = { "api_key": v1_api_key, @@ -597,7 +634,19 @@ class BaseAzureLLM(BaseOpenAILLM): else: # Traditional Azure API uses AzureOpenAI client if _is_async is True: - openai_client = AsyncAzureOpenAI(**azure_client_params) + async_azure_client_params: Final[_AsyncAzureClientParams] = ( + _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(azure_client_params) + ) + async_token_provider_candidate: Final = async_azure_client_params.get("azure_ad_token_provider") + async_azure_client_params_with_threaded_provider: Final[_AsyncAzureClientParams] = ( + { + **async_azure_client_params, + "azure_ad_token_provider": _async_azure_ad_token_provider(async_token_provider_candidate), + } + if async_token_provider_candidate is not None + else async_azure_client_params + ) + openai_client = AsyncAzureOpenAI(**async_azure_client_params_with_threaded_provider) else: openai_client = AzureOpenAI(**azure_client_params) diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py index 4264dae018a..f2bb2ecc68c 100644 --- a/litellm/llms/azure/search/transformation.py +++ b/litellm/llms/azure/search/transformation.py @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal import httpx from pydantic import ConfigDict, ValidationError +from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, @@ -263,9 +264,7 @@ def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | def _default_entra_token_minter() -> str: - from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider - - return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)() + return cached_azure_ad_token_refresh_provider(ENTRA_SCOPE)() class BingGroundingSearchConfig(BaseSearchConfig): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b03b8050299..48858dfa441 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2953,14 +2953,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, - model=model, - litellm_params=litellm_params, - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } # Check if streaming is requested stream = response_api_optional_request_params.get("stream", False) @@ -2976,7 +2977,7 @@ class BaseLLMHTTPHandler: input=input, response_api_optional_request_params=response_api_optional_request_params, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) @@ -3011,7 +3012,7 @@ class BaseLLMHTTPHandler: headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=api_base, @@ -3145,8 +3146,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3399,8 +3403,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3563,8 +3570,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4620,8 +4630,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4795,12 +4808,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model=model, litellm_params=litellm_params - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } api_base: Final = responses_api_provider_config.get_complete_url( api_base=litellm_params.api_base, @@ -4816,14 +4832,14 @@ class BaseLLMHTTPHandler: response_api_optional_request_params=response_api_optional_request_params, api_base=api_base, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=url, @@ -6991,7 +7007,8 @@ class BaseLLMHTTPHandler: **kwargs, } ) - headers: Final = responses_api_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, headers={}, model=model, litellm_params=litellm_params, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 96b4b081c05..ea47d744cae 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -562,7 +562,12 @@ async def _arealtime( configured_realtime_protocol, query_params=query_params, websocket=websocket ) resolved_azure_ad_token: Final = ( - None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), + ) ) await azure_realtime.async_realtime( model=model, @@ -743,15 +748,21 @@ def _azure_realtime_health_protocol( return "GA", query_params -def _realtime_health_check_auth_headers( +async def _realtime_health_check_auth_headers( custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, object] ) -> Mapping[str, str]: if custom_llm_provider == "azure": + azure_ad_token: Final = ( + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams.model_validate(dict(model_params)), + ) + ) return azure_realtime.get_auth_headers( api_key=api_key, - azure_ad_token=( - None if api_key else get_azure_ad_token(GenericLiteLLMParams.model_validate(dict(model_params))) - ), + azure_ad_token=azure_ad_token, ) if api_key is None: return _EMPTY_AUTH_HEADERS @@ -807,7 +818,7 @@ async def realtime_health_check( if custom_llm_provider == LlmProviders.OPENAI.value else resolved_api_key ) - auth_headers: Final = _realtime_health_check_auth_headers( + auth_headers: Final = await _realtime_health_check_auth_headers( custom_llm_provider=custom_llm_provider, api_key=health_check_api_key, model_params=resolved_params, diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 716407d6def..d12e200772f 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -3,6 +3,8 @@ import base64 import os import time import traceback +from collections.abc import Callable +from functools import lru_cache from typing import Final import httpx @@ -23,6 +25,11 @@ oidc_cache: Final = DualCache() _PARSED_LITERAL: Final = TypeAdapter(object) +@lru_cache(maxsize=128) +def _cached_azure_oidc_token_provider(audience: str) -> Callable[[], str]: + return get_azure_ad_token_provider(azure_scope=audience) + + _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS: Final = 60 @@ -283,7 +290,7 @@ def get_secret( verbose_logger.warning( "AZURE_FEDERATED_TOKEN_FILE not found in environment will use Azure AD token provider" ) - azure_token_provider: Final = get_azure_ad_token_provider(azure_scope=oidc_aud) + azure_token_provider: Final = _cached_azure_oidc_token_provider(oidc_aud) try: oidc_token = azure_token_provider() if oidc_token is None: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4f62c6fb096..c23749f7a20 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -534,12 +534,12 @@ async def test_ocr_health_check_sends_the_document_kind_the_provider_config_acce assert document[expected_document_type].startswith(expected_uri_prefix) -def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): +async def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): from litellm.realtime_api import main as realtime_main seen = [] with patch.object(realtime_main, "get_azure_ad_token", lambda params: seen.append(params) or "ad-token"): - headers = realtime_main._realtime_health_check_auth_headers( + headers = await realtime_main._realtime_health_check_auth_headers( "azure", None, MappingProxyType({"api_base": "https://x.openai.azure.com", "self": 1, "params": 2, "__class__": 3}), diff --git a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py index f8d042c14fd..dfd1474211f 100644 --- a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py +++ b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py @@ -1,5 +1,6 @@ import json, litellm from pathlib import Path +from typing import Final from unittest.mock import AsyncMock, Mock, patch import pytest @@ -110,6 +111,27 @@ def test_validate_environment_entra_minter_failure_names_the_options(): assert "no az login" in message +def test_default_entra_minter_reuses_provider(monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + + from litellm.llms.azure import common_utils + from litellm.llms.azure.search.transformation import _default_entra_token_minter + + provider_module: Final = importlib.import_module("litellm.secret_managers.get_azure_ad_token_provider") + provider_factory: Final = Mock(side_effect=lambda **_: lambda: "entra-token") + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + monkeypatch.setattr(common_utils, "get_azure_ad_token_provider", provider_factory) + monkeypatch.setattr(provider_module, "get_azure_ad_token_provider", provider_factory) + + try: + first: Final = _default_entra_token_minter() + second: Final = _default_entra_token_minter() + assert first == second == "entra-token" + assert provider_factory.call_count == 1 + finally: + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + def test_validate_environment_does_not_mutate_and_is_idempotent(): config = _config() caller_headers = {"X-Custom": "keep-me"} diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 80449909b3f..41ad27d6f40 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1,9 +1,12 @@ import json import os +import threading import traceback -from typing import Callable, Optional +from typing import Callable, Final, Optional +from urllib.parse import parse_qs from unittest.mock import MagicMock, patch +import httpx import pytest import litellm @@ -13,6 +16,8 @@ from litellm.llms.azure.common_utils import ( _cached_entra_id_token_provider, get_azure_ad_token, get_azure_ad_token_from_entra_id, + get_azure_ad_token_from_oidc, + get_azure_ad_token_from_username_password, ) from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -2141,6 +2146,132 @@ class TestEntraIdTokenProviderCache: assert mock_credential.call_count == 2 +def test_username_password_token_provider_reuses_same_credentials_and_separates_passwords() -> None: + with ( + patch("azure.identity.UsernamePasswordCredential") as mock_credential, + patch( + "azure.identity.get_bearer_token_provider", + side_effect=lambda credential, scope: lambda: "token", + ), + ): + first: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + second: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + changed_password: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="different-password", + scope="https://cognitiveservices.azure.com/.default", + ) + + assert first() == second() == changed_password() == "token" + assert mock_credential.call_count == 2 + + +@pytest.mark.asyncio +async def test_non_v1_azure_token_refresh_runs_off_the_event_loop_thread() -> None: + from openai import AsyncAzureOpenAI + + def token_provider_naming_its_thread() -> str: + return str(threading.get_ident()) + + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base="https://test.openai.azure.com", + api_version="2024-02-01", + litellm_params={"azure_ad_token_provider": token_provider_naming_its_thread}, + _is_async=True, + ) + + assert isinstance(client, AsyncAzureOpenAI) + try: + token_thread: Final = await client._get_azure_ad_token() + finally: + await client.close() + + assert token_thread is not None + assert token_thread != str(threading.get_ident()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_version", ["2024-02-01", "v1"]) +async def test_async_azure_ad_token_provider_reaches_azure_as_bearer_token( + monkeypatch: pytest.MonkeyPatch, api_version: str +) -> None: + def echo_authorization(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": request.headers["authorization"]}, + } + ], + }, + ) + + async def async_token_provider() -> str: + return "async-token" + + monkeypatch.setattr( + litellm, "aclient_session", httpx.AsyncClient(transport=httpx.MockTransport(echo_authorization)) + ) + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base=f"https://async-token-provider-{api_version}.openai.azure.com", + api_version=api_version, + litellm_params={"azure_ad_token_provider": async_token_provider}, + _is_async=True, + ) + + response: Final = await client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "hi"}]) + + assert response.choices[0].message.content == "Bearer async-token" + + +def test_oidc_token_exchange_closes_its_own_http_client(monkeypatch: pytest.MonkeyPatch) -> None: + class TokenEndpoint(httpx.BaseTransport): + def __init__(self) -> None: + self.closed = False + + def handle_request(self, request: httpx.Request) -> httpx.Response: + form: Final = parse_qs(request.read().decode()) + return httpx.Response( + 200, json={"access_token": f"minted-for-{form['client_assertion'][0]}", "expires_in": 3600} + ) + + def close(self) -> None: + self.closed = True + + token_endpoint: Final = TokenEndpoint() + monkeypatch.setattr(litellm, "sync_transport", token_endpoint, raising=False) + monkeypatch.setenv("AZURE_TEST_OIDC_ASSERTION", "workload-jwt") + + token: Final = get_azure_ad_token_from_oidc( + azure_ad_token="os.environ/AZURE_TEST_OIDC_ASSERTION", + azure_client_id="client", + azure_tenant_id="tenant-closes-its-client", + ) + + assert token == "minted-for-workload-jwt" + assert token_endpoint.closed + + def test_evicting_an_azure_client_built_on_the_callers_session_leaves_it_open(monkeypatch): """`initialize_azure_sdk_client` puts `litellm.aclient_session` on the SDK client. diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 23c142b61ab..4e529764229 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -476,6 +476,42 @@ async def test_async_response_api_handler_streams_when_provider_transform_adds_s assert client.post.call_args.kwargs["json"]["stream"] is True +@pytest.mark.asyncio +async def test_async_response_api_handler_validates_environment_off_the_event_loop() -> None: + def validate_environment(**_kwargs: object) -> dict[str, str]: + return {"validated-on-thread": str(threading.get_ident()), "x-override": "provider"} + + config: Final = Mock() + config.validate_environment.side_effect = validate_environment + config.get_complete_url.return_value = "https://responses.example.com/v1/responses" + config.async_transform_responses_api_request = AsyncMock(return_value={"model": "model", "input": "hi"}) + config.sign_request.return_value = ({}, None) + client: Final = AsyncMock(spec=AsyncHTTPHandler) + client.post.return_value = httpx.Response( + 200, + request=httpx.Request("POST", "https://responses.example.com/v1/responses"), + ) + logging_obj: Final = Mock() + logging_obj.dynamic_success_callbacks = None + + await BaseLLMHTTPHandler().async_response_api_handler( + model="model", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="test", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + extra_headers={"x-override": "caller"}, + ) + + validated_headers: Final = config.async_transform_responses_api_request.await_args.kwargs["headers"] + assert validated_headers["validated-on-thread"] != str(threading.get_ident()) + assert validated_headers["x-override"] == "caller" + client.post.assert_awaited_once() + + _CHATGPT_SSE_BODY = ( "event: response.output_item.done\n" 'data: {"type": "response.output_item.done", "output_index": 0, "item": {"type": "message", ' From 1212260ab15fe2c9ff28f0c465ad2d0833da8143 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 13:38:22 -0700 Subject: [PATCH 008/124] fix(logging): report the cost-map entry billing used in model_map_information (#47182) * fix(logging): report the cost-map entry billing used in model_map_information Billing resolved the deployment's own cost-map entry and logging ran a second, name-only lookup, so callbacks and spend logs showed the catalog rates for a custom-priced deployment. completion_cost now stores the entry it billed from on the logging object and the standard logging payload reads it, falling back to the old lookup only when billing never ran. * fix(logging): pass the deployment id into the logging cost-map lookup Drop the logging-object plumbing and keep the lookup where it was: get_standard_logging_object_payload now hands the deployment id to get_model_cost_information as router_model_id, the same id billing uses, so the deployment's own cost-map entry is reported instead of the catalog entry. * test(logging): mirror the proxy metadata in the deployment-price regression test * fix(logging): keep the catalog model_map_key, report only the deployment's own rates --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 15 +++- .../test_litellm_logging.py | 83 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3a1dafa331c..b065d4dd91a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6471,6 +6471,7 @@ class StandardLoggingPayloadSetup: custom_llm_provider: str | None, init_response_obj: object, api_base: str | None = None, + router_model_id: str | None = None, ) -> StandardLoggingModelInformation: model_cost_name: Final = select_model_name_for_cost_calc( model=base_model if custom_pricing else None, @@ -6478,12 +6479,23 @@ class StandardLoggingPayloadSetup: base_model=base_model, custom_pricing=custom_pricing, ) + pricing_model_name: Final = ( + select_model_name_for_cost_calc( + model=base_model if custom_pricing else None, + completion_response=init_response_obj, + base_model=base_model, + custom_pricing=custom_pricing, + router_model_id=router_model_id, + ) + if router_model_id is not None + else model_cost_name + ) if model_cost_name is None: model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None) else: try: _model_cost_information: Final = litellm.get_model_info( - model=model_cost_name, + model=pricing_model_name or model_cost_name, custom_llm_provider=custom_llm_provider, api_base=api_base, ) @@ -7193,6 +7205,7 @@ def get_standard_logging_object_payload( custom_llm_provider=kwargs.get("custom_llm_provider"), init_response_obj=init_response_obj, api_base=litellm_params.get("api_base"), + router_model_id=logging_obj.get_router_model_id(), ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 37a985264b7..1d3bb2d0f34 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10718,6 +10718,89 @@ def test_uncached_price_leaves_no_pricing_state_on_the_call(): litellm.model_cost.pop(deployment_id, None) +@pytest.mark.parametrize( + ("input_cost_per_token", "output_cost_per_token"), + [(1e-06, 2e-06), (0.0, 0.0)], + ids=["custom_price", "explicit_zero_price"], +) +def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_billed_from( + input_cost_per_token: float, output_cost_per_token: float +): + """A deployment with its own price keeps the catalog model as model_map_key but is logged at its own rates.""" + deployment_id: Final = "lit9660-priced-deployment" + deployment_model_info: Final = { + "id": deployment_id, + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + } + litellm.register_model( + model_cost={ + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = LitellmLogging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="lit9660", + function_id="fn", + ) + litellm_params: Final = { + "metadata": { + "model_group": "lit9660-group", + "deployment": "openai/gpt-5.4-nano", + "model_info": deployment_model_info, + } + } + logging_obj.update_environment_variables( + model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai" + ) + response: Final = ModelResponse( + model="gpt-5.4-nano", + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hi"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=6, total_tokens=16), + ) + + response_cost: Final = logging_obj.response_cost_calculator(result=response) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-5.4-nano", + "custom_llm_provider": "openai", + "litellm_params": litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=logging_obj, + status="success", + ) + + assert response_cost == 10 * input_cost_per_token + 6 * output_cost_per_token + assert payload is not None + assert payload["model_map_information"] == { + "model_map_key": "openai/gpt-5.4-nano", + "model_map_value": litellm.get_model_info(model=f"openai/{deployment_id}", custom_llm_provider="openai"), + } + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert (model_map_value["input_cost_per_token"], model_map_value["output_cost_per_token"]) == ( + input_cost_per_token, + output_cost_per_token, + ) + finally: + litellm.model_cost.pop(deployment_id, None) + + def _assert_zeroed_copy(hit_breakdown: CostBreakdown | None, fresh_breakdown: CostBreakdown) -> None: basis_and_rates: Final = ("service_tier", "data_residency", "vertex_location", "discount_percent", "margin_percent") assert hit_breakdown is not None and hit_breakdown.keys() == fresh_breakdown.keys() From 365d7342c0c1f31533f32b189e60b25aa2cd6fb7 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:38:26 -0700 Subject: [PATCH 009/124] fix(prometheus): label deployment failures with the deployment's provider instead of None (#47384) * fix(prometheus): label deployment failures with the deployment's provider instead of None * test(prometheus): drive every deployment failure surface through the provider label Streaming chat, responses and messages, dropped and stalled upstreams, JSON-configured, custom-provider-map, prompt-management and unprefixed deployments, forged request metadata, an unauthenticated request, the OpenAI and Anthropic SDKs sync and async, and a concurrent burst counted exactly once per deployment --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/prometheus.py | 46 +- ...metheus_deployment_failure_api_provider.py | 527 ++++++++++++++++++ .../integrations/test_prometheus_labels.py | 112 ++++ 3 files changed, 684 insertions(+), 1 deletion(-) create mode 100644 tests/integration/observability/test_prometheus_deployment_failure_api_provider.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 43250fd0fd4..073e545a93f 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -47,9 +47,11 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider from litellm.litellm_core_utils.service_tier_utils import ( get_service_tier_from_standard_logging_payload, ) +from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.team import LiteLLM_TeamTable from litellm.models.user import LiteLLM_UserTable @@ -60,6 +62,7 @@ from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import EndUserRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata from litellm.types.guardrails import GuardrailEventHooks from litellm.types.integrations.prometheus import * from litellm.types.integrations.prometheus import ( @@ -260,6 +263,26 @@ _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, F _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)]) _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0 _ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _logged_fields(value: object) -> Mapping[str, object]: + return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS + + +def _is_llm_provider(name: str) -> bool: + routable: Final = ( + name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name) + ) + return routable and name not in litellm._known_custom_logger_compatible_callbacks # pyright: ignore[reportPrivateUsage] # the router treats these prefixes as prompt-management deployments + + +def _deployment_model_provider(deployment: object) -> str: + if not isinstance(deployment, str): + return "" + prefix, separator, _ = deployment.partition("/") + return prefix if separator and _is_llm_provider(prefix) else "" def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None: @@ -2846,6 +2869,25 @@ class PrometheusLogger(CustomLogger): ) return None + @staticmethod + def _extract_deployment_api_provider(request_kwargs: Mapping[str, object]) -> str: + """ + Provider of the deployment a failed call was routed to: the resolved + ``custom_llm_provider`` on ``litellm_params``, then the one recorded on + ``standard_logging_object``, then the provider prefix of the deployment + model the router wrote into the request metadata (``azure/my-gpt`` -> + ``azure``) when that prefix is an LLM provider the router accepts. + Returns an empty string otherwise. + """ + litellm_params: Final = _logged_fields(request_kwargs.get("litellm_params")) + provider: Final = litellm_params.get("custom_llm_provider") or _logged_fields( + request_kwargs.get("standard_logging_object") + ).get("custom_llm_provider") + if isinstance(provider, str) and provider: + return provider + deployment_metadata: Final = find_deployment_metadata(litellm_params) + return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None) + @staticmethod def _extract_api_provider_from_exception(exception: Exception) -> str | None: if not isinstance(exception, litellm.exceptions.RateLimitError): @@ -3124,7 +3166,9 @@ class PrometheusLogger(CustomLogger): label_litellm_model_name = litellm_model_name label_model_id = model_id label_api_base = api_base - label_api_provider = llm_provider + label_api_provider = self._extract_deployment_api_provider( + _LOGGED_FIELDS.validate_python(request_kwargs) + ) label_requested_model = model_group or litellm_model_name label_model_group = model_group else: diff --git a/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py b/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py new file mode 100644 index 00000000000..190f27f346d --- /dev/null +++ b/tests/integration/observability/test_prometheus_deployment_failure_api_provider.py @@ -0,0 +1,527 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import uuid +from collections import Counter +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.process import owned_proxy +from tests.integration._support.prometheus_series import Sample, scrape +from tests.integration._support.wire import Reply, Request, Wire, wire_server + +_FAILURES: Final = "litellm_deployment_failure_responses_total" +_TOTALS: Final = "litellm_deployment_total_requests_total" +_SERVED: Final = "limited" +_UNREACHABLE: Final = "unreachable" +_STALLED: Final = "stalled" +_NEVER_RELEASED: Final = threading.Event() +_IN_HOUSE_HANDLER: Final = """ +from litellm import CustomLLM + + +class InHouse(CustomLLM): + pass + + +handler = InHouse() +""" + + +@dataclass(frozen=True, slots=True) +class _Deployment: + group: str + model: str + provider: str + extra: Mapping[str, str | int] + model_id: str = "" + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + upstream: Wire + + +def _deployment(group: str, model: str, provider: str, **extra: str | int) -> _Deployment: + return _Deployment(group, model, provider, extra, f"{group}-{uuid.uuid4().hex[:8]}") + + +_CHAT: Final = _deployment("deployment-failure-chat", "openai/refused-chat", "openai") +_EMBEDDING: Final = _deployment("deployment-failure-embedding", "openai/refused-embedding", "openai") +_RESPONSES: Final = _deployment("deployment-failure-responses", "openai/refused-responses", "openai") +_MESSAGES: Final = _deployment("deployment-failure-messages", "anthropic/claude-haiku-4-5", "anthropic") +_UNSUPPORTED: Final = _deployment("deployment-failure-unsupported", "anthropic/claude-haiku-4-5", "anthropic") +_LIMITED: Final = _deployment("deployment-failure-limited", f"openai/{_SERVED}-openai", "openai", rpm=1) +_LIMITED_AZURE: Final = _deployment("gpt-4o-mini", f"azure/{_SERVED}-azure", "azure", rpm=1, api_version="2024-10-21") +_STREAMED_CHAT: Final = _deployment("deployment-failure-streamed-chat", "openai/refused-streamed-chat", "openai") +_STREAMED_RESPONSES: Final = _deployment( + "deployment-failure-streamed-responses", "openai/refused-streamed-responses", "openai" +) +_STREAMED_MESSAGES: Final = _deployment( + "deployment-failure-streamed-messages", "anthropic/claude-haiku-4-5", "anthropic" +) +_UNREACHABLE_EMBEDDING: Final = _deployment( + "deployment-failure-unreachable", f"openai/{_UNREACHABLE}-embedding", "openai" +) +_STALLED_EMBEDDING: Final = _deployment( + "deployment-failure-stalled", f"openai/{_STALLED}-embedding", "openai", timeout=1 +) +_JSON_PROVIDER: Final = _deployment("deployment-failure-json-provider", "abliteration/blocked", "abliteration", rpm=0) +_CUSTOM_PROVIDER: Final = _deployment("deployment-failure-custom-provider", "in-house/blocked", "in-house", rpm=0) +_PROMPT_MANAGEMENT: Final = _deployment( + "deployment-failure-prompt-management", "litellm_agent/anthropic/claude-haiku-4-5", "anthropic" +) +_UNPREFIXED: Final = _deployment("deployment-failure-unprefixed", "gpt-4o-mini", "", rpm=0) +_FORGED_EMBEDDING: Final = _deployment("deployment-failure-forged-embedding", "openai/refused-forged", "openai") +_FORGED_BLOCKED: Final = _deployment("deployment-failure-forged-blocked", "openai/blocked-forged", "openai", rpm=0) +_UNAUTHENTICATED: Final = _deployment("deployment-failure-unauthenticated", "openai/refused-unauthenticated", "openai") +_SDK: Final = { + client: _deployment(f"deployment-failure-{client}", model, provider) + for client, model, provider in ( + ("openai-sync", "openai/refused-sdk-sync", "openai"), + ("openai-async", "openai/refused-sdk-async", "openai"), + ("anthropic-sync", "anthropic/claude-haiku-4-5", "anthropic"), + ("anthropic-async", "anthropic/claude-haiku-4-5", "anthropic"), + ) +} +_BURST_EMBEDDING: Final = _deployment("burst-embedding", "openai/refused-burst-embedding", "openai") +_BURST_RESPONSES: Final = _deployment("burst-responses", "openai/refused-burst-responses", "openai") +_BURST_MESSAGES: Final = _deployment("burst-messages", "anthropic/claude-haiku-4-5", "anthropic") +_BURST_UNSUPPORTED: Final = _deployment("burst-unsupported", "anthropic/claude-haiku-4-5", "anthropic") +_BURST_BLOCKED: Final = _deployment("burst-blocked", "gemini/blocked-burst", "gemini", rpm=0) +_BURST: Final = (_BURST_EMBEDDING, _BURST_RESPONSES, _BURST_MESSAGES, _BURST_UNSUPPORTED, _BURST_BLOCKED) +_DEPLOYMENTS: Final = ( + _CHAT, + _EMBEDDING, + _RESPONSES, + _MESSAGES, + _UNSUPPORTED, + _LIMITED, + _LIMITED_AZURE, + _STREAMED_CHAT, + _STREAMED_RESPONSES, + _STREAMED_MESSAGES, + _UNREACHABLE_EMBEDDING, + _STALLED_EMBEDDING, + _JSON_PROVIDER, + _CUSTOM_PROVIDER, + _PROMPT_MANAGEMENT, + _UNPREFIXED, + _FORGED_EMBEDDING, + _FORGED_BLOCKED, + _UNAUTHENTICATED, + *_SDK.values(), + *_BURST, +) + + +def _respond(request: Request) -> Reply: + model: Final = json.loads(request.body).get("model", "") if request.body else "" + if model.startswith(_UNREACHABLE): + return Reply(drop_connection=True) + if model.startswith(_STALLED): + return Reply(chunks=(b"{}",), gate_before_first=_NEVER_RELEASED) + if _SERVED in request.target or model.startswith(_SERVED): + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "served"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + if request.target.endswith("/v1/messages"): + return Reply( + status=400, + body=json.dumps( + {"type": "error", "error": {"type": "invalid_request_error", "message": "refused upstream"}} + ).encode(), + ) + return Reply( + status=400, + body=json.dumps( + {"error": {"message": "refused upstream", "type": "invalid_request_error", "code": "refused"}} + ).encode(), + ) + + +def _config(directory: Path, upstream: Wire) -> Path: + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *( + { + "model_name": deployment.group, + "litellm_params": { + "model": deployment.model, + "api_key": "sk-fixture", + "api_base": upstream.url if deployment.provider != "openai" else f"{upstream.url}/v1", + **deployment.extra, + }, + "model_info": {"id": deployment.model_id}, + } + for deployment in _DEPLOYMENTS + ), + ] + (directory / "in_house_handler.py").write_text(_IN_HOUSE_HANDLER) + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration["litellm_settings"]["custom_provider_map"] = [ + {"provider": "in-house", "custom_handler": "in_house_handler.handler"} + ] + router_settings: Final = configuration.setdefault("router_settings", {}) + router_settings["num_retries"] = 0 + router_settings["optional_pre_call_checks"] = ["enforce_model_rate_limits"] + path: Final = directory / "prometheus-deployment-failure-api-provider.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-deployment-failure-api-provider") + samples: Final = directory / "prometheus-multiproc" + samples.mkdir() + with ( + wire_server(_respond) as upstream, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + {"PROMETHEUS_MULTIPROC_DIR": str(samples)}, + config=_config(directory, upstream), + workers=2, + ) as owned, + ): + yield _Rig(owned, upstream) + + +def _series(samples: Sequence[Sample], name: str, model_id: str) -> tuple[Sample, ...]: + return tuple(sample for sample in samples if sample.name == name and sample.labels.get("model_id") == model_id) + + +def _providers(rig: _Rig, name: str, model_id: str) -> frozenset[str]: + found: Final = eventually(lambda: _series(scrape(rig.gateway), name, model_id), bool, seconds=30) + return frozenset(sample.labels["api_provider"] for sample in found) + + +def _assert_labeled(rig: _Rig, deployment: _Deployment) -> None: + assert _providers(rig, _FAILURES, deployment.model_id) == {deployment.provider} + assert _providers(rig, _TOTALS, deployment.model_id) == {deployment.provider} + + +def _counted(samples: Sequence[Sample], name: str, model_id: str) -> Mapping[str, float]: + counts: Final[Counter[str]] = Counter() + for sample in _series(samples, name, model_id): + counts[sample.labels["api_provider"]] += sample.value + return counts + + +def _counts(rig: _Rig, name: str, deployment: _Deployment, expected: int) -> Mapping[str, float]: + return eventually( + lambda: _counted(scrape(rig.gateway), name, deployment.model_id), + lambda counts: sum(counts.values()) >= expected, + seconds=60, + ) + + +def _post(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> httpx.Response: + return rig.gateway.request("POST", path, body) + + +def _prompt() -> str: + return f"deployment failure {uuid.uuid4().hex}" + + +def _chat(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": deployment.group, "messages": [{"role": "user", "content": _prompt()}], **extra} + + +def _messages(deployment: _Deployment, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": deployment.group, "max_tokens": 16, "messages": [{"role": "user", "content": _prompt()}], **extra} + + +def test_a_refused_chat_completion_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/chat/completions", _chat(_CHAT)) + assert response.status_code == 400, response.text + _assert_labeled(rig, _CHAT) + + +def test_a_refused_embedding_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/embeddings", {"model": _EMBEDDING.group, "input": _prompt()}) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/embeddings") for request in rig.upstream.drain()) + _assert_labeled(rig, _EMBEDDING) + + +def test_a_refused_responses_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/responses", {"model": _RESPONSES.group, "input": _prompt()}) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/responses") for request in rig.upstream.drain()) + _assert_labeled(rig, _RESPONSES) + + +def test_a_refused_anthropic_messages_call_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/messages", _messages(_MESSAGES)) + assert response.status_code == 400, response.text + assert any(request.target.endswith("/v1/messages") for request in rig.upstream.drain()) + _assert_labeled(rig, _MESSAGES) + + +def test_a_param_the_deployment_does_not_support_labels_the_deployment_failure_with_its_provider(rig: _Rig) -> None: + rig.upstream.drain() + response: Final = _post(rig, "/v1/chat/completions", _chat(_UNSUPPORTED, logit_bias={"50256": -100})) + assert response.status_code == 400, response.text + assert "logit_bias" in response.text + assert rig.upstream.drain() == () + _assert_labeled(rig, _UNSUPPORTED) + + +def _statuses(rig: _Rig, deployment: _Deployment) -> Iterator[int]: + for _ in range(8): + yield _post(rig, "/v1/chat/completions", _chat(deployment)).status_code + + +def _until_rate_limited(rig: _Rig, deployment: _Deployment) -> None: + first_refusal: Final = next((status for status in _statuses(rig, deployment) if status != 200), None) + assert first_refusal == 429, f"expected the rpm limit to refuse a request, first non-200 was {first_refusal}" + + +@pytest.mark.parametrize( + "deployment", [_LIMITED, _LIMITED_AZURE], ids=["alias-group", "group-named-like-another-provider"] +) +def test_a_deployment_over_its_rpm_limit_labels_the_failure_with_the_deployment_provider( + rig: _Rig, deployment: _Deployment +) -> None: + _until_rate_limited(rig, deployment) + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + ("path", "body"), + [ + ("/v1/chat/completions", _chat(_STREAMED_CHAT, stream=True)), + ("/v1/responses", {"model": _STREAMED_RESPONSES.group, "input": _prompt(), "stream": True}), + ("/v1/messages", _messages(_STREAMED_MESSAGES, stream=True)), + ], + ids=["chat", "responses", "messages"], +) +def test_a_refused_streaming_call_labels_the_deployment_failure_with_its_provider( + rig: _Rig, path: str, body: Mapping[str, JsonValue] +) -> None: + deployment: Final = next(deployment for deployment in _DEPLOYMENTS if deployment.group == body["model"]) + response: Final = _post(rig, path, body) + assert response.status_code == 400, response.text + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + ("deployment", "status"), + [(_UNREACHABLE_EMBEDDING, 500), (_STALLED_EMBEDDING, 408)], + ids=["connection-dropped", "read-timeout"], +) +def test_an_upstream_that_never_answers_labels_the_deployment_failure_with_its_provider( + rig: _Rig, deployment: _Deployment, status: int +) -> None: + response: Final = _post(rig, "/v1/embeddings", {"model": deployment.group, "input": _prompt()}) + assert response.status_code == status, response.text + _assert_labeled(rig, deployment) + + +@pytest.mark.parametrize( + "deployment", + [_JSON_PROVIDER, _CUSTOM_PROVIDER, _UNPREFIXED], + ids=["json-configured-provider", "custom-provider-map", "unprefixed-model"], +) +def test_a_call_blocked_before_it_leaves_labels_the_failure_by_the_deployment_model_prefix( + rig: _Rig, deployment: _Deployment +) -> None: + rig.upstream.drain() + response: Final = _post(rig, "/v1/chat/completions", _chat(deployment)) + assert response.status_code == 429, response.text + assert rig.upstream.drain() == () + _assert_labeled(rig, deployment) + + +def test_a_prompt_management_deployment_labels_the_failure_with_the_provider_it_resolved_to(rig: _Rig) -> None: + response: Final = _post(rig, "/v1/chat/completions", _chat(_PROMPT_MANAGEMENT, logit_bias={"50256": -100})) + assert response.status_code == 400, response.text + found: Final = eventually( + lambda: tuple( + sample + for sample in scrape(rig.gateway) + if sample.name == _FAILURES and sample.labels.get("model_group") == _PROMPT_MANAGEMENT.group + ), + bool, + seconds=30, + ) + assert {sample.labels["api_provider"] for sample in found} == {_PROMPT_MANAGEMENT.provider} + + +@pytest.mark.parametrize( + "forged", + ["bedrock/forged", 5, ["bedrock/forged"], "", f"bedrock/{'x' * 5000}"], + ids=["provider-string", "int", "list", "empty", "five-kb"], +) +@pytest.mark.parametrize( + ("path", "deployment", "status"), + [("/v1/embeddings", _FORGED_EMBEDDING, 400), ("/v1/chat/completions", _FORGED_BLOCKED, 429)], + ids=["refused-upstream", "blocked-before-the-call"], +) +def test_a_caller_cannot_pick_the_label_through_request_metadata( + rig: _Rig, path: str, deployment: _Deployment, status: int, forged: JsonValue +) -> None: + bucket: Final[dict[str, JsonValue]] = {"deployment": forged, "model_info": {"id": deployment.model_id}} + body: Final = ( + _chat(deployment, metadata=bucket, litellm_metadata=bucket) + if path == "/v1/chat/completions" + else {"model": deployment.group, "input": _prompt(), "metadata": bucket, "litellm_metadata": bucket} + ) + response: Final = _post(rig, path, body) + assert response.status_code == status, response.text + _assert_labeled(rig, deployment) + every_provider: Final = { + sample.labels.get("api_provider") for sample in scrape(rig.gateway) if sample.name in (_FAILURES, _TOTALS) + } + assert not any(str(provider).startswith("bedrock") for provider in every_provider), every_provider + + +def test_an_unauthenticated_request_leaves_the_deployment_series_alone(rig: _Rig) -> None: + body: Final[dict[str, JsonValue]] = {"model": _UNAUTHENTICATED.group, "input": _prompt()} + rejected: Final = rig.gateway.request("POST", "/v1/embeddings", body, key="sk-not-a-litellm-key") + assert rejected.status_code == 401, rejected.text + refused: Final = _post(rig, "/v1/embeddings", body) + assert refused.status_code == 400, refused.text + assert _counts(rig, _FAILURES, _UNAUTHENTICATED, 1) == {"openai": 1} + assert _counts(rig, _TOTALS, _UNAUTHENTICATED, 1) == {"openai": 1} + + +def _sdk_v1(gateway: Gateway) -> str: + return f"{str(gateway.client.base_url).rstrip('/')}/v1" + + +def _sdk_origin(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _openai_sync(gateway: Gateway, deployment: _Deployment) -> int: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport) as client, + ): + with pytest.raises(openai.APIStatusError) as refusal: + client.embeddings.create(model=deployment.group, input=_prompt()) + return refusal.value.status_code + + +def _openai_async(gateway: Gateway, deployment: _Deployment) -> int: + async def run() -> int: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + openai.AsyncOpenAI( + api_key=gateway.key, base_url=_sdk_v1(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(openai.APIStatusError) as refusal: + await client.embeddings.create(model=deployment.group, input=_prompt()) + return refusal.value.status_code + + return asyncio.run(run()) + + +def _anthropic_sync(gateway: Gateway, deployment: _Deployment) -> int: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + anthropic.Anthropic( + api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(anthropic.APIStatusError) as refusal: + client.messages.create( + model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}] + ) + return refusal.value.status_code + + +def _anthropic_async(gateway: Gateway, deployment: _Deployment) -> int: + async def run() -> int: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + anthropic.AsyncAnthropic( + api_key=gateway.key, base_url=_sdk_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + with pytest.raises(anthropic.APIStatusError) as refusal: + await client.messages.create( + model=deployment.group, max_tokens=16, messages=[{"role": "user", "content": _prompt()}] + ) + return refusal.value.status_code + + return asyncio.run(run()) + + +_SDK_CALLS: Final[Mapping[str, Callable[[Gateway, _Deployment], int]]] = { + "openai-sync": _openai_sync, + "openai-async": _openai_async, + "anthropic-sync": _anthropic_sync, + "anthropic-async": _anthropic_async, +} + + +@pytest.mark.parametrize("client", list(_SDK_CALLS)) +def test_a_refusal_through_the_provider_sdk_labels_the_deployment_failure_with_its_provider( + rig: _Rig, client: str +) -> None: + assert _SDK_CALLS[client](rig.gateway, _SDK[client]) == 400 + _assert_labeled(rig, _SDK[client]) + + +_BURST_SIZE: Final = 10 +_BURST_CALLS: Final[Mapping[str, tuple[str, Callable[[], Mapping[str, JsonValue]], int]]] = { + _BURST_EMBEDDING.model_id: ("/v1/embeddings", lambda: {"model": _BURST_EMBEDDING.group, "input": _prompt()}, 400), + _BURST_RESPONSES.model_id: ("/v1/responses", lambda: {"model": _BURST_RESPONSES.group, "input": _prompt()}, 400), + _BURST_MESSAGES.model_id: ("/v1/messages", lambda: _messages(_BURST_MESSAGES), 400), + _BURST_UNSUPPORTED.model_id: ( + "/v1/chat/completions", + lambda: _chat(_BURST_UNSUPPORTED, logit_bias={"50256": -100}), + 400, + ), + _BURST_BLOCKED.model_id: ("/v1/chat/completions", lambda: _chat(_BURST_BLOCKED), 429), +} + + +def test_a_concurrent_burst_of_failures_counts_each_one_once_under_its_deployment_provider(rig: _Rig) -> None: + calls: Final = [deployment.model_id for deployment in _BURST for _ in range(_BURST_SIZE)] + with ThreadPoolExecutor(max_workers=len(calls)) as pool: + statuses: Final = Counter( + pool.map( + lambda model_id: ( + model_id, + _post(rig, _BURST_CALLS[model_id][0], _BURST_CALLS[model_id][1]()).status_code, + ), + calls, + ) + ) + assert statuses == Counter({(model_id, status): _BURST_SIZE for model_id, (_, _, status) in _BURST_CALLS.items()}) + for deployment in _BURST: + assert _counts(rig, _FAILURES, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE} + assert _counts(rig, _TOTALS, deployment, _BURST_SIZE) == {deployment.provider: _BURST_SIZE} diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index ead0ce8ecff..d49ca57a3de 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -4,6 +4,7 @@ Unit tests for prometheus metric labels configuration import pytest +import litellm from litellm.types.integrations.prometheus import ( PrometheusMetricLabels, UserAPIKeyLabelNames, @@ -928,6 +929,117 @@ def test_deployment_failure_metrics_emit_model_group_from_standard_logging_paylo _clear_prometheus_registry() +def _routed(deployment: object) -> dict[str, object]: + return {"deployment": deployment, "model_info": {"id": "model-123"}} + + +@pytest.mark.parametrize( + ("litellm_params", "logged_provider", "expected"), + [ + pytest.param( + {"custom_llm_provider": "bedrock", "metadata": _routed("azure/gpt-4o")}, + "vertex_ai", + "bedrock", + id="resolved-provider-wins", + ), + pytest.param( + {"custom_llm_provider": None, "metadata": _routed("anthropic/claude-haiku-4-5")}, + "openai", + "openai", + id="provider-on-the-payload-wins", + ), + pytest.param({"metadata": _routed("azure/limited-azure")}, None, "azure", id="deployment-prefix"), + pytest.param( + {"litellm_metadata": _routed("anthropic/claude-haiku-4-5"), "metadata": {}}, + None, + "anthropic", + id="litellm-metadata-deployment", + ), + pytest.param( + {"litellm_metadata": _routed("anthropic/claude-haiku-4-5"), "metadata": _routed("bedrock/converse/model")}, + None, + "anthropic", + id="litellm-metadata-deployment-wins", + ), + pytest.param( + {"litellm_metadata": {"deployment": "bedrock/forged"}, "metadata": _routed("anthropic/claude-haiku-4-5")}, + None, + "anthropic", + id="routed-bucket-wins-over-a-bare-deployment", + ), + pytest.param({"metadata": _routed("abliteration/model")}, None, "abliteration", id="json-configured-provider"), + pytest.param({"metadata": _routed("langfuse/my-prompt")}, None, "", id="prompt-management-deployment"), + pytest.param({"metadata": _routed("gpt-4o")}, None, "", id="unprefixed-deployment"), + pytest.param({"metadata": _routed("meta-llama/Llama-3")}, None, "", id="prefix-is-not-a-provider"), + pytest.param({"metadata": _routed(5)}, None, "", id="non-string-deployment"), + pytest.param({"metadata": {}}, None, "", id="nothing-known"), + ], +) +def test_deployment_failure_metrics_label_the_selected_deployment_provider(litellm_params, logged_provider, expected): + """ + A failure on a selected deployment labels api_provider with that + deployment's provider, never the string "None" and never a guess from the + requested model name (gpt-4o-mini here would read openai). + """ + from litellm.integrations.prometheus import PrometheusLogger + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": litellm_params, + "standard_logging_object": { + "model_group": "gpt-4o-mini", + "model_id": "model-123", + "api_base": "https://example.openai.azure.com", + "custom_llm_provider": logged_provider, + "request_tags": [], + }, + "exception": Exception("boom"), + } + ) + + for metric in ( + logger.litellm_deployment_failure_responses, + logger.litellm_deployment_total_requests, + ): + index = metric._labelnames.index("api_provider") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {expected}, f"expected api_provider={expected!r} on {metric._name}, got {values}" + finally: + _clear_prometheus_registry() + + +def test_deployment_failure_metrics_label_a_custom_provider_deployment(monkeypatch): + from litellm.integrations.prometheus import PrometheusLogger + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "in-house", "custom_handler": object()}]) + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": _routed("in-house/model")}, + "standard_logging_object": { + "model_group": "gpt-4o-mini", + "model_id": "model-123", + "api_base": "https://in-house.example", + "request_tags": [], + }, + "exception": Exception("boom"), + } + ) + + index = logger.litellm_deployment_failure_responses._labelnames.index("api_provider") + values = {sample_key[index] for sample_key in logger.litellm_deployment_failure_responses._metrics} + assert values == {"in-house"} + finally: + _clear_prometheus_registry() + + def test_deployment_tpm_rpm_limit_metrics_emit_model_group_from_enum_values(): """ End-to-end emit wiring for the tpm/rpm limit gauges. From b2f0f4d54c5dace1c1f6ad7b3c2f3bc68cb5982b Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:01:05 -0700 Subject: [PATCH 010/124] fix(vertex-ai): drop floor deprecation_date from gemini-4-argon (#47404) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f5eec206e04..1cadc93969b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75503,7 +75503,6 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { - "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f5eec206e04..1cadc93969b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75503,7 +75503,6 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { - "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, From b51b9b7d43dc7da20b4baa49d431736c50c10600 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:13:46 +0000 Subject: [PATCH 011/124] fix(anthropic)!: answer 400 for a malformed tool_result text on /v1/messages and count_tokens (#47347) * fix(anthropic): answer 400 for a malformed tool_result text on the Responses bridge and count_tokens A tool_result text block whose text is null, a number, or missing made /v1/messages and /v1/messages/count_tokens answer 500 on OpenAI and Azure models. Both routes now answer 400 naming the block, and drop the block when drop_params is on. * fix(anthropic): keep a missing tool_result text as empty text, bound the error, honor body drop_params on count_tokens * fix(anthropic): apply the tool_result text check on the chat/completions bridge and every count route The chat/completions bridge used for Azure, Gemini and other providers now answers the same 400 for a null or numeric tool_result text and drops the block under drop_params, so a model group that mixes OpenAI with another provider answers the same way on every deployment. Gemini and Vertex count_tokens apply the same check, count_tokens turns on drop_params for Claude Code and Codex the way /v1/messages does, and a tool_result without a tool_use_id is checked too * fix(proxy): answer 400 for a malformed tool_result text on count_tokens models without a count API Providers with no token counting API fall back to the local tokenizer, which counted a null or numeric tool_result text as fine while /v1/messages on the same model answers 400. The fallback now runs the same check, unless drop_params applies to the deployment or the request * fix(proxy): honor request drop_params on the local count fallback when no deployment resolves * test(integration): cover non-string tool_result text on both /v1/messages bridges and count routes * fix(anthropic): keep a dropped tool_result on Mistral and reject it before a guardrail rewrite Under drop_params a tool_result whose only text block was null or a number became an empty tool message, and Mistral's empty-message filter removed it, so Mistral answered 400 for a missing tool reply. The filter now only removes empty assistant messages, which is what Mistral rejects. A guardrail that rewrites structured_messages wrote the dropped text back as "", so a null or numeric text was silently blanked even without drop_params. The rewrite path now answers the same 400 as the bridges unless the request or global drop_params is set. * test(integration): cover the Mistral empty tool reply and the guardrail rewrite tool_result text check * fix(anthropic): keep a malformed tool_result text through a guardrail rewrite and keep Mistral's null-message drop A guardrail that rewrites structured_messages now writes the caller's null or numeric tool_result text back in place, so the route's bridge and the deployment's drop_params decide after routing, the same as with no guardrail. The rewrite path no longer answers its own 400 before routing. Mistral drops a null-content system or user message again, as main does, and only a tool reply is exempt from that filter so a dropped tool_result keeps its tool call. * test(mistral): expect empty system and user turns dropped and the empty tool reply kept * perf(anthropic): restore kept tool_result text parts in one linear pass * test(integration): cover the tool_result text restore across tool_results, the chat bridge, and a batch upload behind a rewriting guardrail * fix(anthropic): let guardrails scan malformed tool_result text and honor drop_params on native Anthropic A rewriting guardrail now sees a non-string tool_result text as its JSON text and the write-back restores the caller's value by the sanitized tool_use_id when the guardrail left it unchanged. Native Anthropic drops the bad block under drop_params on /v1/messages and count_tokens, and Mistral keeps an empty tool reply only when it answers a tool call, sent as empty text. * fix(anthropic): compare a kept tool_result text to its scanned form once and align the wire tests with drop_params * test(integration): cover native Anthropic drop_params resolution, guardrail scanning of numeric tool_result text, and the Mistral orphan tool message * fix(count_tokens): honor the router's default drop_params when counting a malformed tool_result text * test(count_tokens): send the tool_result count tests through a real request body instead of a mocked parser * test(count_tokens): send the native Claude count test through a real request body * fix(anthropic): keep each non-string tool_result text when tool_use_ids sanitize to the same id * fix(anthropic): keep non-ASCII characters in the guardrail's view of a non-string tool_result text * test(gemini): mock the count_tokens upstream with respx instead of patching the client factory * fix(mistral): keep an empty tool reply only for an open call, and forward non-object message items on the native drop An empty tool reply re-answering a call the conversation already moved past made Mistral reject the request, and a tool call id that is not a string crashed the filter. Native Anthropic under drop_params assumed every message item was an object. * fix(mistral): find answering tool replies in one pass instead of copying the open call set per reply * fix(mistral): ignore dropped empty placeholders when tracking open tool calls * fix(mistral): drop an empty tool reply when another reply to the same call carries the result * refactor(anthropic): share tool_result text helpers to shrink the diff * fix(anthropic): answer 400 for a non-string tool_result text on the OpenAI Responses bridge --- litellm/litellm_core_utils/core_helpers.py | 5 + .../prompt_templates/factory.py | 16 +- .../chat/guardrail_translation/handler.py | 18 +- .../pass_through/adapters/handler.py | 4 + .../pass_through/adapters/transformation.py | 4 +- .../pass_through/convertible_blocks.py | 255 ++++- .../pass_through/messages/transformation.py | 5 +- .../responses_adapters/handler.py | 6 +- .../responses_adapters/transformation.py | 11 +- litellm/llms/gemini/common_utils.py | 5 +- litellm/llms/mistral/chat/transformation.py | 74 +- .../responses/count_tokens/token_counter.py | 6 +- .../responses/count_tokens/transformation.py | 7 +- litellm/llms/vertex_ai/common_utils.py | 5 +- .../llms/vertex_ai/count_tokens/context.py | 11 + litellm/proxy/proxy_server.py | 53 +- .../transformation.py | 6 +- .../test_tool_result_text_wire.py | 868 ++++++++++++++++++ ...llm_core_utils_prompt_templates_factory.py | 6 +- .../test_anthropic_guardrail_handler.py | 135 ++- ...al_pass_through_adapters_transformation.py | 153 +++ ...al_pass_through_messages_transformation.py | 64 ++ .../test_anthropic_messages_router.py | 85 ++ .../test_responses_adapters_handler.py | 85 ++ .../pass_through/test_convertible_blocks.py | 20 + .../llms/gemini/test_gemini_common_utils.py | 78 ++ .../test_mistral_chat_transformation.py | 205 +++++ .../count_tokens/test_token_counter.py | 87 ++ .../vertex_ai/test_vertex_ai_common_utils.py | 72 ++ tests/unit/proxy/test_proxy_token_counter.py | 200 +++- 30 files changed, 2484 insertions(+), 65 deletions(-) create mode 100644 tests/integration/messages_endpoint/test_tool_result_text_wire.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 65d0089304b..b7bfd5eeeb3 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c import httpx from pydantic import TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_logger from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id @@ -93,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None: return None +def drops_unsupported_params(drop_params: object) -> bool: + return litellm.drop_params is True or normalize_drop_params(drop_params) is True + + def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool: normalized: Final = normalize_drop_params(value) if normalized is None and value is not None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e2fb2be4ec2..481ec6a5076 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1669,7 +1669,7 @@ def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Patte return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK -def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: +def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$.""" return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS) @@ -1815,7 +1815,7 @@ def convert_to_anthropic_tool_result( tool_message: Final[ChatCompletionToolMessage] = message tool_call_id: str = tool_message["tool_call_id"] # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template @@ -1829,7 +1829,7 @@ def convert_to_anthropic_tool_result( function_message: Final[ChatCompletionFunctionMessage] = message tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4()) # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$ - sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id) + sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id) anthropic_tool_result = AnthropicMessagesToolResultParam( type="tool_result", tool_use_id=sanitized_tool_use_id, @@ -2037,7 +2037,7 @@ def convert_to_anthropic_tool_invoke( elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue else: - sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) + sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( type="tool_use", id=sanitized_tool_id, @@ -5900,21 +5900,21 @@ def _optional_tool_name(value: object) -> str | None: def _split_tool_call_ids(calls: Sequence[tuple[str | None, int]]) -> tuple[tuple[str | None, ...], ...]: - taken: Final = frozenset(_sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) + taken: Final = frozenset(sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id) def fresh(call_id: str) -> Iterator[str]: return filter( - lambda candidate: _sanitize_anthropic_tool_use_id(candidate) not in taken, + lambda candidate: sanitize_anthropic_tool_use_id(candidate) not in taken, (f"{call_id}__concat_{n}" for n in itertools.count(1)), ) suffixes: Final = MappingProxyType( - {_sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} + {sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1} ) return tuple( ( call_id, - *(next(suffixes[_sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), + *(next(suffixes[sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)), ) if call_id else (None,) * count diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7654b56ad9a..aa2bee1f31d 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -30,6 +30,10 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + keep_non_string_tool_result_texts, + scannable_tool_result_request, +) from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details from litellm.llms.anthropic.pass_through.stream_assembly import ( build_complete_streaming_response, @@ -493,8 +497,8 @@ class AnthropicMessagesHandler(BaseTranslation): ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle several events separated by a blank line -- and an already-parsed event ``dict``.""" - if isinstance(item, dict): - return (_as_str_mapping(item),) + if (event := as_str_mapping(item)) is not None: + return (event,) if not isinstance(item, (bytes, bytearray)): return () return tuple( @@ -510,7 +514,7 @@ class AnthropicMessagesHandler(BaseTranslation): chat_completion_compatible_request, _tool_name_mapping, ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( - anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()), + anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) return chat_completion_compatible_request @@ -832,7 +836,7 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _write_back_structured_messages( - data: dict, # mutable-ok: API message payload + data: dict[str, object], # mutable-ok: API message payload structured_messages: list, # mutable-ok: API message payload hoisted_system_message: object = None, preserve_system_messages: bool = False, @@ -899,7 +903,7 @@ class AnthropicMessagesHandler(BaseTranslation): for block in content: if isinstance(block, dict) and block.get("type") == "thinking": block.pop("cache_control", None) - data["messages"] = converted + data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages")) @staticmethod def _extract_midturn_system_text( @@ -1601,8 +1605,8 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object: - if isinstance(item, dict): - return _rewritten_event(_as_str_mapping(item), rewrite_event) + if (event := as_str_mapping(item)) is not None: + return _rewritten_event(event, rewrite_event) if isinstance(item, (bytes, bytearray)): return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event) return item diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index a9aa3de69ad..666823d6ec3 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -22,6 +22,7 @@ from litellm.llms.anthropic.pass_through.context_management import ( PolyfillResult, apply_context_management, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text from litellm.llms.anthropic.pass_through.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, @@ -731,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, request_kwargs) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -853,6 +855,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) + reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like # ``clear_tool_uses_20250919``. The dispatcher is async (so the diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 0d3e48c49fd..de929acbfb2 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.hidden_params import get_hidden_params +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, @@ -1334,7 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter: block: Final = cast(Mapping[str, object], item) # cast-ok: untrusted client payload match block.get("type"): case "text": - return ChatCompletionTextObject(type="text", text=str(block.get("text") or "")) + text: Final = tool_result_text_part(block) + return None if text is None else ChatCompletionTextObject(type="text", text=text) case "image" | "document": return self._tool_result_image_part(block.get("source")) case "tool_reference": diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/pass_through/convertible_blocks.py index 543a6efd4c4..fb17910fb97 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/pass_through/convertible_blocks.py @@ -1,11 +1,18 @@ -from collections.abc import Iterator, Mapping, Sequence -from itertools import chain +import json +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from itertools import chain, groupby +from operator import itemgetter from types import MappingProxyType -from typing import Final +from typing import Final, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import assert_never +import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.types.utils import TokenCountMessagesFormat @@ -104,3 +111,245 @@ def unconvertible_blocks_error( ) case _: assert_never(messages_format) + + +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) + + +def _list_items(value: object) -> tuple[object, ...]: + return _OBJECTS.validate_python(value) if isinstance(value, list) else () + + +@dataclass(frozen=True, slots=True) +class _KeptPart: + block: object + scanned_text: str + + +_KeptParts: TypeAlias = Mapping[int, _KeptPart] +_NO_KEPT_PARTS: Final[_KeptParts] = MappingProxyType({}) + + +def _scannable_text(text: object) -> str: + return json.dumps(text, default=str, ensure_ascii=False) + + +def _non_str_text(block: object) -> _KeptPart | None: + match as_str_mapping(block): + case {"type": "text", "text": str()}: + return None + case {"type": "text", "text": text}: + return _KeptPart(block=block, scanned_text=_scannable_text(text)) + case _: + return None + + +def tool_result_text_part(block: object) -> str | None: + match block: + case {"type": "text", "text": str(text)}: + return text + case {"type": "text", "text": object()}: + return None + case {"type": "text"}: + return "" + case _: + return None + + +def tool_result_text_parts(tool_result: Mapping[str, object]) -> tuple[str, ...]: + parts: Final = map(tool_result_text_part, _list_items(tool_result.get("content"))) + return tuple(part for part in parts if part is not None) + + +def _user_content_blocks(message: object) -> tuple[object, ...]: + match as_str_mapping(message): + case {"role": "user", "content": content}: + return _list_items(content) + case _: + return () + + +def _with_user_blocks(message: object, transform: Callable[[object], object]) -> object: + match as_str_mapping(message): + case {"role": "user", "content": list()} as user_message: + return {**user_message, "content": [transform(block) for block in _user_content_blocks(user_message)]} + case _: + return message + + +def _non_string_text_errors(label: str, content: object) -> Iterator[str]: + return ( + f"{label} content.{index}.text: Input should be a valid string" + for index, text_block in enumerate(_list_items(content)) + if _non_str_text(text_block) is not None + ) + + +def _tool_result_text_errors(block: object) -> Iterator[str]: + match block: + case {"type": "tool_result", "tool_use_id": object() as tool_use_id, "content": object() as content}: + return _non_string_text_errors(f"tool_result {tool_use_id}", content) + case {"type": "tool_result", "content": object() as content}: + return _non_string_text_errors("tool_result", content) + case _: + return iter(()) + + +def malformed_tool_result_text_error(messages: Iterable[object]) -> str | None: + blocks: Final = chain.from_iterable(map(_user_content_blocks, messages)) + return next(chain.from_iterable(map(_tool_result_text_errors, blocks)), None) + + +def malformed_tool_result_count_error( + messages: Iterable[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> str | None: + if messages_format != "anthropic" or deployment_drops_params(deployment): + return None + return malformed_tool_result_text_error(messages or ()) + + +def _scannable_part(part: object) -> object: + match as_str_mapping(part): + case {"type": "text", "text": str()}: + return part + case {"type": "text", "text": text} as text_block: + return {**text_block, "text": _scannable_text(text)} + case _: + return part + + +def _scannable_block(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + return { + **tool_result, + "content": [_scannable_part(part) for part in _list_items(tool_result.get("content"))], + } + case _: + return block + + +def _scannable_messages(messages: object) -> object: + items: Final = _list_items(messages) + return [_with_user_blocks(message, _scannable_block) for message in items] if items else messages + + +def scannable_tool_result_request(request: object) -> object: + match as_str_mapping(request): + case {"messages": messages} as request_fields: + return {**request_fields, "messages": _scannable_messages(messages)} + case None: + return request + case request_fields: + return {**request_fields} + + +def _non_string_text_parts(content: object) -> _KeptParts: + return MappingProxyType( + {index: part for index, block in enumerate(_list_items(content)) if (part := _non_str_text(block)) is not None} + ) + + +def _tool_result_non_string_texts(block: object) -> tuple[tuple[str, _KeptParts], ...]: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result: + return ((sanitize_anthropic_tool_use_id(tool_use_id), _non_string_text_parts(tool_result.get("content"))),) + case _: + return () + + +def _content_blocks(content: object) -> tuple[object, ...]: + match content: + case str() if content: + return ({"type": "text", "text": content},) + case _: + return _list_items(content) + + +def _restored_part(block: object, kept: _KeptPart | None) -> object: + return kept.block if kept is not None and tool_result_text_part(block) == kept.scanned_text else block + + +def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _KeptParts) -> object: + if not parts: + return block + blocks: Final = _content_blocks(tool_result.get("content")) + return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]} + + +def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if ( + sanitize_anthropic_tool_use_id(tool_use_id) in kept + ): + return _with_kept_parts( + block, tool_result, next(kept[sanitize_anthropic_tool_use_id(tool_use_id)], _NO_KEPT_PARTS) + ) + case _: + return block + + +def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]: + blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) + by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0))) + if not any(parts for _, parts in by_id): + return rewritten + kept_in_order: Final = MappingProxyType( + { + tool_use_id: iter(tuple(map(itemgetter(1), group))) + for tool_use_id, group in groupby(by_id, key=itemgetter(0)) + } + ) + return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] + + +def _without_non_string_text(block: object) -> object: + match as_str_mapping(block): + case {"type": "tool_result", "content": list()} as tool_result: + parts: Final = _list_items(tool_result.get("content")) + return {**tool_result, "content": [part for part in parts if _non_str_text(part) is None]} + case _: + return block + + +def _without_non_string_texts( + message: dict[str, object], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] +) -> dict[str, object]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + match message: + case {"role": "user", "content": list()}: + return {**message, "content": [_without_non_string_text(block) for block in _user_content_blocks(message)]} + case _: + return message + + +def drop_non_string_tool_result_texts( + messages: Sequence[dict[str, object]], # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + drop_params: object, +) -> Sequence[dict[str, object]]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + if not drops_unsupported_params(drop_params) or malformed_tool_result_text_error(messages) is None: + return messages + return [_without_non_string_texts(message) for message in messages] + + +def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool: + match deployment: + case {"litellm_params": {"drop_params": object() as drop_params}}: + return drops_unsupported_params(drop_params) + case _: + return drops_unsupported_params(None) + + +def reject_malformed_tool_result_text( + messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object] +) -> None: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + return + error: Final = malformed_tool_result_text_error(messages) + if error is None: + return + provider: Final = forwarded_kwargs.get("custom_llm_provider") + raise litellm.BadRequestError( + message=error, model=model, llm_provider=provider if isinstance(provider, str) else "" + ) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 30d45f3cec1..0e8c1b49e8e 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -692,7 +693,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages( + drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None)) + ), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 05ea4e4301d..2cbba168ef4 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS @@ -32,6 +32,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.utils import ProviderConfigManager from ..adapters.handler import request_safeguards_evaluator +from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -96,7 +97,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec return cache_control_as_prompt_cache_breakpoint if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True: + if drops_unsupported_params(forwarded_kwargs.get("drop_params")): return _without_prompt_cache_breakpoint return None @@ -142,6 +143,7 @@ def _build_responses_kwargs( Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + reject_malformed_tool_result_text(messages, model, forwarded_kwargs) block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs) request_messages: Final[AnthropicRequestMessages] = ( [ diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index fb611b64dda..b14a0dc0756 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts from litellm.llms.anthropic.pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, @@ -371,10 +372,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif btype == "tool_result": tool_use_id = block.get("tool_use_id", "") inner = block.get("content") + tool_result_block = cast(Mapping[str, object], block) # cast-ok: TypedDict upcast tool_breakpoint, covered_image_parts, later_image_parts = ( - self._tool_result_images_by_breakpoint( - cast(Mapping[str, object], block) # cast-ok: TypedDict upcast - ) + self._tool_result_images_by_breakpoint(tool_result_block) ) document_candidates = ( tuple( @@ -391,10 +391,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(inner, str): output_text = inner elif isinstance(inner, list): - parts = [ - c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text" - ] - output_text = "\n".join(parts) + output_text = "\n".join(tool_result_text_parts(tool_result_block)) if covered_image_parts or later_image_parts: output_text = ( f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}" diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 4006c89954d..c5ebd84e00f 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -508,6 +508,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -524,7 +525,9 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index a40970ca429..b8a6cb1beb9 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -7,6 +7,7 @@ Docs - https://docs.mistral.ai/api/ """ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload import httpx @@ -53,6 +54,9 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: return accepted +_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) + + class MistralConfig(OpenAIGPTConfig): """ Reference: https://docs.mistral.ai/api/ @@ -273,14 +277,30 @@ class MistralConfig(OpenAIGPTConfig): messages = handle_messages_with_content_list_to_str_conversion(messages) ## 3. Handle name in message - new_messages: Final[list[AllMessageValues]] = [] - for m in messages: - m = MistralConfig._handle_name_in_message(m) - m = MistralConfig._handle_tool_call_message(m) - if MistralConfig._is_empty_assistant_message(m): - continue - m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error - new_messages.append(m) + prepared_messages: Final = tuple( + MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m)) for m in messages + ) + empty_flags: Final = tuple(map(MistralConfig._is_empty_assistant_message, prepared_messages)) + tool_call_turns: Final = accumulate( + zip(count(), prepared_messages, empty_flags, strict=False), + MistralConfig._tool_call_turn, + initial=_NO_TOOL_CALL_TURN, + ) + tool_reply_keys: Final = tuple(map(MistralConfig._tool_reply_key, prepared_messages, tool_call_turns)) + first_reply_positions: Final = { + key: position for position, key in reversed(tuple(enumerate(tool_reply_keys))) if key is not None + } + keys_answered_with_content: Final = frozenset( + key for key, is_empty in zip(tool_reply_keys, empty_flags, strict=True) if not is_empty + ) + answering_positions: Final = frozenset( + position for key, position in first_reply_positions.items() if key not in keys_answered_with_content + ) + new_messages: Final[list[AllMessageValues]] = [ + strip_none_values_from_message(MistralConfig._with_tool_reply_content(m)) + for position, (m, is_empty) in enumerate(zip(prepared_messages, empty_flags, strict=True)) + if not is_empty or position in answering_positions + ] if is_async: return super()._transform_messages(new_messages, model, True) @@ -469,6 +489,44 @@ class MistralConfig(OpenAIGPTConfig): message["tool_calls"] = mistral_tool_calls return message + @staticmethod + def _tool_call_id(tool_call: object) -> str | None: + match tool_call: + case {"id": str(tool_call_id)}: + return tool_call_id + case _: + return None + + @staticmethod + def _tool_call_turn( + turn: tuple[int, frozenset[str]], message_entry: tuple[int, AllMessageValues, bool] + ) -> tuple[int, frozenset[str]]: + match message_entry: + case (position, {"role": "assistant", "tool_calls": list(tool_calls)}, _): + return position, frozenset(filter(None, map(MistralConfig._tool_call_id, tool_calls))) + case (_, {"role": "tool"}, _) | (_, _, True): + return turn + case _: + return _NO_TOOL_CALL_TURN + + @staticmethod + def _tool_reply_key(message: AllMessageValues, turn: tuple[int, frozenset[str]]) -> tuple[int, str] | None: + match message, turn: + case {"role": "tool", "tool_call_id": str(tool_call_id)}, (position, tool_call_ids) if ( + tool_call_id in tool_call_ids + ): + return position, tool_call_id + case _: + return None + + @staticmethod + def _with_tool_reply_content(message: AllMessageValues) -> AllMessageValues: + match message: + case {"role": "tool"} if not message.get("content"): + return {**message, "content": ""} + case _: + return message + @classmethod def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool: """ diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 633f154c4cd..b12fdb95765 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -11,7 +11,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error +from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -110,7 +110,9 @@ class OpenAITokenCounter(BaseTokenCounter): return None malformed_error: Final = ( - OpenAICountTokensConfig.malformed_anthropic_messages_error(messages) + OpenAICountTokensConfig.malformed_anthropic_messages_error( + messages, drop_params=deployment_drops_params(deployment) + ) if messages_format == "anthropic" else None ) diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 90bfa4a8655..438aa05a4e9 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypeAlias from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -295,14 +296,16 @@ class OpenAICountTokensConfig: return input_items, instructions @staticmethod - def malformed_anthropic_messages_error(messages: Sequence[Mapping[str, object]]) -> str | None: + def malformed_anthropic_messages_error( + messages: Sequence[Mapping[str, object]], *, drop_params: bool + ) -> str | None: try: _ANTHROPIC_COUNT_MESSAGES.validate_python(messages) except ValidationError as e: return "; ".join( f"messages.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" for error in e.errors() ) - return None + return None if drop_params else malformed_tool_result_text_error(messages) @staticmethod def anthropic_messages_to_responses_input( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3c9a8429882..8298edd6744 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1375,6 +1375,7 @@ class VertexAITokenCounter(BaseTokenCounter): count_context_error_response, gemini_count_contents, inference_request_view, + malformed_tool_result_text_count_error, provider_error_response, unconvertible_messages_error, vertex_gemini_count_context, @@ -1392,7 +1393,9 @@ class VertexAITokenCounter(BaseTokenCounter): count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools) if isinstance(count_context, VertexGeminiCountContextError): return count_context_error_response(count_context, request_model, model_to_use) - blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format) + blocks_error: Final = unconvertible_messages_error( + request_view.messages, messages_format + ) or malformed_tool_result_text_count_error(messages, messages_format, deployment) if blocks_error is not None: return count_context_error_response(blocks_error, request_model, model_to_use) diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py index c218b4710fd..b13f72e96fe 100644 --- a/litellm/llms/vertex_ai/count_tokens/context.py +++ b/litellm/llms/vertex_ai/count_tokens/context.py @@ -132,6 +132,17 @@ def unconvertible_messages_error( return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) +def malformed_tool_result_text_count_error( + messages: Sequence[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> VertexGeminiCountContextError | None: + from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error + + error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment) + return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) + + def count_context_error_response( error: VertexGeminiCountContextError, request_model: str, model_used: str ) -> TokenCountResponse: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 660d7d529cb..c1102cee8d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -368,6 +368,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.llms.anthropic.pass_through.convertible_blocks import ( + deployment_drops_params, + drop_non_string_tool_result_texts, + malformed_tool_result_count_error, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS @@ -14716,6 +14721,32 @@ def _get_provider_token_counter( return None, None, None +_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) + + +def _deployment_with_request_drop_params( + deployment: dict[str, object] | None, # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + drop_params: object, +) -> dict[str, object] | None: # mutable-ok: BaseTokenCounter.count_tokens takes a dict deployment + if drop_params is None: + return deployment + resolved_deployment: Final = deployment or {} + litellm_params: Final = _DEPLOYMENT_LITELLM_PARAMS.validate_python(resolved_deployment.get("litellm_params") or {}) + return {**resolved_deployment, "litellm_params": {**litellm_params, "drop_params": drop_params}} + + +def _counted_request_messages( + messages: Sequence[dict[str, object]] | None, # mutable-ok: TokenCountRequest.messages is typed list[dict] + messages_format: TokenCountMessagesFormat, + deployment: Mapping[str, object] | None, +) -> list[dict[str, object]] | None: # mutable-ok: BaseTokenCounter.count_tokens takes list messages + if messages is None: + return None + if messages_format != "anthropic": + return list(messages) + return list(drop_non_string_tool_result_texts(messages, deployment_drops_params(deployment))) + + async def _try_provider_token_count( provider_counter: "BaseTokenCounter", custom_llm_provider: str | None, @@ -15082,11 +15113,6 @@ _PROVIDERS_COUNTING_WITHOUT_MESSAGES: Final = frozenset( ) -def _drops_params(deployment: Mapping[str, object] | None) -> bool: - litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) - return litellm.drop_params is True or (litellm_params is not None and litellm_params.get("drop_params") is True) - - _ESTIMABLE_GOOGLE_PART_KEYS: Final = frozenset({"text", "functionCall", "functionResponse", "thoughtSignature"}) @@ -15095,7 +15121,7 @@ def _estimable_contents(contents: Sequence[Mapping[str, object]]) -> bool: return all(set(as_str_mapping(part) or ()) <= _ESTIMABLE_GOOGLE_PART_KEYS for part in parts) -def _router_default_litellm_params(router: Router) -> Mapping[str, object]: +def _router_default_litellm_params(router: Router | None) -> Mapping[str, object]: defaults: Final[Mapping[str, object] | None] = getattr(router, "default_litellm_params", None) if defaults is None: return {} @@ -15216,8 +15242,10 @@ async def count_request_tokens( if _model is not None: model_to_use = _model + sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} + count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) conversation_empty: Final = not messages and not contents - if prompt and not conversation_empty and not _drops_params(deployment): + if prompt and not conversation_empty and not deployment_drops_params(count_deployment): raise ProxyException( message="prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt", type="invalid_request_error", @@ -15225,6 +15253,7 @@ async def count_request_tokens( code=400, ) prompt_only: Final = bool(prompt) and conversation_empty + request_messages: Final = _counted_request_messages(messages, messages_format, count_deployment) contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES ladder: Final = ( @@ -15235,9 +15264,9 @@ async def count_request_tokens( _try_provider_token_count, provider_counter, custom_llm_provider, - messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages, + messages=[{"role": "user", "content": prompt}] if prompt_as_message else request_messages, contents=None if prompt_as_message else contents, - deployment=deployment, + deployment=count_deployment, request_model=request.model, tools=tools, system=system, @@ -15253,6 +15282,10 @@ async def count_request_tokens( if isinstance(ladder, TokenCountResponse): return ladder + local_count_error: Final = malformed_tool_result_count_error(request_messages, messages_format, count_deployment) + if local_count_error is not None: + raise ProxyException(message=local_count_error, type="token_counting_error", param="messages", code=400) + # Check if token counter is disabled before fallback if litellm.disable_token_counter is True: raise ProxyException( @@ -15273,7 +15306,7 @@ async def count_request_tokens( prompt_turn: Final = prompt_only and not (system_message is None and not tools) typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes Sequence[AllMessageValues] | None, - [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else messages), + [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else request_messages), ) counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 842230699b7..a2acbc09784 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,7 +45,7 @@ from litellm.constants import ( REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) @@ -326,10 +326,6 @@ def _dropped_cache_control(_: object) -> None: return None -def drops_unsupported_params(drop_params: object) -> bool: - return litellm.drop_params is True or normalize_drop_params(drop_params) is True - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item diff --git a/tests/integration/messages_endpoint/test_tool_result_text_wire.py b/tests/integration/messages_endpoint/test_tool_result_text_wire.py new file mode 100644 index 00000000000..f8c6eb6b0ca --- /dev/null +++ b/tests/integration/messages_endpoint/test_tool_result_text_wire.py @@ -0,0 +1,868 @@ +import json +import re +import uuid +from collections.abc import Callable, Mapping, Sequence +from email.parser import BytesParser +from email.policy import HTTP +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, string_value +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]) +_TOOL_USE_ID: Final = "toolu_01" +_WEATHER: Final = "sunny, 21C" +_FIELD_ERROR: Final = f"tool_result {_TOOL_USE_ID} content.0.text: Input should be a valid string" +_CLAUDE_CODE: Final = MappingProxyType({"user-agent": "claude-cli/2.1.0 (external, cli)"}) +_OPENAI_COUNT: Final = 4242 +_GOOGLE_COUNT: Final = 777 +_ANTHROPIC_COUNT: Final = 58 +_BEDROCK_COUNT: Final = 1345 +_GEMINI: Final = "gemini-4-argon" +_VERTEX_PROJECT: Final = "scripted-project" +_CITY: Final = "Paris" +_REDACTED_CITY: Final = "[CITY]" +_TOOLS: Final[list[JsonValue]] = [ + { + "name": "get_weather", + "description": "Get the weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] + + +def _messages(*blocks: JsonValue) -> list[JsonValue]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": list(blocks)}]}, + ] + + +_NULL_TEXT: Final = _messages({"type": "text", "text": None}) +_NUMBER_THEN_WEATHER: Final = _messages({"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}) +_WEATHER_ONLY: Final = _messages({"type": "text", "text": _WEATHER}) +_WEATHER_THEN_MISSING: Final = _messages({"type": "text", "text": _WEATHER}, {"type": "text"}) + + +def _body(model: str, messages: list[JsonValue], **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": messages, "tools": _TOOLS, **extra} + + +def _responses_reply() -> bytes: + return json.dumps( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}], + } + ], + "usage": {"input_tokens": 41, "output_tokens": 5, "total_tokens": 46}, + } + ).encode() + + +def _chat_reply() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "mistral-small-latest", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "It is sunny."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}, + } + ).encode() + + +def _anthropic_reply() -> bytes: + return json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "It is sunny."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 9, "output_tokens": 4}, + } + ).encode() + + +def _openai_peer(request: Request) -> Reply: + if request.target.endswith("/responses/input_tokens"): + return Reply(body=json.dumps({"input_tokens": _OPENAI_COUNT}).encode()) + if request.target.endswith("/responses"): + return Reply(body=_responses_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _chat_peer(request: Request) -> Reply: + if request.target.endswith("/chat/completions"): + return Reply(body=_chat_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _google_peer(request: Request) -> Reply: + if request.target.endswith(":countTokens"): + return Reply(body=json.dumps({"totalTokens": _GOOGLE_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _anthropic_peer(request: Request) -> Reply: + if request.target.endswith("/v1/messages/count_tokens"): + return Reply(body=json.dumps({"input_tokens": _ANTHROPIC_COUNT}).encode()) + if request.target.endswith("/v1/messages"): + return Reply(body=_anthropic_reply()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _bedrock_peer(request: Request) -> Reply: + if request.target.endswith("/count-tokens"): + return Reply(body=json.dumps({"inputTokens": _BEDROCK_COUNT}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _openai_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model(model="openai/gpt-5.4-mini", api_base=wire.url, api_key="synthetic-openai-key", **parameters) + + +def _mistral_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="mistral/mistral-small-latest", api_base=wire.url, api_key="synthetic-mistral-key", **parameters + ) + + +def _anthropic_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str: + return scenario.model( + model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-anthropic-key", **parameters + ) + + +def _google_model(gateway: Gateway, scenario: Scenario, wire: Wire, provider: str, **parameters: JsonValue) -> str: + if provider == "gemini": + return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire.url, api_key="scripted-gemini-key", **parameters) + return scenario.model( + model=f"vertex_ai/{_GEMINI}", + api_base=wire.url, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location="us-east5", + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + **parameters, + ) + + +def _json(request: Request) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(request.body) + + +def _only(requests: Sequence[Request], suffix: str) -> dict[str, JsonValue]: + matching: Final = tuple(request for request in requests if request.target.endswith(suffix)) + assert len(matching) == 1, [request.target for request in requests] + return _json(matching[0]) + + +def _function_call_output(body: Mapping[str, JsonValue]) -> JsonValue: + outputs: Final = tuple( + item["output"] + for item in _JSON_OBJECTS.validate_python(body["input"]) + if item.get("type") == "function_call_output" + ) + assert len(outputs) == 1, body + return outputs[0] + + +def _tool_message_text(body: Mapping[str, JsonValue]) -> str: + contents: Final = tuple( + message.get("content") + for message in _JSON_OBJECTS.validate_python(body["messages"]) + if message.get("role") == "tool" + ) + assert len(contents) == 1, body + content: Final = contents[0] + if isinstance(content, str): + return content + return "".join(str(part["text"]) for part in _JSON_OBJECTS.validate_python(content)) + + +def _anthropic_error_message(status: int, text: str, payload: bytes) -> str: + assert status == 400, text + error: Final = _JSON_OBJECT.validate_json(payload) + assert error["type"] == "error", text + detail: Final = _JSON_OBJECT.validate_python(error["error"]) + assert detail["type"] == "invalid_request_error", text + return str(detail["message"]) + + +def _assert_field_rejection(gateway: Gateway, path: str, body: Mapping[str, JsonValue], **headers: str) -> None: + response: Final = gateway.request("POST", path, body, headers=headers) + assert _FIELD_ERROR in _anthropic_error_message(response.status_code, response.text, response.content), ( + response.text + ) + + +def _sdk(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client + ) + + +def _assert_sdk_bad_request(raised: pytest.ExceptionInfo[anthropic.BadRequestError]) -> None: + error: Final = raised.value + assert error.status_code == 400, error + assert _FIELD_ERROR in _anthropic_error_message(error.status_code, error.message, error.response.content), error + + +def test_openai_rejects_a_non_string_tool_result_text_on_both_routes(gateway: Gateway) -> None: + client: Final = _sdk(gateway) + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire) + with pytest.raises(anthropic.BadRequestError) as created: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(created) + with pytest.raises(anthropic.BadRequestError) as streamed: + client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS, stream=True) + _assert_sdk_bad_request(streamed) + with pytest.raises(anthropic.BadRequestError) as counted: + client.messages.count_tokens(model=model, messages=_NULL_TEXT, tools=_TOOLS) + _assert_sdk_bad_request(counted) + assert wire.drain() == () + + +_DROP_SOURCES: Final = MappingProxyType( + { + "deployment": (MappingProxyType({"drop_params": True}), MappingProxyType({}), MappingProxyType({})), + "body": (MappingProxyType({}), MappingProxyType({"drop_params": True}), MappingProxyType({})), + "claude_code": (MappingProxyType({}), MappingProxyType({}), _CLAUDE_CODE), + } +) + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_openai_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _OPENAI_COUNT, counted.text + requests: Final = wire.drain() + assert _function_call_output(_only(requests, "/responses")) == _WEATHER + assert _function_call_output(_only(requests, "/responses/input_tokens")) == _WEATHER + + +def test_explicit_drop_params_false_wins_on_both_routes(gateway: Gateway) -> None: + with wire_server(_openai_peer) as wire, gateway.scenario() as scenario: + model: Final = _openai_model(scenario, wire, drop_params=True) + body: Final = _body(model, _NULL_TEXT, drop_params=False) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}, **_CLAUDE_CODE) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body, **_CLAUDE_CODE) + assert wire.drain() == () + + +def test_chat_bridge_and_local_count_reject_a_non_string_tool_result_text(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}) + _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64, "stream": True}) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", body) + assert wire.drain() == () + + +@pytest.mark.parametrize("source", ("deployment", "body")) +def test_chat_bridge_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, _ = _DROP_SOURCES[source] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _tool_message_text(_only(wire.drain(), "/chat/completions")) == _WEATHER + + +def test_count_without_a_deployment_rejects_and_honors_body_drop_params(gateway: Gateway) -> None: + model: Final = f"integration-unrouted-{uuid.uuid4().hex}" + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(model, _NULL_TEXT)) + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(model, _NULL_TEXT, drop_params=True)) + assert dropped.status_code == 200, dropped.text + tokens: Final = _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] + assert isinstance(tokens, int) and tokens > 0, dropped.text + + +def _counted_contents(body: Mapping[str, JsonValue]) -> JsonValue: + if "contents" in body: + return body["contents"] + return _JSON_OBJECT.validate_python(body["generateContentRequest"])["contents"] + + +@pytest.mark.parametrize("provider", ("gemini", "vertex_ai")) +def test_google_count_rejects_and_drops_a_non_string_tool_result_text(gateway: Gateway, provider: str) -> None: + with wire_server(_google_peer) as wire, gateway.scenario() as scenario: + strict: Final = _google_model(gateway, scenario, wire, provider) + dropping: Final = _google_model(gateway, scenario, wire, provider, drop_params=True) + _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(strict, _NULL_TEXT)) + assert tuple(request for request in wire.drain() if request.target.endswith(":countTokens")) == () + dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _NUMBER_THEN_WEATHER)) + assert dropped.status_code == 200, dropped.text + dropped_body: Final = _only(wire.drain(), ":countTokens") + clean: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _WEATHER_ONLY)) + assert clean.status_code == 200, clean.text + assert _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] == _GOOGLE_COUNT, dropped.text + assert _counted_contents(dropped_body) == _counted_contents(_only(wire.drain(), ":countTokens")) + + +def _forwarded_tool_result(body: Mapping[str, JsonValue]) -> JsonValue: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return _JSON_OBJECTS.validate_python(last["content"])[0]["content"] + + +@pytest.mark.parametrize( + ("headers", "forwarded"), + ((MappingProxyType({}), [{"type": "text", "text": None}]), (_CLAUDE_CODE, [])), + ids=("no-drop-params", "claude-code-drop-params"), +) +def test_native_anthropic_forwards_a_null_tool_result_text_unless_drop_params_drops_it( + gateway: Gateway, headers: Mapping[str, str], forwarded: list[JsonValue] +) -> None: + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire) + body: Final = _body(model, _NULL_TEXT) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +def test_bedrock_count_with_a_claude_code_user_agent_still_counts(gateway: Gateway) -> None: + with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="bedrock/anthropic.claude-sonnet-4-6", + api_base=wire.url, + api_key=None, + aws_access_key_id="AKIAINTEGRATIONTOOLRESULT", + aws_secret_access_key="integration-tool-result-secret", + aws_region_name="us-east-1", + ) + counted: Final = gateway.request( + "POST", "/v1/messages/count_tokens", _body(model, _WEATHER_ONLY), headers=_CLAUDE_CODE + ) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _BEDROCK_COUNT, counted.text + assert len(tuple(request for request in wire.drain() if request.target.endswith("/count-tokens"))) == 1 + + +@pytest.mark.parametrize("provider", ("openai", "mistral")) +def test_valid_tool_result_text_reaches_the_upstream(gateway: Gateway, provider: str) -> None: + peer: Final = _openai_peer if provider == "openai" else _chat_peer + with wire_server(peer) as wire, gateway.scenario() as scenario: + model: Final = (_openai_model if provider == "openai" else _mistral_model)(scenario, wire) + body: Final = _body(model, _WEATHER_THEN_MISSING) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + requests: Final = wire.drain() + if provider == "openai": + assert _function_call_output(_only(requests, "/responses")) == f"{_WEATHER}\n" + assert _function_call_output(_only(requests, "/responses/input_tokens")) == f"{_WEATHER}\n" + else: + assert _tool_message_text(_only(requests, "/chat/completions")) == _WEATHER + + +def _rewritten(content: JsonValue, rewrite: Callable[[str], str]) -> JsonValue: + if isinstance(content, str): + return rewrite(content) + if isinstance(content, list): + return [_rewritten(part, rewrite) for part in content] + if isinstance(content, dict) and isinstance(text := content.get("text"), str): + return {**content, "text": rewrite(text)} + return content + + +def _redacted(content: JsonValue) -> JsonValue: + return _rewritten(content, lambda text: text.replace(_CITY, _REDACTED_CITY)) + + +def _redacted_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {**row, "content": _redacted(row["content"])} if "content" in row else dict(row) + + +def _digits_masked_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + if "content" not in row: + return dict(row) + return {**row, "content": _rewritten(row["content"], lambda text: re.sub(r"\d", "#", text))} + + +def _joined_tool_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + redacted: Final = _redacted_row(row) + content: Final = redacted.get("content") + if row.get("role") != "tool" or not isinstance(content, list): + return redacted + texts: Final = ( + part["text"] for part in _JSON_OBJECTS.validate_python(content) if isinstance(part.get("text"), str) + ) + return {**redacted, "content": " | ".join(str(text) for text in texts)} + + +_ROW_REWRITES: Final = MappingProxyType({"join": _joined_tool_row, "digits": _digits_masked_row}) + + +def _guardrail_peer(request: Request) -> Reply: + rows: Final = _JSON_OBJECTS.validate_python(_json(request).get("structured_messages") or []) + if request.target.startswith("/none/") or not rows: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + rewrite: Final = _ROW_REWRITES.get(request.target.split("/")[1], _redacted_row) + verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [rewrite(row) for row in rows]} + return Reply(body=json.dumps(verdict).encode()) + + +def _delete_guardrail(gateway: Gateway, identity: str) -> None: + deleted: Final = gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def _guardrail(gateway: Gateway, scenario: Scenario, wire: Wire, mode: str) -> str: + name: Final = f"tool-result-{mode}-{uuid.uuid4().hex}" + params: Final[dict[str, JsonValue]] = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{wire.url}/{mode}", + "api_key": "synthetic-guardrail-key", + } + created: Final = gateway.post("/guardrails", {"guardrail": {"guardrail_name": name, "litellm_params": params}}) + scenario.cleanups.callback(_delete_guardrail, gateway, string_value(created["guardrail_id"])) + return name + + +def _first_turn(messages: JsonValue) -> str: + return json.dumps(_JSON_OBJECTS.validate_python(messages)[0]) + + +def test_rewriting_guardrail_leaves_the_rejection_to_the_openai_route(gateway: Gateway) -> None: + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider) + body: Final = _body(model, _NULL_TEXT, max_tokens=64, guardrails=[_guardrail(gateway, scenario, guard, "rows")]) + _assert_field_rejection(gateway, "/v1/messages", body) + _assert_field_rejection(gateway, "/v1/messages", {**body, "stream": True}) + assert provider.drain() == () + + +@pytest.mark.parametrize("source", sorted(_DROP_SOURCES)) +def test_rewriting_guardrail_then_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None: + deployment, extra, headers = _DROP_SOURCES[source] + with ( + wire_server(_openai_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _openai_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail], **extra) + created: Final = gateway.request("POST", "/v1/messages", body, headers=headers) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/responses") + first_turn: Final = _first_turn(forwarded["input"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _function_call_output(forwarded) == _WEATHER, forwarded + + +@pytest.mark.parametrize( + ("deployment", "forwarded"), + ( + (MappingProxyType({}), [{"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}]), + (MappingProxyType({"drop_params": True}), [{"type": "text", "text": _WEATHER}]), + ), + ids=("no-drop-params", "deployment-drop-params"), +) +def test_rewriting_guardrail_forwards_the_malformed_text_to_anthropic_unless_drop_params_drops_it( + gateway: Gateway, deployment: Mapping[str, JsonValue], forwarded: list[JsonValue] +) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider, **deployment) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded_body: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == forwarded + + +_MORNING: Final = f"{_CITY} morning: sunny" +_EVENING: Final = f"{_CITY} evening: 18C" +_TOMORROW: Final = f"{_CITY} tomorrow: rain" +_WEEKEND: Final = f"{_CITY} weekend: clear" + + +def _text(value: JsonValue) -> dict[str, JsonValue]: + return {"type": "text", "text": value} + + +def _tool_use(tool_use_id: str) -> dict[str, JsonValue]: + return {"type": "tool_use", "id": tool_use_id, "name": "get_weather", "input": {"city": _CITY}} + + +def _tool_result(tool_use_id: str, *parts: JsonValue) -> dict[str, JsonValue]: + return {"type": "tool_result", "tool_use_id": tool_use_id, "content": [_text(part) for part in parts]} + + +_SEVERAL_TOOL_RESULTS: Final[list[JsonValue]] = [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use("toolu_01"), _tool_use("toolu_02"), _tool_use("toolu_03")]}, + { + "role": "user", + "content": [ + _tool_result("toolu_01", None, _MORNING, 5, _EVENING), + _tool_result("toolu_02", _TOMORROW, 7.5), + _tool_result("toolu_03", _WEEKEND), + ], + }, +] + + +def _forwarded_tool_results(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1] + return { + string_value(block["tool_use_id"]): block["content"] + for block in _JSON_OBJECTS.validate_python(last["content"]) + if block.get("type") == "tool_result" + } + + +def _texts(content: JsonValue) -> tuple[JsonValue, ...]: + if isinstance(content, str): + return (content,) + return tuple(part.get("text") for part in _JSON_OBJECTS.validate_python(content)) + + +_RESTORED_TOOL_RESULTS: Final = MappingProxyType( + { + "rows": ( + [_text(None), _text(_redacted(_MORNING)), _text(5), _text(_redacted(_EVENING))], + [_text(_redacted(_TOMORROW)), _text(7.5)], + ), + "join": ( + [_text(f"null | {_redacted(_MORNING)} | 5 | {_redacted(_EVENING)}")], + [_text(f"{_redacted(_TOMORROW)} | 7.5")], + ), + } +) + + +@pytest.mark.parametrize("mode", sorted(_RESTORED_TOOL_RESULTS)) +def test_rewriting_guardrail_puts_back_each_malformed_text_it_left_unchanged(gateway: Gateway, mode: str) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, mode) + body: Final = _body(model, _SEVERAL_TOOL_RESULTS, max_tokens=64, guardrails=[guardrail]) + created: Final = gateway.request("POST", "/v1/messages", body) + assert created.status_code == 200, created.text + forwarded: Final = _only(provider.drain(), "/v1/messages") + first_turn: Final = _first_turn(forwarded["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + tool_results: Final = _forwarded_tool_results(forwarded) + assert _CITY not in json.dumps(tool_results), tool_results + assert (tool_results["toolu_01"], tool_results["toolu_02"]) == _RESTORED_TOOL_RESULTS[mode], tool_results + assert _texts(tool_results["toolu_03"]) == (_redacted(_WEEKEND),), tool_results + + +def _null_text_answering(tool_use_id: str) -> list[JsonValue]: + return [ + {"role": "user", "content": f"What is the weather in {_CITY}?"}, + {"role": "assistant", "content": [_tool_use(tool_use_id)]}, + {"role": "user", "content": [_tool_result(tool_use_id, None)]}, + ] + + +@pytest.mark.parametrize("tool_use_id", ("toolu_01", "functions.get_weather:0")) +def test_rewriting_guardrail_leaves_the_rejection_to_the_chat_bridge_for_any_tool_use_id( + gateway: Gateway, tool_use_id: str +) -> None: + with ( + wire_server(_chat_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _mistral_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "rows") + body: Final = _body(model, _null_text_answering(tool_use_id), max_tokens=64, guardrails=[guardrail]) + response: Final = gateway.request("POST", "/v1/messages", body) + message: Final = _anthropic_error_message(response.status_code, response.text, response.content) + assert "content.0.text: Input should be a valid string" in message, message + assert provider.drain() == () + + +def _files_peer(request: Request) -> Reply: + if request.method == "POST" and request.target.endswith("/files"): + uploaded: Final = { + "id": f"file-{uuid.uuid4().hex}", + "object": "file", + "bytes": len(request.body), + "created_at": 1789788253, + "filename": "batch.jsonl", + "purpose": "batch", + "status": "processed", + } + return Reply(body=json.dumps(uploaded).encode()) + if request.target.endswith("/models"): + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _uploaded_records(request: Request) -> tuple[dict[str, JsonValue], ...]: + envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body + parts: Final = BytesParser(policy=HTTP).parsebytes(envelope).iter_parts() + files: Final = tuple(part.get_payload(decode=True) for part in parts if part.get_filename() is not None) + assert len(files) == 1, request.headers + payload: Final = files[0] + assert isinstance(payload, bytes), payload + return tuple(_JSON_OBJECT.validate_json(line) for line in payload.splitlines() if line.strip()) + + +def _listed_models(gateway: Gateway) -> frozenset[str]: + with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client: + listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"}) + assert listed.status_code == 200, listed.text + entries: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(listed.content)["data"]) + return frozenset(string_value(entry["id"]) for entry in entries) + + +def _uploaded_batch_body( + gateway: Gateway, scenario: Scenario, guard: Wire, mode: str, messages: list[JsonValue] +) -> dict[str, JsonValue]: + with wire_server(_files_peer) as provider: + model: Final = _openai_model(scenario, provider) + key: Final = scenario.key(metadata={"guardrails": [_guardrail(gateway, scenario, guard, mode)]}) + eventually( + lambda: tuple(_listed_models(gateway) for _ in range(8)), + lambda listings: all(model in listing for listing in listings), + seconds=90, + ) + record: Final = { + "custom_id": f"weather-{uuid.uuid4().hex}", + "method": "POST", + "url": "/v1/messages", + "body": _body(model, messages, max_tokens=64), + } + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "model": model}, + {"file": ("batch.jsonl", json.dumps(record).encode() + b"\n", "application/jsonl")}, + key=key, + ) + assert uploaded.status_code == 200, uploaded.text + uploads: Final = tuple(request for request in provider.drain() if request.target.endswith("/files")) + assert len(uploads) == 1, [request.target for request in uploads] + (forwarded,) = _uploaded_records(uploads[0]) + assert forwarded["custom_id"] == record["custom_id"], forwarded + return _JSON_OBJECT.validate_python(forwarded["body"]) + + +def test_batch_upload_behind_a_rewriting_guardrail_keeps_the_malformed_text(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "rows", _NUMBER_THEN_WEATHER) + first_turn: Final = _first_turn(forwarded_body["messages"]) + assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn + assert _forwarded_tool_result(forwarded_body) == [_text(5), _text(_WEATHER)], forwarded_body + + +def test_guardrail_that_rewrites_nothing_forwards_a_null_tool_result_text_unchanged(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "none") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + assert len(guard.drain()) == 1 + assert _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) == [{"type": "text", "text": None}] + + +def _forwarded_chat_turns(wire: Wire) -> tuple[dict[str, JsonValue], ...]: + return _JSON_OBJECTS.validate_python(_only(wire.drain(), "/chat/completions")["messages"]) + + +def test_mistral_receives_the_tool_reply_whose_only_text_drop_params_dropped(gateway: Gateway) -> None: + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, drop_params=True) + ) + assert created.status_code == 200, created.text + turns: Final = _forwarded_chat_turns(wire) + assert [turn["role"] for turn in turns] == ["user", "assistant", "tool"], turns + calls: Final = _JSON_OBJECTS.validate_python(turns[1]["tool_calls"]) + assert [call["id"] for call in calls] == [turns[2]["tool_call_id"]], turns + assert _tool_message_text({"messages": list(turns)}) == "", turns + + +_WEATHER_CALL: Final[dict[str, JsonValue]] = { + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": _CITY})}, +} + + +def test_mistral_chat_completions_keeps_an_empty_tool_reply(gateway: Gateway) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "system", "content": ""}, + {"role": "user", "content": ask}, + {"role": "assistant", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": ""}, + {"role": "user", "content": ""}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("content")) for turn in turns] == [ + ("user", ask), + ("assistant", None), + ("tool", ""), + ], turns + assert turns[2]["tool_call_id"] == "call_weather", turns + + +def test_mistral_chat_completions_drops_an_orphan_tool_message_and_sends_a_null_tool_reply_as_empty_text( + gateway: Gateway, +) -> None: + ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}" + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": ask}, + {"role": "tool", "tool_call_id": "call_orphan", "content": ""}, + {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + {"role": "tool", "tool_call_id": "call_weather", "content": None}, + ] + with wire_server(_chat_peer) as wire, gateway.scenario() as scenario: + model: Final = _mistral_model(scenario, wire) + response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages}) + assert response.status_code == 200, response.text + turns: Final = _forwarded_chat_turns(wire) + assert [(turn["role"], turn.get("tool_call_id"), turn.get("content")) for turn in turns] == [ + ("user", None, ask), + ("assistant", None, None), + ("tool", "call_weather", ""), + ], turns + + +_NATIVE_DROP_PARAMS_CASES: Final = MappingProxyType( + { + "body": (_DROP_SOURCES["body"], [_text(_WEATHER)]), + "deployment": (_DROP_SOURCES["deployment"], [_text(_WEATHER)]), + "explicit-false": ( + (MappingProxyType({"drop_params": True}), MappingProxyType({"drop_params": False}), _CLAUDE_CODE), + [_text(5), _text(_WEATHER)], + ), + } +) + + +@pytest.mark.parametrize("case", sorted(_NATIVE_DROP_PARAMS_CASES)) +def test_native_anthropic_resolves_drop_params_the_same_way_on_both_routes(gateway: Gateway, case: str) -> None: + (deployment, extra, headers), forwarded = _NATIVE_DROP_PARAMS_CASES[case] + with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_model(scenario, wire, **deployment) + body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra) + created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers) + counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers) + assert created.status_code == 200, created.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text + requests: Final = wire.drain() + assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded + assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded + + +_CARD_NUMBER: Final = 4111111111111111 +_CARD_NULL_WEATHER: Final = _messages(_text(_CARD_NUMBER), _text(None), _text(_WEATHER)) +_SCANNED_CARD_NULL_WEATHER: Final = ("4111111111111111", "null", _WEATHER) +_MASKED_CARD_NULL_WEATHER: Final = [_text("################"), _text(None), _text("sunny, ##C")] + + +def _scanned_tool_texts(guard: Wire) -> tuple[JsonValue, ...]: + requests: Final = guard.drain() + assert len(requests) == 1, [request.target for request in requests] + rows: Final = _JSON_OBJECTS.validate_python(_json(requests[0])["structured_messages"]) + tool_rows: Final = tuple(row for row in rows if row.get("role") == "tool") + assert len(tool_rows) == 1, rows + return _texts(tool_rows[0]["content"]) + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_before_anthropic_gets_it(gateway: Gateway) -> None: + with ( + wire_server(_anthropic_peer) as provider, + wire_server(_guardrail_peer) as guard, + gateway.scenario() as scenario, + ): + model: Final = _anthropic_model(scenario, provider) + guardrail: Final = _guardrail(gateway, scenario, guard, "digits") + created: Final = gateway.request( + "POST", "/v1/messages", _body(model, _CARD_NULL_WEATHER, max_tokens=64, guardrails=[guardrail]) + ) + assert created.status_code == 200, created.text + scanned: Final = _scanned_tool_texts(guard) + forwarded: Final = _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert forwarded == _MASKED_CARD_NULL_WEATHER, forwarded + + +def test_masking_guardrail_scans_a_numeric_tool_result_text_in_a_batch_upload(gateway: Gateway) -> None: + with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario: + forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "digits", _CARD_NULL_WEATHER) + scanned: Final = _scanned_tool_texts(guard) + assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned + assert _forwarded_tool_result(forwarded_body) == _MASKED_CARD_NULL_WEATHER, forwarded_body diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index ec6af7f27c3..046f448046b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_tools_pt, _rename_duplicate_bedrock_document_names, _convert_to_bedrock_tool_call_invoke, - _sanitize_anthropic_tool_use_id, + sanitize_anthropic_tool_use_id, _convert_to_bedrock_tool_call_result, anthropic_messages_pt, convert_to_anthropic_tool_result, @@ -4503,7 +4503,7 @@ def test_get_tool_calls_from_response_keeps_sanitized_concat_ids_distinct(): ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(sanitized) == len(set(sanitized)) assert ids == ["a:b", "a:b__concat_2", "a_b__concat_1"] @@ -4517,7 +4517,7 @@ def test_get_tool_calls_from_response_bumps_suffix_when_sibling_sanitizes_onto_i ) ids: Final = [call["id"] for call in get_tool_calls_from_response(response)] - sanitized: Final = [_sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] + sanitized: Final = [sanitize_anthropic_tool_use_id(call_id) for call_id in ids if isinstance(call_id, str)] assert len(ids) == len(sanitized) assert len(sanitized) == len(set(sanitized)) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 0aca60351bf..58e3b55773b 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -155,6 +155,17 @@ class MockStructuredMaskingGuardrail(CustomGuardrail): return masked +class MockNumberMaskingGuardrail(MockStructuredMaskingGuardrail): + @staticmethod + def _mask(text: str) -> str: + return ( + text.replace("bob@example.com", "") + .replace("5", "") + .replace("null", "") + .replace("山田太郎", "") + ) + + class TestAnthropicMessagesHandlerStreamingRequestData: """Post-call guardrails on streaming /v1/messages receive the response and identity metadata""" @@ -2499,7 +2510,7 @@ class TestStructuredWriteBackKeepsToolResults: """A guardrail rewrite must never leave a tool_use without its tool_result (Claude Code ToolSearch, LIT-6103).""" @staticmethod - def _claude_code_tool_search_turns(tool_result_content): + def _claude_code_tool_search_turns(tool_result_content, tool_use_id="toolu_01"): return [ {"role": "user", "content": "load WebFetch for bob@example.com"}, { @@ -2507,7 +2518,7 @@ class TestStructuredWriteBackKeepsToolResults: "content": [ { "type": "tool_use", - "id": "toolu_01", + "id": tool_use_id, "name": "ToolSearch", "input": {"query": "select:WebFetch"}, } @@ -2516,7 +2527,7 @@ class TestStructuredWriteBackKeepsToolResults: { "role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}, + {"type": "tool_result", "tool_use_id": tool_use_id, "content": tool_result_content}, {"type": "text", "text": "Now fetch the page."}, ], }, @@ -2563,6 +2574,124 @@ class TestStructuredWriteBackKeepsToolResults: later_blocks = [b for m in messages[tool_use_index + 1 :] for b in self._blocks(m)] assert {"type": "text", "text": "Now fetch the page."} in later_blocks + @pytest.mark.parametrize( + ("tool_result_content", "expected_written_back_content"), + [ + ( + [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}], + [{"type": "text", "text": None}, {"type": "text", "text": "log for "}], + ), + ( + [{"type": "text", "text": "log for bob@example.com"}, {"type": "text", "text": 73}], + [{"type": "text", "text": "log for "}, {"type": "text", "text": 73}], + ), + ([{"type": "text", "text": None}], [{"type": "text", "text": None}]), + ( + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for bob@example.com"}, + {"type": "text", "text": 5}, + {"type": "text", "text": "bob@example.com again"}, + ], + [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + {"type": "text", "text": 5}, + {"type": "text", "text": " again"}, + ], + ), + ], + ids=["null-first", "number-last", "null-only", "interleaved"], + ) + async def test_rewrite_keeps_a_non_string_tool_result_text_in_place( + self, tool_result_content, expected_written_back_content + ): + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert "bob@example.com" not in json.dumps(data["messages"]) + assert data["messages"][2]["content"][0] == { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": expected_written_back_content, + } + + async def test_rewrite_keeps_a_non_string_tool_result_text_whose_tool_use_id_gets_sanitized(self): + tool_result_content = [{"type": "text", "text": None}, {"type": "text", "text": "log for bob@example.com"}] + data = { + "model": "claude-fable-5", + "messages": self._claude_code_tool_search_turns(tool_result_content, tool_use_id="functions.get_weather:0"), + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": None}, + {"type": "text", "text": "log for "}, + ] + + async def test_rewrite_keeps_each_non_string_text_when_two_tool_use_ids_sanitize_alike(self): + data = { + "model": "claude-fable-5", + "messages": [ + {"role": "user", "content": "check both logs for bob@example.com"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "read:log", "name": "Read", "input": {"path": "a.log"}}, + {"type": "tool_use", "id": "read_log", "name": "Read", "input": {"path": "b.log"}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "read:log", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + }, + { + "type": "tool_result", + "tool_use_id": "read_log", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "bob@example.com"}], + }, + ], + }, + ], + } + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockStructuredMaskingGuardrail() + ) + + assert [block["content"] for block in data["messages"][2]["content"]] == [ + [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + [{"type": "text", "text": 5}, {"type": "text", "text": ""}], + ] + + @pytest.mark.parametrize( + ("text", "masked"), + [(5, ""), (None, ""), ({"owner": "山田太郎"}, '{"owner": ""}')], + ids=["number", "null", "non-ascii-object"], + ) + async def test_guardrail_scans_and_masks_a_non_string_tool_result_text(self, text, masked): + tool_result_content = [{"type": "text", "text": text}, {"type": "text", "text": "log for bob@example.com"}] + data = {"model": "claude-fable-5", "messages": self._claude_code_tool_search_turns(tool_result_content)} + + await AnthropicMessagesHandler().process_input_messages( + data=data, guardrail_to_apply=MockNumberMaskingGuardrail() + ) + + assert data["messages"][2]["content"][0]["content"] == [ + {"type": "text", "text": masked}, + {"type": "text", "text": "log for "}, + ] + class TestAnthropicMessagesHandlerStreamingScanKey: """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 9e0bd5c60b8..0e918cd49c5 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,10 +1,13 @@ import base64 import functools import json +from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast +import httpx import pytest +import respx import litellm @@ -5235,3 +5238,153 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta(): assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"] assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA + + +_CHAT_BRIDGE_MODEL: Final = "azure/gpt-5.6" +_CHAT_BRIDGE_API_BASE: Final = "https://tool-result-text.openai.azure.com" +_CHAT_BRIDGE_URL: Final = r".*tool-result-text\.openai\.azure\.com/openai/deployments/gpt-5\.6/chat/completions.*" +_CHAT_BRIDGE_REPLY: Final = { + "id": "chatcmpl-tool-result-text", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "build passed"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, +} +_MALFORMED_TOOL_RESULT_TEXTS: Final = (pytest.param(None, id="null"), pytest.param(5, id="number")) +_MALFORMED_TOOL_RESULT_TEXT_ERROR: Final = ( + "litellm.BadRequestError: tool_result toolu_log content.1.text: Input should be a valid string" +) + + +def _tool_result_with_text_blocks(text_block: Mapping[str, object]) -> list[dict[str, object]]: + blocks: Final = [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}, text_block] + return [ + {"role": "user", "content": "run the build"}, + dict(_anthropic_tool_use_turn("toolu_log")), + dict(_anthropic_tool_result_turn({"toolu_log": blocks})), + ] + + +def _chat_bridge_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(url__regex=_CHAT_BRIDGE_URL).mock(return_value=httpx.Response(200, json=_CHAT_BRIDGE_REPLY)) + + +def _sent_tool_messages(route: respx.Route) -> list[object]: + sent: Final = json.loads(route.calls.last.request.read()) + return [message for message in sent["messages"] if message["role"] == "tool"] + + +@pytest.fixture +def _chat_bridge_httpx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_async_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +def test_chat_bridge_sync_malformed_tool_result_text_answers_400_before_any_provider_call( + respx_mock: respx.MockRouter, text: object, stream: bool +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + with pytest.raises(litellm.BadRequestError) as rejected: + litellm.anthropic.messages.create( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + stream=stream, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + ) + + assert rejected.value.status_code == 400 + assert rejected.value.llm_provider == "azure" + assert rejected.value.message == _MALFORMED_TOOL_RESULT_TEXT_ERROR + assert route.call_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +@pytest.mark.parametrize("text", _MALFORMED_TOOL_RESULT_TEXTS) +async def test_chat_bridge_drop_params_drops_a_malformed_tool_result_text( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, text: object, drop_params_source: str +) -> None: + route: Final = _chat_bridge_route(respx_mock) + if drop_params_source == "global": + monkeypatch.setattr(litellm, "drop_params", True) + + response: Final = await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text", "text": text}), + **({"drop_params": True} if drop_params_source == "request" else {}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": "done"}], + } + ] + assert isinstance(response, dict) + assert response["content"] == [{"type": "text", "text": "build passed"}] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model_as_empty_text( + respx_mock: respx.MockRouter, +) -> None: + route: Final = _chat_bridge_route(respx_mock) + + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_tool_result_with_text_blocks({"type": "text"}), + ) + + assert _sent_tool_messages(route) == [ + { + "role": "tool", + "tool_call_id": "toolu_log", + "content": [ + {"type": "text", "text": "build ok"}, + {"type": "text", "text": ""}, + {"type": "text", "text": "done"}, + {"type": "text", "text": ""}, + ], + } + ] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py index c63bfbdd301..7fbec3ba12e 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py @@ -300,3 +300,67 @@ async def test_a_json_provider_gets_safeguards_and_the_beta_together_unless_drop assert sent.headers["anthropic-beta"] == forwarded_beta assert "safeguard_results" not in response assert response["content"][0]["text"] == "hello" + + +@pytest.mark.parametrize( + ("drop_params", "expected_tool_result_content"), + [ + (True, [{"type": "text", "text": "sunny"}]), + (None, [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}]), + ], + ids=["drop_params", "no_drop_params"], +) +def test_drop_params_drops_a_non_string_tool_result_text_on_the_native_request( + drop_params, expected_tool_result_content +): + messages = [ + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": 5}, {"type": "text", "text": "sunny"}], + } + ], + }, + ] + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request["messages"][2]["content"][0]["content"] == expected_tool_result_content + + +def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_request(): + tool_result_turn = { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": None}, {"type": "text", "text": "sunny"}], + } + ], + } + + request = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "What is the weather?"}, 7, tool_result_turn], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + + assert request["messages"][1] == 7 + assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py index c55fa476c06..409e64635f7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py @@ -617,3 +617,88 @@ async def test_tool_search_forwards_deferred_tools_and_beta_header( assert [block for block in response["content"] if block["type"] == "tool_use"] == [ {"type": "tool_use", "id": "toolu_1", "name": tool_name, "input": tool_input} ] + + +_MIXED_GROUP: Final = "gpt-5.6-mixed" +_MIXED_AZURE_API_BASE: Final = "https://mixed-group.openai.azure.com" +_MIXED_AZURE_CHAT_URL: Final = r".*mixed-group\.openai\.azure\.com/openai/deployments/.*/chat/completions.*" +_MIXED_DEPLOYMENT_IDS: Final = ("mixed-openai-responses-bridge", "mixed-azure-chat-bridge") +_MIXED_AZURE_CHAT_BODY: Final = { + "id": "chatcmpl-mixed", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": _JOKE}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, +} + + +def _mixed_openai_azure_router() -> Router: + return Router( + model_list=[ + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[0]}, + }, + { + "model_name": _MIXED_GROUP, + "litellm_params": {"model": "azure/gpt-5.6", "api_key": "fake-key", "api_base": _MIXED_AZURE_API_BASE}, + "model_info": {"id": _MIXED_DEPLOYMENT_IDS[1]}, + }, + ], + num_retries=0, + ) + + +async def _null_tool_result_text_rejection(router: Router, deployment_id: str) -> litellm.BadRequestError: + with pytest.raises(litellm.BadRequestError) as rejected: + await router.aanthropic_messages( + model=deployment_id, + max_tokens=64, + messages=[ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_build", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_build", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + ) + return rejected.value + + +@pytest.mark.asyncio +async def test_mixed_openai_azure_group_answers_the_same_400_for_a_null_tool_result_text_on_either_deployment( + respx_mock, +): + openai_route: Final = respx_mock.post(url__regex=_OPENAI_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_OPENAI_BODY) + ) + azure_route: Final = respx_mock.post(url__regex=_MIXED_AZURE_CHAT_URL).mock( + return_value=httpx.Response(200, json=_MIXED_AZURE_CHAT_BODY) + ) + router: Final = _mixed_openai_azure_router() + + rejections: Final = [ + await _null_tool_result_text_rejection(router, deployment_id) for deployment_id in _MIXED_DEPLOYMENT_IDS + ] + + assert [(rejection.status_code, rejection.llm_provider) for rejection in rejections] == [ + (400, "openai"), + (400, "azure"), + ] + assert {rejection.message.partition("\n\n")[0] for rejection in rejections} == { + "litellm.BadRequestError: tool_result toolu_build content.1.text: Input should be a valid string" + } + assert openai_route.call_count == 0 + assert azure_route.call_count == 0 diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index b6f831d4bef..8fe6bd31082 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -576,6 +576,91 @@ def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_rea assert marked == unmarked +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turn_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + } + ] + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_malformed_tool_result_text_answers_400_naming_the_block(text_block: Mapping[str, object]) -> None: + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert rejected.value.status_code == 400 + assert "tool_result toolu_log content.1.text: Input should be a valid string" in rejected.value.message + + +def test_many_malformed_tool_result_texts_answer_one_bounded_error() -> None: + tool_use_id: Final = "toolu_" + "x" * 4096 + messages: Final = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": [{"type": "text", "text": None}] * 2048, + } + ], + } + ] + + with pytest.raises(litellm.BadRequestError) as rejected: + _build_responses_kwargs(model="openai/gpt-5.6", extra_kwargs=OPENAI_API, max_tokens=1024, messages=messages) + + assert rejected.value.status_code == 400 + assert f"tool_result {tool_use_id} content.0.text: Input should be a valid string" in rejected.value.message + assert rejected.value.message.count(tool_use_id) == 1 + + +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +def test_drop_params_drops_a_malformed_tool_result_text(text_block: Mapping[str, object]) -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs={**OPENAI_API, "drop_params": True}, + max_tokens=1024, + messages=_tool_result_turn_with(text_block), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"}, + ] + + +def test_tool_result_text_block_without_text_still_reaches_the_model_as_empty_text() -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=OPENAI_API, + max_tokens=1024, + messages=_tool_result_turn_with({"type": "text"}), + ) + + assert responses_kwargs["input"] == [ + {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"}, + ] + + REASONING_MODEL: Final = "gpt-bridge-reasoning-cost" REASONING_RATES: Final = { "input_cost_per_token": 2e-06, diff --git a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py index b5301b413ff..eac75486da4 100644 --- a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py @@ -8,6 +8,7 @@ from pydantic import JsonValue, TypeAdapter from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.llms.anthropic.pass_through.convertible_blocks import ( + malformed_tool_result_text_error, unconvertible_anthropic_blocks, unconvertible_blocks_error, ) @@ -190,3 +191,22 @@ def test_an_encrypted_web_search_inference_cannot_send_is_named() -> None: error: Final = unconvertible_blocks_error(_replayed_web_search("opaque"), "anthropic") assert error is not None assert error.endswith("server_tool_use, web_search_tool_result") + + +@pytest.mark.parametrize( + ("tool_result", "error"), + ( + ( + {"type": "tool_result", "tool_use_id": "toolu_01", "content": [{"type": "text", "text": None}]}, + "tool_result toolu_01 content.0.text: Input should be a valid string", + ), + ( + {"type": "tool_result", "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": 5}]}, + "tool_result content.1.text: Input should be a valid string", + ), + ), +) +def test_a_non_string_tool_result_text_is_named_with_or_without_a_tool_use_id( + tool_result: Mapping[str, object], error: str +) -> None: + assert malformed_tool_result_text_error([{"role": "user", "content": [tool_result]}]) == error diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index e54023c6596..918a7475eec 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, patch import httpx import pytest +import respx from pydantic import JsonValue, TypeAdapter import litellm @@ -153,6 +154,83 @@ async def test_google_ai_studio_count_tokens_answers_with_the_status_google_reje assert "Invalid function name" in (result.error_message or "") +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +_COUNT_TOKENS_URL: Final = "https://count-tokens.test/v1beta/models/gemini-2.5-flash:countTokens" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_google_ai_studio_count_tokens_rejects_a_non_string_tool_result_text_without_request( + text: JsonValue, respx_mock: respx.MockRouter +) -> None: + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment={"litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test"}}, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert not count_route.called + + +@pytest.mark.asyncio +async def test_google_ai_studio_count_tokens_leaves_out_a_numeric_tool_result_text_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + count_route: Final = respx_mock.post(_COUNT_TOKENS_URL).respond(200, json={"totalTokens": 11}) + + result: Final = await GoogleAIStudioTokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": {"api_key": "test-key", "api_base": "https://count-tokens.test", "drop_params": True} + }, + request_model="gemini/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 11 + assert _JSON_BODY.validate_json(count_route.calls.last.request.content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("system_instruction", "tools", "expected_body"), diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 2e88bccd49b..10f57be33fe 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -920,6 +920,85 @@ class TestMistralStripsOutputOnlyFields: assert "reasoning_content" not in result[-1] +def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_call(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + result = request["messages"] + assert [m.get("role") for m in result] == ["user", "assistant", "tool"] + assert result[2].get("tool_call_id") == "call_1" + + +@pytest.mark.parametrize("reply", [{"content": None}, {}], ids=["null", "missing"]) +def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", **reply}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""} + + +def test_mistral_transform_request_drops_an_empty_tool_message_no_tool_call_asked_for(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "hi"}, + {"role": "tool", "tool_call_id": "call_9", "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + +@pytest.mark.parametrize("role", ["system", "user"]) +def test_mistral_transform_request_drops_a_message_with_null_content(role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [{"role": role, "content": None}, {"role": "user", "content": "hi"}], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert request["messages"] == [{"role": "user", "content": "hi"}] + + def test_mistral_transform_request_hoists_tool_message_image(): """Images inside role:"tool" messages must be moved to a following user message (Mistral rejects/ignores non-text tool content), including when @@ -957,3 +1036,129 @@ def test_mistral_transform_request_hoists_tool_message_image(): {"type": "text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, {"type": "image_url", "image_url": {"url": data_uri}}, ] + + +def test_mistral_transform_request_drops_an_empty_tool_message_for_a_call_already_answered_earlier(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "assistant", "content": "It is sunny"}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "assistant", "user"] + + +def test_mistral_transform_request_drops_an_empty_tool_message_when_a_later_reply_carries_the_result(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "sunny" + + +def test_mistral_transform_request_keeps_an_empty_tool_message_answering_a_call_in_the_first_message(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["assistant", "tool", "user"] + assert request["messages"][1]["content"] == "" + + +@pytest.mark.parametrize("placeholder_role", ["assistant", "user"]) +def test_mistral_transform_request_keeps_an_empty_tool_message_after_a_dropped_empty_placeholder(placeholder_role): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + {"role": placeholder_role, "content": ""}, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + {"role": "user", "content": "thanks"}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant", "tool", "user"] + assert request["messages"][2]["content"] == "" + + +def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_not_a_string(): + messages: List[AllMessageValues] = cast( + List[AllMessageValues], + [ + {"role": "user", "content": "read the log"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": {"nested": "call_1"}, "type": "function", "function": {"name": "read", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": {"nested": "call_1"}, "content": ""}, + ], + ) + + request = MistralConfig().transform_request( + model="mistral-small-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index be394b71ec7..7495d98b42a 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -1,5 +1,6 @@ import json import operator +from collections.abc import Mapping from typing import Final import httpx @@ -395,6 +396,92 @@ async def test_undecodable_count_body_becomes_a_500_error_response_carrying_the_ ) +MALFORMED_TEXT_BLOCKS: Final = [ + pytest.param({"type": "text", "text": None}, id="null"), + pytest.param({"type": "text", "text": 7}, id="number"), +] + + +def _tool_result_turns_with(text_block: Mapping[str, object]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, text_block, {"type": "text", "text": "done"}], + } + ], + }, + ] + + +async def _count_tool_result_turns( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object], litellm_params: dict[str, object] +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=_tool_result_turns_with(text_block), + contents=None, + deployment={"litellm_params": litellm_params}, + request_model="openai/gpt-4o", + messages_format="anthropic", + ) + return result, route + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_malformed_tool_result_text_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, text_block, {"api_key": "sk-test"}) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code, result.error_message) == ( + True, + 400, + "tool_result toolu_log content.1.text: Input should be a valid string", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text_block", MALFORMED_TEXT_BLOCKS) +async def test_anthropic_format_drop_params_counts_the_tool_result_without_the_malformed_text( + monkeypatch: pytest.MonkeyPatch, text_block: Mapping[str, object] +) -> None: + result, route = await _count_tool_result_turns( + monkeypatch, text_block, {"api_key": "sk-test", "drop_params": True} + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\ndone"} in sent["input"] + + +@pytest.mark.asyncio +async def test_anthropic_format_tool_result_text_block_without_text_still_counts_as_empty_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_tool_result_turns(monkeypatch, {"type": "text"}, {"api_key": "sk-test"}) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 20dba332c08..7d380205ca1 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -117,6 +117,7 @@ def _vertex_count_tokens_peer(status_code: int) -> Iterator[_VertexCountTokensPe _VERTEX_DEPLOYMENT: Final = {"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-central1"}} +_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue]) @pytest.mark.asyncio @@ -164,6 +165,77 @@ async def test_vertex_gemini_count_rejects_an_image_url_it_refuses_to_fetch_with assert peer.requests == () +def _weather_tool_result_messages(text: JsonValue) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [{"type": "text", "text": text}, {"type": "text", "text": "partly cloudy"}], + } + ], + }, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("text", (None, 5)) +async def test_vertex_gemini_count_rejects_a_non_string_tool_result_text_without_request(text: JsonValue) -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(text), + contents=None, + deployment=_VERTEX_DEPLOYMENT, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.error_message == "tool_result toolu_01 content.0.text: Input should be a valid string" + assert peer.requests == () + + +@pytest.mark.asyncio +async def test_vertex_gemini_count_leaves_out_a_numeric_tool_result_text_when_the_deployment_drops_params() -> None: + with _vertex_count_tokens_peer(200) as peer: + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="gemini-2.5-flash", + messages=_weather_tool_result_messages(5), + contents=None, + deployment={ + "litellm_params": { + "vertex_project": "test-project", + "vertex_location": "us-central1", + "drop_params": True, + } + }, + request_model="vertex_ai/gemini-2.5-flash", + messages_format="anthropic", + ) + + assert result is not None + assert result.error is False + assert result.total_tokens == 37 + assert _JSON_BODY.validate_json(peer.requests[0].content)["contents"] == [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {"city": "Paris"}}}]}, + { + "role": "user", + "parts": [{"function_response": {"name": "get_weather", "response": {"content": "partly cloudy"}}}], + }, + ] + + _TOOL_OUTPUT: Final = "line of a long file that the agent read earlier. " * 180 diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 720739a730b..18fe5a2ad32 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import anyio import httpx import pytest +import respx import tiktoken from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict, JsonValue @@ -36,6 +37,7 @@ load_dotenv() from fastapi import Request from fastapi.responses import JSONResponse +from starlette.types import Message import litellm from litellm import Router @@ -56,7 +58,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter -from litellm.proxy._types import ProxyException, TokenCountRequest +from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import ( count_tokens as anthropic_count_tokens, ) @@ -1275,6 +1277,202 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): proxy_server.count_request_tokens = original_token_counter +def _json_request(body: Mapping[str, object], user_agent: str) -> Request: + payload: Final = json.dumps(body).encode() + + async def receive() -> Message: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/messages/count_tokens", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"user-agent", user_agent.encode())], + }, + receive, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_drop_params", "user_agent"), + (({"drop_params": True}, "python-httpx/0.28.1"), ({}, "claude-cli/2.1.0 (external, cli)")), +) +async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_drop_params_applies( + monkeypatch: pytest.MonkeyPatch, request_drop_params: dict[str, bool], user_agent: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + body: Final = { + "model": "gpt-tool-result", + **request_drop_params, + "messages": [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": None}], + } + ], + }, + ], + } + router: Final = Router( + model_list=[ + {"model_name": "gpt-tool-result", "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-test"}} + ] + ) + request: Final = _json_request(body, user_agent) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 9}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 9 + sent: Final = json.loads(route.calls.last.request.content) + assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"} in sent["input"] + + +def _build_log_messages(log_text: JsonValue) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": "run the build"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_log", "name": "bash", "input": {"cmd": "make"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_log", + "content": [{"type": "text", "text": "build ok"}, {"type": "text", "text": log_text}], + } + ], + }, + ] + + +async def _count_on_a_model_without_a_count_api( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], router_defaults: dict[str, JsonValue] | None = None +) -> JSONResponse | dict[str, JsonValue]: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "mistral-tool-result", + "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "sk-test"}, + } + ], + default_litellm_params=router_defaults, + ) + request: Final = _json_request(body, "python-httpx/0.28.1") + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + with respx.mock: + return await anthropic_count_tokens(request, UserAPIKeyAuth()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_text", (None, 5)) +async def test_count_tokens_without_a_count_api_rejects_a_non_string_tool_result_text( + monkeypatch: pytest.MonkeyPatch, log_text: JsonValue +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": "mistral-tool-result", "messages": _build_log_messages(log_text)} + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(bytes(response.body))["error"] == { + "type": "invalid_request_error", + "message": "tool_result toolu_log content.1.text: Input should be a valid string", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("mistral-tool-result", "model-not-in-the-router")) +async def test_count_tokens_without_a_count_api_counts_a_non_string_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, {"model": model, "drop_params": True, "messages": _build_log_messages(None)} + ) + + assert isinstance(response, dict) + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("body_drop_params", "expected_status"), (({}, None), ({"drop_params": False}, 400))) +async def test_count_tokens_honors_router_default_drop_params_unless_the_body_overrides_it( + monkeypatch: pytest.MonkeyPatch, body_drop_params: dict[str, JsonValue], expected_status: int | None +) -> None: + response: Final = await _count_on_a_model_without_a_count_api( + monkeypatch, + {"model": "mistral-tool-result", "messages": _build_log_messages(None), **body_drop_params}, + router_defaults={"drop_params": True}, + ) + + match response: + case JSONResponse(): + assert response.status_code == expected_status + case _: + assert expected_status is None + assert isinstance(response["input_tokens"], int) and response["input_tokens"] > 0 + + +@pytest.mark.asyncio +async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_tool_result_text_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "claude-native-drop", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": "sk-ant-test", + "drop_params": True, + }, + } + ] + ) + request: Final = _json_request( + {"model": "claude-native-drop", "messages": _build_log_messages(5)}, "python-httpx/0.28.1" + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock: + route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 7}) + ) + response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth()) + + assert response["input_tokens"] == 7 + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "build ok"}] + + class _ScriptedCounter(BaseTokenCounter): def __init__(self, replies: dict[str, TokenCountResponse], burst: int = 1) -> None: self._replies = replies From 53821f665417c0d02be0c4495e188ea9c0c3584e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 14:19:14 -0700 Subject: [PATCH 012/124] fix(decisions): log every request field the caller sent on /v1/systemone and /v1/decisions (#47183) * fix(decisions): log every request field the caller sent on /v1/systemone and /v1/decisions * fix(decisions): serialize the logged request with model_dump_json so no mutable dict is built * refactor(decisions): drop the state-only messages pick in function_setup, prepare_call logs the request * fix(decisions): return the single logged message so no mutable list type is annotated * fix(decisions): log the caller's raw request fields before validation so a rejected request is still recorded * fix(decisions): serialize the raw request fields with pydantic_core so question objects and mapping proxies log as JSON * fix(decisions): log the caller request at the SDK entry so requests rejected before prepare_call are recorded too * test(decisions): assert the exact BadRequestError for the rejected-questions case --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/decisions/call.py | 27 +++++++ litellm/decisions/main.py | 11 +++ litellm/systemone/main.py | 9 ++- litellm/utils.py | 7 -- tests/unit/decisions/test_call.py | 114 ++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 tests/unit/decisions/test_call.py diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 4943b22e1fb..b31cb1f69db 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -1,3 +1,4 @@ +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from functools import partial @@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias import httpx from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python from typing_extensions import assert_never import litellm @@ -56,6 +58,7 @@ from litellm.types.decisions import ( OpenAIDecisionResponse, UnsupportedDecisionsRequest, ) +from litellm.types.llms.openai import ChatCompletionUserMessage from litellm.types.utils import LlmProviders, ProviderSpecificHeader from litellm.utils import ProviderConfigManager, get_model_cost_key @@ -415,6 +418,30 @@ def _outbound_headers( return MappingProxyType({**(extra_headers or {}), **forwarded}) +_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _jsonable_fallback(value: object) -> object: + return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value) + + +def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage: + caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python( + to_jsonable_python(caller_request, fallback=_jsonable_fallback) + ) + sent_fields: Final = json.dumps( + dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {})) + ) + return ChatCompletionUserMessage(role="user", content=sent_fields) + + +def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None: + """Record every request field the caller sent as the logged `messages`, before any validation.""" + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_messages([_logged_request_message(caller_request)]) + + def prepare_call( *, model: str, diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 7d2ee03da0e..4e609fd2ab8 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -17,6 +17,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_images_next_to_input, send, @@ -78,6 +79,16 @@ def _prepare_decisions_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + log_caller_request( + kwargs, + { + "input": decision_input, + "questions": questions, + "audio": audio, + "videos": videos, + **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs}, + }, + ) return prepare_call( model=model, fields=_request_fields(model, decision_input, questions, audio, videos, kwargs), diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 01f151ef4d0..27fa84e3f9b 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -14,6 +14,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_other_format, send, @@ -39,12 +40,14 @@ def _prepare_systemone_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + fields: Final = SystemOneRequestFields( + state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos + ) + log_caller_request(kwargs, fields) reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields( - state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos - ), + fields=fields, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, diff --git a/litellm/utils.py b/litellm/utils.py index cf3b8756e2f..143c2470d33 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -998,10 +998,6 @@ def _ocr_document_summary(document: object) -> str: # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. -_DECISION_CALL_TYPES: Final[frozenset[str]] = frozenset( - (CallTypes.decisions.value, CallTypes.adecisions.value, CallTypes.systemone.value, CallTypes.asystemone.value) -) - def function_setup( original_function: str, @@ -1249,9 +1245,6 @@ def function_setup( if isinstance(search_query, list) else search_query ) - elif call_type in _DECISION_CALL_TYPES: - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py new file mode 100644 index 00000000000..03c135369b8 --- /dev/null +++ b/tests/unit/decisions/test_call.py @@ -0,0 +1,114 @@ +import json +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm.decisions.main import _prepare_decisions_call +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.systemone.main import _prepare_systemone_call + +QUESTION_TEXT: Final = "Is this about a billing problem?" +CONNECTION: Final = { + "model": "hosted_vllm/jev-fake", + "safety_identifier": None, + "api_key": "fake-provider-key", + "api_base": "http://127.0.0.1:4010", + "timeout": None, + "custom_llm_provider": None, + "extra_headers": None, +} + + +def _logging_obj(call_type: str) -> Logging: + return Logging( + model="hosted_vllm/jev-fake", + messages="placeholder", + stream=False, + call_type=call_type, + start_time=datetime.now(timezone.utc), + litellm_call_id="lit-9659", + function_id="lit-9659", + ) + + +def _logged_request(logging_obj: Logging) -> Mapping[str, object]: + messages: Final = logging_obj.model_call_details["messages"] + assert len(messages) == 1 and messages[0]["role"] == "user" + return json.loads(messages[0]["content"]) + + +def test_systemone_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("asystemone") + + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["state"] == {"message": "My card was charged twice."} + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + assert set(logged) == {"state", "questions"} + + +def test_decisions_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("adecisions") + + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["input"] == "My card was charged twice." + assert logged["questions"][0]["instructions"] == QUESTION_TEXT + assert set(logged) == {"input", "questions"} + + +def test_request_is_logged_even_when_it_is_rejected_before_the_provider_call(): + logging_obj: Final = _logging_obj("asystemone") + + with pytest.raises(litellm.BadRequestError): + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj, "input": "sent to the wrong function"}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + + +def test_decisions_request_is_logged_even_when_its_questions_fail_validation(): + logging_obj: Final = _logging_obj("adecisions") + + with pytest.raises(litellm.BadRequestError): + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "not-a-question-type", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"][0]["instructions"] == QUESTION_TEXT From 62ce515d15fd61f15985d54120265f5976fbce4d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 14:27:29 -0700 Subject: [PATCH 013/124] feat(router): let a deployment set a timeout per service_tier (#47406) * feat(router): let a deployment set a timeout per service_tier Adds timeout_by_service_tier and stream_timeout_by_service_tier to a deployment's litellm_params. A request whose service_tier matches a key uses that value in place of the deployment's timeout or stream_timeout. A timeout the client sends still wins and requests without a matching tier are unchanged Co-authored-by: adc77 * style(router): sort the request_timeout_resolver import * fix(router): read the service tier maps without the untyped params get * fix(router): look the service tier timeout up through a typed get --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: adc77 --- .../request_timeout_resolver.py | 23 +++++++- litellm/passthrough/timeout_utils.py | 20 +++++-- litellm/router.py | 3 + litellm/types/litellm_params.py | 2 + litellm/types/router.py | 5 ++ .../test_pass_through_endpoints.py | 26 +++++++++ tests/unit/test_router/test_router_timeout.py | 56 +++++++++++++++++++ tests/unit/types/test_litellm_params.py | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 16 ++++++ 9 files changed, 147 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py index a32651fedee..0e011dc06a0 100644 --- a/litellm/litellm_core_utils/request_timeout_resolver.py +++ b/litellm/litellm_core_utils/request_timeout_resolver.py @@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that: from __future__ import annotations -from typing import Final +from collections.abc import Mapping +from typing import Final, Literal, Protocol + +from pydantic import TypeAdapter from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None) + def get_configured_request_timeout() -> float | None: """Return the explicitly-configured ``litellm.request_timeout``, else ``None``.""" @@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None: if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS): return timeout return None + + +class _DeploymentParams(Protocol): + def get(self, key: str, /) -> object: ... + + +def get_service_tier_timeout( + service_tier: object, + litellm_params: _DeploymentParams, + key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"], +) -> float | None: + """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``.""" + if not isinstance(service_tier, str): + return None + timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key)) + return None if timeouts is None else timeouts.get(service_tier) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index f600c7817f2..ce13552ff49 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -5,7 +5,10 @@ from typing import Final from pydantic import TypeAdapter -from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, + get_service_tier_timeout, +) DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 @@ -50,21 +53,27 @@ def resolve_llm_passthrough_timeout( Anthropic /v1/messages). Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params - timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout, + timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout, when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default. Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: - kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the - non-streaming chain above. + kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] -> + litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above. Only the first set value is validated as seconds, so a value in a lower-precedence field never fails the call. """ request: Final = kwargs if kwargs is not None else _NO_PARAMS deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS + service_tier: Final = request.get("service_tier") stream_candidates: Final = ( - (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout) + ( + request.get("stream_timeout"), + get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"), + deployment.get("stream_timeout"), + router_stream_timeout, + ) if request.get("stream") else () ) @@ -72,6 +81,7 @@ def resolve_llm_passthrough_timeout( *stream_candidates, request.get("timeout"), request.get("request_timeout"), + get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"), deployment.get("timeout"), deployment.get("request_timeout"), router_timeout, diff --git a/litellm/router.py b/litellm/router.py index af79fa763c4..5bf46c68b20 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( ) from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, + get_service_tier_timeout, ) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.sensitive_data_masker import ( @@ -4773,6 +4774,7 @@ class Router: """Helper to get stream timeout from kwargs or deployment params""" return ( kwargs.get("stream_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment or self.stream_timeout # timeout set on router or self.request_timeout # litellm_settings.request_timeout (per-attempt) @@ -4786,6 +4788,7 @@ class Router: return ( kwargs.get("timeout", None) # the params dynamically set by user or kwargs.get("request_timeout", None) # the params dynamically set by user + or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") or data.get("timeout", None) # timeout set on litellm_params for this deployment or data.get("request_timeout", None) # timeout set on litellm_params for this deployment or self.request_timeout # litellm_settings.request_timeout (per-attempt) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index b05d25462aa..978eb9c4557 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -77,6 +77,8 @@ class ProviderConnection: request_timeout: float | None = None force_timeout: float | None = None stream_timeout: float | str | None = None + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None tenant_id: str | None = None client_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 46617e1b753..f83127806db 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -505,6 +505,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): timeout: float | str | httpx.Timeout | SDKTimeout | None = None # if str, pass in as os.environ/ # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/ stream_timeout: float | str | None = None + # per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key + timeout_by_service_tier: Mapping[str, float] | None = None + stream_timeout_by_service_tier: Mapping[str, float] | None = None max_retries: int | None = None drop_params: bool | str | None = None organization: str | None = None # for openai orgs @@ -709,6 +712,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): responses_api_version: ReadOnly[str | None] timeout: float | str | httpx.Timeout | SDKTimeout | None # writable-ok: preserve timeout updates stream_timeout: float | str | None + timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] + stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None] max_retries: int | None organization: list | str | None # for openai orgs provider_affinity_header: ReadOnly[str | None] diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 5412ea56ac0..4c3c7ae0f0a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1857,6 +1857,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value( assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected +def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier(): + litellm_params = { + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + } + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0 + assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0 + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params) + == 20.0 + ) + assert ( + resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params) + == 950.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"service_tier": "flex", "stream": True}, + litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}}, + ) + == 30.0 + ) + + def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner(): with pytest.raises(ValidationError): resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)}) diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index 3265179a513..8fc32620f71 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -153,6 +153,62 @@ def test_unit_test_streaming_timeout(stream): assert stream_timeout_val == normal_timeout +@pytest.mark.parametrize( + "request_kwargs, expected_timeout", + [ + ({"service_tier": "flex"}, 900.0), + ({"service_tier": "default"}, 610.0), + ({}, 610.0), + ({"service_tier": "flex", "timeout": 20}, 20), + ({"service_tier": "flex", "stream": True}, 950.0), + ({"service_tier": "default", "stream": True}, 610.0), + ({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30), + ], +) +def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout( + request_kwargs: dict[str, object], expected_timeout: float +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 610, + "timeout_by_service_tier": {"flex": 900}, + "stream_timeout_by_service_tier": {"flex": 950}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout + + +def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai-gpt-5.1", + "litellm_params": { + "model": "openai/gpt-5.1", + "api_key": "sk-test", + "timeout": 610, + "stream_timeout": 30, + "timeout_by_service_tier": {"flex": 900}, + }, + } + ] + ) + deployment_params: Final = router.model_list[0]["litellm_params"] + + assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0 + assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0 + + async def _drain_responses_stream(router: Router) -> None: response: Final = await router.aresponses(model="gpt", input="hi", stream=True) _ = [event async for event in response] diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index ce763378656..e88d757519c 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -75,6 +75,8 @@ CONNECTION_NAMES: Final = ( "request_timeout", "force_timeout", "stream_timeout", + "timeout_by_service_tier", + "stream_timeout_by_service_tier", "max_retries", "tenant_id", "client_id", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index daffc3c0c41..733a093bcac 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35424,6 +35424,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -35436,6 +35440,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Token Exchange Audience */ token_exchange_audience?: string | null; /** Token Exchange Endpoint */ @@ -50666,6 +50674,10 @@ export interface components { } | null; /** Stream Timeout */ stream_timeout?: number | string | null; + /** Stream Timeout By Service Tier */ + stream_timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Tag Regex */ tag_regex?: string[] | null; /** Tags */ @@ -50678,6 +50690,10 @@ export interface components { }[] | null; /** Timeout */ timeout?: number | string | null; + /** Timeout By Service Tier */ + timeout_by_service_tier?: { + [key: string]: number; + } | null; /** Token Exchange Audience */ token_exchange_audience?: string | null; /** Token Exchange Endpoint */ From 6162184b1ce2435daf7996241b349361d4342d08 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 14:40:15 -0700 Subject: [PATCH 014/124] fix(managed_files): send provider file ids when creating a fine-tuning job from managed files (#46552) * fix(managed_files): send provider file ids when creating a fine-tuning job from managed files Fine-tuning job creation with a managed training file sent the gateway's managed id and the provider-prefixed deployment model to OpenAI, which rejected the job. The pre-call hook now builds the file id mapping for training_file and validation_file, the deployment hook swaps both, the SDK strips the provider prefix from the model, and the create response returns the caller's own file ids * fix(managed_files): route managed fine-tuning creates only to deployments holding every file Fine-tuning creates now narrow routing the same way managed batches do, so a deployment added after the upload is never picked. Adds a two-worker chaos integration test and runs tests/unit/fine_tuning in the endpoints shard. * fix(managed_files): keep fine-tuning creates on the model group when no deployment holds the files A managed file used on a model group it was never uploaded for kept failing with the provider's 400 before the routing filter. The filter turned that into a 429 no-deployments error, so fine-tuning now falls back to the whole group in that case while batch input files keep routing only to deployments that hold them. * test(fine_tuning): cover the provider prefix strip for OpenAI and Azure creates through the proxy * test(fine_tuning): start each multi-deployment case from an empty upstream log * test(fine_tuning): boot both proxies of the late deployment cell on one scratch database --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + .../proxy/hooks/managed_files.py | 63 ++-- litellm/fine_tuning/main.py | 7 +- .../proxy/fine_tuning_endpoints/endpoints.py | 3 +- .../compatibility/test_fine_tuning_jobs.py | 99 +++++- .../test_fine_tuning_managed_files_chaos.py | 289 ++++++++++++++++++ .../security/test_fine_tuning_managed_ids.py | 8 +- .../proxy/hooks/test_managed_files.py | 162 +++++++++- tests/unit/fine_tuning/__init__.py | 0 tests/unit/fine_tuning/test_main.py | 54 ++++ .../vertex_ai/fine_tuning/test_handler.py | 5 +- .../fine_tuning_endpoints/test_endpoints.py | 33 +- 12 files changed, 668 insertions(+), 56 deletions(-) create mode 100644 tests/integration/compatibility/test_fine_tuning_managed_files_chaos.py create mode 100644 tests/unit/fine_tuning/__init__.py create mode 100644 tests/unit/fine_tuning/test_main.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index cb1b595f3f0..5e42f182345 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -410,6 +410,7 @@ jobs: tests/unit/embeddings tests/unit/endpoints tests/unit/files + tests/unit/fine_tuning tests/unit/images tests/unit/interactions tests/unit/messages diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 439770390f8..bc35c4a0aba 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -6,6 +6,7 @@ import base64 import json import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -479,6 +480,13 @@ def _with_managed_background_marker(request: Mapping[str, object]) -> dict[str, _MANAGED_FILES_TARGET: Final = "managed_files" +_PROVIDER_FILE_ID_KWARGS: Final = MappingProxyType( + { + CallTypes.acreate_batch: ("input_file_id",), + CallTypes.acreate_fine_tuning_job: ("training_file", "validation_file"), + } +) +_PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDER_FILE_ID_KWARGS.values()))) _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0) @@ -1562,6 +1570,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if isinstance(file_id, str) and file_id ) await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict) + if fine_tuning_file_ids: + data["model_file_id_mapping"] = await self.get_model_file_id_mapping( + fine_tuning_file_ids, user_api_key_dict.parent_otel_span + ) return data @@ -1576,24 +1588,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if request_kwargs is None: return healthy_deployments - input_file_id = cast(Optional[str], request_kwargs.get("input_file_id")) - model_file_id_mapping = cast( - Optional[Dict[str, Dict[str, str]]], - request_kwargs.get("model_file_id_mapping"), + request: Final = cast(Mapping[str, object], request_kwargs) + model_file_id_mapping: Final = cast(Mapping[str, Mapping[str, str]], request.get("model_file_id_mapping") or {}) + requested_file_ids: Final = (request.get(key) for key in _PROVIDER_FILE_ID_KEYS) + model_ids_per_file: Final = tuple( + frozenset(model_file_id_mapping[file_id]) + for file_id in requested_file_ids + if isinstance(file_id, str) and file_id in model_file_id_mapping + ) + allowed_model_ids: Final[frozenset[str]] = ( + model_ids_per_file[0].intersection(*model_ids_per_file[1:]) if model_ids_per_file else frozenset() ) - allowed_model_ids = [] - if input_file_id and model_file_id_mapping: - model_id_dict = model_file_id_mapping.get(input_file_id, {}) - allowed_model_ids = list(model_id_dict.keys()) if len(allowed_model_ids) == 0: return healthy_deployments - return [ + holders: Final[list[dict[str, object]]] = [ deployment for deployment in healthy_deployments if deployment.get("model_info", {}).get("id") in allowed_model_ids ] + return holders if holders or "input_file_id" in request else healthy_deployments async def async_pre_call_deployment_hook( self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] @@ -1601,25 +1616,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ Allow modifying the request just before it's sent to the deployment. """ - accessor_key: Optional[str] = None - if call_type and call_type == CallTypes.acreate_batch: - accessor_key = "input_file_id" - elif call_type and call_type == CallTypes.acreate_fine_tuning_job: - accessor_key = "training_file" - else: + accessor_keys: Final = _PROVIDER_FILE_ID_KWARGS.get(call_type) if call_type else None + if accessor_keys is None: return kwargs - if accessor_key: + model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")) + # model_info may be at top-level or nested under litellm_metadata + # (batch/file operations use litellm_metadata) + model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None)) + if model_id is None: + model_id = cast( + Optional[str], + kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None), + ) + for accessor_key in accessor_keys: input_file_id = cast(Optional[str], kwargs.get(accessor_key)) - model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")) - # model_info may be at top-level or nested under litellm_metadata - # (batch/file operations use litellm_metadata) - model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None)) - if model_id is None: - model_id = cast( - Optional[str], - kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None), - ) mapped_file_id: Optional[str] = None if input_file_id and model_file_id_mapping and model_id: mapped_file_id = model_file_id_mapping.get(input_file_id, {}).get(model_id, None) @@ -1813,7 +1824,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ), ) - async def get_model_file_id_mapping(self, file_ids: List[str], litellm_parent_otel_span: Span) -> dict: + async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict: """ Get model-specific file IDs for a list of proxy file IDs. Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 5297a8215be..f49f069b83f 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -173,6 +173,7 @@ def create_fine_tuning_job( try: _is_async: Final = kwargs.pop("acreate_fine_tuning_job", False) is True optional_params: Final = GenericLiteLLMParams(**kwargs) + provider_model: Final = model.removeprefix(f"{custom_llm_provider}/") # handle hyperparameters hyperparameters = hyperparameters or {} # original hyperparameters @@ -218,7 +219,7 @@ def create_fine_tuning_job( ) create_fine_tuning_job_data_dict = _build_fine_tuning_job_data( - model, + provider_model, training_file, _oai_hyperparameters, suffix, @@ -263,7 +264,7 @@ def create_fine_tuning_job( extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams) create_fine_tuning_job_data_dict = _build_fine_tuning_job_data( - model, + provider_model, training_file, _oai_hyperparameters, suffix, @@ -298,7 +299,7 @@ def create_fine_tuning_job( response = vertex_fine_tuning_apis_instance.create_fine_tuning_job( _is_async=_is_async, create_fine_tuning_job_data=_build_fine_tuning_job_data( - model, + provider_model, training_file, _oai_hyperparameters, suffix, diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index d015112cf3a..030f8d7a46a 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -164,7 +164,8 @@ async def create_fine_tuning_job( ) response = cast(LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data)) - response.training_file = unified_file_id + response.training_file = training_file + response.validation_file = fine_tuning_request.validation_file set_hidden_param(response, "unified_file_id", unified_file_id) ## ELSE, Route based on custom_llm_provider elif fine_tuning_request.custom_llm_provider: diff --git a/tests/integration/compatibility/test_fine_tuning_jobs.py b/tests/integration/compatibility/test_fine_tuning_jobs.py index 9678198352f..1eff386b8ba 100644 --- a/tests/integration/compatibility/test_fine_tuning_jobs.py +++ b/tests/integration/compatibility/test_fine_tuning_jobs.py @@ -27,7 +27,6 @@ MANAGED_MODEL: Final = "gpt-4.1-openai" DEPLOYMENT_KEYS: Final = ("sk-fine-tuning-deployment-a", "sk-fine-tuning-deployment-b") PROVIDER_JOB: Final = "ftjob-provider-123" RUN: Final = uuid.uuid4().hex[:12] -MANAGED_JOB: Final = f"ftjob-managed-{RUN}" TRAINING_LINE: Final = b'{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}\n' JOB_PATH: Final = re.compile(r"^/(?:openai/)?(?:v1/)?fine_tuning/jobs/([^/]+)(/cancel)?$") JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -88,8 +87,18 @@ def _job(job_id: str, status: str, training_file: str = "file-abc123", **fields: } -def _provider_file_id(bearer: str) -> str: - return f"file-{RUN}-" + bearer.removeprefix("sk-fine-tuning-") +def _provider_file_id(bearer: str, filename: str) -> str: + return f"file-{RUN}-" + bearer.removeprefix("sk-fine-tuning-") + "-" + filename.removesuffix(".jsonl") + + +def _managed_job_id(provider_training_file: str) -> str: + return provider_training_file.replace("file-", "ftjob-managed-", 1) + + +def _uploaded_filename(request: Request) -> str: + names: Final = [part.get_filename() for part in _multipart_parts(request) if part.get_filename()] + assert len(names) == 1, request.body[:300] + return str(names[0]) def _bearer(request: Request) -> str: @@ -118,19 +127,27 @@ def _respond(request: Request) -> Reply: if (request.method, path) == ("POST", "/v1/files"): return _reply( { - "id": _provider_file_id(_bearer(request)), + "id": _provider_file_id(_bearer(request), _uploaded_filename(request)), "object": "file", "bytes": len(TRAINING_LINE), "created_at": 1700000000, - "filename": "train.jsonl", + "filename": _uploaded_filename(request), "purpose": "fine-tune", "status": "processed", } ) if (request.method, path) in {("POST", "/v1/fine_tuning/jobs"), ("POST", "/openai/fine_tuning/jobs")}: sent: Final = JSON_OBJECT.validate_json(request.body) - job_id: Final = MANAGED_JOB if _bearer(request) in DEPLOYMENT_KEYS else PROVIDER_JOB - return _reply(_job(job_id, "validating_files", str(sent["training_file"]))) + managed: Final = _bearer(request) in DEPLOYMENT_KEYS + echoed: Final = {"validation_file": sent.get("validation_file")} if managed else {} + return _reply( + _job( + _managed_job_id(str(sent["training_file"])) if managed else PROVIDER_JOB, + "validating_files", + str(sent["training_file"]), + **echoed, + ) + ) matched: Final = JOB_PATH.match(path) if matched and matched.group(1) and request.method == "GET": return _reply(_job(matched.group(1), "running")) @@ -329,6 +346,30 @@ def test_azure_create_merges_finetune_settings_and_keeps_azure_fields(rig: Rig) assert Job.model_validate_json(raw.text) == _client_job(PROVIDER_JOB, "validating_files", "file-abc"), raw.text +@pytest.mark.parametrize( + ("provider", "target", "provider_defaults"), + [ + ("openai", "/v1/fine_tuning/jobs", {}), + ("azure", f"/openai/fine_tuning/jobs?api-version={AZURE_API_VERSION}", {"trainingType": 1}), + ], +) +def test_create_sends_the_model_without_its_provider_prefix( + rig: Rig, provider: str, target: str, provider_defaults: Mapping[str, JsonValue] +) -> None: + with rig.sdk("/v1") as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model=f"{provider}/gpt-4o-mini", training_file="file-abc123", extra_body={"custom_llm_provider": provider} + ) + sent: Final = _only(_drained(rig), "POST", target) + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4o-mini", + "training_file": "file-abc123", + "hyperparameters": {}, + **provider_defaults, + }, sent.body + assert raw.status_code == 200, raw.text + + def test_azure_cancel_merges_finetune_settings(rig: Rig) -> None: with rig.sdk("/v1") as client: raw: Final = client.fine_tuning.jobs.with_raw_response.cancel( @@ -400,10 +441,6 @@ def test_retrieve_reads_the_documented_custom_llm_provider_header(rig: Rig) -> N def test_managed_create_sends_the_provider_file_id_of_the_pinned_deployment(rig: Rig) -> None: - pytest.skip( - "BUG: managed create sends the base64 LiteLLM file id and the prefixed model openai/gpt-4.1 to the provider" - " and returns the decoded unified id as training_file" - ) with rig.sdk("/v1") as client: uploaded: Final = client.files.with_raw_response.create( file=("train.jsonl", TRAINING_LINE, "application/jsonl"), @@ -431,12 +468,48 @@ def test_managed_create_sends_the_provider_file_id_of_the_pinned_deployment(rig: assert deployment_key in DEPLOYMENT_KEYS, sent.headers assert JSON_OBJECT.validate_json(sent.body) == { "model": "gpt-4.1", - "training_file": _provider_file_id(deployment_key), + "training_file": _provider_file_id(deployment_key, "train.jsonl"), "hyperparameters": {}, }, sent.body assert raw.status_code == 200, raw.text job: Final = Job.model_validate_json(raw.text) assert re.fullmatch( - rf"litellm_proxy;model_id:[0-9a-f]{{64}};generic_response_id:{MANAGED_JOB}", _decoded(job.id) + rf"litellm_proxy;model_id:[0-9a-f]{{64}};generic_response_id:" + rf"{_managed_job_id(_provider_file_id(deployment_key, 'train.jsonl'))}", + _decoded(job.id), ), raw.text assert job.training_file == managed_file, raw.text + + +def _managed_upload(rig: Rig, client: OpenAI, filename: str) -> str: + uploaded: Final = client.files.with_raw_response.create( + file=(filename, TRAINING_LINE, "application/jsonl"), + purpose="fine-tune", + extra_body={"target_model_names": MANAGED_MODEL}, + ) + assert uploaded.status_code == 200, uploaded.text + assert sorted((request.method, request.target, _bearer(request)) for request in _drained(rig)) == [ + ("POST", "/v1/files", key) for key in DEPLOYMENT_KEYS + ] + return str(JSON_OBJECT.validate_json(uploaded.text)["id"]) + + +def test_managed_create_sends_the_provider_ids_of_managed_training_and_validation_files(rig: Rig) -> None: + with rig.sdk("/v1") as client: + training_file: Final = _managed_upload(rig, client, "tune.jsonl") + validation_file: Final = _managed_upload(rig, client, "validate.jsonl") + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model=MANAGED_MODEL, training_file=training_file, validation_file=validation_file + ) + sent: Final = _only(_drained(rig), "POST", "/v1/fine_tuning/jobs") + deployment_key: Final = _bearer(sent) + assert deployment_key in DEPLOYMENT_KEYS, sent.headers + assert JSON_OBJECT.validate_json(sent.body) == { + "model": "gpt-4.1", + "training_file": _provider_file_id(deployment_key, "tune.jsonl"), + "validation_file": _provider_file_id(deployment_key, "validate.jsonl"), + "hyperparameters": {}, + }, sent.body + assert raw.status_code == 200, raw.text + job: Final = Job.model_validate_json(raw.text) + assert (job.training_file, job.validation_file) == (training_file, validation_file), raw.text diff --git a/tests/integration/compatibility/test_fine_tuning_managed_files_chaos.py b/tests/integration/compatibility/test_fine_tuning_managed_files_chaos.py new file mode 100644 index 00000000000..54794fe29b9 --- /dev/null +++ b/tests/integration/compatibility/test_fine_tuning_managed_files_chaos.py @@ -0,0 +1,289 @@ +"""Managed fine-tuning creates on a two-worker proxy whose model group has two deployments: a concurrent burst from +sync and async clients, traffic pinned to each worker in turn, and a worker killed and replaced, all send every +deployment its own provider file ids and hand the caller back its own managed ids. A deployment added to the group +after the upload holds no copy of the files, so it never receives the create even when the router prefers it. A model +group the files were never uploaded for still gets the create and the provider's own 400, as before routing checked +which deployments hold the files.""" + +from __future__ import annotations + +import asyncio +import json +import uuid +from collections.abc import Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import pytest +import yaml +from integration._support.client import eventually, gateway_from_environment +from integration._support.database import scratch_database +from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import AsyncOpenAI, BadRequestError, OpenAI +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(480) + +MODEL: Final = "gpt-4.1-chaos" +DEPLOYMENT_KEYS: Final = ("sk-chaos-deployment-a", "sk-chaos-deployment-b") +LATE_DEPLOYMENT_KEY: Final = "sk-chaos-deployment-late" +OTHER_MODEL: Final = "gpt-4.1-chaos-other" +OTHER_DEPLOYMENT_KEY: Final = "sk-chaos-deployment-other" +RUN: Final = uuid.uuid4().hex[:12] +TRAINING_LINE: Final = b'{"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}\n' +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +BURST: Final = 20 +CREATE: Final = ("POST", "/v1/fine_tuning/jobs") +WORKER_BOOT_DISCOVERY: Final = ("GET", "/v1/models") + + +@dataclass(frozen=True, slots=True) +class Rig: + owned: OwnedProxy + wire: Wire + training_file: str + validation_file: str + + def base_url(self) -> str: + return f"{str(self.owned.gateway.client.base_url).rstrip('/')}/v1" + + def sdk(self) -> OpenAI: + return OpenAI(base_url=self.base_url(), api_key=self.owned.gateway.key, max_retries=0) + + def async_sdk(self) -> AsyncOpenAI: + return AsyncOpenAI(base_url=self.base_url(), api_key=self.owned.gateway.key, max_retries=0) + + +def _bearer(request: Request) -> str: + return request.headers.get("authorization", "").removeprefix("Bearer ") + + +def _provider_file_id(bearer: str, purpose: str) -> str: + return f"file-{RUN}-{bearer.removeprefix('sk-chaos-')}-{purpose}" + + +def _uploaded_purpose(request: Request) -> str: + return "validate" if b'filename="validate.jsonl"' in request.body else "train" + + +def _invalid_training_file(training_file: str) -> Reply: + error: Final = { + "message": f"invalid training_file: {training_file}", + "type": "invalid_request_error", + "param": "training_file", + "code": None, + } + return Reply(status=400, body=json.dumps({"error": error}).encode()) + + +def _respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + if (request.method, path) == ("POST", "/v1/files"): + return Reply( + body=json.dumps( + { + "id": _provider_file_id(_bearer(request), _uploaded_purpose(request)), + "object": "file", + "bytes": len(TRAINING_LINE), + "created_at": 1700000000, + "filename": f"{_uploaded_purpose(request)}.jsonl", + "purpose": "fine-tune", + "status": "processed", + } + ).encode() + ) + if (request.method, path) == ("POST", "/v1/fine_tuning/jobs"): + sent: Final = JSON_OBJECT.validate_json(request.body) + if not str(sent["training_file"]).startswith(f"file-{RUN}-"): + return _invalid_training_file(str(sent["training_file"])) + return Reply( + body=json.dumps( + { + "id": f"ftjob-{uuid.uuid4().hex}", + "object": "fine_tuning.job", + "created_at": 1700000000, + "error": None, + "fine_tuned_model": None, + "finished_at": None, + "hyperparameters": {"n_epochs": 3}, + "model": sent["model"], + "organization_id": "org-chaos", + "result_files": [], + "seed": 42, + "status": "validating_files", + "trained_tokens": None, + "training_file": sent["training_file"], + "validation_file": sent.get("validation_file"), + } + ).encode() + ) + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + + +def _deployment(wire: Wire, key: str, routing: Mapping[str, JsonValue], model: str = MODEL) -> dict[str, JsonValue]: + return { + "model_name": model, + "litellm_params": {"model": "openai/gpt-4.1", "api_key": key, "api_base": f"{wire.url}/v1", **routing}, + "model_info": {"supported_endpoints": ["/chat/completions", "/fine_tuning"]}, + } + + +def _config(wire: Wire, late_preferred_deployment: bool = False) -> dict[str, JsonValue]: + late: Final = (_deployment(wire, LATE_DEPLOYMENT_KEY, {"order": 1}),) if late_preferred_deployment else () + return { + "model_list": [ + *(_deployment(wire, key, {}) for key in DEPLOYMENT_KEYS), + *late, + _deployment(wire, OTHER_DEPLOYMENT_KEY, {}, OTHER_MODEL), + ], + "router_settings": {"num_retries": 0, "disable_cooldowns": True}, + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "database_url": "os.environ/DATABASE_URL"}, + } + + +def _config_file(directory: Path, wire: Wire, late_preferred_deployment: bool = False) -> Path: + directory.mkdir(parents=True, exist_ok=True) + config: Final = directory / "config.yaml" + config.write_text(yaml.safe_dump(_config(wire, late_preferred_deployment))) + return config + + +def _upload(client: OpenAI, purpose: str) -> str: + uploaded: Final = client.files.with_raw_response.create( + file=(f"{purpose}.jsonl", TRAINING_LINE, "application/jsonl"), + purpose="fine-tune", + extra_body={"target_model_names": MODEL}, + ) + assert uploaded.status_code == 200, uploaded.text + return str(JSON_OBJECT.validate_json(uploaded.text)["id"]) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("fine-tuning-chaos") + with gateway_from_environment() as shared, wire_server(_respond) as wire: + config: Final = _config_file(directory, wire) + with owned_proxy_process( + shared, directory, {}, config=config, workers=2, healthcheck=WorkerHealthcheck.LENIENT + ) as owned: + eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=120) + unloaded: Final = Rig(owned, wire, "", "") + with unloaded.sdk() as client: + training_file: Final = _upload(client, "train") + validation_file: Final = _upload(client, "validate") + wire.drain() + yield Rig(owned, wire, training_file, validation_file) + + +def _create(rig: Rig) -> tuple[int, Mapping[str, JsonValue]]: + with rig.sdk() as client: + raw: Final = client.fine_tuning.jobs.with_raw_response.create( + model=MODEL, training_file=rig.training_file, validation_file=rig.validation_file + ) + return raw.status_code, JSON_OBJECT.validate_json(raw.text) + + +async def _async_creates(rig: Rig, count: int) -> tuple[tuple[int, Mapping[str, JsonValue]], ...]: + async with rig.async_sdk() as client: + raws: Final = await asyncio.gather( + *( + client.fine_tuning.jobs.with_raw_response.create( + model=MODEL, training_file=rig.training_file, validation_file=rig.validation_file + ) + for _ in range(count) + ) + ) + return tuple((raw.status_code, JSON_OBJECT.validate_json(raw.text)) for raw in raws) + + +def _sent_creates(rig: Rig, expected: int) -> tuple[Request, ...]: + collected: Final[list[Request]] = [] + + def creates() -> tuple[Request, ...]: + collected.extend(rig.wire.drain()) + return tuple(request for request in collected if (request.method, request.target) == CREATE) + + sent: Final = eventually(creates, lambda found: len(found) >= expected, seconds=30) + assert len(sent) == expected, [(request.method, request.target) for request in collected] + assert {(request.method, request.target) for request in collected} <= {CREATE, WORKER_BOOT_DISCOVERY}, collected + return sent + + +def _assert_each_deployment_got_its_own_ids(creates: tuple[Request, ...]) -> None: + for request in creates: + bearer = _bearer(request) + assert bearer in DEPLOYMENT_KEYS, request.headers + assert JSON_OBJECT.validate_json(request.body) == { + "model": "gpt-4.1", + "training_file": _provider_file_id(bearer, "train"), + "validation_file": _provider_file_id(bearer, "validate"), + "hyperparameters": {}, + }, request.body + + +def _assert_caller_gets_its_own_ids(rig: Rig, outcomes: tuple[tuple[int, Mapping[str, JsonValue]], ...]) -> None: + received: Final = [(status, body.get("training_file"), body.get("validation_file")) for status, body in outcomes] + assert received == [(200, rig.training_file, rig.validation_file)] * len(outcomes), outcomes + + +def test_concurrent_sync_and_async_burst_sends_each_deployment_its_own_file_ids(rig: Rig) -> None: + rig.wire.drain() + with ThreadPoolExecutor(max_workers=BURST // 2) as pool: + synced: Final = tuple(pool.map(_create, (rig,) * (BURST // 2))) + awaited: Final = asyncio.run(_async_creates(rig, BURST // 2)) + creates: Final = _sent_creates(rig, BURST) + _assert_each_deployment_got_its_own_ids(creates) + assert {_bearer(request) for request in creates} == set(DEPLOYMENT_KEYS) + _assert_caller_gets_its_own_ids(rig, synced + awaited) + + +def test_each_worker_alone_sends_the_provider_file_ids(rig: Rig) -> None: + rig.wire.drain() + for paused in sorted(rig.owned.worker_pids()): + with rig.owned.suspended(paused): + outcomes = tuple(_create(rig) for _ in range(4)) + _assert_caller_gets_its_own_ids(rig, outcomes) + _assert_each_deployment_got_its_own_ids(_sent_creates(rig, 4)) + + +def test_a_replaced_worker_still_sends_the_provider_file_ids(rig: Rig) -> None: + rig.wire.drain() + for victim in sorted(rig.owned.worker_pids()): + rig.owned.kill_worker(victim) + outcomes: Final = tuple(_create(rig) for _ in range(6)) + _assert_caller_gets_its_own_ids(rig, outcomes) + _assert_each_deployment_got_its_own_ids(_sent_creates(rig, 6)) + + +def test_a_deployment_added_after_the_upload_never_receives_the_create(tmp_path: Path) -> None: + with gateway_from_environment() as shared, wire_server(_respond) as wire, scratch_database() as database_url: + one_database: Final = {"DATABASE_URL": database_url} + before: Final = tmp_path / "before" + with owned_proxy_process(shared, before, one_database, config=_config_file(before, wire)) as uploader: + with Rig(uploader, wire, "", "").sdk() as client: + training_file: Final = _upload(client, "train") + validation_file: Final = _upload(client, "validate") + after: Final = tmp_path / "after" + with owned_proxy_process( + shared, after, one_database, config=_config_file(after, wire, late_preferred_deployment=True), workers=2 + ) as owned: + eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=120) + wire.drain() + rig: Final = Rig(owned, wire, training_file, validation_file) + outcomes: Final = tuple(_create(rig) for _ in range(6)) + creates: Final = _sent_creates(rig, 6) + _assert_caller_gets_its_own_ids(rig, outcomes) + _assert_each_deployment_got_its_own_ids(creates) + + +def test_a_model_group_without_a_copy_of_the_files_still_gets_the_providers_400(rig: Rig) -> None: + rig.wire.drain() + with rig.sdk() as client, pytest.raises(BadRequestError) as rejected: + client.fine_tuning.jobs.create(model=OTHER_MODEL, training_file=rig.training_file) + creates: Final = _sent_creates(rig, 1) + assert rejected.value.status_code == 400, rejected.value.response.text + assert [_bearer(request) for request in creates] == [OTHER_DEPLOYMENT_KEY], creates diff --git a/tests/integration/security/test_fine_tuning_managed_ids.py b/tests/integration/security/test_fine_tuning_managed_ids.py index 8f179e5a25f..791dd1d1e75 100644 --- a/tests/integration/security/test_fine_tuning_managed_ids.py +++ b/tests/integration/security/test_fine_tuning_managed_ids.py @@ -294,13 +294,9 @@ def test_owner_create_returns_the_unified_job_id(fixture: str, request: pytest.F built: Final[Rig] = request.getfixturevalue(fixture) created: Final = Job.model_validate_json(built.created_text) expected: Final = _client_job( - _unified_job_id(built.deployment_id, built.provider_job), "validating_files", built.provider_file - ) - assert created.model_dump(exclude={"training_file"}) == expected.model_dump(exclude={"training_file"}), ( - "training_file is excluded because test_managed_create_sends_the_provider_file_id_of_the_pinned_deployment " - "documents the known managed-create bug", - built.created_text, + _unified_job_id(built.deployment_id, built.provider_job), "validating_files", built.managed_file ) + assert created == expected, built.created_text def test_another_key_cannot_create_with_or_cancel_the_owners_managed_ids(rig: Rig) -> None: diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 91f6593575e..20ed8ac0c10 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -3,6 +3,7 @@ import json import logging import time import asyncio +import uuid from collections.abc import Awaitable, Callable, Mapping from types import SimpleNamespace from typing import TYPE_CHECKING, Final, cast @@ -13,6 +14,7 @@ import pytest import respx from fastapi import HTTPException from openai import APIConnectionError +from pydantic import TypeAdapter if TYPE_CHECKING: from litellm.types.utils import LiteLLMBatch @@ -22,7 +24,7 @@ from litellm_enterprise.proxy.hooks.managed_files import ( _provider_file_retrieve_credentials, ) from litellm.caching import DualCache -from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable +from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import ( BATCH_CREATE_HIDDEN_PARAM, is_base64_encoded_unified_file_id, @@ -6050,6 +6052,164 @@ async def test_create_from_raw_provider_file_id_skips_managed_ownership_lookup(c assert not any("unified_file_id" in where for where in table.find_first_calls), table.find_first_calls +def _owned_managed_file(unified_uuid: str, model_mappings: Mapping[str, str]) -> tuple[str, LiteLLM_ManagedFileTable]: + raw: Final = f"litellm_proxy:application/jsonl;unified_id,{unified_uuid};target_model_names,gpt-4o-mini" + unified_file_id: Final = base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + return unified_file_id, LiteLLM_ManagedFileTable( + unified_file_id=unified_file_id, + file_object=None, + model_mappings=dict(model_mappings), + flat_model_file_ids=list(model_mappings.values()), + created_by="user_a", + team_id="team_a", + ) + + +async def _provider_kwargs( + managed_files: PROXY_LiteLLMManagedFiles, call_type: str, request: Mapping[str, str], deployment_id: str +) -> Mapping[str, object]: + prepared: Final = await managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(user_id="user_a", team_id="team_a", parent_otel_span=MagicMock()), + cache=MagicMock(), + data=dict(request), + call_type=call_type, + ) + assert isinstance(prepared, dict), prepared + sent: Final = await managed_files.async_pre_call_deployment_hook( + kwargs={**prepared, "litellm_metadata": {"model_info": {"id": deployment_id}}}, call_type=CallTypes(call_type) + ) + assert sent is not None + return sent + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("call_type", "file_field"), _INPUT_FILE_CREATE_CALLS) +@pytest.mark.parametrize(("deployment_id", "provider_file_id"), [("model-a", "file-a"), ("model-b", "file-b")]) +async def test_create_from_managed_file_sends_the_routed_deployments_provider_file_id( + call_type: str, file_field: str, deployment_id: str, provider_file_id: str +) -> None: + unified_file_id, row = _owned_managed_file( + "0b6f2c1e-9a4d-4c3e-8f71-2d5a6b7c8d90", {"model-a": "file-a", "model-b": "file-b"} + ) + managed_files, _ = _managed_files_with_fake_prisma(row) + + sent: Final = await _provider_kwargs( + managed_files, call_type, {"model": "gpt-4o-mini", file_field: unified_file_id}, deployment_id + ) + + assert sent[file_field] == provider_file_id, sent + + +@pytest.mark.asyncio +async def test_fine_tuning_create_swaps_managed_training_and_validation_files_independently() -> None: + training_file, training_row = _owned_managed_file("5c1d9e0a-3b7f-4e2a-9c6d-1f8e7a6b5c40", {"model-a": "file-train"}) + validation_file, validation_row = _owned_managed_file( + "8e2f4a6c-1d3b-4f5e-a7c9-0b2d4f6a8c10", {"model-a": "file-validate"} + ) + managed_files, _ = _managed_files_with_fake_prisma(training_row, validation_row) + + sent: Final = await _provider_kwargs( + managed_files, + "acreate_fine_tuning_job", + {"model": "gpt-4o-mini", "training_file": training_file, "validation_file": validation_file}, + "model-a", + ) + + assert (sent["training_file"], sent["validation_file"]) == ("file-train", "file-validate"), sent + + +@pytest.mark.asyncio +async def test_fine_tuning_create_keeps_a_raw_validation_file_next_to_a_managed_training_file() -> None: + training_file, training_row = _owned_managed_file("5c1d9e0a-3b7f-4e2a-9c6d-1f8e7a6b5c40", {"model-a": "file-train"}) + managed_files, _ = _managed_files_with_fake_prisma(training_row) + + sent: Final = await _provider_kwargs( + managed_files, + "acreate_fine_tuning_job", + {"model": "gpt-4o-mini", "training_file": training_file, "validation_file": "file-raw-validate"}, + "model-a", + ) + + assert (sent["training_file"], sent["validation_file"]) == ("file-train", "file-raw-validate"), sent + + +_REQUEST_KWARGS: Final = TypeAdapter(dict[str, object]) +_DEPLOYMENTS: Final = TypeAdapter(list[dict[str, dict[str, str]]]) + + +async def _routable_deployment_ids( + managed_files: PROXY_LiteLLMManagedFiles, call_type: str, request: Mapping[str, str] +) -> tuple[str, ...]: + prepared: Final = _REQUEST_KWARGS.validate_python( + await managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(user_id="user_a", team_id="team_a", parent_otel_span=MagicMock()), + cache=MagicMock(), + data=dict(request), + call_type=call_type, + ) + ) + routable: Final = _DEPLOYMENTS.validate_python( + await managed_files.async_filter_deployments( + model="gpt-4o-mini", + healthy_deployments=[{"model_info": {"id": model_id}} for model_id in ("model-a", "model-b", "model-c")], + messages=None, + request_kwargs=prepared, + ) + ) + return tuple(deployment["model_info"]["id"] for deployment in routable) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "files", "expected"), + [ + pytest.param( + "acreate_fine_tuning_job", + {"training_file": {"model-a": "f-1", "model-b": "f-2"}, "validation_file": {"model-b": "f-3"}}, + ("model-b",), + id="fine-tune-routes-to-the-deployment-holding-both-files", + ), + pytest.param( + "acreate_fine_tuning_job", + {"training_file": {"model-a": "f-1", "model-b": "f-2"}, "validation_file": "file-raw-validate"}, + ("model-a", "model-b"), + id="fine-tune-raw-validation-file-does-not-narrow", + ), + pytest.param( + "acreate_fine_tuning_job", + {"training_file": {"model-a": "f-1"}, "validation_file": {"model-b": "f-3"}}, + ("model-a", "model-b", "model-c"), + id="fine-tune-files-on-disjoint-deployments-keep-every-deployment", + ), + pytest.param( + "acreate_fine_tuning_job", + {"training_file": {"model-z": "f-5"}}, + ("model-a", "model-b", "model-c"), + id="fine-tune-file-of-another-model-group-keeps-every-deployment", + ), + pytest.param("acreate_batch", {"input_file_id": {"model-c": "f-4"}}, ("model-c",), id="batch-input-file"), + pytest.param( + "acreate_batch", {"input_file_id": {"model-z": "f-6"}}, (), id="batch-input-file-of-another-model-group" + ), + ], +) +async def test_create_from_managed_files_routes_only_to_deployments_holding_every_file( + call_type: str, files: Mapping[str, Mapping[str, str] | str], expected: tuple[str, ...] +) -> None: + owned: Final = { + field: _owned_managed_file(str(uuid.uuid5(uuid.NAMESPACE_URL, field)), mappings) + for field, mappings in files.items() + if not isinstance(mappings, str) + } + managed_files, _ = _managed_files_with_fake_prisma(*(row for _, row in owned.values())) + request: Final = { + "model": "gpt-4o-mini", + **{field: owned[field][0] if field in owned else str(mappings) for field, mappings in files.items()}, + } + + assert await _routable_deployment_ids(managed_files, call_type, request) == expected + + _MANAGED_READ_SALT_KEY: Final = "managed-read-unit-test-salt-key" diff --git a/tests/unit/fine_tuning/__init__.py b/tests/unit/fine_tuning/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/fine_tuning/test_main.py b/tests/unit/fine_tuning/test_main.py new file mode 100644 index 00000000000..57c845c6e1a --- /dev/null +++ b/tests/unit/fine_tuning/test_main.py @@ -0,0 +1,54 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +API_BASE: Final = "https://fine-tuning.test" +BASE_MODEL: Final = "gpt-4.1-nano-2025-04-14" +PROVIDER_JOB: Final = { + "id": "ftjob-abc123", + "object": "fine_tuning.job", + "created_at": 1700000000, + "error": None, + "fine_tuned_model": None, + "finished_at": None, + "hyperparameters": {"n_epochs": 3}, + "model": BASE_MODEL, + "organization_id": "org-test", + "result_files": [], + "seed": 42, + "status": "validating_files", + "trained_tokens": None, + "training_file": "file-abc123", + "validation_file": None, +} + + +@pytest.mark.asyncio +@respx.mock +@pytest.mark.parametrize( + ("custom_llm_provider", "route"), + [("openai", f"{API_BASE}/v1/fine_tuning/jobs"), ("azure", f"{API_BASE}/openai/fine_tuning/jobs")], +) +@pytest.mark.parametrize("prefixed", [True, False], ids=["provider-prefixed", "bare"]) +async def test_create_sends_the_model_without_the_provider_prefix( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, route: str, prefixed: bool +) -> None: + sent: Final = respx.post(url__startswith=route).mock(return_value=httpx.Response(200, json=PROVIDER_JOB)) + + async with httpx.AsyncClient() as session: + monkeypatch.setattr(litellm, "aclient_session", session) + await litellm.acreate_fine_tuning_job( + model=f"{custom_llm_provider}/{BASE_MODEL}" if prefixed else BASE_MODEL, + training_file="file-abc123", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=f"{API_BASE}/v1" if custom_llm_provider == "openai" else API_BASE, + **({"api_version": "2024-10-21"} if custom_llm_provider == "azure" else {}), + ) + + assert json.loads(sent.calls.last.request.content)["model"] == BASE_MODEL diff --git a/tests/unit/llms/vertex_ai/fine_tuning/test_handler.py b/tests/unit/llms/vertex_ai/fine_tuning/test_handler.py index 5447d553f30..79c96b8970a 100644 --- a/tests/unit/llms/vertex_ai/fine_tuning/test_handler.py +++ b/tests/unit/llms/vertex_ai/fine_tuning/test_handler.py @@ -45,7 +45,8 @@ class TestCustomLogger(CustomLogger): @pytest.mark.asyncio() -async def test_create_vertex_fine_tune_jobs_mocked(): +@pytest.mark.parametrize("model_prefix", ["", "vertex_ai/"], ids=["bare", "provider-prefixed"]) +async def test_create_vertex_fine_tune_jobs_mocked(model_prefix: str): # Define reusable variables for the test project_id = "633608382793" location = "us-central1" @@ -90,7 +91,7 @@ async def test_create_vertex_fine_tune_jobs_mocked(): ), ): create_fine_tuning_response = await litellm.acreate_fine_tuning_job( - model=base_model, + model=f"{model_prefix}{base_model}", custom_llm_provider="vertex_ai", training_file=training_file, vertex_project=project_id, diff --git a/tests/unit/proxy/fine_tuning_endpoints/test_endpoints.py b/tests/unit/proxy/fine_tuning_endpoints/test_endpoints.py index 99a79801698..06ac56b09ad 100644 --- a/tests/unit/proxy/fine_tuning_endpoints/test_endpoints.py +++ b/tests/unit/proxy/fine_tuning_endpoints/test_endpoints.py @@ -37,9 +37,9 @@ RAW_FILE_ID = "file-victim-abc123" RAW_JOB_ID = "ftjob-victim-abc123" -def _unified_file_id() -> str: +def _unified_file_id(unified_id: str = "victim-unified-id", provider_file_id: str = RAW_FILE_ID) -> str: unified = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( - "application/json", "victim-unified-id", "gpt-4o-mini", RAW_FILE_ID, "gpt-4o-mini-id" + "application/json", unified_id, "gpt-4o-mini", provider_file_id, "gpt-4o-mini-id" ) return base64.urlsafe_b64encode(unified.encode()).decode().rstrip("=") @@ -54,7 +54,7 @@ def _decode_unified_id(encoded_id: str) -> str: return base64.urlsafe_b64decode(f"{encoded_id}{padding}").decode() -def _job() -> LiteLLMFineTuningJob: +def _job(validation_file: str | None = None) -> LiteLLMFineTuningJob: job = LiteLLMFineTuningJob( id=RAW_JOB_ID, created_at=1234567890, @@ -69,7 +69,7 @@ def _job() -> LiteLLMFineTuningJob: status="running", trained_tokens=None, training_file=RAW_FILE_ID, - validation_file=None, + validation_file=validation_file, ) job._hidden_params = {} return job @@ -225,6 +225,31 @@ async def test_create__unified_training_file_allowed_when_managed_files_required assert response.hidden_params["unified_file_id"] == _decode_unified_id(unified_file_id) +MANAGED_VALIDATION_FILE_ID: Final = _unified_file_id("validation-unified-id", "file-provider-validation") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("validation_file", "provider_validation_file", "expected_validation_file"), + [ + (None, None, None), + (RAW_FILE_ID, RAW_FILE_ID, RAW_FILE_ID), + (MANAGED_VALIDATION_FILE_ID, "file-provider-validation", MANAGED_VALIDATION_FILE_ID), + ], + ids=["no-validation-file", "raw-validation-file", "managed-validation-file"], +) +async def test_create__managed_job_returns_the_callers_file_ids( + seams, validation_file: str | None, provider_validation_file: str | None, expected_validation_file: str | None +): + seams.logging.get_proxy_hook.return_value = ManagedResourceAccessCheckerStub() + seams.router.acreate_fine_tuning_job.return_value = _job(validation_file=provider_validation_file) + training_file: Final = _unified_file_id() + + response: Final = await _create(training_file, validation_file=validation_file) + + assert (response.training_file, response.validation_file) == (training_file, expected_validation_file) + + @pytest.mark.asyncio async def test_create__other_teams_unified_training_file_rejected(seams): seams.logging.get_proxy_hook.return_value = ManagedResourceAccessCheckerStub(file_access=False) From cf9ebeda073bd03e6479558f7b683904054f2301 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:04:25 +0000 Subject: [PATCH 015/124] fix(proxy): stop billing every OpenAI pass-through /responses sub-route as a Responses call (#47171) * test(proxy): cover OpenAI pass-through billing for each /responses sub-route * fix(proxy): bill only the OpenAI pass-through Responses calls OpenAI bills * fix(proxy): fail closed on unknown /responses sub-routes and label unbilled spend rows Bill every OpenAI pass-through /responses call except input_tokens, input_items, delete, and a non-background retrieve or cancel, so an encoded or new sub-route is never free. Unbilled calls keep the response's model on their spend row, and a resumed background stream takes its model from the stream's final event. * fix(proxy): give billed background Responses reads their own spend log row A billed retrieve or cancel of a finished background response reused the create's resp_ id, so the spend log insert dropped the row that carried the cost. It now takes the native path's _background_cost suffix. Adds Azure path coverage and types the test recorder. * fix(proxy): bill a background Responses read only once it carries usage An in-progress poll took the _background_cost spend log id with $0, so the finished read's row that carried the cost was dropped again. Polls with no usage are now unbilled like any other read. * fix(proxy): keep OpenAI's response id for callbacks on a billed background read * fix(openai): give a streamed background create the settlement spend row id * fix(proxy): keep a request body from giving its spend row the background settlement id * test(proxy): give the forged settlement marker test a fixed time * fix(passthrough): move the background settlement marker into the pass-through types module A test that reloads spend_tracking_utils gave it a new enum class, so the handler's marker no longer matched by identity and the settlement row lost its _background_cost id. The types module is never reloaded. --- .../openai_passthrough_logging_handler.py | 88 +++- .../spend_tracking/spend_tracking_utils.py | 17 +- .../pass_through_endpoints.py | 6 + ...test_openai_passthrough_logging_handler.py | 432 +++++++++++++++++- .../test_spend_tracking_utils.py | 27 ++ 5 files changed, 550 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 2e9d92ed309..c763b5dc26e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -6,7 +6,7 @@ Handles cost tracking and logging for OpenAI passthrough endpoints, specifically from collections.abc import Mapping, Sequence from datetime import datetime -from typing import Final +from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx @@ -14,6 +14,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param +from litellm.litellm_core_utils.internal_call_metadata import is_background_response from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -31,6 +32,8 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( ) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, EndpointType, PassthroughStandardLoggingPayload, ) @@ -56,7 +59,7 @@ _AZURE_OPENAI_HOSTNAMES: Final = ("openai.azure.com", "cognitiveservices.azure.c _AZURE_OPENAI_PATH_MARKERS: Final = ("/openai/", "/v1/") -def _hostname_matches(hostname: str, suffixes: tuple) -> bool: +def _hostname_matches(hostname: str, suffixes: tuple[str, ...]) -> bool: """True if hostname equals one of `suffixes` or is a subdomain of it. Uses suffix matching (not a bare substring test) so look-alikes such as @@ -102,6 +105,19 @@ def is_openai_compatible_url(url_route: str | None) -> bool: _is_openai_compatible_url: Final = is_openai_compatible_url +ResponsesCallBilling: TypeAlias = Literal["unbilled", "billed", "background_settlement"] + + +def _is_finished_background_response(response_body: Mapping[str, object]) -> bool: + return is_background_response(response_body) and response_body.get("usage") is not None + + +def _responses_sub_route(path: str) -> tuple[str, ...] | None: + segments: Final = path.rstrip("/").split("/") + anchor: Final = max((index for index, segment in enumerate(segments) if segment == "responses"), default=None) + return None if anchor is None else tuple(segments[anchor + 1 :]) + + def _is_remote_high_detail_image(part: object) -> bool: if not isinstance(part, Mapping) or part.get("type") != "image_url": return False @@ -186,6 +202,30 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): "/v1/responses" in parsed_url.path or "/responses" in parsed_url.path ) + @staticmethod + def openai_responses_call_billing( + method: str, url_route: str, response_body: Mapping[str, object] + ) -> ResponsesCallBilling: + if not OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route): + return "unbilled" + match (method, _responses_sub_route(urlparse(url_route).path)): + case ("POST", ("input_tokens",)) | ("GET", (_, "input_items")) | ("DELETE", (_,)): + return "unbilled" + case ("GET", (_,)) | ("POST", (_, "cancel")): + return "background_settlement" if _is_finished_background_response(response_body) else "unbilled" + case _: + return "billed" + + @staticmethod + def _label_unbilled_responses_call( + logging_obj: LiteLLMLoggingObj, + url_route: str, + response_body: Mapping[str, object], + ) -> None: + model: Final = response_body.get("model") + if OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) and isinstance(model, str): + logging_obj.model_call_details["model"] = model # rebind-ok: spend rows read the model from here + @staticmethod def is_openai_embeddings_route(url_route: str) -> bool: """Check if the URL route is an OpenAI embeddings endpoint.""" @@ -348,9 +388,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): is_embeddings: Final = OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route) is_image_generation: Final = OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) is_image_editing: Final = OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) - is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + responses_billing: Final = OpenAIPassthroughLoggingHandler.openai_responses_call_billing( + httpx_response.request.method, url_route, response_body + ) + is_responses: Final = responses_billing != "unbilled" if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): + OpenAIPassthroughLoggingHandler._label_unbilled_responses_call(logging_obj, url_route, response_body) return { "result": None, "kwargs": kwargs, @@ -505,6 +549,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): logging_obj.model_call_details["model"] = model logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider logging_obj.model_call_details["response_cost"] = response_cost + logging_obj.model_call_details[SETTLES_BACKGROUND_RESPONSE_KWARG] = ( # rebind-ok: spend rows read it here + BackgroundResponseSettlement.SETTLES if responses_billing == "background_settlement" else None + ) endpoint_type: Final = ( "chat_completions" @@ -625,16 +672,24 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): Handle logging for collected OpenAI streaming chunks with cost tracking. """ try: - # Extract model from request body - model: Final = request_body.get("model", "gpt-4o") - is_responses: Final = OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + terminal_response: Final = ( + OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks(all_chunks=all_chunks) + if is_responses + else None + ) + requested_model: Final[object] = request_body.get("model") + model: Final = ( + requested_model + if isinstance(requested_model, str) and requested_model + else (terminal_response.model if terminal_response is not None else None) or "gpt-4o" + ) # Build complete response from chunks using our streaming handler handler: Final = OpenAIPassthroughLoggingHandler() handler_instance: Final = handler complete_response: Final = ( - OpenAIResponsesAPIConfig.parse_terminal_response_from_stream_chunks(all_chunks=all_chunks) + terminal_response if is_responses else handler._build_complete_streaming_response( all_chunks=all_chunks, @@ -667,17 +722,27 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): ) ) + settles_background_response: Final = ( + BackgroundResponseSettlement.SETTLES + if terminal_response is not None + and is_background_response(terminal_response) + and terminal_response.usage is not None + else None + ) + # Preserve existing litellm_params to maintain metadata tags existing_litellm_params: Final = litellm_logging_obj.model_call_details.get("litellm_params", {}) or {} + litellm_params: Final = existing_litellm_params.copy() # Prepare kwargs for logging kwargs: Final = { "response_cost": response_cost, + SETTLES_BACKGROUND_RESPONSE_KWARG: settles_background_response, "model": model, "custom_llm_provider": custom_llm_provider, "call_type": litellm_logging_obj.call_type, "messages": litellm_logging_obj.model_call_details.get("messages"), - "litellm_params": existing_litellm_params.copy(), + "litellm_params": litellm_params, } # Extract user information for tracking @@ -689,9 +754,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): passthrough_logging_payload=passthrough_logging_payload, ) if user: - kwargs["litellm_params"].setdefault("proxy_server_request", {}).setdefault("body", {})["user"] = ( - user - ) + litellm_params.setdefault("proxy_server_request", {}).setdefault("body", {})["user"] = user # Attach the payload to kwargs so the success handler adopts it; # its later rebuild runs on a copy whose Responses usage was @@ -712,6 +775,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): litellm_logging_obj.model_call_details["model"] = model litellm_logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider litellm_logging_obj.model_call_details["response_cost"] = response_cost + litellm_logging_obj.model_call_details[SETTLES_BACKGROUND_RESPONSE_KWARG] = ( # rebind-ok: spend row id + settles_background_response + ) verbose_proxy_logger.debug( f"OpenAI streaming passthrough cost tracking - Model: {model}, Cost: ${response_cost:.6f}" diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 78818cde266..091fb5d1305 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -69,6 +69,10 @@ from litellm.proxy._types import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.router import DeploymentTypedDict, LiteLLM_Params from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, @@ -315,10 +319,12 @@ BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost" BACKGROUND_RESPONSE_COST_REQUEST_ID_SUFFIX: Final = "_background_cost" -def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object]) -> str: +def _cost_row_request_id_suffix( + call_type: str, metadata: Mapping[str, object], settles_background_response: bool +) -> str: if call_type == CallTypes.aretrieve_batch.value: return BATCH_COST_REQUEST_ID_SUFFIX - if ( + if settles_background_response or ( call_type in NON_INFERENCE_CALL_TYPES and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN ): @@ -327,7 +333,7 @@ def _cost_row_request_id_suffix(call_type: str, metadata: Mapping[str, object]) def get_spend_logs_id( - call_type: str, response_obj: dict, kwargs: dict, metadata: Mapping[str, object] = EMPTY_MAPPING + call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING ) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") candidate_ids: Final = ( @@ -340,7 +346,10 @@ def get_spend_logs_id( ) if resolved_id is None: return None - return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata)}" + settles_background_response: Final = ( + kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES + ) + return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}" _MISSING_ATTRIBUTE: Final = object() diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index e634354408f..0d99cd54a6b 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -25,6 +25,12 @@ LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_thro # on a collision FastAPI dispatches the built-in handler, which does not carry this flag. LITELLM_PASS_THROUGH_ENDPOINT_MARKER: Final = "__litellm_pass_through_endpoint__" +SETTLES_BACKGROUND_RESPONSE_KWARG: Final = "settles_background_response" + + +class BackgroundResponseSettlement(str, Enum): + SETTLES = "settles" + class EndpointType(str, Enum): VERTEX_AI = "vertex-ai" diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 86a670bcb3e..c9064caf929 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -1,19 +1,26 @@ +from dataclasses import dataclass +import asyncio import json +import logging +from collections.abc import Mapping from datetime import datetime -from typing import Any, Dict, List +from typing import Any, Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from pydantic import BaseModel, ConfigDict import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.token_counter import high_detail_image_token_upper_bound from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, count_relayed_prompt_tokens, ) +from litellm.proxy._types import SpendLogsPayload from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -1837,10 +1844,7 @@ class TestOpenAIPassthroughResponsesStreamingSpendLog: def _expected_spend(self) -> float: rates = litellm.model_cost[self.MODEL_MAP_KEY] - return ( - self.INPUT_TOKENS * rates["input_cost_per_token"] - + self.OUTPUT_TOKENS * rates["output_cost_per_token"] - ) + return self.INPUT_TOKENS * rates["input_cost_per_token"] + self.OUTPUT_TOKENS * rates["output_cost_per_token"] def _responses_stream_chunks(self) -> List[str]: """Usage arrives only on the terminal `response.completed` event, nested under @@ -1953,6 +1957,28 @@ class TestOpenAIPassthroughResponsesStreamingSpendLog: row_metadata = json.loads(spend_log_row["metadata"]) assert row_metadata["model_map_information"]["model_map_key"] == self.MODEL_MAP_KEY + def test_resumed_responses_stream_spend_log_is_labeled_with_the_streamed_model(self, local_model_cost_map): + result: Final = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks( + litellm_logging_obj=self._logging_obj(), + passthrough_success_handler_obj=None, + url_route=f"https://api.openai.com/v1/responses/{self.RESPONSE_ID}?stream=true", + request_body={}, + endpoint_type=None, + start_time=self.start_time, + all_chunks=self._responses_stream_chunks(), + end_time=self.end_time, + ) + + spend_log_row: Final = get_logging_payload( + kwargs=result["kwargs"], + response_obj=result["result"], + start_time=self.start_time, + end_time=self.end_time, + ) + + assert spend_log_row["model"] == self.MODEL_MAP_KEY + assert spend_log_row["spend"] == pytest.approx(self._expected_spend()) + class TestOpenAIPassthroughEmbeddingsSpendLog: """An OpenAI-passthrough `/v1/embeddings` call must write a priced spend log row @@ -2043,6 +2069,402 @@ class TestOpenAIPassthroughEmbeddingsSpendLog: assert spend_log_row["request_id"] == self.CALL_ID +RESPONSES_URL: Final = "https://api.openai.com/v1/responses" +STORED_RESPONSE_ID: Final = "resp_080b10ff5d740c69006ad3de732e5087" +DISPATCH_TIME: Final = datetime(2026, 10, 17, 13, 47, 3) +RESPONSES_USAGE: Final = { + "input_tokens": 11, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 16, +} +CREATE_REQUEST: Final = {"model": "gpt-5.4-mini", "input": "Say hi in one word"} +COMPACT_REQUEST: Final = {"model": "gpt-5.4-mini", "input": "Remember the number 42."} +COMPACTION: Final = { + "id": STORED_RESPONSE_ID, + "object": "response.compaction", + "created_at": 1792270025, + "output": [{"id": "cmp_082492f9", "type": "compaction", "encrypted_content": "gAAAA"}], + "usage": RESPONSES_USAGE, +} +ASSISTANT_MESSAGE: Final = { + "id": "msg_080b10ff5d740c69", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hi", "annotations": []}], +} + + +def _stored_response(status: str, background: bool, usage: Mapping[str, object] | None) -> dict[str, object]: + return { + "id": STORED_RESPONSE_ID, + "object": "response", + "created_at": 1792269939, + "status": status, + "background": background, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": {}, + "model": "gpt-5.4-mini-2026-03-17", + "output": [ASSISTANT_MESSAGE] if usage else [], + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 0.98, + "usage": usage, + } + + +@dataclass(frozen=True, slots=True) +class _LoggedCall: + spend_row: SpendLogsPayload + callback_response_id: str + + +class _CallbackPayloadId(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + + +class _SpendRowRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.call: Final[asyncio.Future[_LoggedCall]] = asyncio.get_running_loop().create_future() + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.call.set_result( + _LoggedCall( + spend_row=get_logging_payload( + kwargs=kwargs, response_obj=response_obj, start_time=start_time, end_time=end_time + ), + callback_response_id=_CallbackPayloadId.model_validate(kwargs.get("standard_logging_object")).id, + ) + ) + + +class TestOpenAIPassthroughResponsesSubRouteBilling: + def _dispatch( + self, + method: str, + sub_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + base_url: str = RESPONSES_URL, + ): + url_route: Final = f"{base_url}{sub_route}" + logging_obj: Final = LiteLLMLoggingObj( + model="gpt-5.4-mini", + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=DISPATCH_TIME, + litellm_call_id="6c1f3a52-93d4-4b1e-9f0e-6f2f4f6b9a10", + function_id="1234", + ) + return PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=httpx.Response(200, json=response_body, request=httpx.Request(method, url_route)), + response_body=response_body, + request_body=request_body, + logging_obj=logging_obj, + url_route=url_route, + result="", + start_time=DISPATCH_TIME, + end_time=DISPATCH_TIME, + cache_hit=False, + custom_llm_provider="openai", + litellm_params={}, + call_type="pass_through_endpoint", + ) + + @pytest.mark.parametrize( + ("method", "sub_route", "request_body", "response_body"), + [ + pytest.param( + "POST", "", CREATE_REQUEST, _stored_response("completed", False, RESPONSES_USAGE), id="create" + ), + pytest.param( + "POST", + "/", + CREATE_REQUEST, + _stored_response("completed", False, RESPONSES_USAGE), + id="create_trailing_slash", + ), + pytest.param("POST", "/compact", COMPACT_REQUEST, COMPACTION, id="compact"), + pytest.param("POST", "/%2563ompact", COMPACT_REQUEST, COMPACTION, id="percent_encoded_compact"), + pytest.param( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + _stored_response("completed", True, RESPONSES_USAGE), + id="background_retrieve_completed", + ), + pytest.param( + "POST", + f"/{STORED_RESPONSE_ID}/cancel", + {}, + _stored_response("cancelled", True, RESPONSES_USAGE), + id="background_cancel", + ), + ], + ) + def test_billed_responses_call_records_cost( + self, + local_model_cost_map, + method: str, + sub_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + ): + dispatched: Final = self._dispatch(method, sub_route, request_body, response_body) + + assert dispatched["standard_logging_response_object"] is not None + assert dispatched["kwargs"]["response_cost"] > 0 + + @pytest.mark.parametrize( + ("method", "sub_route", "request_body", "response_body"), + [ + pytest.param( + "POST", + "/input_tokens", + {"model": "gpt-5.4-mini", "input": "hi"}, + {"object": "response.input_tokens", "input_tokens": 7}, + id="input_tokens", + ), + pytest.param( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + _stored_response("completed", False, RESPONSES_USAGE), + id="retrieve", + ), + pytest.param( + "GET", + f"/{STORED_RESPONSE_ID}/input_items", + {}, + { + "object": "list", + "data": [ + { + "id": "msg_074176e3", + "type": "message", + "status": "completed", + "role": "user", + "content": [{"type": "input_text", "text": "Say hi in one word"}], + } + ], + "first_id": "msg_074176e3", + "last_id": "msg_074176e3", + "has_more": False, + }, + id="input_items", + ), + pytest.param( + "DELETE", + f"/{STORED_RESPONSE_ID}", + {}, + {"id": STORED_RESPONSE_ID, "object": "response.deleted", "deleted": True}, + id="delete", + ), + pytest.param( + "POST", + f"/{STORED_RESPONSE_ID}/cancel", + {}, + _stored_response("cancelled", True, None), + id="background_cancel_before_any_output", + ), + ], + ) + def test_unbilled_responses_sub_route_records_no_cost_and_no_error( + self, + local_model_cost_map, + caplog: pytest.LogCaptureFixture, + method: str, + sub_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + ): + with caplog.at_level(logging.WARNING): + dispatched: Final = self._dispatch(method, sub_route, request_body, response_body) + + assert not dispatched["kwargs"].get("response_cost") + assert [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] == [] + + async def _logged_call( + self, + method: str, + sub_route: str, + request_body: dict[str, object], + response_body: dict[str, object], + litellm_call_id: str, + ) -> _LoggedCall: + url_route: Final = f"{RESPONSES_URL}{sub_route}" + recorder: Final = _SpendRowRecorder() + logging_obj: Final = LiteLLMLoggingObj( + model="unknown", + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=DISPATCH_TIME, + litellm_call_id=litellm_call_id, + function_id="1234", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="unknown", user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint" + ) + + await PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=httpx.Response(200, json=response_body, request=httpx.Request(method, url_route)), + response_body=response_body, + logging_obj=logging_obj, + url_route=url_route, + result="", + start_time=DISPATCH_TIME, + end_time=DISPATCH_TIME, + cache_hit=False, + request_body=request_body, + passthrough_logging_payload=PassthroughStandardLoggingPayload( + url=url_route, request_body=request_body, request_method=method + ), + custom_llm_provider="openai", + ) + return await asyncio.wait_for(recorder.call, timeout=5) + + @pytest.mark.asyncio + async def test_unbilled_retrieve_spend_log_keeps_the_response_model(self, local_model_cost_map): + response_body: Final = _stored_response("completed", False, RESPONSES_USAGE) + + spend_log_row: Final = ( + await self._logged_call( + "GET", f"/{STORED_RESPONSE_ID}", {}, response_body, "0f6f3d2e-6a51-4c4e-9a43-6f1b0d1e9c21" + ) + ).spend_row + + assert spend_log_row["model"] == response_body["model"] + assert not spend_log_row["spend"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("method", "sub_route", "response_body"), + [ + pytest.param( + "GET", + f"/{STORED_RESPONSE_ID}", + _stored_response("completed", True, RESPONSES_USAGE), + id="background_retrieve_completed", + ), + pytest.param( + "POST", + f"/{STORED_RESPONSE_ID}/cancel", + _stored_response("cancelled", True, RESPONSES_USAGE), + id="background_cancel", + ), + ], + ) + async def test_billed_background_read_spend_log_survives_next_to_its_create_and_poll_rows( + self, local_model_cost_map, method: str, sub_route: str, response_body: dict[str, object] + ): + create: Final = await self._logged_call( + "POST", + "", + {**CREATE_REQUEST, "background": True}, + _stored_response("queued", True, None), + "3a4f0c1e-2b7d-4c55-8e0e-0d7a51d9f6a1", + ) + poll: Final = await self._logged_call( + "GET", + f"/{STORED_RESPONSE_ID}", + {}, + _stored_response("in_progress", True, None), + "5d8c2f71-0e4b-4a9d-b3c6-7f1e2a9d0c44", + ) + read: Final = await self._logged_call( + method, sub_route, {}, response_body, "9b2e6d40-71c3-4f8a-a5d2-4e3c8f0b7d12" + ) + + assert read.spend_row["request_id"] not in {create.spend_row["request_id"], poll.spend_row["request_id"]} + assert read.spend_row["request_id"].startswith(STORED_RESPONSE_ID) + assert read.spend_row["spend"] > 0 + assert not poll.spend_row["spend"] + assert read.callback_response_id == STORED_RESPONSE_ID + + @pytest.mark.asyncio + async def test_streamed_background_create_and_its_finished_read_share_one_spend_row(self, local_model_cost_map): + finished: Final = _stored_response("completed", True, RESPONSES_USAGE) + stream_logging_obj: Final = LiteLLMLoggingObj( + model="gpt-5.4-mini", + messages=[], + stream=True, + call_type="pass_through_endpoint", + start_time=DISPATCH_TIME, + litellm_call_id="7e3b9a14-5c2d-4f60-8d1a-2b6c0e9f4a37", + function_id="1234", + ) + stream_logging_obj.model_call_details["custom_llm_provider"] = "openai" + streamed_create: Final = OpenAIPassthroughLoggingHandler._handle_logging_openai_collected_chunks( + litellm_logging_obj=stream_logging_obj, + passthrough_success_handler_obj=None, + url_route=RESPONSES_URL, + request_body={**CREATE_REQUEST, "background": True, "stream": True}, + endpoint_type=None, + start_time=DISPATCH_TIME, + all_chunks=[ + f"data: {json.dumps({'type': 'response.completed', 'sequence_number': 1, 'response': finished})}", + "data: [DONE]", + ], + end_time=DISPATCH_TIME, + ) + create_row: Final = get_logging_payload( + kwargs=streamed_create["kwargs"], + response_obj=streamed_create["result"], + start_time=DISPATCH_TIME, + end_time=DISPATCH_TIME, + ) + + read: Final = await self._logged_call( + "GET", f"/{STORED_RESPONSE_ID}", {}, finished, "1c8d5e2a-9f47-4b3e-a6d0-5e7f2c1b8a93" + ) + + assert create_row["spend"] > 0 + assert read.spend_row["request_id"] == create_row["request_id"] + assert read.callback_response_id == STORED_RESPONSE_ID + + @pytest.mark.parametrize( + "base_url", + [ + pytest.param("https://my-resource.openai.azure.com/openai/v1/responses", id="azure_openai_v1"), + pytest.param("https://my-resource.cognitiveservices.azure.com/openai/responses", id="azure_openai_native"), + ], + ) + def test_azure_responses_sub_routes_bill_like_openai(self, local_model_cost_map, base_url: str): + retrieve: Final = self._dispatch( + "GET", f"/{STORED_RESPONSE_ID}", {}, _stored_response("completed", False, RESPONSES_USAGE), base_url + ) + input_tokens: Final = self._dispatch( + "POST", + "/input_tokens", + {"model": "gpt-5.4-mini", "input": "hi"}, + {"object": "response.input_tokens", "input_tokens": 7}, + base_url, + ) + background_retrieve: Final = self._dispatch( + "GET", f"/{STORED_RESPONSE_ID}", {}, _stored_response("completed", True, RESPONSES_USAGE), base_url + ) + compact: Final = self._dispatch("POST", "/compact", COMPACT_REQUEST, COMPACTION, base_url) + + assert not retrieve["kwargs"].get("response_cost") + assert not input_tokens["kwargs"].get("response_cost") + assert background_retrieve["kwargs"]["response_cost"] > 0 + assert compact["kwargs"]["response_cost"] > 0 + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 34e17b3cda8..11a881527e6 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -47,6 +47,10 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( spend_log_row_with_retained_metadata, ) from litellm.proxy.utils import hash_token +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, @@ -4810,6 +4814,29 @@ def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_th assert request_ids == ("resp_shared", "resp_shared_background_cost") +@pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value]) +def test_a_request_body_cannot_move_its_spend_row_onto_a_background_settlement_id(forged_marker): + """A failed call logs its raw request body as kwargs, so a body field named like the settlement + marker must not give the failure row the id a background response's billing row owns.""" + request_body: Final = { + "model": "gpt-5.4-mini", + "input": "hi", + SETTLES_BACKGROUND_RESPONSE_KWARG: forged_marker, + "litellm_call_id": "call-failed", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + } + failed_at: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) + + row: Final = get_logging_payload( + kwargs=request_body, + response_obj=Exception("upstream rejected the request"), + start_time=failed_at, + end_time=failed_at, + ) + + assert row["request_id"] == "call-failed" + + def test_get_spend_logs_id_prefers_the_response_id_over_the_standard_logging_id(): """An unredacted response keeps deciding its own row key, so cache-hit ids and every other call type behave exactly as they did before.""" From b110c8cfa7422be35753eb550807e1a7bdf9375b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 15:06:02 -0700 Subject: [PATCH 016/124] test(integration): check SSO sign-in removal without reaching the public internet (#47409) The SSO removal test configured Google sign-in, which fetches its discovery document from accounts.google.com before redirecting. CircleCI blocks that egress, so the test failed on every main run since #46709 added it. Microsoft sign-in builds its endpoints locally and goes through the same SSO environment variable handling, so the test now uses it --- tests/integration/configuration/test_effective_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/configuration/test_effective_settings.py b/tests/integration/configuration/test_effective_settings.py index acdaacfcfb5..07477f1f048 100644 --- a/tests/integration/configuration/test_effective_settings.py +++ b/tests/integration/configuration/test_effective_settings.py @@ -1547,8 +1547,8 @@ def test_an_sso_client_removed_from_db_environment_variables_stops_sign_in_on_ev "DATABASE_URL_READ_REPLICA", "GENERIC_CLIENT_ID", "GOOGLE_CLIENT_ID", - "GOOGLE_CLIENT_SECRET", "MICROSOFT_CLIENT_ID", + "MICROSOFT_CLIENT_SECRET", "PROXY_BASE_URL", ) with ( @@ -1567,7 +1567,7 @@ def test_an_sso_client_removed_from_db_environment_variables_stops_sign_in_on_ev response: Final = first.request( "POST", "/config/update", - {"environment_variables": {"GOOGLE_CLIENT_ID": client_id, "GOOGLE_CLIENT_SECRET": secret}}, + {"environment_variables": {"MICROSOFT_CLIENT_ID": client_id, "MICROSOFT_CLIENT_SECRET": secret}}, ) assert response.status_code == 200, response.text @@ -1584,7 +1584,7 @@ def test_an_sso_client_removed_from_db_environment_variables_stops_sign_in_on_ev moved: Final = first.request( "PATCH", "/update/sso_settings", - {"google_client_id": sso_table_client, "google_client_secret": "sso-table-secret"}, + {"microsoft_client_id": sso_table_client, "microsoft_client_secret": "sso-table-secret"}, ) assert moved.status_code == 200, moved.text eventually(sign_in_clients, lambda clients: clients == (sso_table_client,) * 2, seconds=30) From 77bd7d6de841bdc746d6b45a0ba98d5f60eb7af1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 15:06:17 -0700 Subject: [PATCH 017/124] test(integration): expect the requested model in streamed Responses events on the arize v2 rig (#47407) #46662 made streamed /v1/responses lifecycle events report the requested model in the nested response.model, matching the non-streamed body. The arize v2 caller-stream helper still expected the upstream deployment name there, so f1 and a6 failed on every main run since bbd446b476 --- tests/integration/observability/_openinference_support.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/observability/_openinference_support.py b/tests/integration/observability/_openinference_support.py index a57e25f88ae..b46503703c7 100644 --- a/tests/integration/observability/_openinference_support.py +++ b/tests/integration/observability/_openinference_support.py @@ -386,13 +386,14 @@ def _responses_caller_body(body: dict[str, JsonValue], model: str) -> dict[str, } -def _responses_stream_caller_body(body: dict[str, JsonValue]) -> dict[str, JsonValue]: +def _responses_stream_caller_body(body: dict[str, JsonValue], model: str) -> dict[str, JsonValue]: response_id: Final = body.get("id") assert isinstance(response_id, str), body usage: Final = _json_object_value(body["usage"]) if body.get("status") == "completed" else {} return { **body, "id": "", + "model": model, **({"usage": {**usage, "cost": 4.05e-06}} if usage else {}), } @@ -413,7 +414,7 @@ def _responses_caller_stream(reply: Reply, model: str) -> tuple[dict[str, JsonVa **event, "model": model, **( - {"response": _responses_stream_caller_body(_json_object_value(event["response"]))} + {"response": _responses_stream_caller_body(_json_object_value(event["response"]), model)} if "response" in event else {} ), From 1ef59fab8b7621730384ba88ce37f5dd05c0bc65 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 15:06:35 -0700 Subject: [PATCH 018/124] test(ui-e2e): read the virtual key from its own pre in the Save your Key dialog (#47408) #47199 added a Gateway URL pre to the created-key dialog, so the bare pre locator matched two elements and failed strict mode in four specs --- tests/e2e/ui/tests/internal-user/internalUser.spec.ts | 2 +- tests/e2e/ui/tests/proxy-admin/keys.spec.ts | 2 +- tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts | 2 +- tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts index d34bc034453..cb7d9320744 100644 --- a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts @@ -67,7 +67,7 @@ test.describe("Internal User", () => { await page.getByRole("button", { name: "Create Key", exact: true }).click(); await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); - const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim(); + const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim(); expect(apiKey).toMatch(/^sk-/); await page.keyboard.press("Escape"); diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index deb7ae70d07..d3d67bde1cf 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -254,7 +254,7 @@ test.describe("Proxy Admin - Keys", () => { // verify it can call /chat/completions for the model it was scoped to. // The mock LLM server (fixtures/mock_llm_server/server.py) replies with // a fixed "This is a mock response." body. - const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim(); + const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim(); expect(apiKey).toMatch(/^sk-/); const response = await page.request.post("/chat/completions", { diff --git a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts index 4572f71b3db..835a26af0dc 100644 --- a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts @@ -69,7 +69,7 @@ test.describe("Second proxy admin", () => { await page.getByRole("button", { name: "Create Key", exact: true }).click(); await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); - const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim(); + const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim(); expect(apiKey).toMatch(/^sk-/); await page.keyboard.press("Escape"); diff --git a/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts b/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts index b7978fae7fb..95000b0ff18 100644 --- a/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts +++ b/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts @@ -130,7 +130,7 @@ test.describe("Team Admin - Member permissions", () => { await memberPage.getByRole("button", { name: "Create Key", exact: true }).click(); const saveDialog = memberPage.getByRole("dialog", { name: "Save your Key" }); await expect(saveDialog).toBeVisible({ timeout: 15_000 }); - const apiKey = (await saveDialog.locator("pre").innerText()).trim(); + const apiKey = (await saveDialog.locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim(); expect(apiKey).toMatch(/^sk-/); createdKeys.push(apiKey); await memberPage.keyboard.press("Escape"); From 09e919351437851ea0ba689f7e878a7395627f57 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 15:14:46 -0700 Subject: [PATCH 019/124] fix(cost): bill Gemini audio/image/video prompt tokens at the resolved input rate when no modality rate exists (#47373) * fix(cost): bill Gemini audio/image/video prompt tokens at the resolved input rate when no modality rate exists Co-authored-by: shivam * style(cost): ruff format the modality cost calls --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: shivam --- .../litellm_core_utils/llm_cost_calc/utils.py | 52 +++++++++++---- .../cost_calculation/cases/vertex_ai.py | 64 +++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 60 +++++++++++++++++ 3 files changed, 164 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2afb6ed4c58..c6141be008e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1088,6 +1088,25 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu ) +def _modality_token_cost( + model_info: ModelInfo, + modality_cost_key: str, + tokens: int, + prompt_base_cost: float, + service_tier: str | None, +) -> float: + """ + Audio, image and video prompt tokens bill at their modality rate when the model has one. + Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which + `_get_token_base_cost` has already resolved for the service tier and long-context tier. + """ + tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier) + cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key + if model_info.get(cost_key) is None: + return float(tokens) * prompt_base_cost + return calculate_cost_component(model_info, cost_key, tokens) + + def _calculate_input_cost( prompt_tokens_details: PromptTokensDetailsResult, model_info: ModelInfo, @@ -1118,28 +1137,37 @@ def _calculate_input_cost( if prompt_tokens_details["audio_tokens"] and not ( prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None ): - audio_cost_key: Final = get_service_tier_cost_key("input_cost_per_audio_token", service_tier) - prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_audio_token", + prompt_tokens_details["audio_tokens"], + prompt_base_cost, + service_tier, + ) ### IMAGE TOKEN COST if prompt_tokens_details["image_tokens"] and not ( prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None ): - # For image token costs: - # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. - image_token_cost_key = "input_cost_per_image_token" - if model_info.get(image_token_cost_key) is None: - image_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_image_token", + prompt_tokens_details["image_tokens"], + prompt_base_cost, + service_tier, + ) ### VIDEO TOKEN COST if prompt_tokens_details["video_tokens"] and not ( prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None ): - video_token_cost_key = "input_cost_per_video_token" - if model_info.get(video_token_cost_key) is None: - video_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"]) + prompt_cost += _modality_token_cost( + model_info, + "input_cost_per_video_token", + prompt_tokens_details["video_tokens"], + prompt_base_cost, + service_tier, + ) ### CACHE WRITING COST - Now uses tiered pricing if ( diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 3b825bfe7b2..305c10535b7 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -946,6 +946,69 @@ GEMINI_3_1_PRO_FALLBACK_VIDEO_TOKENS_AT_INPUT_RATE: Final = CostTrackingTestCase ), ) +GEMINI_3_1_PRO_FALLBACK_VIDEO_TOKENS_ABOVE_200K: Final = CostTrackingTestCase( + name="gemini-3.1-pro-fallback_video_tokens_above_200k", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gemini-3.1-pro", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a7c1e02d9b44 summarize the attached hour-long recording in one line", + }, + { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAGGZ0eXBpc29tAAACAGlzb21pc282AAACCG1kYXQNFBsiKTA3PkVMU1phaG92fYSLkpmgp661vMPK0djf5u30+wIJEBceJSwzOkFIT1ZdZGtyeYCHjpWco6qxuL/GzdTb4unw9/4FDBMaISgvNj1ES1JZYGdudXyDipGYn6attLvCydDX3uXs8/oBCA8WHSQrMjlAR05VXGNqcXh/ho2Um6KpsLe+xczT2uHo7/b9BAsSGSAnLjU8Q0pRWF9mbXR7gomQl56lrLO6wcjP1t3k6/L5AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/AMKERgfJi00O0JJUFdeZWxzeoGIj5adpKuyucDHztXc4+rx+P8GDRQbIikwNz5FTFNaYWhvdn2Ei5KZoKeutbzDytHY3+bt9PsCCRAXHiUsMzpBSE9WXWRrcnmAh46VnKOqsbi/xs3U2+Lp8Pf+BQwTGiEoLzY9REtSWWBnbnV8g4qRmJ+mrbS7wsnQ197l7PP6AQgPFh0kKzI5QEdOVVxjanF4f4aNlJuiqbC3vsXM09rh6O/2/QQLEhkgJy41PENKUVhfZm10e4KJkJeepayzusHIz9bd5Ovy+QAHDhUcIyoxOD9GTVRbYmlwd36FjJOaoaivtr3Ey9LZ4Ofu9fwDChEYHyYtNDtCSVBXXmVsc3qBiI+WnaSrsrnAx87V3OPq8fj/Bg==", + "format": "mp4", + }, + }, + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "candidates": [ + { + "content": {"parts": [{"text": "scripted answer a7c1e02d9b44"}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 250000, + "candidatesTokenCount": 300, + "totalTokenCount": 250300, + "promptTokensDetails": [ + {"modality": "TEXT", "tokenCount": 10000}, + {"modality": "VIDEO", "tokenCount": 240000}, + ], + }, + "modelVersion": "gemini-3.1-pro", + }, + ), + expected=ExactExpected( + spend=1.05567, input_cost=1.05, output_cost=0.00567, prompt_tokens=250000, completion_tokens=300 + ), +) + GEMINI_3_1_PRO_STREAM: Final = CostTrackingTestCase( name="gemini-3.1-pro-stream", covers="quota_management.spend_tracking.scripted_wire.logs_cost", @@ -3587,6 +3650,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_1_PRO_WEB_SEARCH_MEDIUM, GEMINI_3_1_PRO_GOOGLE_MAPS_GROUNDING, GEMINI_3_1_PRO_FALLBACK_VIDEO_TOKENS_AT_INPUT_RATE, + GEMINI_3_1_PRO_FALLBACK_VIDEO_TOKENS_ABOVE_200K, GEMINI_3_1_PRO_STREAM, GEMINI_3_1_PRO_STREAM_NO_USAGE, GEMINI_3_1_PRO_STREAM_NO_USAGE_TOOL_CALL, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 67a3e2ddd5c..15712cc7d9c 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -450,6 +450,66 @@ def test_video_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12) +def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_input_rate(): + """Prompt audio/video tokens with no modality rate are prompt tokens, so a >200k prompt bills them above-200k.""" + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens": 4e-6, + "output_cost_per_token": 1.2e-5, + "output_cost_per_token_above_200k_tokens": 1.8e-5, + } + + usage = Usage( + prompt_tokens=816032, + completion_tokens=4559, + total_tokens=820591, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=9033, audio_tokens=14999, video_tokens=792000), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=4559), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, completion_cost = generic_cost_per_token( + model="test-model", usage=usage, custom_llm_provider="vertex_ai" + ) + + assert round(prompt_cost, 9) == round(816032 * 4e-6, 9) + assert round(completion_cost, 9) == round(4559 * 1.8e-5, 9) + + +def test_prompt_modality_rates_still_win_over_tiered_input_rate(): + """An explicit input_cost_per_audio_token / input_cost_per_video_token beats the fallback, even above 200k.""" + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_above_200k_tokens": 4e-6, + "input_cost_per_audio_token": 3e-6, + "input_cost_per_video_token": 1e-6, + "output_cost_per_token": 1.2e-5, + } + + usage = Usage( + prompt_tokens=300000, + completion_tokens=10, + total_tokens=300010, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100000, audio_tokens=100000, video_tokens=100000), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=10), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai") + + assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9) + + def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" From d7b28840eb2d7c7f33ad831badee05ba02ae6974 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 15:19:40 -0700 Subject: [PATCH 020/124] test(integration): read stream stall spend rows from the proxy that served them (#47410) The cooldown cases in test_hosted_vllm_stream_stall_wire.py boot an owned proxy, which writes to a private database since #47213. The file was written on a base without #47213 and read its rows from the worker database through read_rows, so every cooldown case that waits for a failure row timed out on main with State did not converge: [] Every row read now goes through Gateway.rows on the gateway that served the request --- .../test_hosted_vllm_stream_stall_wire.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py index fb58885e011..00e5388b474 100644 --- a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py +++ b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py @@ -16,7 +16,6 @@ from integration._support.client import ( object_value, string_value, ) -from integration._support.database import read_rows from integration._support.openai_wire import ( answering_model_discovery, chat_stream, @@ -159,14 +158,14 @@ def _error_class(message: JsonValue) -> str: return string_value(message).split(":", 1)[0] -def _failure_row(query: str, groups: tuple[str, ...]) -> dict[str, JsonValue]: - rows: Final = eventually(lambda: read_rows(query, groups), lambda found: len(found) >= 1, seconds=_ROW_SECONDS) +def _failure_row(gateway: Gateway, query: str, groups: tuple[str, ...]) -> dict[str, JsonValue]: + rows: Final = eventually(lambda: gateway.rows(query, groups), lambda found: len(found) >= 1, seconds=_ROW_SECONDS) assert [row["status"] for row in rows] == ["failure"], rows return object_value(object_value(rows[0]["metadata"])["error_information"]) -def _logged_failure(model: str) -> tuple[JsonValue, JsonValue]: - error: Final = _failure_row(_ROWS, (model,)) +def _logged_failure(gateway: Gateway, model: str) -> tuple[JsonValue, JsonValue]: + error: Final = _failure_row(gateway, _ROWS, (model,)) return error["error_code"], error["error_class"] @@ -188,7 +187,7 @@ def test_a_chat_stream_that_stalls_past_its_stream_timeout_ends_with_a_408_timeo events: Final = parse_sse(response.text) assert _chat_errors(events[:-1]) == () and "error" in events[-1].data, response.text error: Final = object_value(events[-1].data["error"]) - observed: Final = (error["code"], _error_class(error["message"]), *_logged_failure(model)) + observed: Final = (error["code"], _error_class(error["message"]), *_logged_failure(gateway, model)) assert observed == ("408", "litellm.Timeout", "408", "Timeout"), (error, observed) assert _upstream_targets(wire) == (target,) @@ -205,7 +204,7 @@ def test_a_native_responses_stream_that_stalls_past_its_stream_timeout_fails_wit assert "response.output_text.delta" in types, response.text assert types[-1] == "response.failed" and "response.completed" not in types, response.text failed: Final = object_value(object_value(events[-1].data["response"])["error"]) - observed: Final = (failed["code"], *_logged_failure(model)) + observed: Final = (failed["code"], *_logged_failure(gateway, model)) assert observed == ("request_timeout", "408", "Timeout"), (failed, observed) assert _upstream_targets(wire) == (_RESPONSES_TARGET,) @@ -222,7 +221,7 @@ def test_a_messages_stream_that_stalls_past_its_stream_timeout_ends_with_a_timeo assert "content_block_delta" in types and "message_stop" not in types, response.text assert types[-1] == "error", response.text error: Final = object_value(events[-1].data["error"]) - observed: Final = (_error_class(error["message"]), *_logged_failure(model)) + observed: Final = (_error_class(error["message"]), *_logged_failure(gateway, model)) assert observed == ("litellm.Timeout", "408", "Timeout"), (error, observed) assert _upstream_targets(wire) == (_CHAT_TARGET,) @@ -238,7 +237,7 @@ def test_a_chat_stream_that_never_pauses_is_answered_in_full_under_the_same_stre assert _chat_errors(events) == (), response.text assert _finish_reasons(events) == ("stop",), response.text assert _upstream_targets(wire) == (_CHAT_TARGET,) - rows: Final = eventually(lambda: read_rows(_ROWS, (model,)), lambda found: len(found) >= 1, _ROW_SECONDS) + rows: Final = eventually(lambda: gateway.rows(_ROWS, (model,)), lambda found: len(found) >= 1, _ROW_SECONDS) assert [row["status"] for row in rows] == ["success"], rows @@ -302,7 +301,7 @@ def test_a_deployment_whose_stream_stalls_stays_out_of_cooldown( with _peer(shape) as wire, cooldown_rig.gateway.scenario() as scenario: group, identity = _deployment(scenario, wire, backend) response: Final = _streamed(cooldown_rig.gateway, endpoint, group) - error: Final = _failure_row(_ROWS, (group,)) + error: Final = _failure_row(cooldown_rig.gateway, _ROWS, (group,)) assert not cooled(cooldown_rig.cache, identity), (error, response.text) @@ -324,7 +323,7 @@ def _fallback_cooled(cooldown_rig: _CooldownRig, fallback_shape: Shape) -> tuple primary, _ = _deployment(scenario, failing) fallback, identity = _deployment(scenario, fallback_wire) response: Final = _streamed(cooldown_rig.gateway, "chat", primary, fallbacks=[fallback]) - error: Final = _failure_row(_EITHER_GROUP_ROWS, (primary, fallback)) + error: Final = _failure_row(cooldown_rig.gateway, _EITHER_GROUP_ROWS, (primary, fallback)) assert _upstream_targets(fallback_wire) == (_CHAT_TARGET,), response.text return cooled(cooldown_rig.cache, identity), error, response.text From 1525e4bce50cbe80eb811541129529819e0fd4cd Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 15:33:29 -0700 Subject: [PATCH 021/124] feat(cli): keep the claude.ai login with `lite claude --subscription` and save pkce in config (#47385) `lite claude --subscription` sends the LiteLLM key in the x-litellm-api-key header through ANTHROPIC_CUSTOM_HEADERS and leaves ANTHROPIC_AUTH_TOKEN and ANTHROPIC_API_KEY unset, so Claude Code stays on its claude.ai login and its connectors keep working. The pre-launch key check sends the key the same way. `claude_subscription` and `pkce` are now `lite config` keys that default the new `--subscription/--no-subscription` flag on `lite claude` and the `--pkce/--no-pkce` flag on `lite login`. The flags still win for a single run. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/client/cli/commands/agents.py | 237 ++++++++- litellm/proxy/client/cli/commands/auth.py | 12 +- litellm/proxy/client/cli/commands/config.py | 38 ++ litellm/proxy/client/cli/commands/up.py | 3 +- tests/unit/proxy/client/cli/test_agents.py | 496 +++++++++++++++++- .../proxy/client/cli/test_auth_commands.py | 37 +- .../proxy/client/cli/test_config_commands.py | 54 +- .../unit/proxy/client/cli/test_up_commands.py | 26 +- 8 files changed, 873 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 03540a7395e..52e36964bc1 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -7,13 +7,14 @@ import sys import tempfile from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final, TypeAlias, TypeVar import click import requests -from pydantic import TypeAdapter, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError from litellm.proxy.common_utils.codex_model_catalog import ( CODEX_BASE_INSTRUCTIONS_PATH, @@ -25,8 +26,15 @@ from litellm.proxy.common_utils.codex_model_catalog import ( from litellm.types.llms.base_model import LiteLLMBaseModel from .auth import cli_context_obj, context_secret_vault, get_stored_api_key, login -from .claude_settings import ClaudeSettingsError, install_statusline_script +from .claude_settings import ( + ENV_KEY, + ClaudeSettingsError, + claude_settings_path, + install_statusline_script, + load_json_or_empty, +) from .cmd_quoting import quote_for_cmd +from .config import claude_subscription_default from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, @@ -41,6 +49,8 @@ from .pi import ( ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL" ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_API_KEY_ENV: Final = "ANTHROPIC_API_KEY" +ANTHROPIC_CUSTOM_HEADERS_ENV: Final = "ANTHROPIC_CUSTOM_HEADERS" +LITELLM_KEY_HEADER: Final = "x-litellm-api-key" ENABLE_TOOL_SEARCH_ENV: Final = "ENABLE_TOOL_SEARCH" ENABLE_TOOL_SEARCH_VALUE: Final = "true" ENABLE_GATEWAY_MODEL_DISCOVERY_ENV: Final = "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY" @@ -73,6 +83,7 @@ _INSTALL_DOCS: Final[dict[str, str]] = { } _HIDDEN_AGENTS: Final = frozenset({"pi"}) +_SUBSCRIPTION_AGENTS: Final = frozenset({"claude"}) CODEX_PROXY_PROVIDER: Final = "litellm" CODEX_HOME_ENV: Final = "CODEX_HOME" @@ -96,11 +107,24 @@ def agent_profile(command: str) -> tuple[str, frozenset[str]]: return base, frozenset({PROFILE_ANTHROPIC, PROFILE_OPENAI}) +def _custom_headers_with_litellm_key(existing: str, api_key: str) -> str: + kept: Final = tuple( + line for line in existing.splitlines() if line.strip() and not _is_header_line(line, LITELLM_KEY_HEADER) + ) + return "\n".join((*kept, f"{LITELLM_KEY_HEADER}: Bearer {api_key}")) + + +def _is_header_line(line: str, name: str) -> bool: + return line.partition(":")[0].strip().lower() == name + + def build_agent_env( base_env: Mapping[str, str], base_url: str, api_key: str, profiles: frozenset[str], + *, + subscription: bool = False, ) -> dict[str, str]: """Return a copy of base_env wired to route the agent through the proxy. @@ -115,12 +139,22 @@ def build_agent_env( proxy's /v1/models; likewise left alone when already set. pi ignores both base URL variables and instead resolves $LITELLM_PROXY_API_KEY from its synced models.json provider entry. + With subscription=True the Anthropic profile keeps Claude Code on its claude.ai login: the key + travels in the x-litellm-api-key header through ANTHROPIC_CUSTOM_HEADERS, and no + ANTHROPIC_AUTH_TOKEN or ANTHROPIC_API_KEY is left in the environment, since either one makes + Claude Code stop using the subscription (and its connectors). """ env: Final = dict(base_env) root: Final = base_url.rstrip("/") if PROFILE_ANTHROPIC in profiles: env[ANTHROPIC_BASE_URL_ENV] = root - env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key + if subscription: + env[ANTHROPIC_CUSTOM_HEADERS_ENV] = _custom_headers_with_litellm_key( + env.get(ANTHROPIC_CUSTOM_HEADERS_ENV, ""), api_key + ) + env.pop(ANTHROPIC_AUTH_TOKEN_ENV, None) + else: + env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key env.pop(ANTHROPIC_API_KEY_ENV, None) if ENABLE_TOOL_SEARCH_ENV not in env: env[ENABLE_TOOL_SEARCH_ENV] = ENABLE_TOOL_SEARCH_VALUE @@ -146,6 +180,130 @@ def codex_proxy_provider(base_url: str) -> Mapping[str, str | bool]: ) +_SETTINGS_FLAG: Final = "--settings" +_SUBSCRIPTION_SETTINGS_ENV: Final = MappingProxyType({ANTHROPIC_AUTH_TOKEN_ENV: "", ANTHROPIC_API_KEY_ENV: ""}) +_SETTINGS_OVERRIDE_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) + + +def _settings_flag_spans(args: Sequence[str]) -> tuple[tuple[int, int, str], ...]: + """(start, end, value) of every `--settings X` or `--settings=X` in args, in order.""" + return tuple( + (i, i + 1, arg.partition("=")[2]) if arg != _SETTINGS_FLAG else (i, i + 2, "".join(args[i + 1 : i + 2])) + for i, arg in enumerate(args) + if arg == _SETTINGS_FLAG or arg.startswith(_SETTINGS_FLAG + "=") + ) + + +def _split_settings_flags(args: Sequence[str]) -> tuple[str | None, tuple[str, ...]]: + """The value of the last `--settings` the user passed (the one Claude Code would keep), and args without any.""" + spans: Final = _settings_flag_spans(args) + covered: Final = frozenset(chain.from_iterable(range(start, end) for start, end, _ in spans)) + rest: Final = tuple(arg for i, arg in enumerate(args) if i not in covered) + return (spans[-1][2] if spans else None), rest + + +def _load_settings_override(value: str) -> Mapping[str, JsonValue]: + """The user's `--settings` value: inline JSON when it starts with `{`, else a settings file that must exist.""" + if value.lstrip().startswith("{"): + try: + return _SETTINGS_OVERRIDE_ADAPTER.validate_json(value) + except ValidationError as e: + raise AgentRunError(f"`{_SETTINGS_FLAG}` is not a JSON object: {value}") from e + path: Final = Path(value).expanduser() + try: + return load_json_or_empty(path) if path.is_file() else _missing_settings_file(path) + except ClaudeSettingsError as e: + raise AgentRunError(str(e)) from e + except OSError as e: + raise AgentRunError(f"`{_SETTINGS_FLAG}` must be a settings file or a JSON object, got: {value}") from e + + +def _missing_settings_file(path: Path) -> Mapping[str, JsonValue]: + raise AgentRunError(f"`{_SETTINGS_FLAG}` file not found: {path}") + + +def _settings_env(settings: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + env: Final = settings.get(ENV_KEY) + return env if isinstance(env, dict) else {} + + +def _settings_json(base: Mapping[str, JsonValue], api_key: str, inherited_headers: str) -> str: + base_env: Final = _settings_env(base) + own_headers: Final = base_env.get(ANTHROPIC_CUSTOM_HEADERS_ENV) + headers: Final = _custom_headers_with_litellm_key( + own_headers if isinstance(own_headers, str) else inherited_headers, api_key + ) + overlay: Final = {**base_env, **_SUBSCRIPTION_SETTINGS_ENV, ANTHROPIC_CUSTOM_HEADERS_ENV: headers} + return json.dumps({**base, ENV_KEY: overlay}, separators=(",", ":")) + + +def _inherited_custom_headers(settings_files: Sequence[Mapping[str, JsonValue]], environ: Mapping[str, str]) -> str: + """The ANTHROPIC_CUSTOM_HEADERS Claude Code would apply without the overlay: the first settings file in + precedence order that sets it, else the process environment.""" + from_files: Final = (_settings_env(settings).get(ANTHROPIC_CUSTOM_HEADERS_ENV) for settings in settings_files) + return next( + (value for value in from_files if isinstance(value, str)), environ.get(ANTHROPIC_CUSTOM_HEADERS_ENV, "") + ) + + +def claude_settings_files( + read_settings: Callable[[Path], Mapping[str, JsonValue]], environ: Mapping[str, str], project_dir: Path +) -> tuple[Mapping[str, JsonValue], ...]: + """The settings files Claude Code merges below `--settings`, highest precedence first: project local, project, + then user. A missing file reads as empty, a malformed one is a clean CLI error.""" + paths: Final = ( + project_dir / ".claude" / "settings.local.json", + project_dir / ".claude" / "settings.json", + claude_settings_path(environ), + ) + try: + return tuple(read_settings(path) for path in paths) + except ClaudeSettingsError as e: + raise AgentRunError(str(e)) from e + + +OVERLAY_FILE_NAME: Final = "litellm-subscription-overlay.json" + + +def write_settings_overlay(text: str, environ: Mapping[str, str]) -> Path: + """Write the overlay next to Claude Code's own settings.json (so under CLAUDE_CONFIG_DIR when set) and return + its path. One file per user, rewritten on every launch, mode 0600 whether or not it already existed. It carries + the LiteLLM key, which is why it is a private file and not an inline `--settings` argument that every local + user could read from the process list, and why it does not pile up in the temp dir.""" + path: Final = claude_settings_path(environ).parent / OVERLAY_FILE_NAME + path.parent.mkdir(parents=True, exist_ok=True) + fd: Final = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + if os.name == "posix": + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(text) + return path + + +def subscription_launch_args( + command: Sequence[str], + api_key: str, + environ: Mapping[str, str], + settings_files: Sequence[Mapping[str, JsonValue]], + write_overlay: Callable[[str, Mapping[str, str]], Path], +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The `--settings` that keeps Claude Code on its claude.ai login, plus the user's remaining args. + + Claude Code applies `env` from its settings files over the process environment, so a key that `lite up` or + `lite configure claude` left in ANTHROPIC_AUTH_TOKEN would land back in `Authorization`, and an + ANTHROPIC_CUSTOM_HEADERS there would drop the x-litellm-api-key header. The overlay blanks both credential + keys (read as unset for this session only) and carries the custom headers with the LiteLLM key appended, + taking the headers from the user's `--settings`, else the settings files in Claude Code's precedence order, + else the process environment. Any `--settings` the user passed is stripped and the last one (inline JSON or a + file, the one Claude Code would keep) is merged in, since Claude Code keeps only the last `--settings`. + """ + inherited: Final = _inherited_custom_headers(settings_files, environ) + user_value, rest = _split_settings_flags(tuple(command[1:])) + base: Final[Mapping[str, JsonValue]] = _load_settings_override(user_value) if user_value is not None else {} + overlay: Final = write_overlay(_settings_json(base, api_key, inherited), environ) + return (_SETTINGS_FLAG, str(overlay)), rest + + def _codex_proxy_args(base_url: str) -> list[str]: """Codex `-c` overrides that point it at the proxy. @@ -570,6 +728,7 @@ def verify_proxy_key( api_key: str, *, get: Callable[..., requests.Response] = requests.get, + key_header: str = "Authorization", ) -> None: """Probe the proxy with the key so bad creds fail here, not inside the agent. @@ -578,7 +737,7 @@ def verify_proxy_key( """ url: Final = base_url.rstrip("/") + "/v1/models" try: - resp: Final = get(url, headers={"Authorization": f"Bearer {api_key}"}, timeout=10) + resp: Final = get(url, headers={key_header: f"Bearer {api_key}"}, timeout=10) except requests.RequestException as e: raise AgentRunError( f"Could not reach the LiteLLM proxy at {base_url.rstrip('/')}: {e}. " @@ -591,6 +750,15 @@ def verify_proxy_key( ) +def verify_subscription_key( + base_url: str, + api_key: str, + *, + get: Callable[..., requests.Response] = requests.get, +) -> None: + verify_proxy_key(base_url, api_key, get=get, key_header=LITELLM_KEY_HEADER) + + _WINDOWS_SHIM_SUFFIXES: Final[frozenset[str]] = frozenset({".cmd", ".bat"}) _CMD_LINE_BREAKS: Final = ("\r", "\n") @@ -686,14 +854,19 @@ def run_agent( command: Sequence[str], *, skip_verify: bool = False, + subscription: bool = False, base_env: Mapping[str, str] | None = None, which: Callable[[str], str | None] = shutil.which, verify: Callable[[str, str], None] = verify_proxy_key, + verify_subscription: Callable[[str, str], None] = verify_subscription_key, sync_models: Callable[[str, Mapping[str, str], str, str, bool], ModelSyncResult] = agent_model_sync_env, warn: Callable[[str], None] = _warn, launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off, reattach_terminal: Callable[[], None] | None = None, preparers: Mapping[str, _Preparer] = MappingProxyType(_PREPARERS), + read_settings: Callable[[Path], Mapping[str, JsonValue]] = load_json_or_empty, + write_overlay: Callable[[str, Mapping[str, str]], Path] = write_settings_overlay, + project_dir: Path | None = None, ) -> None: """Validate, wire the environment, and hand off to the agent. @@ -713,7 +886,7 @@ def run_agent( raise AgentRunError(f"Could not find `{command[0]}` on your PATH.{hint}") if not skip_verify: - verify(base_url, api_key) + (verify_subscription if subscription else verify)(base_url, api_key) env_before_sync: Final = base_env if base_env is not None else os.environ synced: Final = sync_models(binary, env_before_sync, base_url, api_key, skip_verify) @@ -725,15 +898,28 @@ def run_agent( env: Final = MappingProxyType( { - **build_agent_env(env_before_sync, base_url, api_key, profiles), + **build_agent_env(env_before_sync, base_url, api_key, profiles, subscription=subscription), **(synced if isinstance(synced, Mapping) else _NO_EXTRA_ENV), } ) synced_args: Final = synced.args if isinstance(synced, ModelSyncArgs) else () extra_args: Final = (*agent_launch_args(command[0], base_url), *synced_args, *prepared_args) + settings_args, forwarded = ( + subscription_launch_args( + command, + api_key, + env_before_sync, + claude_settings_files( + read_settings, env_before_sync, project_dir if project_dir is not None else Path.cwd() + ), + write_overlay, + ) + if subscription and os.path.basename(command[0]) in _SUBSCRIPTION_AGENTS + else ((), tuple(command[1:])) + ) if reattach_terminal is not None: reattach_terminal() - launcher(binary, [command[0], *extra_args, *command[1:]], env) + launcher(binary, [command[0], *extra_args, *settings_args, *forwarded], env) def _is_interactive() -> bool: @@ -762,16 +948,27 @@ def resolve_api_key(ctx: click.Context) -> str: _SKIP_VERIFY_HELP: Final = "Skip the pre-launch key check against the proxy." +_SUBSCRIPTION_HELP: Final = ( + "Keep using your claude.ai login (and its connectors). The LiteLLM key is sent in the " + "x-litellm-api-key header instead of replacing Claude Code's own credential. " + "Make it the default with `lite config set claude_subscription true`." +) -def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None: +def _launch( + ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool, subscription: bool | None = None +) -> None: ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"] started_interactive: Final = _is_interactive() api_key: Final = resolve_api_key(ctx) display_name, _profiles = agent_profile(binary) - click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}") + use_subscription: Final = ( + subscription if subscription is not None else binary in _SUBSCRIPTION_AGENTS and claude_subscription_default() + ) + login_note: Final = " (keeping your claude.ai login)" if use_subscription else "" + click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}{login_note}") try: run_agent( @@ -779,12 +976,22 @@ def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify api_key, [binary, *args], skip_verify=skip_verify, + subscription=use_subscription, reattach_terminal=(_restore_controlling_terminal if started_interactive else None), ) except AgentRunError as e: raise click.ClickException(str(e)) +_Decorated: Final = TypeVar("_Decorated", bound=Callable[..., object] | click.Command) + + +def _subscription_option(binary: str) -> Callable[[_Decorated], _Decorated]: + if binary not in _SUBSCRIPTION_AGENTS: + return lambda func: func + return click.option("--subscription/--no-subscription", default=None, help=_SUBSCRIPTION_HELP) + + def _make_agent_command(binary: str, display_name: str) -> click.Command: @click.command( name=binary, @@ -793,10 +1000,11 @@ def _make_agent_command(binary: str, display_name: str) -> click.Command: hidden=binary in _HIDDEN_AGENTS, ) @click.option("--skip-verify", is_flag=True, default=False, help=_SKIP_VERIFY_HELP) + @_subscription_option(binary) @click.argument("args", nargs=-1, type=click.UNPROCESSED) @click.pass_context - def _command(ctx: click.Context, skip_verify: bool, args: Sequence[str]) -> None: - _launch(ctx, binary, list(args), skip_verify=skip_verify) + def _command(ctx: click.Context, skip_verify: bool, args: Sequence[str], subscription: bool | None = None) -> None: + _launch(ctx, binary, list(args), skip_verify=skip_verify, subscription=subscription) _command.help = ( f"Run {display_name} routed through your LiteLLM proxy.\n\n" @@ -813,6 +1021,7 @@ def agent_commands() -> tuple[click.Command, ...]: __all__ = [ + "OVERLAY_FILE_NAME", "AgentRunError", "ListedModel", "ModelSyncSkipped", @@ -821,10 +1030,14 @@ __all__ = [ "agent_model_sync_env", "agent_profile", "build_agent_env", + "claude_settings_files", "opencode_model_sync_env", "opencode_provider_config", "prepare_pi", "resolve_api_key", "run_agent", + "subscription_launch_args", "verify_proxy_key", + "verify_subscription_key", + "write_settings_overlay", ] diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index ad54cab6460..b5302d1f561 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -52,6 +52,7 @@ from .claude_settings import ( refuse_while_owned, settings_file_owners, ) +from .config import pkce_default from .pkce_login import ( Http, PkceFailure, @@ -884,17 +885,16 @@ def _pkce_login(base_url: str, config_claude: bool, vault: SecretVault) -> None: ), ) @click.option( - "--pkce", - is_flag=True, - default=False, + "--pkce/--no-pkce", + default=None, help=( "Sign in with OAuth authorization code + PKCE through your system browser (loopback redirect), " "with a refresh token that renews the key automatically. Requires a proxy that serves " - "/.well-known/litellm-cli-auth." + "/.well-known/litellm-cli-auth. Make it the default with `lite config set pkce true`." ), ) @click.pass_context -def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: +def login(ctx: click.Context, config_claude: bool, pkce: bool | None) -> None: """Login to LiteLLM proxy using SSO authentication""" from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER @@ -908,7 +908,7 @@ def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: raise click.ClickException(f"Cannot configure Claude Code, so not logging in: {e}") try: - if pkce: + if pkce if pkce is not None else pkce_default(): _pkce_login(base_url, config_claude, context_secret_vault(ctx)) return cli_sso_flow: Final = _start_cli_sso_flow(base_url=base_url) diff --git a/litellm/proxy/client/cli/commands/config.py b/litellm/proxy/client/cli/commands/config.py index de22251ea8c..539d88dce4a 100644 --- a/litellm/proxy/client/cli/commands/config.py +++ b/litellm/proxy/client/cli/commands/config.py @@ -13,6 +13,8 @@ from pydantic import TypeAdapter from litellm.litellm_core_utils.private_json import ensure_private_dir, write_private_json HIDDEN_COMMANDS_KEY: Final = "hidden_commands" +CLAUDE_SUBSCRIPTION_KEY: Final = "claude_subscription" +PKCE_KEY: Final = "pkce" _config_adapter: Final[TypeAdapter[Mapping[str, str]]] = TypeAdapter(Mapping[str, str]) @@ -88,10 +90,46 @@ def _normalize_hidden_commands(value: str) -> str: return ",".join(sorted(names)) +_BOOLEAN_WORDS: Final[Mapping[str, str]] = MappingProxyType( + { + "true": "true", + "on": "true", + "1": "true", + "yes": "true", + "false": "false", + "off": "false", + "0": "false", + "no": "false", + } +) + + +def _normalize_boolean(key: str) -> Callable[[str], str]: + def normalize(value: str) -> str: + normalized: Final = _BOOLEAN_WORDS.get(value.strip().lower()) + if normalized is None: + raise click.UsageError(f"{key} must be true or false, e.g. `lite config set {key} true`") + return normalized + + return normalize + + +def claude_subscription_default() -> bool: + """Whether `lite claude` keeps the claude.ai login unless told otherwise.""" + return get_config_value(CLAUDE_SUBSCRIPTION_KEY) == "true" + + +def pkce_default() -> bool: + """Whether `lite login` signs in with browser PKCE unless told otherwise.""" + return get_config_value(PKCE_KEY) == "true" + + _NORMALIZERS: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType( { "base_url": normalize_base_url, HIDDEN_COMMANDS_KEY: _normalize_hidden_commands, + CLAUDE_SUBSCRIPTION_KEY: _normalize_boolean(CLAUDE_SUBSCRIPTION_KEY), + PKCE_KEY: _normalize_boolean(PKCE_KEY), } ) diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index d1e966d6eae..4043761f255 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -30,6 +30,7 @@ from .claude_settings import ( merge_claude_settings, write_claude_settings, ) +from .config import pkce_default class UpError(ClaudeSettingsError): @@ -131,7 +132,7 @@ def ensure_fresh_login(ctx: click.Context) -> None: if _usable_login(_key_resolved_on_the_way_in(ctx_obj, base_url, vault), vault): return - pkce: Final = _stored_login_is_pkce(vault) + pkce: Final = _stored_login_is_pkce(vault) or pkce_default() login_command: Final = "lite login --pkce" if pkce else "lite login" if not sys.stdin.isatty(): raise UpError(f"No fresh LiteLLM login found for this proxy. Run `{login_command}` first.") diff --git a/tests/unit/proxy/client/cli/test_agents.py b/tests/unit/proxy/client/cli/test_agents.py index 7be98521f8e..3481346076f 100644 --- a/tests/unit/proxy/client/cli/test_agents.py +++ b/tests/unit/proxy/client/cli/test_agents.py @@ -1,6 +1,7 @@ import inspect import json import os +import stat import subprocess import sys from pathlib import Path @@ -27,7 +28,9 @@ from litellm.proxy.client.cli.commands.agents import ( opencode_model_sync_env, run_agent, verify_proxy_key, + verify_subscription_key, ) +from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError from litellm.proxy.client.cli.main import cli as lite AGENTS_MODULE = "litellm.proxy.client.cli.commands.agents" @@ -173,6 +176,19 @@ class _FakeJsonResponse: return self._payload +def _overlay_writer(tmp_path): + def write(text, environ): + target = tmp_path / "overlay.json" + target.write_text(text) + return target + + return write + + +def _overlay(args): + return json.loads(Path(args[2]).read_text()) + + class TestAgentProfile: def test_claude_is_anthropic(self): name, profiles = agent_profile("claude") @@ -263,6 +279,55 @@ class TestBuildAgentEnv: assert env["PATH"] == "/usr/bin" assert base == {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"} + def test_subscription_sends_the_key_in_a_header_and_sets_no_credential(self): + env = build_agent_env({}, "http://localhost:4000/", "sk-key", frozenset({"anthropic"}), subscription=True) + assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000" + assert env["ANTHROPIC_CUSTOM_HEADERS"] == "x-litellm-api-key: Bearer sk-key" + assert "ANTHROPIC_AUTH_TOKEN" not in env + assert "ANTHROPIC_API_KEY" not in env + + def test_subscription_clears_inherited_credentials(self): + base = {"ANTHROPIC_AUTH_TOKEN": "stale-token", "ANTHROPIC_API_KEY": "real-key"} + env = build_agent_env(base, "http://localhost:4000", "sk-key", frozenset({"anthropic"}), subscription=True) + assert "ANTHROPIC_AUTH_TOKEN" not in env + assert "ANTHROPIC_API_KEY" not in env + assert base == {"ANTHROPIC_AUTH_TOKEN": "stale-token", "ANTHROPIC_API_KEY": "real-key"} + + def test_subscription_keeps_the_users_other_custom_headers(self): + env = build_agent_env( + {"ANTHROPIC_CUSTOM_HEADERS": "x-team: blue\n\nx-trace: 1\n"}, + "http://localhost:4000", + "sk-key", + frozenset({"anthropic"}), + subscription=True, + ) + assert env["ANTHROPIC_CUSTOM_HEADERS"].splitlines() == [ + "x-team: blue", + "x-trace: 1", + "x-litellm-api-key: Bearer sk-key", + ] + + def test_subscription_replaces_a_stale_litellm_key_header_whatever_its_case(self): + env = build_agent_env( + {"ANTHROPIC_CUSTOM_HEADERS": "X-LiteLLM-Api-Key: Bearer old\nx-team: blue"}, + "http://localhost:4000", + "sk-new", + frozenset({"anthropic"}), + subscription=True, + ) + assert env["ANTHROPIC_CUSTOM_HEADERS"].splitlines() == ["x-team: blue", "x-litellm-api-key: Bearer sk-new"] + + def test_subscription_leaves_openai_agents_on_the_bearer_key(self): + env = build_agent_env({}, "http://localhost:4000", "sk-key", frozenset({"openai"}), subscription=True) + assert env["OPENAI_API_KEY"] == "sk-key" + assert "ANTHROPIC_CUSTOM_HEADERS" not in env + + def test_default_mode_does_not_touch_custom_headers(self): + env = build_agent_env( + {"ANTHROPIC_CUSTOM_HEADERS": "x-team: blue"}, "http://localhost:4000", "sk-key", frozenset({"anthropic"}) + ) + assert env["ANTHROPIC_CUSTOM_HEADERS"] == "x-team: blue" + class TestAgentLaunchArgs: def test_claude_and_opencode_get_no_extra_args(self): @@ -330,6 +395,32 @@ class TestVerifyProxyKey: get=lambda *a, **k: _FakeResponse(500), ) + def test_key_header_moves_the_bearer_off_authorization(self): + captured = {} + + def fake_get(url, headers, timeout): + captured["headers"] = headers + return _FakeResponse(200) + + verify_proxy_key("http://localhost:4000", "sk-key", get=fake_get, key_header="x-litellm-api-key") + + assert captured["headers"] == {"x-litellm-api-key": "Bearer sk-key"} + + def test_subscription_probe_sends_the_key_only_in_the_litellm_header(self): + captured = {} + + def fake_get(url, headers, timeout): + captured["headers"] = headers + return _FakeResponse(200) + + verify_subscription_key("http://localhost:4000", "sk-key", get=fake_get) + + assert captured["headers"] == {"x-litellm-api-key": "Bearer sk-key"} + + def test_subscription_probe_reports_a_rejected_key(self): + with pytest.raises(AgentRunError, match="rejected your key"): + verify_subscription_key("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(401)) + class TestOpencodeModelSync: @staticmethod @@ -1195,6 +1286,327 @@ class TestRunAgent: with pytest.raises(AgentRunError): run_agent("http://localhost:4000", "sk-key", []) + def test_subscription_launch_probes_with_the_header_and_exports_it(self): + probed = [] + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + subscription=True, + base_env={"ANTHROPIC_AUTH_TOKEN": "stale"}, + which=lambda name: "/usr/local/bin/claude", + verify=lambda *a: probed.append(("bearer", a)), + verify_subscription=lambda *a: probed.append(("header", a)), + launcher=lambda p, a, e: launched.update(env=dict(e)), + read_settings=lambda path: {}, + ) + assert probed == [("header", ("http://localhost:4000", "sk-key"))] + assert launched["env"]["ANTHROPIC_CUSTOM_HEADERS"] == "x-litellm-api-key: Bearer sk-key" + assert "ANTHROPIC_AUTH_TOKEN" not in launched["env"] + + def test_subscription_launch_blanks_the_credentials_claude_code_would_load_from_settings_json(self, tmp_path): + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--resume"], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=tuple(a)), + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + assert launched["args"][:2] == ("claude", "--settings") + assert _overlay(launched["args"]) == { + "env": { + "ANTHROPIC_AUTH_TOKEN": "", + "ANTHROPIC_API_KEY": "", + "ANTHROPIC_CUSTOM_HEADERS": "x-litellm-api-key: Bearer sk-key", + } + } + assert launched["args"][3:] == ("--resume",) + + def test_subscription_launch_keeps_the_key_out_of_argv_and_in_one_private_file_next_to_settings_json( + self, tmp_path + ): + config_dir = tmp_path / "cfg" + config_dir.mkdir() + stale = config_dir / "litellm-subscription-overlay.json" + stale.write_text('{"env": {"ANTHROPIC_CUSTOM_HEADERS": "x-litellm-api-key: Bearer old-key"}}') + stale.chmod(0o644) + launched = {} + for _ in range(2): + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + subscription=True, + base_env={"CLAUDE_CONFIG_DIR": str(config_dir)}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a)), + read_settings=lambda path: {}, + ) + assert "sk-key" not in " ".join(launched["args"]) + assert Path(launched["args"][2]) == stale + assert sorted(path.name for path in config_dir.iterdir()) == ["litellm-subscription-overlay.json"] + assert json.loads(stale.read_text())["env"]["ANTHROPIC_CUSTOM_HEADERS"] == "x-litellm-api-key: Bearer sk-key" + if os.name == "posix": + assert stat.S_IMODE(stale.stat().st_mode) == 0o600 + + def test_subscription_launch_accepts_inline_settings_json_longer_than_a_file_name(self, tmp_path): + long_value = "a" * 400 + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--settings", json.dumps({"env": {"FOO": long_value}})], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a)), + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + assert _overlay(launched["args"])["env"]["FOO"] == long_value + + def test_subscription_launch_reports_a_missing_settings_file_as_a_clean_cli_error(self, tmp_path): + with pytest.raises(AgentRunError, match="file not found"): + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--settings", str(tmp_path / "nope.json")], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda *a: None, + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + + def test_subscription_launch_reads_the_settings_files_claude_code_merges_below_the_flag(self, tmp_path): + seen = [] + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + subscription=True, + base_env={"CLAUDE_CONFIG_DIR": str(tmp_path / "cfg")}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda *a: None, + read_settings=lambda path: seen.append(path) or {}, + write_overlay=_overlay_writer(tmp_path), + project_dir=tmp_path / "repo", + ) + assert seen == [ + tmp_path / "repo" / ".claude" / "settings.local.json", + tmp_path / "repo" / ".claude" / "settings.json", + tmp_path / "cfg" / "settings.json", + ] + + @pytest.mark.parametrize("separate_flag", [True, False]) + def test_subscription_launch_merges_the_blanks_into_the_users_settings_flag(self, tmp_path, separate_flag): + settings_file = tmp_path / "mine.json" + settings_file.write_text('{"model": "opus", "env": {"ANTHROPIC_AUTH_TOKEN": "mine", "FOO": "bar"}}') + user_flag = ["--settings", str(settings_file)] if separate_flag else [f"--settings={settings_file}"] + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--resume", *user_flag, "-p", "hi"], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a)), + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + assert launched["args"][:2] == ["claude", "--settings"] + assert _overlay(launched["args"]) == { + "model": "opus", + "env": { + "ANTHROPIC_AUTH_TOKEN": "", + "FOO": "bar", + "ANTHROPIC_API_KEY": "", + "ANTHROPIC_CUSTOM_HEADERS": "x-litellm-api-key: Bearer sk-key", + }, + } + assert launched["args"][3:] == ["--resume", "-p", "hi"] + + def test_subscription_launch_merges_the_blanks_into_inline_settings_json(self, tmp_path): + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--settings", '{"env": {"ANTHROPIC_API_KEY": "mine"}, "theme": "dark"}'], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a)), + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + assert launched["args"][:2] == ["claude", "--settings"] + assert _overlay(launched["args"]) == { + "theme": "dark", + "env": { + "ANTHROPIC_API_KEY": "", + "ANTHROPIC_AUTH_TOKEN": "", + "ANTHROPIC_CUSTOM_HEADERS": "x-litellm-api-key: Bearer sk-key", + }, + } + assert launched["args"][3:] == [] + + @pytest.mark.parametrize( + ("flag_headers", "local_headers", "project_headers", "user_headers", "expected"), + [ + ("X-Flag: f", "X-Local: l", "X-Proj: p", "X-User: u", "X-Flag: f\nx-litellm-api-key: Bearer sk-key"), + (None, "X-Local: l", "X-Proj: p", "X-User: u", "X-Local: l\nx-litellm-api-key: Bearer sk-key"), + ( + None, + None, + "X-Proj: p\nx-litellm-api-key: Bearer stale", + "X-User: u", + "X-Proj: p\nx-litellm-api-key: Bearer sk-key", + ), + (None, None, None, "X-User: u", "X-User: u\nx-litellm-api-key: Bearer sk-key"), + (None, None, None, None, "X-Proc: h\nx-litellm-api-key: Bearer sk-key"), + ], + ) + def test_subscription_overlay_keeps_the_custom_headers_claude_code_would_apply_and_adds_the_key( + self, tmp_path, flag_headers, local_headers, project_headers, user_headers, expected + ): + """Claude Code applies settings `env` over the process env, in precedence order, so the overlay must too.""" + user_flag = ( + ["--settings", json.dumps({"env": {"ANTHROPIC_CUSTOM_HEADERS": flag_headers}})] if flag_headers else [] + ) + files = { + tmp_path / "repo" / ".claude" / "settings.local.json": local_headers, + tmp_path / "repo" / ".claude" / "settings.json": project_headers, + tmp_path / "cfg" / "settings.json": user_headers, + } + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", *user_flag], + subscription=True, + base_env={"ANTHROPIC_CUSTOM_HEADERS": "X-Proc: h", "CLAUDE_CONFIG_DIR": str(tmp_path / "cfg")}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a), env=dict(e)), + read_settings=lambda path: {"env": {"ANTHROPIC_CUSTOM_HEADERS": files[path]}} if files[path] else {}, + write_overlay=_overlay_writer(tmp_path), + project_dir=tmp_path / "repo", + ) + assert _overlay(launched["args"])["env"]["ANTHROPIC_CUSTOM_HEADERS"] == expected + assert launched["env"]["ANTHROPIC_CUSTOM_HEADERS"] == "X-Proc: h\nx-litellm-api-key: Bearer sk-key" + + def test_subscription_launch_strips_every_settings_flag_and_merges_the_last_one_claude_code_would_keep( + self, tmp_path + ): + launched = {} + run_agent( + "http://localhost:4000", + "sk-key", + [ + "claude", + "--settings", + '{"env": {"ANTHROPIC_AUTH_TOKEN": "first"}, "theme": "light"}', + "--resume", + '--settings={"theme": "dark"}', + ], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda p, a, e: launched.update(args=list(a)), + read_settings=lambda path: {}, + write_overlay=_overlay_writer(tmp_path), + ) + assert launched["args"][:2] == ["claude", "--settings"] + assert _overlay(launched["args"]) == { + "theme": "dark", + "env": { + "ANTHROPIC_AUTH_TOKEN": "", + "ANTHROPIC_API_KEY": "", + "ANTHROPIC_CUSTOM_HEADERS": "x-litellm-api-key: Bearer sk-key", + }, + } + assert launched["args"][3:] == ["--resume"] + + def test_subscription_launch_reports_a_malformed_settings_file_as_a_clean_cli_error(self): + def broken(path): + raise ClaudeSettingsError(f"{path} contains invalid JSON") + + with pytest.raises(AgentRunError, match="contains invalid JSON"): + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda *a: None, + read_settings=broken, + ) + + def test_subscription_launch_rejects_a_settings_flag_that_is_not_a_json_object(self): + with pytest.raises(AgentRunError, match="is not a JSON object"): + run_agent( + "http://localhost:4000", + "sk-key", + ["claude", "--settings", '{"env": 1, '], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=lambda *a: None, + launcher=lambda *a: None, + read_settings=lambda path: {}, + ) + + def test_default_launch_still_probes_with_the_bearer(self): + probed = [] + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify=lambda *a: probed.append(("bearer", a)), + verify_subscription=lambda *a: probed.append(("header", a)), + launcher=lambda *a: None, + ) + assert probed == [("bearer", ("http://localhost:4000", "sk-key"))] + + def test_subscription_rejection_aborts_before_launch(self): + launched = [] + + def boom(*a): + raise AgentRunError("rejected") + + with pytest.raises(AgentRunError): + run_agent( + "http://localhost:4000", + "sk-key", + ["claude"], + subscription=True, + base_env={}, + which=lambda name: "/usr/local/bin/claude", + verify_subscription=boom, + launcher=lambda *a: launched.append(a), + ) + assert launched == [] + def test_reattach_terminal_runs_just_before_launch(self): order = [] run_agent( @@ -1552,6 +1964,71 @@ class TestAgentCommands: assert captured["command"] == ["opencode"] assert "routing OpenCode through proxy at http://localhost:4000" in result.output + def test_claude_subscription_flag_reaches_run_agent_and_is_not_forwarded(self): + captured = {} + + def fake_run_agent(base_url, api_key, command, **kwargs): + captured["command"] = list(command) + captured["subscription"] = kwargs.get("subscription") + + with patch(f"{AGENTS_MODULE}.run_agent", side_effect=fake_run_agent): + result = self.runner.invoke( + _agent_command("claude"), + ["--subscription", "--resume"], + obj={"base_url": "http://localhost:4000", "api_key": "sk-key"}, + ) + + assert result.exit_code == 0, result.output + assert captured["subscription"] is True + assert captured["command"] == ["claude", "--resume"] + assert "keeping your claude.ai login" in result.output + + @pytest.mark.parametrize( + ("configured", "args", "expected"), + [ + (False, [], False), + (True, [], True), + (True, ["--no-subscription"], False), + (False, ["--subscription"], True), + ], + ) + def test_subscription_comes_from_the_flag_else_the_saved_setting(self, configured, args, expected): + captured = {} + with ( + patch(f"{AGENTS_MODULE}.claude_subscription_default", return_value=configured), + patch( + f"{AGENTS_MODULE}.run_agent", + side_effect=lambda b, k, c, **kw: captured.update(sub=kw.get("subscription"), command=list(c)), + ), + ): + result = self.runner.invoke( + _agent_command("claude"), args, obj={"base_url": "http://localhost:4000", "api_key": "sk-key"} + ) + assert result.exit_code == 0, result.output + assert captured["sub"] is expected + assert captured["command"] == ["claude"] + assert ("keeping your claude.ai login" in result.output) is expected + + @pytest.mark.parametrize("agent", ["codex", "opencode"]) + def test_the_saved_claude_setting_does_not_change_other_agents(self, agent): + captured = {} + with ( + patch(f"{AGENTS_MODULE}.claude_subscription_default", return_value=True), + patch( + f"{AGENTS_MODULE}.run_agent", + side_effect=lambda b, k, c, **kw: captured.update(sub=kw.get("subscription")), + ), + ): + result = self.runner.invoke( + _agent_command(agent), [], obj={"base_url": "http://localhost:4000", "api_key": "sk-key"} + ) + assert result.exit_code == 0, result.output + assert captured["sub"] is False + + @pytest.mark.parametrize("agent", ["codex", "opencode", "pi"]) + def test_only_claude_offers_the_subscription_flag(self, agent): + assert "--subscription" not in {opt for param in _agent_command(agent).params for opt in param.opts} + def test_skip_verify_is_consumed_not_forwarded(self): captured = {} @@ -1676,7 +2153,9 @@ class TestPrepareCodex: def test_registers_the_installed_script_as_a_session_scoped_stop_hook(self): from litellm.proxy.client.cli.commands.agents import prepare_codex - args = prepare_codex("http://localhost:4000", "sk-key", {}, install=lambda: "/py /home/me/.litellm/statusline.py") + args = prepare_codex( + "http://localhost:4000", "sk-key", {}, install=lambda: "/py /home/me/.litellm/statusline.py" + ) assert args == ( "-c", 'hooks.Stop=[{hooks=[{type="command",command="/py /home/me/.litellm/statusline.py"}]}]', @@ -1697,11 +2176,20 @@ class TestPrepareCodex: warnings = [] env = {"CODEX_HOME": str(tmp_path)} - for body in ('[[hooks.Stop]]\nhooks = [{ type = "command", command = "mine" }]\n', 'hooks.Stop = []\n', "[hooks]\n"): + for body in ( + '[[hooks.Stop]]\nhooks = [{ type = "command", command = "mine" }]\n', + "hooks.Stop = []\n", + "[hooks]\n", + ): (tmp_path / "config.toml").write_text(body) - assert prepare_codex("http://localhost:4000", "sk", env, install=lambda: "/py /s.py", warn=warnings.append) == () + assert ( + prepare_codex("http://localhost:4000", "sk", env, install=lambda: "/py /s.py", warn=warnings.append) + == () + ) (tmp_path / "config.toml").write_text('model = "gpt-5.6-sol"\n[projects."/x"]\ntrust_level = "trusted"\n') - assert prepare_codex("http://localhost:4000", "sk", env, install=lambda: "/py /s.py", warn=warnings.append) != () + assert ( + prepare_codex("http://localhost:4000", "sk", env, install=lambda: "/py /s.py", warn=warnings.append) != () + ) assert len(warnings) == 3 and "already declares hooks" in warnings[0] def test_a_config_that_cannot_be_read_or_decoded_still_lets_codex_launch(self, tmp_path): diff --git a/tests/unit/proxy/client/cli/test_auth_commands.py b/tests/unit/proxy/client/cli/test_auth_commands.py index 27edf9f3eb3..cb1fcba9df7 100644 --- a/tests/unit/proxy/client/cli/test_auth_commands.py +++ b/tests/unit/proxy/client/cli/test_auth_commands.py @@ -5,7 +5,6 @@ import time from pathlib import Path from unittest.mock import Mock, patch - import pytest import responses from click.testing import CliRunner @@ -26,7 +25,6 @@ from litellm.litellm_core_utils.cli_token_utils import ( ) from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands import claude_settings as claude_settings_module -from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner from litellm.proxy.client.cli.commands.auth import ( get_stored_api_key, login, @@ -34,6 +32,7 @@ from litellm.proxy.client.cli.commands.auth import ( print_token, whoami, ) +from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner @pytest.fixture @@ -1718,6 +1717,35 @@ class TestPkceLoginCommand: save.assert_called_once() assert _FakeSession.instances[0].posts == [] + @pytest.mark.parametrize( + ("saved", "args", "uses_pkce"), + [ + ("true", [], True), + ("false", [], False), + (None, [], False), + ("true", ["--no-pkce"], False), + ("false", ["--pkce"], True), + ], + ) + def test_login_uses_pkce_when_the_flag_or_the_saved_setting_says_so(self, saved, args, uses_pkce): + from litellm.proxy.client.cli.commands.config import save_config + + if saved is not None: + save_config({"pkce": saved}) + with ( + patch("litellm.proxy.client.cli.commands.auth.run_pkce_login", return_value=_pkce_credential()), + patch( + "litellm.proxy.client.cli.commands.auth._start_cli_sso_flow", + side_effect=RuntimeError("classic sso flow"), + ), + patch("litellm.proxy.client.cli.commands.auth.save_token", return_value=SecretStored()), + patch("litellm.proxy.client.cli.interface.show_commands"), + ): + result = self.runner.invoke(login, args, obj={"base_url": PKCE_BASE_URL}) + + assert ("Login successful!" in result.output) is uses_pkce + assert ("Authentication failed: classic sso flow" in result.output) is (not uses_pkce) + def test_pkce_login_saves_the_refreshable_record_and_skips_the_sso_poll(self): with ( patch("litellm.proxy.client.cli.commands.auth.run_pkce_login", return_value=_pkce_credential()) as run, @@ -2050,7 +2078,10 @@ class TestGetStoredApiKeyRefresh: @pytest.mark.parametrize( "body, expected_detail", [ - ('{"detail": "Too many CLI login attempts.", "retry": {"after": [30, null]}}', ": Too many CLI login attempts."), + ( + '{"detail": "Too many CLI login attempts.", "retry": {"after": [30, null]}}', + ": Too many CLI login attempts.", + ), ('{"detail": 5}', ""), ('{"detail": ""}', ""), ('{"detail": {"message": "nested"}}', ""), diff --git a/tests/unit/proxy/client/cli/test_config_commands.py b/tests/unit/proxy/client/cli/test_config_commands.py index 611307635e0..2cd7a76ab95 100644 --- a/tests/unit/proxy/client/cli/test_config_commands.py +++ b/tests/unit/proxy/client/cli/test_config_commands.py @@ -6,16 +6,16 @@ from unittest.mock import patch import pytest from click.testing import CliRunner - - +from litellm.litellm_core_utils.private_json import write_private_json from litellm.proxy.client.cli import cli from litellm.proxy.client.cli.commands.config import ( + claude_subscription_default, get_config_file_path, get_config_value, load_config, + pkce_default, save_config, ) -from litellm.litellm_core_utils.private_json import write_private_json from litellm.proxy.client.cli.interface import show_commands @@ -360,3 +360,51 @@ class TestWritePrivateJson: assert not target.exists() assert list(tmp_path.glob(".tmp-*")) == [] + + +class TestClaudeSubscriptionSetting: + @pytest.mark.parametrize("word", ["true", "TRUE", "on", "1", "yes"]) + def test_truthy_words_are_stored_as_true(self, cli_runner, isolated_home, word): + result = cli_runner.invoke(cli, ["config", "set", "claude_subscription", word]) + + assert result.exit_code == 0, result.output + assert load_config()["claude_subscription"] == "true" + assert claude_subscription_default() is True + + @pytest.mark.parametrize("word", ["false", "Off", "0", "no"]) + def test_falsy_words_are_stored_as_false(self, cli_runner, isolated_home, word): + cli_runner.invoke(cli, ["config", "set", "claude_subscription", "true"]) + result = cli_runner.invoke(cli, ["config", "set", "claude_subscription", word]) + + assert result.exit_code == 0, result.output + assert load_config()["claude_subscription"] == "false" + assert claude_subscription_default() is False + + def test_anything_else_is_rejected_and_nothing_is_saved(self, cli_runner, isolated_home): + result = cli_runner.invoke(cli, ["config", "set", "claude_subscription", "maybe"]) + + assert result.exit_code != 0 + assert "true or false" in result.output + assert "claude_subscription" not in load_config() + + def test_unset_means_off(self, isolated_home): + assert claude_subscription_default() is False + + +class TestPkceSetting: + @pytest.mark.parametrize(("word", "expected"), [("true", True), ("on", True), ("false", False), ("off", False)]) + def test_the_saved_value_drives_the_default(self, cli_runner, isolated_home, word, expected): + result = cli_runner.invoke(cli, ["config", "set", "pkce", word]) + + assert result.exit_code == 0, result.output + assert pkce_default() is expected + + def test_a_non_boolean_is_rejected_and_nothing_is_saved(self, cli_runner, isolated_home): + result = cli_runner.invoke(cli, ["config", "set", "pkce", "sometimes"]) + + assert result.exit_code != 0 + assert "true or false" in result.output + assert "pkce" not in load_config() + + def test_unset_means_off(self, isolated_home): + assert pkce_default() is False diff --git a/tests/unit/proxy/client/cli/test_up_commands.py b/tests/unit/proxy/client/cli/test_up_commands.py index ddd54dd1374..5c2e726315b 100644 --- a/tests/unit/proxy/client/cli/test_up_commands.py +++ b/tests/unit/proxy/client/cli/test_up_commands.py @@ -15,8 +15,8 @@ from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsErro from litellm.proxy.client.cli.commands.up import ( BackupRecord, UpError, - ensure_fresh_login, down, + ensure_fresh_login, load_json_or_empty, merge_claude_settings, read_backup, @@ -372,6 +372,30 @@ class TestEnsureFreshLogin: assert login_calls == [("http://proxy-a:4000", True)] + def test_logs_in_with_pkce_when_the_saved_pkce_default_is_on(self, monkeypatch): + """`lite config set pkce true` is honoured by the login `lite up` starts, not only by `lite login`.""" + monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: True) + store = _FakeTokenStore(monkeypatch, {"key": "sk-a", "base_url": "http://proxy-a:4000"}, {}) + monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) + monkeypatch.setattr(up_module, "pkce_default", lambda: True) + login_calls = _capture_login( + monkeypatch, + on_login=lambda: store.log_in(_pkce_record("http://proxy-b:4000", seconds_left=86_400), "sk-pkce-fresh"), + ) + + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + + assert login_calls == [("http://proxy-b:4000", True)] + + def test_names_the_pkce_login_in_the_non_interactive_error_when_the_saved_pkce_default_is_on(self, monkeypatch): + monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: False) + _FakeTokenStore(monkeypatch, {"key": "sk-a", "base_url": "http://proxy-a:4000"}, {}) + monkeypatch.setattr(up_module, "is_cli_token_fresh", lambda token_data: True) + monkeypatch.setattr(up_module, "pkce_default", lambda: True) + + with pytest.raises(UpError, match="Run `lite login --pkce` first"): + ensure_fresh_login(_make_ctx("http://proxy-b:4000")) + def test_names_the_pkce_login_in_the_non_interactive_error_for_a_pkce_credential(self, monkeypatch): monkeypatch.setattr(up_module.sys.stdin, "isatty", lambda: False) _FakeTokenStore(monkeypatch, _pkce_record("http://proxy-a:4000", seconds_left=-10), {}) From b67003162a6f6ffb97576da42cb434f947b43f29 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 15:35:29 -0700 Subject: [PATCH 022/124] test(integration): wait for every worker to report the websocket session cap instead of sleeping (#47412) The override test slept a fixed 10 s after writing responses_websocket_session_limit_seconds and then opened sockets on both workers. A worker whose config resync ran later than that kept the old 7200 s cap, so its socket never closed and the test failed after 75 s A test-only /health/integration-general-setting route returns the value a worker holds along with its pid. The test polls it over fresh connections until every worker reports the new cap, and after the delete until every worker reports it gone --- tests/integration/_support/runtime.py | 9 +++++ .../test_responses_websocket_session_limit.py | 34 ++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/tests/integration/_support/runtime.py b/tests/integration/_support/runtime.py index a657ce4dff0..c53c23d42b3 100644 --- a/tests/integration/_support/runtime.py +++ b/tests/integration/_support/runtime.py @@ -33,8 +33,17 @@ async def worker_deployment_ids(model_name: str) -> JSONResponse: return JSONResponse(content={"model_ids": model_ids}, headers={"x-integration-worker-pid": str(os.getpid())}) +async def worker_general_setting(name: str) -> JSONResponse: + from litellm.proxy.proxy_server import proxy_config + + return JSONResponse( + content={"value": proxy_config.settings.get(name)}, headers={"x-integration-worker-pid": str(os.getpid())} + ) + + def configure_readiness() -> None: from litellm.proxy.proxy_server import app app.add_api_route("/health/integration-readiness", worker_readiness, include_in_schema=False) app.add_api_route("/health/integration-deployment-ids", worker_deployment_ids, include_in_schema=False) + app.add_api_route("/health/integration-general-setting", worker_general_setting, include_in_schema=False) diff --git a/tests/integration/providers/test_responses_websocket_session_limit.py b/tests/integration/providers/test_responses_websocket_session_limit.py index f2b3a48c6d1..4369a84e1cc 100644 --- a/tests/integration/providers/test_responses_websocket_session_limit.py +++ b/tests/integration/providers/test_responses_websocket_session_limit.py @@ -9,6 +9,7 @@ import threading import time import uuid from collections.abc import Generator, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path @@ -53,7 +54,7 @@ JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) SESSION_LIMIT_FIELD: Final = "responses_websocket_session_limit_seconds" LIMIT_CLOSE_REASON: Final = "Session duration limit reached" RELOAD_INTERVAL_SECONDS: Final = 3 -WORKER_SYNC_SECONDS: Final = RELOAD_INTERVAL_SECONDS + 7 +WORKER_SYNC_DEADLINE_SECONDS: Final = 60 CAP_SECONDS: Final = 60 CAP_CLOSE_SLACK_SECONDS: Final = 15 DEAF_CLOSE_SLACK_SECONDS: Final = 3 @@ -839,11 +840,28 @@ def _holding_worker(workers: frozenset[int], client_port: int) -> int | None: return next((pid for pid in sorted(workers) if holds(pid)), None) -def _settled_on_every_worker(written_at: float) -> None: +def _session_limit_on_a_worker(candidate: Gateway) -> tuple[int, JsonValue]: + response: Final = candidate.request( + "GET", + "/health/integration-general-setting", + params={"name": SESSION_LIMIT_FIELD}, + headers={"connection": "close"}, + ) + assert response.status_code == 200, f"Session limit on a worker: {response.status_code} {response.text}" + return int(response.headers["x-integration-worker-pid"]), JSON_OBJECT.validate_json(response.content)["value"] + + +def _session_limits_by_worker(candidate: Gateway) -> tuple[tuple[int, JsonValue], ...]: + with ThreadPoolExecutor(max_workers=8) as pool: + probes: Final = tuple(pool.submit(_session_limit_on_a_worker, candidate) for _ in range(16)) + return tuple(probe.result() for probe in probes) + + +def _settled_on_every_worker(candidate: Gateway, workers: frozenset[int], limit: JsonValue) -> None: eventually( - lambda: time.monotonic() - written_at, - lambda elapsed: elapsed >= WORKER_SYNC_SECONDS, - seconds=WORKER_SYNC_SECONDS + 5, + lambda: _session_limits_by_worker(candidate), + lambda answers: frozenset(pid for pid, _ in answers) >= workers and all(value == limit for _, value in answers), + seconds=WORKER_SYNC_DEADLINE_SECONDS, ) @@ -939,16 +957,14 @@ async def _override_workload(owned: OwnedProxy, key: str, model: str, database_u ) as earlier: accepted_at: Final = time.monotonic() updates: Final = await _update_sequence(candidate, (7200, CAP_SECONDS, CAP_SECONDS)) - written_at: Final = time.monotonic() stored_after_updates: Final = await asyncio.to_thread(_stored_limit, database_url) - await asyncio.to_thread(_settled_on_every_worker, written_at) + await asyncio.to_thread(_settled_on_every_worker, candidate, workers, CAP_SECONDS) opened: Final = await _sockets_on_every_worker(proxy, key, workers) capped: Final = await asyncio.gather(*tuple(_capped_close(socket) for socket in opened)) earlier_result: Final = await _held_turn(earlier, model, accepted_at) delete: Final = await asyncio.to_thread(_delete_limit, candidate) - deleted_at: Final = time.monotonic() stored_after_delete: Final = await asyncio.to_thread(_general_settings_row, database_url) - await asyncio.to_thread(_settled_on_every_worker, deleted_at) + await asyncio.to_thread(_settled_on_every_worker, candidate, workers, None) restored: Final = await _idle_turn( proxy, key, From ca0ec627b60b2b3db3ad9644c45a1f86141ac2c3 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 15:45:56 -0700 Subject: [PATCH 023/124] fix(exa): return highlights, highlight scores, score and the response envelope from Exa search (#47423) Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../websearch_interception/transformation.py | 10 +- litellm/llms/exa_ai/search/transformation.py | 14 +- .../test_websearch_interception_wire.py | 78 ++++++++++ .../integration/providers/test_search_wire.py | 133 +++++++++++++++++- .../sdk/test_exa_search_envelope_sdk.py | 82 +++++++++++ .../test_websearch_native_blocks.py | 58 ++++++++ .../llms/exa_ai/search/test_transformation.py | 77 +++++++++- 7 files changed, 442 insertions(+), 10 deletions(-) create mode 100644 tests/integration/sdk/test_exa_search_envelope_sdk.py diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index bbb1b4d494b..58ae8bd6e48 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -5,7 +5,7 @@ Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. """ import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final from pydantic import TypeAdapter @@ -539,7 +539,8 @@ class WebSearchTransformation: result: SearchResponse from litellm.asearch() Returns: - Formatted text with Title, URL, Snippet for each result + Formatted text with Title, URL, Snippet for each result, or the response + without its provider_metadata (the caller's billing envelope) when there are no results """ # Convert SearchResponse to string if hasattr(result, "results") and result.results: @@ -548,6 +549,9 @@ class WebSearchTransformation: [f"Title: {r.title}\nURL: {r.url}\nSnippet: {r.snippet}" for r in result.results] ) else: - search_result_text = str(result) + fields: Final[Mapping[str, object]] = dict(result) + search_result_text = " ".join( + f"{key}={value!r}" for key, value in fields.items() if key != "provider_metadata" + ) return search_result_text diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index fbde1377180..89073409599 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -21,6 +21,7 @@ from litellm.secret_managers.main import get_secret_str _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True)) _STRS: Final = TypeAdapter(Iterable[str], config=ConfigDict(hide_input_in_errors=True)) +_RESULT_EXTRAS: Final = (("highlights", "highlights"), ("highlightScores", "highlight_scores"), ("score", "score")) class _ExaAISearchRequestRequired(TypedDict): @@ -173,6 +174,9 @@ class ExaAISearchConfig(BaseSearchConfig): - results[].summary → SearchResult.snippet (fallback when "text" and "highlights" are absent) - results[].publishedDate → SearchResult.date - No last_updated field in Exa AI response (set to None) + - results[].highlights / highlightScores / score → SearchResult.highlights / highlight_scores / score, + each set only when Exa returned it + - every top-level key except results (requestId, costDollars, ...) → SearchResponse.provider_metadata Args: raw_response: Raw httpx response from Exa AI API @@ -197,11 +201,15 @@ class ExaAISearchConfig(BaseSearchConfig): or "", "date": result.get("publishedDate"), # ISO 8601 datetime string "last_updated": None, # Exa AI doesn't provide last_updated in response + **{field: result[key] for key, field in _RESULT_EXTRAS if key in result}, } ) results.append(search_result) - return SearchResponse( - results=results, - object="search", + return SearchResponse.model_validate( + { + "results": results, + "object": "search", + "provider_metadata": {key: value for key, value in response_json.items() if key != "results"}, + } ) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py index 4e5ef521c0e..5ec860910e5 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py @@ -409,3 +409,81 @@ def test_database_created_search_tool_backend_receives_the_intercepted_query_ove assert targets[-3:] == (("POST", "/v1/messages"), ("GET", "/database/search"), ("POST", "/v1/messages")), ( targets ) + + +def test_empty_exa_search_reaches_the_model_without_the_exa_response_envelope(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = "websearch-exa-" + uuid.uuid4().hex + tool_name: Final = "integration-exa-" + uuid.uuid4().hex + envelope: Final = { + "requestId": "exa-request-" + identity, + "resolvedSearchType": "neural", + "searchTime": 98.5, + "costDollars": {"total": 0.005, "search": {"neural": 0.005}}, + } + + def respond(request: Request) -> Reply: + parts: Final = urlsplit(request.target) + if parts.path == "/exa/search": + assert (request.method, json.loads(request.body)["query"]) == ("POST", _QUERY), request.body + return Reply(body=json.dumps({**envelope, "results": []}).encode()) + assert request.method == "POST" and parts.path == "/v1/messages", request.target + body: Final = json.loads(request.body) + assert [tool["name"] for tool in body["tools"]] == ["litellm_web_search"], body["tools"] + results: Final = [ + block + for message in body["messages"] + if isinstance(message["content"], list) + for block in message["content"] + if block["type"] == "tool_result" + ] + if not results: + return _anthropic_reply(identity, [_TEXT_BLOCK, _search_tool_use(identity)], "tool_use") + assert results == [{"type": "tool_result", "tool_use_id": identity, "content": "results=[] object='search'"}], ( + results + ) + return _anthropic_reply(identity, [_FINAL_BLOCK], "end_turn") + + with wire_server(respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["search_tools"] = [ + { + "search_tool_name": tool_name, + "litellm_params": { + "search_provider": "exa_ai", + "api_key": "synthetic-exa-key", + "api_base": wire.url + "/exa", + }, + } + ] + config["litellm_settings"].update( + { + "callbacks": ["websearch_interception"], + "websearch_interception_params": { + "enabled": True, + "enabled_providers": ["anthropic"], + "search_tool_name": tool_name, + }, + } + ) + path: Final = tmp_path / "websearch-exa.yaml" + path.write_text(yaml.safe_dump(config)) + environment: Final = {"ANTHROPIC_API_BASE": wire.url} + with owned_proxy(gateway, tmp_path, environment, config=path) as candidate, candidate.scenario() as models: + model: Final = models.model( + model="anthropic/claude-sonnet-4-5-20250929", api_base=wire.url, api_key="synthetic-anthropic-key" + ) + response: Final = candidate.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": [{"role": "user", "content": identity}], + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 3}], + }, + ) + assert response.status_code == 200, response.text + assert response.json()["content"][-1] == _FINAL_BLOCK, response.text + assert envelope["requestId"] not in response.text, response.text + targets: Final = tuple((request.method, urlsplit(request.target).path) for request in wire.drain()) + assert targets[-3:] == (("POST", "/v1/messages"), ("POST", "/exa/search"), ("POST", "/v1/messages")), targets diff --git a/tests/integration/providers/test_search_wire.py b/tests/integration/providers/test_search_wire.py index 5ee80f7f5db..005539cc331 100644 --- a/tests/integration/providers/test_search_wire.py +++ b/tests/integration/providers/test_search_wire.py @@ -1,11 +1,12 @@ import json import uuid +from concurrent.futures import ThreadPoolExecutor from typing import Final import pytest from integration._support.client import Gateway, Scenario from integration._support.wire import Reply, Request, wire_server -from pydantic import BaseModel +from pydantic import BaseModel, JsonValue _EXA_KEY: Final = "synthetic-exa-key" _QUERY: Final = "latest AI developments 2024" @@ -43,6 +44,72 @@ class _SearchResponse(BaseModel): results: tuple[_SearchResult, ...] +_EXA_HIGHLIGHTS: Final = {"highlights": {"numSentences": 2}} +_EXA_ENVELOPE: Final = { + "requestId": "exa-request-5d1f0c", + "resolvedSearchType": "neural", + "searchTime": 412.5, + "costDollars": {"total": 0.007, "search": {"neural": 0.005}, "contents": {"highlights": 0.002}}, +} +_EXA_SCORED_RESULTS: Final = ( + { + "id": "https://result.invalid/margins", + "title": "Margin outlook", + "url": "https://result.invalid/margins", + "publishedDate": "2026-01-15T00:00:00.000Z", + "score": 0.91, + "highlights": ["Margins hold steady.", "Deposit costs ease."], + "highlightScores": [0.82, 0.64], + }, + { + "id": "https://result.invalid/deposits", + "title": "Deposit pricing", + "url": "https://result.invalid/deposits", + "score": 0.47, + "highlights": ["Deposit betas fall."], + "highlightScores": [0.55], + }, +) + + +class _ScoredSearchResult(_SearchResult): + highlights: tuple[str, ...] | None = None + highlight_scores: tuple[float, ...] | None = None + score: float | None = None + + +class _ScoredSearchResponse(BaseModel): + object: str + results: tuple[_ScoredSearchResult, ...] + provider_metadata: JsonValue = None + + +_EXPECTED_SCORED_RESPONSE: Final = _ScoredSearchResponse( + object="search", + results=( + _ScoredSearchResult( + title="Margin outlook", + url="https://result.invalid/margins", + snippet="Margins hold steady.\n\nDeposit costs ease.", + date="2026-01-15T00:00:00.000Z", + last_updated=None, + highlights=("Margins hold steady.", "Deposit costs ease."), + highlight_scores=(0.82, 0.64), + score=0.91, + ), + _ScoredSearchResult( + title="Deposit pricing", + url="https://result.invalid/deposits", + snippet="Deposit betas fall.", + date=None, + last_updated=None, + highlights=("Deposit betas fall.",), + highlight_scores=(0.55,), + score=0.47, + ), + ), + provider_metadata=_EXA_ENVELOPE, +) _EXPECTED_RESPONSE: Final = _SearchResponse( object="search", results=( @@ -162,6 +229,70 @@ def test_list_query_reaches_exa_as_one_space_joined_query(gateway: Gateway) -> N assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/exa/search")] +def test_exa_highlights_scores_and_response_envelope_reach_the_caller_on_every_route_spelling( + gateway: Gateway, +) -> None: + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/exa/search"), request.target + assert json.loads(request.body) == {"query": _QUERY, "numResults": 2, "contents": _EXA_HIGHLIGHTS}, request.body + return Reply(body=json.dumps({**_EXA_ENVELOPE, "results": list(_EXA_SCORED_RESULTS)}).encode()) + + tool: Final = f"exa-tool-{uuid.uuid4().hex}" + body: Final = {"query": _QUERY, "max_results": 2, "contents": _EXA_HIGHLIGHTS} + with wire_server(respond) as wire, gateway.scenario() as scenario: + _create_exa_tool(gateway, scenario, tool, f"{wire.url}/exa") + spellings: Final = ( + (f"/v1/search/{tool}", body), + (f"/search/{tool}", body), + ("/v1/search", {"search_tool_name": tool, **body}), + ("/search", {"model": tool, **body}), + ) + for path, payload in spellings: + response = gateway.request("POST", path, payload) + assert response.status_code == 200, f"{path}: {response.text}" + assert _ScoredSearchResponse.model_validate_json(response.content) == _EXPECTED_SCORED_RESPONSE, ( + f"{path}: {response.text}" + ) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/exa/search")] * len( + spellings + ) + + +def test_concurrent_exa_searches_each_get_their_own_response_envelope(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + query: Final = json.loads(request.body)["query"] + results: Final = [] if query.endswith("-empty") else [_EXA_SCORED_RESULTS[1]] + return Reply( + body=json.dumps({"requestId": f"exa-{query}", "costDollars": {"total": 0.005}, "results": results}).encode() + ) + + tool: Final = f"exa-tool-{uuid.uuid4().hex}" + queries: Final = tuple(f"burst-{uuid.uuid4().hex}{'-empty' if index % 2 else ''}" for index in range(16)) + with wire_server(respond) as wire, gateway.scenario() as scenario: + _create_exa_tool(gateway, scenario, tool, f"{wire.url}/exa") + with ThreadPoolExecutor(max_workers=len(queries)) as pool: + responses: Final = tuple( + pool.map(lambda query: gateway.request("POST", f"/v1/search/{tool}", {"query": query}), queries) + ) + observed: Final = tuple( + ( + response.status_code, + _ScoredSearchResponse.model_validate_json(response.content).provider_metadata, + tuple(result.score for result in _ScoredSearchResponse.model_validate_json(response.content).results), + ) + for response in responses + ) + assert observed == tuple( + ( + 200, + {"requestId": f"exa-{query}", "costDollars": {"total": 0.005}}, + () if query.endswith("-empty") else (0.47,), + ) + for query in queries + ), [response.text for response in responses] + assert len(wire.drain()) == len(queries) + + def _not_found(name: str) -> str: return f"Search tool '{name}' not found in router.search_tools" diff --git a/tests/integration/sdk/test_exa_search_envelope_sdk.py b/tests/integration/sdk/test_exa_search_envelope_sdk.py new file mode 100644 index 00000000000..73c3538fa45 --- /dev/null +++ b/tests/integration/sdk/test_exa_search_envelope_sdk.py @@ -0,0 +1,82 @@ +import json +import uuid +from typing import Final + +import pytest +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +import litellm +from litellm.llms.base_llm.search.transformation import SearchResponse + +pytestmark: Final = pytest.mark.timeout(60) + +_ENVELOPE: Final[dict[str, JsonValue]] = { + "requestId": "exa-request-sdk", + "resolvedSearchType": "neural", + "searchTime": 211.0, + "costDollars": {"total": 0.005, "search": {"neural": 0.005}}, +} +_RESULT: Final[dict[str, JsonValue]] = { + "id": "https://result.invalid/sdk", + "title": "SDK result", + "url": "https://result.invalid/sdk", + "publishedDate": "2026-02-01T00:00:00.000Z", + "score": 0.73, + "highlights": ["First line.", "Second line."], + "highlightScores": [0.61, 0.42], +} +_EXPECTED: Final[dict[str, JsonValue]] = { + "object": "search", + "results": [ + { + "title": "SDK result", + "url": "https://result.invalid/sdk", + "snippet": "First line.\n\nSecond line.", + "date": "2026-02-01T00:00:00.000Z", + "last_updated": None, + "highlights": ["First line.", "Second line."], + "highlight_scores": [0.61, 0.42], + "score": 0.73, + } + ], + "provider_metadata": _ENVELOPE, +} + + +def _respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/exa/search"), request.target + return Reply(body=json.dumps({**_ENVELOPE, "results": [_RESULT]}).encode()) + + +def test_sync_exa_search_returns_highlights_scores_and_the_response_envelope() -> None: + with wire_server(_respond) as wire: + response: Final = litellm.search( + query=f"sdk sync {uuid.uuid4().hex}", + search_provider="exa_ai", + api_key="synthetic-exa-key", + api_base=f"{wire.url}/exa", + max_results=1, + contents={"highlights": {"numSentences": 2}}, + ) + assert len(wire.drain()) == 1 + + assert isinstance(response, SearchResponse), type(response) + assert response.model_dump(mode="json") == _EXPECTED + + +@pytest.mark.asyncio +async def test_async_exa_search_returns_highlights_scores_and_the_response_envelope() -> None: + with wire_server(_respond) as wire: + response: Final = await litellm.asearch( + query=f"sdk async {uuid.uuid4().hex}", + search_provider="exa_ai", + api_key="synthetic-exa-key", + api_base=f"{wire.url}/exa", + max_results=1, + contents={"highlights": {"numSentences": 2}}, + ) + assert len(wire.drain()) == 1 + + assert isinstance(response, SearchResponse), type(response) + assert response.model_dump(mode="json") == _EXPECTED diff --git a/tests/unit/integrations/websearch_interception/test_websearch_native_blocks.py b/tests/unit/integrations/websearch_interception/test_websearch_native_blocks.py index 92feab86dc0..c43a51cfd52 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_native_blocks.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_native_blocks.py @@ -670,3 +670,61 @@ class TestLegacyPathMatchesNewPath: "text", ] assert out["content"][1]["tool_use_id"] == out["content"][0]["id"] + + +@pytest.mark.parametrize( + ("response", "expected_text"), + [ + (SearchResponse(results=[], object="search"), "results=[] object='search'"), + ( + SearchResponse.model_validate( + { + "results": [], + "object": "search", + "provider_metadata": {"requestId": "exa-request", "costDollars": {"total": 0.005}}, + } + ), + "results=[] object='search'", + ), + ( + SearchResponse.model_validate( + { + "results": [], + "object": "search", + "search_id": "search-1", + "provider_metadata": {"requestId": "exa-request"}, + } + ), + "results=[] object='search' search_id='search-1'", + ), + ( + SearchResponse.model_validate({"results": [], "object": "search", "query": "q", "cls": 1}), + "results=[] object='search' query='q' cls=1", + ), + ( + SearchResponse.model_validate( + { + "results": [ + SearchResult.model_validate( + { + "title": "T", + "url": "https://example.com", + "snippet": "S", + "highlights": ["S"], + "highlight_scores": [0.5], + "score": 0.9, + } + ) + ], + "object": "search", + "provider_metadata": {"requestId": "exa-request"}, + } + ), + "Title: T\nURL: https://example.com\nSnippet: S", + ), + ], +) +def test_format_search_response_keeps_the_provider_metadata_out_of_the_model_text( + response: SearchResponse, expected_text: str +): + assert WebSearchTransformation.format_search_response(response) == expected_text diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 456ed26cc86..a5a5d5e22dc 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -123,12 +123,83 @@ def test_transform_search_response_maps_every_exa_result_in_provider_order(): assert response.object == "search" assert response.results == [ - SearchResult( - title="First", url="https://example.com/first", snippet="First text", date="2024-01-05T00:00:00.000Z" + SearchResult.model_validate( + { + "title": "First", + "url": "https://example.com/first", + "snippet": "First text", + "date": "2024-01-05T00:00:00.000Z", + "highlights": ["ignored when text is present"], + "highlight_scores": [0.5], + "score": 0.42, + } + ), + SearchResult.model_validate( + { + "title": "Second", + "url": "https://example.com/second", + "snippet": "one\n\ntwo", + "highlights": ["one", "two"], + } ), - SearchResult(title="Second", url="https://example.com/second", snippet="one\n\ntwo"), SearchResult(title="Third", url="https://example.com/third", snippet="Summary"), ] + assert response.model_extra == { + "provider_metadata": { + "requestId": "b5947044c4b78efa9552a7c89b306d95", + "resolvedSearchType": "neural", + "costDollars": {"total": 0.005}, + } + } + + +@pytest.mark.parametrize( + ("exa_fields", "expected_extras"), + [ + ({"text": "full text"}, {}), + ({"score": 0.3}, {"score": 0.3}), + ({"highlights": ["a highlight"]}, {"highlights": ["a highlight"]}), + ({"highlightScores": [0.1]}, {"highlight_scores": [0.1]}), + ({"score": None}, {"score": None}), + ({"highlights": [], "highlightScores": [], "score": 0}, {"highlights": [], "highlight_scores": [], "score": 0}), + ({"id": "https://example.com", "author": "Ada", "image": None, "summary": "a summary"}, {}), + ], +) +def test_transform_search_response_attaches_only_the_exa_result_fields_it_returned( + exa_fields: dict[str, object], expected_extras: dict[str, object] +): + response: Final = _transform( + { + "results": [ + {"title": "Plain", "url": "https://example.com/plain", "text": "plain"}, + {"title": "Title", "url": "https://example.com", **exa_fields}, + ] + } + ) + + assert [result.model_extra for result in response.results] == [{}, expected_extras] + + +@pytest.mark.parametrize( + ("payload", "expected_metadata"), + [ + ({}, {}), + ({"results": []}, {}), + ({"requestId": "abc"}, {"requestId": "abc"}), + ({"context": "joined result text", "results": []}, {"context": "joined result text"}), + ( + {"requestId": "abc", "searchTime": 412.5, "costDollars": {"total": 0.007}, "results": [{"text": "t"}]}, + {"requestId": "abc", "searchTime": 412.5, "costDollars": {"total": 0.007}}, + ), + ], +) +def test_transform_search_response_keeps_the_exa_envelope_as_provider_metadata( + payload: dict[str, object], expected_metadata: dict[str, object] +): + response: Final = _transform(payload) + + assert response.model_extra == {"provider_metadata": expected_metadata} + assert response.model_dump(mode="json")["provider_metadata"] == expected_metadata def test_transform_search_response_result_without_optional_fields_gets_empty_defaults(): From 9b4f44135dfe25c7de077d51280a60fc761f2af0 Mon Sep 17 00:00:00 2001 From: hamzashah-dev Date: Tue, 20 Oct 2026 03:50:33 +0500 Subject: [PATCH 024/124] fix(proxy): refresh cached team on /team/block and /team/unblock (#35643) * fix(proxy): refresh cached team on /team/block and /team/unblock Both endpoints wrote `blocked` to Postgres and returned without touching the management-object cache. Auth resolves teams through `team_id:{team_id}` before reading the DB and `common_checks` enforces `blocked` off that cached object, so a team whose traffic had already warmed the cache kept serving requests after being blocked, and kept getting rejected after being unblocked, until the entry expired. `_refresh_cached_team` already exists for exactly this and its docstring says every endpoint mutating `litellm_teamtable` must call it; these two were the only mutating endpoints that didn't. The update now also asks for `object_permission` so the refreshed cache entry doesn't drop the relation, matching `team_model_add` and `team_model_delete`. * fix(proxy): keep a team block from being undone by in-flight spend writes update_cache read the cached team when a request finished and wrote that whole copy back in a background task, so a team blocked or invalidated while requests were in flight came back unblocked on every busy worker until its 60 s cache entry expired. The team spend now goes onto the team held in this worker's memory at write time, and a missing entry is left missing for the next auth read to load. Block and unblock responses keep litellm_model_table and object_permission null, as before this PR, while the cache still gets both relations. * test(proxy): type the team spend cache test parameters * test(proxy): hold a team block under in-flight team spend and keep team aliases routing after block and unblock --------- Co-authored-by: Hamza Shah Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../proxy/hooks/proxy_track_cost_callback.py | 1 - .../management_endpoints/team_endpoints.py | 96 +++++------ litellm/proxy/proxy_server.py | 41 ++--- .../integration/management/test_team_block.py | 150 +++++++++++++++++- tests/unit/proxy/auth/test_jwt.py | 29 ---- .../test_team_endpoints.py | 104 +++++++++++- .../proxy/proxy_server/test_spend_counters.py | 73 ++++++++- 7 files changed, 381 insertions(+), 113 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 3195394b9aa..4434fe57ab6 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -457,7 +457,6 @@ class _ProxyDBLogger(CustomLogger): update_cache_read_keys=update_cache_read_keys( user_id=user_id, end_user_id=end_user_id, - team_id=team_id, tags=tags, response_cost=response_cost, ), diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0f40662ad83..0595d23bd9a 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -5343,6 +5343,52 @@ async def team_member_me( ) +async def _set_team_blocked( + team_id: str, + blocked: bool, + user_api_key_dict: UserAPIKeyAuth, +) -> "prisma_models.LiteLLM_TeamTable": + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if prisma_client is None: + raise Exception("No DB Connected.") + + existing_team: Final = await _team_db(prisma_client).find_unique(where={"team_id": team_id}) + if existing_team is None: + raise HTTPException( + status_code=404, + detail={"error": f"Team not found, passed team_id={team_id}"}, + ) + + if not await get_team_access().allows( + user_api_key_dict, LiteLLM_TeamTable.model_validate(existing_team.model_dump()), TEAM_OR_ORG_ADMIN + ): + team_access_denied() + + record: Final = await _team_db(prisma_client).update( + where={"team_id": team_id}, + data={"blocked": blocked}, + include=_TEAM_UPDATE_INCLUDE, + ) + if record is None: + raise HTTPException( + status_code=404, + detail={"error": f"Team not found, passed team_id={team_id}"}, + ) + + await refresh_cached_team( + team_row=record, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + return record.model_copy(update=dict.fromkeys(_TEAM_UPDATE_INCLUDE)) + + @router.post("/team/block", tags=["team management"], dependencies=[Depends(user_api_key_auth)]) @management_endpoint_wrapper async def block_team( @@ -5372,30 +5418,7 @@ async def block_team( """ - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - raise Exception("No DB Connected.") - - existing_team: Final = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) - if existing_team is None: - raise HTTPException( - status_code=404, - detail={"error": f"Team not found, passed team_id={data.team_id}"}, - ) - - # Verify caller has access to manage this team - if not await get_team_access().allows( - user_api_key_dict, LiteLLM_TeamTable.model_validate(existing_team.model_dump()), TEAM_OR_ORG_ADMIN - ): - team_access_denied() - - record: Final = await _team_db(prisma_client).update( - where={"team_id": data.team_id}, - data={"blocked": True}, - ) - - return record + return await _set_team_blocked(team_id=data.team_id, blocked=True, user_api_key_dict=user_api_key_dict) @router.post("/team/unblock", tags=["team management"], dependencies=[Depends(user_api_key_auth)]) @@ -5421,30 +5444,7 @@ async def unblock_team( }' ``` """ - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - raise Exception("No DB Connected.") - - existing_team: Final = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id}) - if existing_team is None: - raise HTTPException( - status_code=404, - detail={"error": f"Team not found, passed team_id={data.team_id}"}, - ) - - # Verify caller has access to manage this team - if not await get_team_access().allows( - user_api_key_dict, LiteLLM_TeamTable.model_validate(existing_team.model_dump()), TEAM_OR_ORG_ADMIN - ): - team_access_denied() - - record: Final = await _team_db(prisma_client).update( - where={"team_id": data.team_id}, - data={"blocked": False}, - ) - - return record + return await _set_team_blocked(team_id=data.team_id, blocked=False, user_api_key_dict=user_api_key_dict) @router.get( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c1102cee8d1..4ea3100f869 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4194,7 +4194,6 @@ async def run_spend_counter_pipeline(pending: Sequence[PendingSpendIncrement]) - def update_cache_read_keys( user_id: str | None, end_user_id: str | None, - team_id: str | None, tags: Sequence[object] | None, response_cost: float | None, ) -> tuple[str, ...]: @@ -4202,9 +4201,8 @@ def update_cache_read_keys( return () user_keys: tuple[str, ...] = (user_id, GLOBAL_PROXY_SPEND_CACHE_KEY) if user_id is not None else () end_user_keys: tuple[str, ...] = (end_user_cache_key(end_user_id),) if end_user_id is not None else () - team_keys: tuple[str, ...] = (f"team_id:{team_id}",) if team_id is not None else () tag_keys: tuple[str, ...] = tuple(tag_cache_key(tag) for tag in tags or () if isinstance(tag, str) and tag) - return user_keys + end_user_keys + team_keys + tag_keys + return user_keys + end_user_keys + tag_keys _UPDATE_CACHE_PREFETCH_SLOT: Final = "update_cache_read" @@ -4281,7 +4279,7 @@ async def update_cache( with service_target(AUTH_OBJECTS_TARGET): cached_values: Final = await _read_update_cache_values( keys=update_cache_read_keys( - user_id=user_id, end_user_id=end_user_id, team_id=team_id, tags=tags, response_cost=response_cost + user_id=user_id, end_user_id=end_user_id, tags=tags, response_cost=response_cost ), parent_otel_span=parent_otel_span, ) @@ -4431,35 +4429,28 @@ async def update_cache( ) ### UPDATE TEAM SPEND ### - async def _update_team_cache(): + def _update_team_cache(): if team_id is None or response_cost is None: return _id: Final = f"team_id:{team_id}" try: - cached_team: Final = cached_values.get(_id) - if cached_team is None: - # do nothing if team not in api key cache - return - existing_spend_obj: Final[LiteLLM_TeamTableCachedObj | None] = CacheCodec.deserialize( - cached_team, LiteLLM_TeamTableCachedObj + current_team: Final = CacheCodec.deserialize( + user_api_key_cache.get_cache(key=_id, local_only=True), LiteLLM_TeamTableCachedObj ) - if existing_spend_obj is None: + if current_team is None: return verbose_proxy_logger.debug( - "_update_team_db: existing spend: %s; response_cost: %s", existing_spend_obj, response_cost + "_update_team_db: existing spend: %s; response_cost: %s", current_team, response_cost ) - - existing_spend: Final[float] = existing_spend_obj.spend or 0.0 - # Calculate the new cost by adding the existing cost and response_cost - new_spend: Final = existing_spend + response_cost - - existing_spend_obj.spend = new_spend - values_to_update_in_cache.append( - ( - _id, - CacheCodec.serialize(existing_spend_obj, model_type=LiteLLM_TeamTableCachedObj), - ) + user_api_key_cache.set_cache( + key=_id, + value=CacheCodec.serialize( + current_team.model_copy(update={"spend": (current_team.spend or 0.0) + response_cost}), + model_type=LiteLLM_TeamTableCachedObj, + ), + local_only=True, + ttl=get_management_object_ttl(user_api_key_cache), ) except Exception as e: verbose_proxy_logger.warning( @@ -4536,7 +4527,7 @@ async def update_cache( await _update_end_user_cache() if team_id is not None: - await _update_team_cache() + _update_team_cache() if tags is not None: await _update_tag_cache() diff --git a/tests/integration/management/test_team_block.py b/tests/integration/management/test_team_block.py index 3d1f9e430ea..9a340944569 100644 --- a/tests/integration/management/test_team_block.py +++ b/tests/integration/management/test_team_block.py @@ -1,14 +1,21 @@ +import itertools +import threading import uuid from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from typing import Final import httpx import pytest -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue -from tests.integration._support.client import Gateway, string_value +from tests.integration._support.client import Gateway, eventually, held, string_value from tests.integration._support.database import read_rows +IN_FLIGHT_CALLERS_PER_PROXY: Final = 8 +BLOCK_CYCLES: Final = 4 +PROBES_PER_PROXY: Final = 6 + class _ObservedRequest(BaseModel): body: dict[str, JsonValue] @@ -35,6 +42,16 @@ class _ErrorResponse(BaseModel): error: _ProxyError +class _Message(BaseModel): + model_config = ConfigDict(frozen=True) + content: str + + +class _ChatBody(BaseModel): + model_config = ConfigDict(frozen=True) + messages: tuple[_Message, ...] + + @pytest.fixture def upstream(gateway: Gateway) -> Iterator[httpx.Client]: with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client: @@ -58,6 +75,87 @@ def _models(client: httpx.Client) -> tuple[str, ...]: return tuple(string_value(request.body["model"]) for request in observations.requests) +def _texts(client: httpx.Client) -> tuple[str, ...]: + response: Final = client.get("/__observations") + assert response.status_code == 200, response.text + observations: Final = _Observations.model_validate_json(response.text) + return tuple( + message.content + for message in itertools.chain.from_iterable( + _ChatBody.model_validate(request.body).messages for request in observations.requests + ) + ) + + +def _keep_calling(proxy: Gateway, model: str, key: str, stop: threading.Event) -> tuple[int, ...]: + return tuple( + _chat(proxy, model, key, f"in flight {uuid.uuid4().hex}").status_code + for _ in itertools.takewhile(lambda _: not stop.is_set(), itertools.count()) + ) + + +def _statuses_on_fresh_connections(proxies: tuple[Gateway, ...], model: str, key: str, marker: str) -> tuple[int, ...]: + return tuple( + proxy.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"{marker} {uuid.uuid4().hex}"}]}, + key=key, + headers={"connection": "close"}, + ).status_code + for proxy in proxies * PROBES_PER_PROXY + ) + + +def _block_while_team_traffic_is_in_flight( + gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str +) -> tuple[httpx.Response, tuple[int, ...]]: + stop: Final = threading.Event() + with ThreadPoolExecutor(max_workers=2 * IN_FLIGHT_CALLERS_PER_PROXY) as pool: + callers: Final = tuple( + pool.submit(_keep_calling, proxy, model, key, stop) + for proxy in (gateway, peer) * IN_FLIGHT_CALLERS_PER_PROXY + ) + try: + eventually( + lambda: len(_models(upstream)), lambda served: served >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, seconds=30 + ) + blocked: Final = gateway.request("POST", "/team/block", {"team_id": team}) + finally: + stop.set() + statuses: Final = tuple(itertools.chain.from_iterable(caller.result() for caller in callers)) + return blocked, statuses + + +def _block_and_unblock_under_team_traffic( + gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str, cycle: int +) -> None: + upstream.delete("/__observations").raise_for_status() + blocked, in_flight = _block_while_team_traffic_is_in_flight(gateway, peer, upstream, model, key, team) + assert blocked.status_code == 200, blocked.text + assert in_flight.count(200) >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, repr(in_flight) + assert set(in_flight) <= {200, 401}, repr(in_flight) + + marker: Final = f"after block {cycle} {uuid.uuid4().hex}" + refused: Final = held( + lambda: _statuses_on_fresh_connections((gateway, peer), model, key, marker), + lambda statuses: set(statuses) == {401}, + holding=5, + seconds=15, + ) + assert refused == (401,) * (2 * PROBES_PER_PROXY), repr(refused) + leaked: Final = tuple(text for text in _texts(upstream) if text.startswith(marker)) + assert leaked == (), repr(leaked) + + unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) + assert unblocked.status_code == 200, unblocked.text + eventually( + lambda: _statuses_on_fresh_connections((gateway, peer), model, key, f"after unblock {cycle}"), + lambda statuses: set(statuses) == {200}, + seconds=15, + ) + + def test_team_block_and_unblock_update_endpoint_info_and_db_state( gateway: Gateway, peer: Gateway, upstream: httpx.Client ) -> None: @@ -126,8 +224,6 @@ def test_team_block_and_unblock_update_endpoint_info_and_db_state( def test_unblocking_a_team_restores_warmed_keys_on_both_proxies( gateway: Gateway, peer: Gateway, upstream: httpx.Client ) -> None: - pytest.skip("BUG: SLP-605 /team/unblock does not restore warmed team-key access after blocked auth is cached") - with gateway.scenario() as scenario: provider_model: Final = f"team-block-{uuid.uuid4().hex}" model: Final = scenario.model(model=f"openai/{provider_model}") @@ -166,8 +262,6 @@ def test_unblocking_a_team_restores_warmed_keys_on_both_proxies( def test_warmed_team_keys_are_refused_on_both_proxies_after_block( gateway: Gateway, peer: Gateway, upstream: httpx.Client ) -> None: - pytest.skip("BUG: SLP-605 blocking a team does not refresh warmed team-key auth state") - with gateway.scenario() as scenario: provider_model: Final = f"team-block-{uuid.uuid4().hex}" model: Final = scenario.model(model=f"openai/{provider_model}") @@ -202,8 +296,6 @@ def test_warmed_team_keys_are_refused_on_both_proxies_after_block( def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block( gateway: Gateway, peer: Gateway, upstream: httpx.Client ) -> None: - pytest.skip("BUG: SLP-605 a peer proxy that cached the team before /team/block admits a never-seen team key") - with gateway.scenario() as scenario: provider_model: Final = f"team-block-{uuid.uuid4().hex}" model: Final = scenario.model(model=f"openai/{provider_model}") @@ -229,3 +321,45 @@ def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block( assert peer_denied.status_code == 401, peer_denied.text assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text assert after_block == (), repr(after_block) + + +def test_a_team_model_alias_still_routes_on_both_proxies_after_block_and_unblock( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model(model=f"openai/{provider_model}") + alias: Final = f"alias-{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model], model_aliases={alias: model}) + key: Final = scenario.key(team_id=team, models=[model]) + + warmed: Final = tuple( + _chat(proxy, alias, key, f"team alias warm {uuid.uuid4().hex}").status_code for proxy in (gateway, peer) + ) + assert warmed == (200, 200), repr(warmed) + assert _models(upstream) == (provider_model, provider_model) + upstream.delete("/__observations").raise_for_status() + + blocked: Final = gateway.request("POST", "/team/block", {"team_id": team}) + assert blocked.status_code == 200, blocked.text + unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team}) + assert unblocked.status_code == 200, unblocked.text + + after_unblock: Final = _statuses_on_fresh_connections((gateway, peer), alias, key, "team alias after unblock") + assert after_unblock == (200,) * (2 * PROBES_PER_PROXY), repr(after_unblock) + assert _models(upstream) == (provider_model,) * (2 * PROBES_PER_PROXY) + + +def test_a_block_issued_while_team_traffic_is_in_flight_holds_on_every_worker( + gateway: Gateway, peer: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + provider_model: Final = f"team-block-{uuid.uuid4().hex}" + model: Final = scenario.model( + model=f"openai/{provider_model}", input_cost_per_token=0.000001, output_cost_per_token=0.000002 + ) + team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team) + + for cycle in range(BLOCK_CYCLES): + _block_and_unblock_under_team_traffic(gateway, peer, upstream, model, key, team, cycle) diff --git a/tests/unit/proxy/auth/test_jwt.py b/tests/unit/proxy/auth/test_jwt.py index a7aa379ad37..fb512539126 100644 --- a/tests/unit/proxy/auth/test_jwt.py +++ b/tests/unit/proxy/auth/test_jwt.py @@ -860,35 +860,6 @@ async def test_allowed_routes_admin( import pytest -@pytest.mark.asyncio -async def test_team_cache_update_called(): - import litellm - from litellm.proxy.proxy_server import user_api_key_cache - - # Use setattr to replace the method on the user_api_key_cache object - cache = DualCache() - - setattr( - litellm.proxy.proxy_server, - "user_api_key_cache", - cache, - ) - - with patch.object(cache, "async_batch_get_cache", new=AsyncMock(return_value=[None])) as mock_call_cache: - # Call the function under test - await litellm.proxy.proxy_server.update_cache( - token=None, - user_id=None, - end_user_id=None, - team_id="1234", - response_cost=20, - parent_otel_span=None, - ) # type: ignore - - await asyncio.sleep(3) - mock_call_cache.assert_awaited_once_with(keys=["team_id:1234"], parent_otel_span=None, throttle_redis=False) - - @pytest.fixture def public_jwt_key(): import json diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index a379b424aeb..5063ae0b357 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -2636,10 +2636,99 @@ async def test_team_model_add_delete_keep_model_aliases_in_team_cache(endpoint_n assert team_model_aliases(cached_team) == {"fast": "gpt-4o"} +@pytest.mark.asyncio +@pytest.mark.parametrize(("endpoint_name", "blocked"), [("block_team", True), ("unblock_team", False)]) +async def test_team_block_unblock_reach_warmed_auth_on_every_worker( + endpoint_name: str, blocked: bool, monkeypatch: pytest.MonkeyPatch +): + import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module + from litellm.proxy._types import BlockTeamRequest + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.auth.team_grants import team_model_aliases + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints import team_endpoints + + team_id: Final = "team-kill-switch" + columns: Final = {"team_id": team_id, "team_alias": "kill-switch", "blocked": not blocked} + alias_table: Final = {"id": 1, "model_aliases": '{"fast": "gpt-4o"}', "created_by": "admin", "updated_by": "admin"} + + async def update( + where: Mapping[str, str], data: Mapping[str, bool], include: Mapping[str, bool] | None = None + ) -> LiteLLM_TeamTable: + relations: Final = {"litellm_model_table": alias_table} if (include or {}).get("litellm_model_table") else {} + return LiteLLM_TeamTable.model_validate({**columns, **data, **relations}) + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=LiteLLM_TeamTable(**columns)) + prisma_client.db.litellm_teamtable.update = AsyncMock(side_effect=update) + cache: Final = UserApiKeyCache() + redis: Final = _BroadcastRecordingRedis() + monkeypatch.setattr(pubsub_module, "coordination_redis_cache", lambda: redis) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", None) + + warmed: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache) + assert warmed.blocked is not blocked + + response: Final = await getattr(team_endpoints, endpoint_name)( + data=BlockTeamRequest(team_id=team_id), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks + + seen_by_auth: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache) + assert seen_by_auth.blocked is blocked + assert team_model_aliases(seen_by_auth) == {"fast": "gpt-4o"} + assert redis.broadcast_keys == [f"team_id:{team_id}", "team_alias:kill-switch"] + assert (response.blocked, response.litellm_model_table, response.object_permission) == (blocked, None, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint_name", ["block_team", "unblock_team"]) +@pytest.mark.parametrize( + ("caller", "stored_team", "status_code"), + [ + ( + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="alice"), + LiteLLM_TeamTable(team_id="team-1234", members_with_roles=[Member(user_id="alice", role="user")]), + 403, + ), + (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), None, 404), + ], +) +async def test_team_block_unblock_refuse_before_writing( + endpoint_name: str, + caller: UserAPIKeyAuth, + stored_team: LiteLLM_TeamTable | None, + status_code: int, + monkeypatch: pytest.MonkeyPatch, +): + from litellm.proxy._types import BlockTeamRequest + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints import team_endpoints + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=stored_team) + prisma_client.db.litellm_teamtable.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", UserApiKeyCache()) + + with pytest.raises(HTTPException) as exc_info: + await getattr(team_endpoints, endpoint_name)( + data=BlockTeamRequest(team_id="team-1234"), http_request=MagicMock(), user_api_key_dict=caller + ) + + assert exc_info.value.status_code == status_code + prisma_client.db.litellm_teamtable.update.assert_not_awaited() + + @pytest.mark.asyncio @pytest.mark.parametrize( "endpoint_name", - ["team_model_add", "team_model_delete", "update_team_member_permissions"], + ["team_model_add", "team_model_delete", "update_team_member_permissions", "block_team", "unblock_team"], ) async def test_team_write_404s_when_row_vanishes_before_update(endpoint_name): """A team deleted between the read and the write must 404. @@ -2662,9 +2751,12 @@ async def test_team_write_404s_when_row_vanishes_before_update(endpoint_name): TeamModelDeleteRequest, UserAPIKeyAuth, ) + from litellm.proxy._types import BlockTeamRequest from litellm.proxy.management_endpoints.team_endpoints import ( + block_team, team_model_add, team_model_delete, + unblock_team, update_team_member_permissions, ) @@ -2701,6 +2793,16 @@ async def test_team_write_404s_when_row_vanishes_before_update(endpoint_name): http_request=mock_request, user_api_key_dict=mock_user_api_key_dict, ), + "block_team": lambda: block_team( + data=BlockTeamRequest(team_id="team-1234"), + http_request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + ), + "unblock_team": lambda: unblock_team( + data=BlockTeamRequest(team_id="team-1234"), + http_request=mock_request, + user_api_key_dict=mock_user_api_key_dict, + ), }[endpoint_name] with ( diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index 01400384842..3ff4293ba93 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -22,6 +22,7 @@ Pins covered: from __future__ import annotations import asyncio +from collections.abc import Awaitable, Callable from datetime import datetime from typing import Final, cast from unittest.mock import AsyncMock, MagicMock @@ -29,6 +30,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm.proxy.proxy_server as ps +from litellm.models.team import LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_local +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team from .conftest import normalize @@ -1575,7 +1580,6 @@ async def test_update_cache_no_cached_entities_schedules_pipeline_flush(monkeypa "u1", f"{ps.litellm_proxy_admin_name}:spend", "end_user_id:eu1", - "team_id:t1", "tag:x", ] @@ -1600,3 +1604,70 @@ async def test_update_cache_user_cache_failure_invalid_state_is_swallowed(monkey ) assert result is None + + +async def _drain_spend_cache_writes() -> None: + for _ in range(3): + await asyncio.sleep(0) + + +async def _update_team_spend(team_id: str, response_cost: float) -> None: + await ps.update_cache( + token=None, + user_id=None, + end_user_id=None, + team_id=team_id, + response_cost=response_cost, + parent_otel_span=None, + ) + + +@pytest.mark.asyncio +async def test_update_cache_adds_spend_to_the_cached_team_only(monkeypatch: pytest.MonkeyPatch) -> None: + cache: Final = UserApiKeyCache() + cache.set_cache(key="team_id:warm", value=LiteLLM_TeamTableCachedObj(team_id="warm", spend=5.0, blocked=True)) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + await _update_team_spend(team_id="warm", response_cost=2.5) + await _update_team_spend(team_id="cold", response_cost=2.5) + await _drain_spend_cache_writes() + + warm: Final = cache.get_cache(key="team_id:warm", model_type=LiteLLM_TeamTableCachedObj) + assert warm is not None + assert (warm.spend, warm.blocked) == (7.5, True) + assert cache.get_cache(key="team_id:cold") is None + + +async def _block_on_this_worker(cache: UserApiKeyCache, team_id: str) -> None: + await refresh_cached_team( + team_row=LiteLLM_TeamTable(team_id=team_id, blocked=True), + user_api_key_cache=cache, + proxy_logging_obj=ps.proxy_logging_obj, + ) + + +async def _invalidated_by_a_peer(cache: UserApiKeyCache, team_id: str) -> None: + evict_local(cache_keys=(f"team_id:{team_id}",), user_api_key_cache=cache) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("management_write", "expected_blocked"), + [(_block_on_this_worker, True), (_invalidated_by_a_peer, None)], +) +async def test_update_cache_never_restores_a_team_a_management_write_replaced( + management_write: Callable[[UserApiKeyCache, str], Awaitable[None]], + expected_blocked: bool | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + team_id: Final = "kill-switch" + cache: Final = UserApiKeyCache() + cache.set_cache(key=f"team_id:{team_id}", value=LiteLLM_TeamTableCachedObj(team_id=team_id, blocked=False)) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + await _update_team_spend(team_id=team_id, response_cost=1.0) + await management_write(cache, team_id) + await _drain_spend_cache_writes() + + seen_by_auth: Final = cache.get_cache(key=f"team_id:{team_id}", model_type=LiteLLM_TeamTableCachedObj) + assert (seen_by_auth.blocked if seen_by_auth is not None else None) == expected_blocked From 48763ccb505a98b04933d72f50cfdd89ae56f5da Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 15:58:29 -0700 Subject: [PATCH 025/124] fix(scim)!: store PATCH enterprise extension attributes in metadata.scim_enterprise (#46738) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/scim/scim_v2.py | 259 +++++++- .../scim/test_scim_patch_user.py | 556 ++++++++++++++++++ 2 files changed, 805 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 97a6a29ef66..ec905a9f187 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -5,13 +5,13 @@ This is an enterprise feature and requires a premium license. """ import re -from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass -from functools import partial +from functools import partial, reduce from itertools import chain, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final, NamedTuple, Protocol, TypeAlias, overload +from typing import TYPE_CHECKING, Final, NamedTuple, NoReturn, Protocol, TypeAlias, overload from fastapi import ( APIRouter, @@ -24,6 +24,7 @@ from fastapi import ( Response, ) from pydantic import TypeAdapter, ValidationError +from pydantic.fields import FieldInfo from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -2271,6 +2272,217 @@ def _handle_multi_valued_attribute_update(path: str, op_type: str, value: object metadata[metadata_key] = dumped +def _enterprise_extension_sub_path(path: str) -> str | None: + schema: Final = SCIM_ENTERPRISE_USER_SCHEMA + if path.casefold() == schema.casefold(): + return "" + prefix: Final = f"{schema}:" + if path.casefold().startswith(prefix.casefold()): + return path[len(prefix) :] + return None + + +def _scim_model_field_names(model_fields: Mapping[str, FieldInfo]) -> Mapping[str, str]: + return { + **{field_name.casefold(): field_name for field_name in model_fields}, + **{ + field.alias.casefold(): field_name + for field_name, field in model_fields.items() + if isinstance(field.alias, str) + }, + } + + +def _canonicalize_scim_model_data( + value: Mapping[str, object], + model_fields: Mapping[str, FieldInfo], +) -> Mapping[str, object]: + field_names: Final = _scim_model_field_names(model_fields) + return {field_names.get(key.casefold(), key): field_value for key, field_value in value.items()} + + +def _canonicalize_enterprise_data(value: Mapping[str, object]) -> Mapping[str, object]: + enterprise_data: Final = _canonicalize_scim_model_data(value, SCIMEnterpriseUser.model_fields) + manager: Final = enterprise_data.get("manager") + if isinstance(manager, str): + return {**enterprise_data, "manager": {"value": manager}} + if not isinstance(manager, dict): + return enterprise_data + return { + **enterprise_data, + "manager": _canonicalize_scim_model_data( + _JSON_OBJECT_ADAPTER.validate_python(manager), + SCIMUserManager.model_fields, + ), + } + + +@dataclass(frozen=True, slots=True) +class _InvalidEnterpriseValue: + detail: str + + +def _merge_enterprise_data( + existing: Mapping[str, object], + incoming: Mapping[str, object], + incoming_manager_is_object: bool, +) -> Mapping[str, object]: + merged: Final[Mapping[str, object]] = {**existing, **incoming} + existing_manager: Final = existing.get("manager") + incoming_manager: Final = incoming.get("manager") + if incoming_manager_is_object and isinstance(existing_manager, dict) and isinstance(incoming_manager, dict): + existing_manager_data: Final = _canonicalize_scim_model_data( + _JSON_OBJECT_ADAPTER.validate_python(existing_manager), + SCIMUserManager.model_fields, + ) + incoming_manager_data: Final = _JSON_OBJECT_ADAPTER.validate_python(incoming_manager) + return {**merged, "manager": {**existing_manager_data, **incoming_manager_data}} + return merged + + +def _validate_enterprise_update( + value: Mapping[str, object], +) -> Mapping[str, object] | None | _InvalidEnterpriseValue: + try: + enterprise: Final = SCIMEnterpriseUser.model_validate(value) + except ValidationError: + return _InvalidEnterpriseValue(detail="Invalid value for the enterprise extension") + dumped: Final = enterprise.model_dump(by_alias=True, exclude_none=True) + return dumped if dumped else None + + +def _raise_invalid_enterprise_value(error: _InvalidEnterpriseValue) -> NoReturn: + raise HTTPException(status_code=400, detail={"error": error.detail}) + + +def _handle_enterprise_update( + sub_path: str, + op_type: str, + value: object, + existing_enterprise: Mapping[str, object], +) -> Mapping[str, object] | None | _InvalidEnterpriseValue: + if op_type == "remove" and not sub_path: + return None + + if not sub_path: + if not isinstance(value, dict): + return _InvalidEnterpriseValue(detail="The enterprise extension value must be an object") + incoming_value: Final = _JSON_OBJECT_ADAPTER.validate_python(value) + incoming: Final = _canonicalize_enterprise_data(incoming_value) + incoming_manager_is_object: Final = isinstance( + _canonicalize_scim_model_data(incoming_value, SCIMEnterpriseUser.model_fields).get("manager"), + dict, + ) + return _validate_enterprise_update( + _merge_enterprise_data(existing_enterprise, incoming, incoming_manager_is_object) + ) + + path_parts: Final = sub_path.split(".") + if len(path_parts) == 1: + field_name: Final = _scim_model_field_names(SCIMEnterpriseUser.model_fields).get(sub_path.casefold()) + if field_name is None: + return _validate_enterprise_update(existing_enterprise) + if op_type == "remove": + return _validate_enterprise_update( + {key: attr for key, attr in existing_enterprise.items() if key != field_name} + ) + if value is None: + return _InvalidEnterpriseValue(detail=f"The {op_type} operation requires a 'value' member") + field_incoming: Final = _canonicalize_enterprise_data({field_name: value}) + field_incoming_manager_is_object: Final = field_name == "manager" and isinstance(value, dict) + return _validate_enterprise_update( + _merge_enterprise_data( + existing_enterprise, + field_incoming, + field_incoming_manager_is_object, + ) + ) + + if len(path_parts) == 2 and path_parts[0].casefold() == "manager": + manager_field: Final = _scim_model_field_names(SCIMUserManager.model_fields).get(path_parts[1].casefold()) + if manager_field is None: + return _validate_enterprise_update(existing_enterprise) + existing_manager: Final = existing_enterprise.get("manager") + manager_data: Final[Mapping[str, object]] = ( + _canonicalize_scim_model_data( + _JSON_OBJECT_ADAPTER.validate_python(existing_manager), + SCIMUserManager.model_fields, + ) + if isinstance(existing_manager, dict) + else {} + ) + if op_type == "remove": + removed_manager: Final[Mapping[str, object]] = { + key: attr for key, attr in manager_data.items() if key != manager_field + } + updated_enterprise: Final[Mapping[str, object]] = ( + {**existing_enterprise, "manager": removed_manager} + if removed_manager + else {key: attr for key, attr in existing_enterprise.items() if key != "manager"} + ) + return _validate_enterprise_update(updated_enterprise) + if value is None: + return _InvalidEnterpriseValue(detail=f"The {op_type} operation requires a 'value' member") + updated_manager: Final[Mapping[str, object]] = {**manager_data, manager_field: value} + return _validate_enterprise_update({**existing_enterprise, "manager": updated_manager}) + + return _validate_enterprise_update(existing_enterprise) + + +def _enterprise_update_for_pathless_key( + key_and_value: tuple[str, object], + *, + op_type: str, +) -> Iterator[tuple[str, str, object]]: + key, value = key_and_value + sub_path: Final = _enterprise_extension_sub_path(key) + if sub_path is not None: + yield sub_path, op_type, value + + +def _enterprise_updates_for_operation( + op: SCIMPatchOperation, +) -> Iterator[tuple[str, str, object]]: + path: Final = op.path.lower() if isinstance(op.path, str) else "" + value: Final[object] = TypeAdapter(object).validate_python(op.value) + if not path and isinstance(value, dict): + pathless_values: Final[Mapping[str, object]] = _JSON_OBJECT_ADAPTER.validate_python(value) + yield from chain.from_iterable( + map(partial(_enterprise_update_for_pathless_key, op_type=op.op), pathless_values.items()) + ) + return + sub_path: Final = _enterprise_extension_sub_path(path) + if sub_path is not None: + yield sub_path, op.op, value + + +def _enterprise_updates(operations: Sequence[SCIMPatchOperation]) -> Iterator[tuple[str, str, object]]: + yield from chain.from_iterable(map(_enterprise_updates_for_operation, operations)) + + +def _apply_enterprise_update( + state: Mapping[str, object] | None | _InvalidEnterpriseValue, + update: tuple[str, str, object], +) -> Mapping[str, object] | None | _InvalidEnterpriseValue: + if isinstance(state, _InvalidEnterpriseValue): + return state + sub_path, op_type, value = update + existing_enterprise: Final[Mapping[str, object]] = {} if state is None else state + return _handle_enterprise_update(sub_path, op_type, value, existing_enterprise) + + +def _apply_enterprise_updates( + updates: Sequence[tuple[str, str, object]], + stored_enterprise: object, +) -> Mapping[str, object] | None | _InvalidEnterpriseValue: + initial_enterprise: Final[Mapping[str, object] | None] = ( + _canonicalize_enterprise_data(_JSON_OBJECT_ADAPTER.validate_python(stored_enterprise)) + if isinstance(stored_enterprise, dict) + else None + ) + return reduce(_apply_enterprise_update, updates, initial_enterprise) + + def _handle_generic_metadata(path: str, op_type: str, value: object, metadata: dict[str, object]) -> None: """Handle generic metadata operations for unknown paths.""" if op_type == "remove": @@ -2285,20 +2497,37 @@ def _apply_patch_ops( ) -> tuple[dict[str, object], set[str]]: """Apply patch operations and return update data and final team set.""" update_data: Final[dict[str, object]] = {} - metadata: Final = existing_user.metadata or {} - scim_metadata: Final = metadata.get("scim_metadata", {}) + stored_metadata: Final[object] = TypeAdapter(object).validate_python(existing_user.model_dump().get("metadata")) + metadata: Final = _JSON_OBJECT_ADAPTER.validate_python(stored_metadata) if isinstance(stored_metadata, dict) else {} + stored_scim_metadata: Final = metadata.get("scim_metadata") + scim_metadata: Final = ( + _JSON_OBJECT_ADAPTER.validate_python(stored_scim_metadata) if isinstance(stored_scim_metadata, dict) else {} + ) + enterprise_updates: Final = tuple(_enterprise_updates(patch_ops.Operations)) + enterprise: Final[Mapping[str, object] | None | _InvalidEnterpriseValue] = ( + _apply_enterprise_updates( + enterprise_updates, + metadata.get(SCIM_ENTERPRISE_METADATA_KEY), + ) + if enterprise_updates + else None + ) + if isinstance(enterprise, _InvalidEnterpriseValue): + _raise_invalid_enterprise_value(enterprise) teams_set: Final[set[str]] = set(existing_user.teams or []) replace_team_set: set[str] | None = None for op in patch_ops.Operations: - path = (op.path or "").lower() + path = op.path.lower() if isinstance(op.path, str) else "" value = op.value op_type = op.op # Handle SCIM operations without path where value contains the fields if not path and isinstance(value, dict): - for key, val in value.items(): + for key, val in _JSON_OBJECT_ADAPTER.validate_python(value).items(): + if _enterprise_extension_sub_path(key) is not None: + continue key_lower = key.lower() if key_lower == "active": _handle_active_update(op_type, val, metadata) @@ -2309,7 +2538,7 @@ def _apply_patch_ops( elif key_lower in SCIM_MULTI_VALUED_ATTRIBUTE_METADATA_KEYS: _handle_multi_valued_attribute_update(key_lower, op_type, val, metadata) elif key_lower == "name" and isinstance(val, dict): - for name_key, name_val in val.items(): + for name_key, name_val in _JSON_OBJECT_ADAPTER.validate_python(val).items(): name_key_lower = name_key.lower() if name_key_lower in ("givenname", "familyname"): _handle_name_update( @@ -2334,12 +2563,22 @@ def _apply_patch_ops( new_replace_set = _handle_group_operations(op_type, value, teams_set, op.path) if new_replace_set is not None: replace_team_set = new_replace_set + elif _enterprise_extension_sub_path(path) is not None: + continue else: _handle_generic_metadata(path, op_type, value, metadata) final_team_set: Final = replace_team_set if replace_team_set is not None else teams_set - metadata["scim_metadata"] = scim_metadata - update_data["metadata"] = metadata + final_metadata: Final = ( + { + **{key: metadata_value for key, metadata_value in metadata.items() if key != SCIM_ENTERPRISE_METADATA_KEY}, + "scim_metadata": scim_metadata, + **({SCIM_ENTERPRISE_METADATA_KEY: enterprise} if enterprise is not None else {}), + } + if enterprise_updates + else {**metadata, "scim_metadata": scim_metadata} + ) + update_data["metadata"] = final_metadata return update_data, final_team_set diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py b/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py index edcce16ab41..31c5a54a753 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_patch_user.py @@ -1,11 +1,17 @@ +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException +from pydantic import TypeAdapter from litellm.proxy._types import LiteLLM_UserTable from litellm.proxy.management_endpoints.scim.scim_v2 import _apply_patch_ops, patch_user from litellm.types.proxy.management_endpoints.scim_v2 import ( + SCIM_ENTERPRISE_METADATA_KEY, + SCIM_ENTERPRISE_USER_SCHEMA, + SCIMEnterpriseUser, SCIMPatchOp, SCIMPatchOperation, SCIMUser, @@ -536,3 +542,553 @@ def test_apply_patch_ops_replace_groups_empty_value_does_not_use_path_filter(): _, final_team_set = _apply_patch_ops(existing_user=user, patch_ops=patch_ops) assert final_team_set == set() + + +@pytest.mark.asyncio +async def test_patch_user_pathless_enterprise_extension_merges_existing_metadata(): + mock_user: Final = _user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: {"employeeNumber": "42"}}) + mock_client: Final = MagicMock() + mock_db: Final = MagicMock() + mock_client.db = mock_db + mock_db.litellm_usertable.find_unique = AsyncMock(return_value=mock_user) + + async def mock_update(*, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_UserTable: + assert where == {"user_id": "user-mva"} + metadata_json: Final = data["metadata"] + assert isinstance(metadata_json, str) + metadata: Final = TypeAdapter(dict[str, object]).validate_json(metadata_json) + return _user_with_metadata(metadata) + + mock_db.litellm_usertable.update = AsyncMock(side_effect=mock_update) + mock_db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + value={ + SCIM_ENTERPRISE_USER_SCHEMA: { + "department": "Eng", + "costCenter": "CC-10", + } + }, + ) + ] + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_client), + ): + result: Final = await patch_user(user_id="user-mva", patch_ops=patch_ops) + + update_data: Final = mock_db.litellm_usertable.update.call_args.kwargs["data"] + metadata_json: Final = update_data["metadata"] + assert isinstance(metadata_json, str) + metadata: Final = TypeAdapter(dict[str, object]).validate_json(metadata_json) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "department": "Eng", + "costCenter": "CC-10", + } + assert not any(isinstance(key, str) and key.casefold().startswith("urn:") for key in metadata) + assert result.enterprise_user == SCIMEnterpriseUser( + employeeNumber="42", + department="Eng", + costCenter="CC-10", + ) + + +def test_apply_patch_ops_replace_enterprise_urn_path_from_idp(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + path="urn:ietf:params:scim:schemas:extension:enterprise:2.0:User:department", + value="Eng", + ) + ] + ) + + update_data, _ = _apply_patch_ops(existing_user=_user_with_metadata({}), patch_ops=patch_ops) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == {"department": "Eng"} + assert not any(isinstance(key, str) and key.casefold().startswith("urn:") for key in metadata) + + +def test_apply_patch_ops_pathless_enterprise_manager_sub_attribute_merges(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + value={ + f"{SCIM_ENTERPRISE_USER_SCHEMA.upper()}:MANAGER.VALUE": "manager-1", + }, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + {SCIM_ENTERPRISE_METADATA_KEY: {"manager": {"displayName": "Engineering Manager"}}} + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "manager": {"value": "manager-1", "displayName": "Engineering Manager"} + } + + +def test_apply_patch_ops_replace_enterprise_manager_normalizes_case_insensitive_keys(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + value={"VALUE": "mgr-2"}, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": {"value": "mgr-1", "displayName": "Boss"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": {"value": "mgr-2", "displayName": "Boss"}, + } + + +@pytest.mark.parametrize( + ("path", "value"), + ( + pytest.param( + f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + {"displayName": "Q"}, + id="pathed", + ), + pytest.param( + None, + {SCIM_ENTERPRISE_USER_SCHEMA: {"manager": {"displayName": "Q"}}}, + id="pathless", + ), + pytest.param( + None, + {SCIM_ENTERPRISE_USER_SCHEMA: {"MANAGER": {"displayName": "Q"}}}, + id="pathless-uppercase-key", + ), + ), +) +def test_apply_patch_ops_replace_manager_object_preserves_unspecified_sub_attributes( + path: str | None, + value: object, +) -> None: + patch_ops: Final = SCIMPatchOp( + Operations=[SCIMPatchOperation(op="replace", path=path, value=value)] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "department": "A", + "manager": {"value": "m", "displayName": "B"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "department": "A", + "manager": {"value": "m", "displayName": "Q"}, + } + + +@pytest.mark.parametrize( + ("manager_path", "manager_value"), + ( + pytest.param( + f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + {"$ref": "new"}, + id="pathed", + ), + pytest.param( + None, + {SCIM_ENTERPRISE_USER_SCHEMA: {"manager": {"$ref": "new"}}}, + id="pathless", + ), + ), +) +def test_apply_patch_ops_batched_manager_ref_update_replaces_previous_link( + manager_path: str | None, + manager_value: object, +) -> None: + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:department", + value="B", + ), + SCIMPatchOperation( + op="replace", + path=manager_path, + value=manager_value, + ), + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "department": "A", + "manager": {"value": "m", "$ref": "old"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "department": "B", + "manager": {"value": "m", "$ref": "new"}, + } + + +def test_apply_patch_ops_add_enterprise_manager_merges_existing_sub_attributes(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="add", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + value={"displayName": "New Boss"}, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": { + "value": "mgr-1", + "$ref": "https://idp.example/Users/mgr-1", + }, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": { + "value": "mgr-1", + "$ref": "https://idp.example/Users/mgr-1", + "displayName": "New Boss", + }, + } + + +def test_apply_patch_ops_pathless_add_enterprise_manager_merges_existing_sub_attributes(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="add", + value={ + SCIM_ENTERPRISE_USER_SCHEMA: { + "manager": {"displayName": "New Boss"}, + } + }, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": { + "value": "mgr-1", + "$ref": "https://idp.example/Users/mgr-1", + }, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": { + "value": "mgr-1", + "$ref": "https://idp.example/Users/mgr-1", + "displayName": "New Boss", + }, + } + + +def test_apply_patch_ops_remove_enterprise_manager_sub_attribute_preserves_other_fields(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="remove", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager.value", + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": {"value": "mgr-1", "displayName": "Boss"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": {"displayName": "Boss"}, + } + + +def test_apply_patch_ops_remove_enterprise_attribute_preserves_other_attributes(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="remove", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:costCenter", + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: {"costCenter": "CC-10", "department": "Eng"}}), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == {"department": "Eng"} + + +def test_apply_patch_ops_remove_last_enterprise_attribute_drops_metadata_key(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="remove", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:employeeNumber", + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: {"employeeNumber": "42"}}), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert SCIM_ENTERPRISE_METADATA_KEY not in metadata + + +def test_apply_patch_ops_unknown_enterprise_attribute_preserves_existing_and_applies_other_ops() -> None: + existing_enterprise: Final = { + "employeeNumber": "42", + "manager": {"value": "mgr-1", "displayName": "Boss"}, + } + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:shoeSize", + value="10", + ), + SCIMPatchOperation(op="replace", path="displayName", value="Updated User"), + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: existing_enterprise}), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == existing_enterprise + assert not any(isinstance(key, str) and key.casefold().startswith("urn:") for key in metadata) + assert update_data["user_alias"] == "Updated User" + + +def test_apply_patch_ops_display_name_only_preserves_enterprise_value_unchanged() -> None: + existing_enterprise: Final = {"Department": "Eng", "manager": "mgr-1"} + patch_ops: Final = SCIMPatchOp( + Operations=[SCIMPatchOperation(op="replace", path="displayName", value="Ada")] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: existing_enterprise}), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == {"Department": "Eng", "manager": "mgr-1"} + + +@pytest.mark.parametrize("path", ("manager.foo", "manager.a.b", "department.x")) +def test_apply_patch_ops_unknown_enterprise_nested_attribute_preserves_existing(path: str) -> None: + existing_enterprise: Final = { + "department": "Engineering", + "manager": {"value": "mgr-1", "displayName": "Boss"}, + } + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:{path}", + value="ignored", + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata({SCIM_ENTERPRISE_METADATA_KEY: existing_enterprise}), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == existing_enterprise + + +@pytest.mark.parametrize("manager_id", ("mgr-id", "")) +def test_apply_patch_ops_add_enterprise_manager_string_replaces_existing_manager(manager_id: str) -> None: + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="add", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + value=manager_id, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": {"value": "old", "displayName": "Old Boss"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": {"value": manager_id}, + } + + +def test_apply_patch_ops_pathless_add_enterprise_manager_string_replaces_existing_manager() -> None: + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="add", + value={SCIM_ENTERPRISE_USER_SCHEMA: {"manager": "mgr-id"}}, + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + { + SCIM_ENTERPRISE_METADATA_KEY: { + "employeeNumber": "42", + "manager": {"value": "old", "displayName": "Old Boss"}, + } + } + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert metadata[SCIM_ENTERPRISE_METADATA_KEY] == { + "employeeNumber": "42", + "manager": {"value": "mgr-id"}, + } + + +def test_apply_patch_ops_remove_enterprise_manager_drops_manager() -> None: + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="remove", + path=f"{SCIM_ENTERPRISE_USER_SCHEMA}:manager", + ) + ] + ) + + update_data, _ = _apply_patch_ops( + existing_user=_user_with_metadata( + {SCIM_ENTERPRISE_METADATA_KEY: {"manager": {"value": "mgr-1", "displayName": "Boss"}}} + ), + patch_ops=patch_ops, + ) + + metadata: Final = update_data["metadata"] + assert isinstance(metadata, dict) + assert SCIM_ENTERPRISE_METADATA_KEY not in metadata + + +def test_apply_patch_ops_pathless_enterprise_value_must_be_a_dict(): + patch_ops: Final = SCIMPatchOp( + Operations=[ + SCIMPatchOperation( + op="replace", + value={SCIM_ENTERPRISE_USER_SCHEMA: "not-a-dict"}, + ) + ] + ) + + with pytest.raises(HTTPException) as exc_info: + _apply_patch_ops(existing_user=_user_with_metadata({}), patch_ops=patch_ops) + + assert exc_info.value.status_code == 400 From 24d92f77d5816ddfdf4cbd1e76794207c47e9360 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:58:35 +0000 Subject: [PATCH 026/124] test(perf): stop flagging load-balanced worker reads as restarts in database upgrade test (#47425) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/test_database_upgrade.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/perf/test_database_upgrade.py b/tests/e2e/perf/test_database_upgrade.py index f165b9b738f..8079aa0274d 100644 --- a/tests/e2e/perf/test_database_upgrade.py +++ b/tests/e2e/perf/test_database_upgrade.py @@ -55,9 +55,9 @@ def test_database_upgrade(proxy: ProxyClient, idp: Keycloak, resources: Resource monitor.assert_complete(state) assert_finished(connection, state) rss_after: Final = read_rss_everywhere(proxy) - assert {r.worker for r in rss_before.readings} == {r.worker for r in rss_after.readings}, ( - "Gateway workers restarted" - ) + assert {(r.replica, r.hostname) for r in rss_before.readings} == { + (r.replica, r.hostname) for r in rss_after.readings + }, "Gateway pods replaced during the upgrade" observed: Final = Observation( records=outcome.records, mock_requests=tuple(seen for url in urls for seen in read_mock_requests(url)), From ad1f3c5e91b8c7fcb2bcb067e05bafb3dd4bd1be Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 16:01:23 -0700 Subject: [PATCH 027/124] fix(mcp): retry upstream 429 on tools/list and error when every server fails (#46649) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 7 + .../proxy/_experimental/mcp_server/catalog.py | 28 +- .../mcp_server/faults/list_outcomes.py | 121 +++++++- .../mcp_server/mcp_server_manager.py | 13 +- .../_experimental/mcp_server/operations.py | 22 +- .../_experimental/mcp_server/subscriptions.py | 12 +- .../mcp_server/faults/test_list_outcomes.py | 285 ++++++++++++++++++ .../mcp_server/test_mcp_server_manager.py | 150 ++++++++- .../test_mcp_server_tool_calls_and_headers.py | 201 +++++++++++- .../mcp_server/test_operations.py | 39 +++ .../mcp_server/test_subscriptions.py | 50 ++- 11 files changed, 903 insertions(+), 25 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c7c1576bc20..2d54a873b89 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -252,6 +252,13 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0) MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0) +MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3) +MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float( + "LITELLM_MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS", 0.5 +) +MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS: Final = get_env_float( + "LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS", 5.0 +) MCP_METADATA_TIMEOUT: Final = get_env_float("LITELLM_MCP_METADATA_TIMEOUT", 10.0) MCP_HEALTH_CHECK_TIMEOUT: Final = get_env_float("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", 10.0) AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", 5.0) diff --git a/litellm/proxy/_experimental/mcp_server/catalog.py b/litellm/proxy/_experimental/mcp_server/catalog.py index d0383be61a8..71eb2112bd5 100644 --- a/litellm/proxy/_experimental/mcp_server/catalog.py +++ b/litellm/proxy/_experimental/mcp_server/catalog.py @@ -17,6 +17,7 @@ from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final, Generic, ParamSpec, TypeAlias, TypeVar, cast +from mcp.shared.exceptions import MCPError from mcp.types import CacheableResult from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter @@ -1055,12 +1056,20 @@ async def aggregate_gateway_tools( ) +class ToolListingFailedOutright(MCPError): + pass + + async def list_gateway_tools( context: OperationContext, params: PaginatedRequestParams, *, log_list_tools_to_spendlogs: bool = True ) -> ListToolsResult: - from mcp.types import ListToolsResult + from mcp.types import INTERNAL_ERROR, ListToolsResult - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import SERVER_OUTCOMES_META_KEY, outcome_wire_value + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + SERVER_OUTCOMES_META_KEY, + listing_failed_outright, + outcome_wire_value, + ) from litellm.proxy._experimental.mcp_server.operations import _list_mcp_tools caller, auth, servers, server_headers, oauth_headers, headers, client_ip = context.legacy_auth() @@ -1078,16 +1087,21 @@ async def list_gateway_tools( record_listing=True, list_tools_log_source="mcp_protocol", ) + wire_outcomes: Final = ( + {key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()} if listing.outcomes else None + ) + if params.cursor is None and listing_failed_outright(listing.outcomes): + raise ToolListingFailedOutright( + code=INTERNAL_ERROR, + message="Every MCP server in scope failed to list tools", + data={SERVER_OUTCOMES_META_KEY: wire_outcomes}, + ) return ListToolsResult( tools=listing.tools, next_cursor=listing.next_cursor, ttl_ms=listing.ttl_ms, cache_scope="private", - _meta={ - SERVER_OUTCOMES_META_KEY: {key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()} - } - if listing.outcomes - else None, + _meta={SERVER_OUTCOMES_META_KEY: wire_outcomes} if wire_outcomes else None, ) diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py index b94ed1ee251..84673c8c738 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py @@ -10,17 +10,27 @@ becomes an outcome, never a second failure. from __future__ import annotations +import email.utils +import random import re -from collections.abc import Iterator -from typing import Final, Literal, NamedTuple, NoReturn, TypeAlias +from collections.abc import Awaitable, Callable, Iterator, Mapping +from datetime import datetime, timezone +from typing import Final, Literal, NamedTuple, NoReturn, TypeAlias, TypeVar from urllib.request import parse_http_list, parse_keqv_list +import anyio import httpx import httpx2 from mcp.types import Tool as MCPTool from pydantic import ConfigDict, TypeAdapter from typing_extensions import assert_never +from litellm._logging import verbose_logger +from litellm.constants import ( + MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS, + MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS, + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES, +) from litellm.proxy._experimental.mcp_server.exceptions import ( MCPServerListError, MCPUpstreamAuthError, @@ -176,6 +186,113 @@ def classify_list_exception(exc: BaseException) -> ServerListFault: return ServerListFault(tag="internal") +class UpstreamRateLimit(NamedTuple): + retry_after_seconds: float | None + + +def _parse_retry_after(value: str | None) -> float | None: + if value is None: + return None + stripped: Final = value.strip() + if re.fullmatch(r"\d+", stripped): + return float(stripped) + try: + parsed: Final = email.utils.parsedate_to_datetime(stripped) + except (TypeError, ValueError): + return None + aware: Final = parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + return max(0.0, (aware - datetime.now(timezone.utc)).total_seconds()) + + +def upstream_rate_limit(exc: BaseException) -> UpstreamRateLimit | None: + """The upstream 429 signal for the tools/list retry loop: an auth challenge wins over rate + limiting so a 401/403 anywhere in the tree keeps its own channel, then the first 429 response + in deliberate order carries its Retry-After hint. Anything else is not retryable here.""" + if upstream_auth_challenge(exc) is not None: + return None + for response in _iter_upstream_responses(exc): + if response.status_code == 429: + return UpstreamRateLimit(_parse_retry_after(response.headers.get("retry-after"))) + return None + + +def rate_limit_retry_delay( + attempt: int, + retry_after_seconds: float | None, + *, + base_delay: float, + max_delay: float, + jitter: float, +) -> float: + if retry_after_seconds is not None: + return min(retry_after_seconds, max_delay) + return min(max_delay, base_delay * 2**attempt * (1 + jitter)) + + +T: Final = TypeVar("T") + + +async def retry_upstream_rate_limit( + fetch: Callable[[], Awaitable[T]], + server_name: str, + *, + max_retries: int = MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES, + base_delay: float = MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS, + max_delay: float = MCP_UPSTREAM_RATE_LIMIT_MAX_DELAY_SECONDS, + sleep: Callable[[float], Awaitable[None]] = anyio.sleep, + jitter: Callable[[], float] = random.random, + deadline_seconds: float | None = None, + clock: Callable[[], float] = anyio.current_time, +) -> T: + attempts: Final = max(0, max_retries) + deadline: Final = None if deadline_seconds is None else clock() + deadline_seconds + last_rate_limit: BaseException | None = None # rebind-ok: each 429 replaces the cause a retry deadline re-raises + for attempt in range(attempts + 1): + try: + if attempt > 0 and deadline is not None: + try: + with anyio.fail_after(max(0.0, deadline - clock())): + return await fetch() + except TimeoutError as timeout: + if last_rate_limit is None: + raise + raise last_rate_limit from timeout + return await fetch() + except Exception as exc: + limit = upstream_rate_limit(exc) + if limit is not None: + last_rate_limit = exc + if limit is None or attempt == attempts: + raise + delay = rate_limit_retry_delay( + attempt, + limit.retry_after_seconds, + base_delay=base_delay, + max_delay=max_delay, + jitter=jitter(), + ) + if deadline is not None and clock() + delay >= deadline: + raise + verbose_logger.warning( + "MCP server %s answered 429 to tools/list; retry %d/%d in %.2fs", + server_name, + attempt + 1, + max_retries, + delay, + ) + await sleep(delay) + raise AssertionError("unreachable") + + +def listing_failed_outright(outcomes: Mapping[str, ServerOutcome]) -> bool: + """True only when every server in scope contributed a non-auth fault: an auth outcome means a + caller can fix the listing by authenticating, so the aggregate stays a 200 with outcomes.""" + return bool(outcomes) and all( + isinstance(outcome, ServerListFault) and outcome.tag not in ("auth_required", "forbidden") + for outcome in outcomes.values() + ) + + def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]: """The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST response: category plus status code only, never upstream prose or URLs.""" diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 96cf6ee11aa..aba5161d56d 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -104,6 +104,7 @@ from litellm.proxy._experimental.mcp_server.exceptions import ( from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( ServerListFault, raise_classified_list_failure, + retry_upstream_rate_limit, upstream_auth_challenge, upstream_insufficient_scope, ) @@ -4531,7 +4532,11 @@ class MCPServerManager: ) else: page: Final = ( - await self._fetch_tools_page_with_timeout(client, server, params) + await retry_upstream_rate_limit( + lambda: self._fetch_tools_page_with_timeout(client, server, params), + server.name, + deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, + ) if params is not None else ListToolsResult(tools=await self._fetch_tools_with_timeout(client, server.name)) ) @@ -5600,7 +5605,11 @@ class MCPServerManager: """ try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): - tools: Final = await client.list_tools(raise_on_error=True) + tools: Final = await retry_upstream_rate_limit( + lambda: client.list_tools(raise_on_error=True), + server_name, + deadline_seconds=MCP_TOOL_LISTING_TIMEOUT, + ) verbose_logger.debug("Tools from %s: %s", server_name, tools) return tools except TimeoutError as e: diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py index 90f01368127..c45faf38f51 100644 --- a/litellm/proxy/_experimental/mcp_server/operations.py +++ b/litellm/proxy/_experimental/mcp_server/operations.py @@ -3041,6 +3041,19 @@ async def _execute_handle_list_tools( return ListToolsResult(tools=[]) +async def _discover_tools_listing( + context: OperationContext, host_progress_callback: ProgressCallback | None +) -> ListToolsResult: + from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + + try: + return await _execute_handle_list_tools( + context, PaginatedRequestParams(), host_progress_callback, log_list_tools_to_spendlogs=False + ) + except ToolListingFailedOutright: + return ListToolsResult(tools=[]) + + async def _execute_mcp_server_tool_call( context: OperationContext, params: CallToolRequestParams, host_progress_callback: ProgressCallback | None = None ) -> CallToolResult | InputRequiredResult: @@ -3517,14 +3530,7 @@ class GatewayOperations: ) memo_token: Final = _mcp_server_admission_memo.set({}) tasks: Final = ( - asyncio.create_task( - _execute_handle_list_tools( - context, - PaginatedRequestParams(), - self._host_progress_callback, - log_list_tools_to_spendlogs=False, - ) - ), + asyncio.create_task(_discover_tools_listing(context, self._host_progress_callback)), *(asyncio.create_task(self.execute(listing, context)) for listing in listings), ) try: diff --git a/litellm/proxy/_experimental/mcp_server/subscriptions.py b/litellm/proxy/_experimental/mcp_server/subscriptions.py index db622ea3cbf..e9de5692d83 100644 --- a/litellm/proxy/_experimental/mcp_server/subscriptions.py +++ b/litellm/proxy/_experimental/mcp_server/subscriptions.py @@ -50,6 +50,9 @@ from mcp.types import ( from litellm.constants import MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT from litellm.experimental_mcp_client.client import MCPClient from litellm.proxy._experimental.mcp_server import operations +from litellm.proxy._experimental.mcp_server.catalog import ( + ToolListingFailedOutright, +) from litellm.proxy._experimental.mcp_server.catalog import ( _configuration_identity as configuration_identity, # pyright: ignore[reportPrivateUsage] # reuse catalog-owned configuration identity without treating discovered metadata as configuration ) @@ -283,9 +286,12 @@ async def _listing(event: ServerEvent, context: OperationContext) -> str | None: cursor: str | None = None # rebind-ok: advance the signed gateway cursor for each catalog page with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): for _ in range(MCP_TOOL_LISTING_MAX_PAGES): - result = await gateway.execute( - operation.model_copy(update={"params": PaginatedRequestParams(cursor=cursor)}), context - ) + try: + result = await gateway.execute( + operation.model_copy(update={"params": PaginatedRequestParams(cursor=cursor)}), context + ) + except ToolListingFailedOutright: + return digest.hexdigest() items = ( result.tools if isinstance(result, ListToolsResult) diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py index ee33c41afb6..f6902c778e5 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py +++ b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py @@ -326,3 +326,288 @@ def test_managed_scope_step_up_only_accepts_unambiguous_bearer_challenges(status wrapped = RuntimeError("transport failure") wrapped.__cause__ = cause assert upstream_insufficient_scope(wrapped) == expected + + +def _rate_limited_error(retry_after: str | None = None) -> httpx.HTTPStatusError: + request: Final = httpx.Request("POST", "https://mcp.example.com/mcp") + headers: Final = {"retry-after": retry_after} if retry_after is not None else {} + return httpx.HTTPStatusError( + "rate limited", + request=request, + response=httpx.Response(429, headers=headers, request=request), + ) + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_backs_off_then_reraises() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + calls: Final[list[httpx.HTTPStatusError]] = [] + failure: Final = _rate_limited_error() + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def always_limited() -> None: + calls.append(failure) + raise failure + + expected_error: Final = pytest.raises(httpx.HTTPStatusError) + with expected_error as exc_info: + await retry_upstream_rate_limit( + always_limited, + "srv", + max_retries=3, + base_delay=0.5, + max_delay=5.0, + sleep=record, + jitter=lambda: 0.0, + ) + + assert exc_info.value is failure + assert sleeps == [0.5, 1.0, 2.0] + assert len(calls) == 4 + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_honors_retry_after_up_to_cap() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def always_limited() -> None: + raise _rate_limited_error("120") + + with pytest.raises(httpx.HTTPStatusError): + await retry_upstream_rate_limit( + always_limited, + "srv", + max_retries=3, + base_delay=0.5, + max_delay=5.0, + sleep=record, + jitter=lambda: 0.0, + ) + + assert sleeps == [5.0, 5.0, 5.0] + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_parses_http_date_retry_after() -> None: + import email.utils + import time + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + attempts: Final[list[None]] = [] + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def limited() -> str: + attempts.append(None) + if len(attempts) == 1: + raise _rate_limited_error(email.utils.formatdate(time.time() + 3, usegmt=True)) + raise _rate_limited_error("abc") + + with pytest.raises(httpx.HTTPStatusError): + await retry_upstream_rate_limit( + limited, + "srv", + max_retries=3, + base_delay=0.5, + max_delay=5.0, + sleep=record, + jitter=lambda: 0.0, + ) + + assert 0 < sleeps[0] <= 3.5 + assert sleeps[1:] == [1.0, 2.0] + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_passes_through_non_429() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + calls = 0 + failure: Final = ValueError("broken") + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def broken() -> None: + nonlocal calls + calls += 1 + raise failure + + expected_error: Final = pytest.raises(ValueError, match="broken") + with expected_error as exc_info: + await retry_upstream_rate_limit( + broken, "srv", max_retries=3, base_delay=0.5, max_delay=5.0, sleep=record, jitter=lambda: 0.0 + ) + + assert exc_info.value is failure + assert calls == 1 + assert sleeps == [] + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_negative_max_retries_is_a_single_attempt() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + calls = 0 + failure: Final = _rate_limited_error("0") + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def always_limited() -> None: + nonlocal calls + calls += 1 + raise failure + + expected_error: Final = pytest.raises(httpx.HTTPStatusError) + with expected_error as exc_info: + await retry_upstream_rate_limit( + always_limited, + "srv", + max_retries=-1, + base_delay=0.5, + max_delay=5.0, + sleep=record, + jitter=lambda: 0.0, + ) + + assert exc_info.value is failure + assert calls == 1 + assert sleeps == [] + + +@pytest.mark.asyncio +async def test_retry_deadline_skips_sleep_that_would_pass_deadline() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + now: Final[list[float]] = [100.0] + sleeps: Final[list[float]] = [] + calls = 0 + failure: Final = _rate_limited_error() + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def always_limited() -> None: + nonlocal calls + calls += 1 + raise failure + + expected_error: Final = pytest.raises(httpx.HTTPStatusError) + with expected_error as exc_info: + await retry_upstream_rate_limit( + always_limited, + "srv", + max_retries=3, + base_delay=2.0, + max_delay=5.0, + sleep=record, + jitter=lambda: 0.0, + deadline_seconds=1.0, + clock=lambda: now[0], + ) + + assert exc_info.value is failure + assert calls == 1 + assert sleeps == [] + + +@pytest.mark.asyncio +async def test_retry_attempt_is_interrupted_at_deadline() -> None: + import anyio + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + calls = 0 + failure: Final = _rate_limited_error("0") + + async def fetch() -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise failure + await anyio.sleep(5) + + expected_error: Final = pytest.raises(httpx.HTTPStatusError) + with anyio.fail_after(5): + with expected_error as exc_info: + await retry_upstream_rate_limit( + fetch, "srv", max_retries=3, base_delay=0.0, max_delay=5.0, deadline_seconds=0.2 + ) + + assert exc_info.value is failure + + +@pytest.mark.asyncio +async def test_retry_deadline_does_not_bound_first_attempt() -> None: + import anyio + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + async def slow() -> str: + await anyio.sleep(0.15) + return "tools" + + result: Final = await retry_upstream_rate_limit( + slow, "srv", max_retries=3, base_delay=0.5, max_delay=5.0, deadline_seconds=0.05 + ) + + assert result == "tools" + + +@pytest.mark.asyncio +async def test_retry_upstream_rate_limit_never_retries_cancellation() -> None: + import asyncio + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import retry_upstream_rate_limit + + sleeps: Final[list[float]] = [] + calls = 0 + + async def record(delay: float) -> None: + sleeps.append(delay) + + async def cancelled() -> None: + nonlocal calls + calls += 1 + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await retry_upstream_rate_limit( + cancelled, "srv", max_retries=3, base_delay=0.5, max_delay=5.0, sleep=record, jitter=lambda: 0.0 + ) + + assert calls == 1 + assert sleeps == [] + + +def test_listing_failed_outright_only_when_every_server_faulted_non_auth() -> None: + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + ServerListOk, + listing_failed_outright, + ) + + assert not listing_failed_outright({}) + assert listing_failed_outright({"a": ServerListFault(tag="upstream_error", status_code=429)}) + assert listing_failed_outright( + {"a": ServerListFault(tag="timeout"), "b": ServerListFault(tag="rate_limited", status_code=429)} + ) + assert not listing_failed_outright( + {"a": ServerListFault(tag="upstream_error", status_code=500), "b": ServerListOk(tool_count=2)} + ) + assert not listing_failed_outright({"a": ServerListFault(tag="auth_required", status_code=401)}) + assert not listing_failed_outright({"a": ServerListFault(tag="forbidden", status_code=403)}) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index d47b2f73a48..d314291ef74 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -7,6 +7,7 @@ import logging import os import sys import time +import anyio from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from datetime import datetime from pathlib import Path @@ -37,6 +38,8 @@ from mcp.types import ( CallToolRequestParams as MCPCallToolRequestParams, CallToolResult, GetPromptResult, + ListToolsResult, + PaginatedRequestParams, Prompt, ResourceTemplate, TextResourceContents, @@ -44,7 +47,7 @@ from mcp.types import ( from mcp.types import Tool as MCPTool from pydantic import AnyUrl, TypeAdapter -from litellm.constants import MCP_METADATA_TIMEOUT +from litellm.constants import MCP_METADATA_TIMEOUT, MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES from litellm.proxy._experimental.mcp_server import discoverable_endpoints from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var @@ -20427,3 +20430,148 @@ async def test_legacy_upstream_elicitation_rejects_modern_downstream_without_con finally: active_mcp_request_ctx_var.reset(request_token) legacy_server.active_mcp_session_var.reset(session_token) + + +def _upstream_http_status_error(status_code: int, headers: dict[str, str] | None = None) -> httpx.HTTPStatusError: + request: Final = httpx.Request("POST", "https://upstream.example/mcp") + return httpx.HTTPStatusError( + "upstream rejected", + request=request, + response=httpx.Response(status_code, headers=headers or {}, request=request), + ) + + +class TestMCPUpstreamRateLimitRetry: + """An upstream 429 on tools/list is retried inside the listing deadline instead of folding + into a silent empty outcome on the first attempt.""" + + @pytest.mark.asyncio + async def test_fetch_tools_with_timeout_retries_429_then_returns_tools(self) -> None: + manager: Final = MCPServerManager() + tool: Final = MCPTool(name="ping", inputSchema={"type": "object"}) + client: Final = MagicMock() + client.list_tools = AsyncMock( + side_effect=[ + _upstream_http_status_error(429, {"Retry-After": "0"}), + _upstream_http_status_error(429, {"Retry-After": "0"}), + [tool], + ] + ) + + tools: Final = await manager._fetch_tools_with_timeout(client, "flaky-server") + + assert tools == [tool] + assert client.list_tools.await_count == 3 + + @pytest.mark.asyncio + async def test_fetch_tools_with_timeout_gives_up_after_max_retries(self) -> None: + manager: Final = MCPServerManager() + client: Final = MagicMock() + client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "0"})) + + expected_error: Final = pytest.raises(MCPServerListError) + with expected_error as exc_info: + await manager._fetch_tools_with_timeout(client, "saturated-server") + + assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429) + assert client.list_tools.await_count == 1 + MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES + + @pytest.mark.asyncio + async def test_fetch_tools_with_timeout_does_not_retry_auth_error(self) -> None: + manager: Final = MCPServerManager() + client: Final = MagicMock() + client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(401)) + + with pytest.raises(MCPUpstreamAuthError): + await manager._fetch_tools_with_timeout(client, "srv") + + assert client.list_tools.await_count == 1 + + @pytest.mark.asyncio + async def test_fetch_tools_with_timeout_does_not_retry_server_error(self) -> None: + manager: Final = MCPServerManager() + client: Final = MagicMock() + client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(500)) + + expected_error: Final = pytest.raises(MCPServerListError) + with expected_error as exc_info: + await manager._fetch_tools_with_timeout(client, "srv") + + assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=500) + assert client.list_tools.await_count == 1 + + @pytest.mark.asyncio + async def test_fetch_tools_with_timeout_reports_429_when_backoff_would_pass_listing_timeout( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.5 + ) + manager: Final = MCPServerManager() + client: Final = MagicMock() + client.list_tools = AsyncMock(side_effect=_upstream_http_status_error(429, {"Retry-After": "5"})) + + with anyio.fail_after(3): + expected_error: Final = pytest.raises(MCPServerListError) + with expected_error as exc_info: + await manager._fetch_tools_with_timeout(client, "srv") + + assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429) + assert client.list_tools.await_count == 1 + + @pytest.mark.asyncio + async def test_get_tools_page_retries_429_then_returns_page(self) -> None: + tool: Final = MCPTool(name="list_notes", inputSchema={"type": "object"}) + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="notes", name="notes", transport=MCPTransport.http) + manager.registry = {server.server_id: server} + manager.create_mcp_client = AsyncMock( + return_value=SimpleNamespace( + _last_initialize_instructions=None, + list_tools_page=AsyncMock( + side_effect=[ + _upstream_http_status_error(429, {"Retry-After": "0"}), + ListToolsResult(tools=[tool]), + ] + ), + ) + ) + + result: Final = await manager.get_tools_page(server, params=PaginatedRequestParams()) + + assert [t.name for t in result.tools] == ["notes-list_notes"] + client: Final = manager.create_mcp_client.return_value + assert client.list_tools_page.await_count == 2 + + @pytest.mark.asyncio + async def test_get_tools_page_retry_attempt_is_bounded_by_listing_timeout( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_TOOL_LISTING_TIMEOUT", 0.05 + ) + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="notes", name="notes", transport=MCPTransport.http) + manager.registry = {server.server_id: server} + first_seen: Final = anyio.Event() + never: Final = anyio.Event() + error: Final = _upstream_http_status_error(429, {"Retry-After": "0"}) + + async def _list_tools_page(params: PaginatedRequestParams) -> ListToolsResult: + if not first_seen.is_set(): + first_seen.set() + raise error + await never.wait() + raise AssertionError("unreachable") + + list_tools_page: Final = AsyncMock(side_effect=_list_tools_page) + client: Final = SimpleNamespace(_last_initialize_instructions=None, list_tools_page=list_tools_page) + manager.create_mcp_client = AsyncMock(return_value=client) + + with anyio.fail_after(5): + expected_error: Final = pytest.raises(MCPServerListError) + with expected_error as exc_info: + await manager.get_tools_page(server, params=PaginatedRequestParams()) + + assert exc_info.value.fault == ServerListFault(tag="upstream_error", status_code=429) + assert list_tools_page.await_count == 2 diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index 48106b1de5f..66d66271f71 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -6,19 +6,22 @@ import json import os from datetime import datetime, timedelta from types import SimpleNamespace -from typing import Final, Literal +from typing import Callable, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest from fastapi import HTTPException from mcp import ReadResourceResult, Resource +from mcp.server.context import ServerRequestContext from mcp.server.models import InitializationOptions from mcp.types import ( INVALID_REQUEST, METHOD_NOT_FOUND, BlobResourceContents, CallToolResult, + ListToolsResult, + PaginatedRequestParams, Prompt, ResourceTemplate, TextContent, @@ -32,6 +35,7 @@ from starlette.types import Message, Receive, Scope, Send from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var from litellm.proxy._experimental.mcp_server.mcp_server_manager import ListedToolsCaller +from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy._types import ( LiteLLM_MCPServerTable, MCPTransport, @@ -5876,6 +5880,76 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["per_server_list_outcomes"] == {"server_a": {"status": "ok", "tool_count": 1}} +@pytest.mark.asyncio +async def test_get_tools_from_mcp_servers_all_failed_listing_never_runs_proxy_failure_hooks() -> None: + from litellm.proxy._types import UserAPIKeyAuth + + servers: Final = ( + MCPServer( + server_id="server_a", + name="server_a", + alias="server_a", + server_name="server_a", + transport=MCPTransport.http, + ), + MCPServer( + server_id="server_b", + name="server_b", + alias="server_b", + server_name="server_b", + transport=MCPTransport.http, + ), + ) + user_auth: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user") + logging_obj: Final = MagicMock() + logging_obj.model_call_details = {"metadata": {"spend_logs_metadata": {}}} + logging_obj.async_success_handler = AsyncMock() + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock() + manager: Final = MagicMock() + manager.get_tools_from_server = AsyncMock(side_effect=RuntimeError("server unavailable")) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", + new=AsyncMock(return_value=servers), + ), + patch( + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", + return_value=(None, None), + ), + patch("litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", manager), + patch("litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", return_value=[]), + patch( + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", + new=AsyncMock(return_value=[]), + ), + patch( + "litellm.proxy._experimental.mcp_server.operations.function_setup", + return_value=(logging_obj, None), + ), + patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj), + ): + listing: Final = await mcp_operations._get_tools_from_mcp_servers( + user_api_key_auth=user_auth, + mcp_auth_header=None, + mcp_servers=["server_a", "server_b"], + mcp_server_auth_headers=None, + params=None, + log_list_tools_to_spendlogs=True, + enforce_rate_limits=False, + ) + + assert listing.tools == [] + logging_obj.async_success_handler.assert_awaited_once() + spend_meta: Final = logging_obj.model_call_details["metadata"]["spend_logs_metadata"] + assert spend_meta["per_server_list_outcomes"] == { + "server_a": {"status": "internal"}, + "server_b": {"status": "internal"}, + } + proxy_logging_obj.post_call_failure_hook.assert_not_awaited() + + @pytest.mark.asyncio async def test_get_tools_from_mcp_servers_returns_tools_when_success_logging_fails(): """ @@ -10071,6 +10145,131 @@ async def test_handle_list_tools_attaches_outcome_meta(_mcp_request_ctx): assert outcomes_meta["broken"] == {"status": "unreachable"} +async def _list_tools_with_listing( + _mcp_request_ctx: Callable[..., ServerRequestContext], + listing: AggregateToolListing, + params: PaginatedRequestParams, +) -> ListToolsResult: + try: + from litellm.proxy._experimental.mcp_server.server import handle_list_tools + except ImportError: + pytest.skip("MCP server not available") + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", + new=AsyncMock(return_value=(None, None, None, None, None, None, None)), + ), + patch( + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", + new=AsyncMock(return_value=listing), + ), + ): + return await handle_list_tools(_mcp_request_ctx(), params) + + +@pytest.mark.asyncio +async def test_handle_list_tools_errors_when_every_server_failed( + _mcp_request_ctx: Callable[..., ServerRequestContext], +) -> None: + from mcp.shared.exceptions import MCPError + from mcp.types import INTERNAL_ERROR + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + SERVER_OUTCOMES_META_KEY, + AggregateToolListing, + ServerListFault, + ) + + listing: Final = AggregateToolListing( + tools=[], + outcomes={ + "a": ServerListFault(tag="upstream_error", status_code=429), + "b": ServerListFault(tag="timeout"), + }, + ) + + expected_error: Final = pytest.raises(MCPError) + with expected_error as exc_info: + await _list_tools_with_listing(_mcp_request_ctx, listing, _paged_params()) + + assert exc_info.value.error.code == INTERNAL_ERROR + assert exc_info.value.error.data[SERVER_OUTCOMES_META_KEY] == { + "a": {"status": "upstream_error", "http_status": 429}, + "b": {"status": "timeout"}, + } + + +@pytest.mark.asyncio +async def test_handle_list_tools_auth_fault_keeps_degraded_result( + _mcp_request_ctx: Callable[..., ServerRequestContext], +) -> None: + from mcp.types import ListToolsResult + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + SERVER_OUTCOMES_META_KEY, + AggregateToolListing, + ServerListFault, + ) + + listing: Final = AggregateToolListing( + tools=[], + outcomes={"a": ServerListFault(tag="auth_required", status_code=401), "b": ServerListFault(tag="timeout")}, + ) + + result: Final = await _list_tools_with_listing(_mcp_request_ctx, listing, _paged_params()) + + assert isinstance(result, ListToolsResult) + assert result.tools == [] + wire: Final = result.model_dump(by_alias=True) + assert wire["_meta"][SERVER_OUTCOMES_META_KEY]["a"] == {"status": "auth_required", "http_status": 401} + + +@pytest.mark.asyncio +async def test_handle_list_tools_all_faults_on_continuation_stays_degraded( + _mcp_request_ctx: Callable[..., ServerRequestContext], +) -> None: + from mcp.types import ListToolsResult, PaginatedRequestParams + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + SERVER_OUTCOMES_META_KEY, + AggregateToolListing, + ServerListFault, + ) + + listing: Final = AggregateToolListing( + tools=[], + outcomes={"a": ServerListFault(tag="upstream_error", status_code=429)}, + ) + + result: Final = await _list_tools_with_listing( + _mcp_request_ctx, listing, PaginatedRequestParams(cursor="cursor-1") + ) + + assert isinstance(result, ListToolsResult) + assert result.tools == [] + wire: Final = result.model_dump(by_alias=True) + assert wire["_meta"][SERVER_OUTCOMES_META_KEY]["a"] == {"status": "upstream_error", "http_status": 429} + + +@pytest.mark.asyncio +async def test_handle_list_tools_empty_outcomes_is_a_clean_empty_result( + _mcp_request_ctx: Callable[..., ServerRequestContext], +) -> None: + from mcp.types import ListToolsResult + + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + + listing: Final = AggregateToolListing(tools=[], outcomes={}) + + result: Final = await _list_tools_with_listing(_mcp_request_ctx, listing, _paged_params()) + + assert isinstance(result, ListToolsResult) + assert result.tools == [] + wire: Final = result.model_dump(by_alias=True) + assert wire.get("_meta") is None + + def _make_oauth2_server( alias: str, *, diff --git a/tests/unit/proxy/_experimental/mcp_server/test_operations.py b/tests/unit/proxy/_experimental/mcp_server/test_operations.py index 3957f486c4c..6f7437c90dc 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_operations.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_operations.py @@ -1263,6 +1263,45 @@ async def test_discovery_preserves_caller_scope_and_proxy_restrictions(): assert listed.await_args.args[0].mcp_servers == ("only-this",) +@pytest.mark.asyncio +async def test_discovery_outright_tools_failure_still_advertises_other_capabilities() -> None: + from mcp.types import DiscoverRequest, ListPromptsResult, ListToolsResult, Prompt + + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + + tools: Final = AsyncMock(side_effect=ToolListingFailedOutright(code=-32603, message="every server failed")) + prompts: Final = AsyncMock(return_value=ListPromptsResult(prompts=[Prompt(name="p")])) + context: Final = prepare_context(UserAPIKeyAuth(user_id="scoped"), mcp_servers=["broken"]) + with ( + patch.object(operations, "_execute_handle_list_tools", tools), + patch.object(operations, "_execute_list_prompts", prompts), + patch.object(operations, "_execute_list_resources", AsyncMock(return_value=ListToolsResult(tools=[]))), + patch.object(operations, "_execute_list_resource_templates", AsyncMock(return_value=ListToolsResult(tools=[]))), + ): + result: Final = await GatewayOperations().execute(DiscoverRequest(), context) + assert result.capabilities.tools is None + assert result.capabilities.prompts is not None + + +@pytest.mark.asyncio +async def test_discovery_propagates_non_outright_mcp_errors_from_tools_listing() -> None: + from mcp import MCPError + from mcp.types import INVALID_PARAMS, DiscoverRequest + + from litellm.proxy._experimental.mcp_server import operations + + tools: Final = AsyncMock(side_effect=MCPError(code=INVALID_PARAMS, message="bad cursor")) + context: Final = prepare_context( + UserAPIKeyAuth(user_id="scoped"), mcp_servers=["broken"], mcp_proxy_mode=True, protocol_version="2025-06-18" + ) + with patch.object(operations, "_execute_handle_list_tools", tools): + expected_error: Final = pytest.raises(MCPError) + with expected_error as exc_info: + await GatewayOperations().execute(DiscoverRequest(), context) + assert exc_info.value.error.code == INVALID_PARAMS + + @pytest.mark.asyncio async def test_discovery_denial_cannot_advertise_tools(): from mcp.types import DiscoverRequest diff --git a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py b/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py index a97ab04c506..fadb709e00f 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py @@ -10,7 +10,7 @@ from mcp.server.context import ServerRequestContext from mcp.server.session import ServerSession from mcp.server.subscriptions import ListenHandler from mcp.shared.subscriptions import PromptsListChanged, ToolsListChanged -from mcp.types import SubscriptionFilter, SubscriptionsListenRequestParams +from mcp.types import ListToolsRequest, ListToolsResult, SubscriptionFilter, SubscriptionsListenRequestParams from litellm.experimental_mcp_client.client import MCPClient from litellm.proxy._experimental.mcp_server import subscriptions @@ -45,6 +45,18 @@ def context(session: ServerSession, revision: str = "2026-07-28") -> ServerReque ) +class _ListingGateway: + def __init__(self, *, result: ListToolsResult | None = None, error: MCPError | None = None) -> None: + self._result: ListToolsResult | None = result + self._error: MCPError | None = error + + async def execute(self, _operation: ListToolsRequest, _context: OperationContext) -> ListToolsResult: + if self._error is not None: + raise self._error + assert self._result is not None + return self._result + + @pytest.mark.asyncio async def test_queued_notification_rechecks_authority_before_delivery() -> None: bus: Final = subscriptions._ListenerBus() @@ -486,3 +498,39 @@ async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limi finally: listener.cancel() await asyncio.gather(listener, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_all_failed_tools_listing_keeps_empty_catalog_digest(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._experimental.mcp_server.catalog import ToolListingFailedOutright + + listener_context: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="listener")) + empty_gateway: Final = _ListingGateway(result=ListToolsResult(tools=[])) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: empty_gateway + ) + empty_digest: Final = await subscriptions._listing(ToolsListChanged(), listener_context) + + failed_gateway: Final = _ListingGateway( + error=ToolListingFailedOutright(code=-32603, message="Every MCP server in scope failed") + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: failed_gateway + ) + failed_digest: Final = await subscriptions._listing(ToolsListChanged(), listener_context) + + assert failed_digest == empty_digest + + +@pytest.mark.asyncio +async def test_tools_listing_propagates_other_mcp_errors(monkeypatch: pytest.MonkeyPatch) -> None: + failure: Final = MCPError(code=-32602, message="Bad cursor") + gateway: Final = _ListingGateway(error=failure) + monkeypatch.setattr("litellm.proxy._experimental.mcp_server.operations.GatewayOperations", lambda: gateway) + listener_context: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="listener")) + + expected_error: Final = pytest.raises(MCPError) + with expected_error as exc_info: + await subscriptions._listing(ToolsListChanged(), listener_context) + + assert exc_info.value is failure From 6f66b7a2014e673f60d886a32d5915e99d349e26 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 16:01:40 -0700 Subject: [PATCH 028/124] feat(proxy)!: personal keys inherit MCP server, MCP tool and agent access from the owner's teams (#46293) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/auth/user_api_key_auth_mcp.py | 648 +++++++-- .../_experimental/mcp_server/contracts.py | 1 + .../mcp_server/mcp_server_manager.py | 71 +- .../_experimental/mcp_server/operations.py | 375 ++++- .../mcp_server/rest_endpoints.py | 249 +++- .../_experimental/mcp_server/tool_search.py | 31 +- .../proxy/agent_endpoints/a2a_endpoints.py | 14 +- .../auth/agent_permission_handler.py | 255 +++- litellm/proxy/auth/auth_checks.py | 259 +++- litellm/proxy/auth/team_grants.py | 86 ++ litellm/proxy/common_request_processing.py | 13 +- .../hooks/parallel_request_limiter_v3.py | 23 +- litellm/proxy/litellm_pre_call_utils.py | 4 +- litellm/proxy/proxy_server.py | 6 +- .../auth/test_user_api_key_auth_mcp.py | 1020 +++++++++++++- .../mcp_server/test_mcp_server_manager.py | 1067 +++++++++++--- .../test_mcp_server_tool_calls_and_headers.py | 217 ++- .../mcp_server/test_rest_endpoints.py | 274 +++- .../auth/test_agent_permission_handler.py | 1245 ++++++++++++++--- .../agent_endpoints/test_a2a_endpoints.py | 118 ++ ...st_auth_checks_object_access_and_lookup.py | 264 ++++ .../hooks/test_parallel_request_limiter_v3.py | 120 ++ 22 files changed, 5600 insertions(+), 760 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 4090666e16f..53b56c7b31d 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1,3 +1,4 @@ +import asyncio import functools import re import secrets @@ -5,8 +6,9 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from datetime import datetime, timezone +from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal from fastapi import HTTPException from pydantic import SecretStr, TypeAdapter @@ -41,11 +43,15 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( is_session_bearer_shaped, ) -from litellm.proxy._experimental.mcp_server.utils import asgi_scope_fields +from litellm.proxy._experimental.mcp_server.utils import ( + asgi_scope_fields, + strip_known_server_prefix, +) from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_AccessGroupTable, LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, LiteLLM_TeamTable, ProxyException, SpecialHeaders, @@ -466,6 +472,24 @@ class MCPServerAccess: scope: Literal["unscoped", "scoped", "unresolved"] = "unscoped" +def _mcp_server_access_after_error( + error: Exception, + key_object_permission: LiteLLM_ObjectPermissionTable | None, + *, + require_readable_team_policy: bool, +) -> MCPServerAccess: + if require_readable_team_policy: + raise error + if isinstance(error, UnloadableEntitlementError): + verbose_logger.warning("Denying MCP access, entitlement unreadable: %s", error) + else: + verbose_logger.warning("Failed to get allowed MCP servers: %s", error) + return MCPServerAccess( + server_ids=(), + scope="scoped" if getattr(key_object_permission, "mcp_servers", None) is not None else "unresolved", + ) + + @dataclass(frozen=True, slots=True) class DcrBridgeTarget: """The single DCR-bridge server a request targets, paired with the exact name the caller @@ -1804,11 +1828,28 @@ class MCPRequestHandler: ) return list(access.server_ids) + @staticmethod + async def _apply_agent_caller_ceiling_to_multi_source_server_access( + access: MCPServerAccess, + user_api_key_auth: UserAPIKeyAuth | None, + ) -> MCPServerAccess: + if user_api_key_auth is None or not MCPRequestHandler.is_personal_key_inheriting_team_grants(user_api_key_auth): + return access + caller_capped, caller_restricts = await MCPRequestHandler.apply_agent_caller_ceiling( + access.server_ids, + user_api_key_auth, + ) + return MCPServerAccess( + server_ids=caller_capped, + scope="scoped" if caller_restricts else access.scope, + ) + @staticmethod async def get_mcp_server_access( user_api_key_auth: UserAPIKeyAuth | None = None, *, keyless_source: bool = False, + require_readable_team_policy: bool = False, ) -> MCPServerAccess: """ Get list of allowed MCP servers for the given user/key based on permissions. @@ -1844,9 +1885,11 @@ class MCPRequestHandler: # matters: the no_mcp_servers opt-out below reads the caller's own object_permission, so above # this branch a user's own opt-out would wrongly zero their TEAMS' grants too (each source is # independent; an opt-out silences only its own source, inside the recursive call). - if is_mcp_admitted_user_subject(user_api_key_auth) and user_api_key_auth is not None: - return MCPServerAccess( - server_ids=tuple(await MCPRequestHandler.resolve_admitted_subject_servers(user_api_key_auth)), + multi_source_access: Final = await MCPRequestHandler._multi_source_server_access(user_api_key_auth) + if multi_source_access is not None: + return await MCPRequestHandler._apply_agent_caller_ceiling_to_multi_source_server_access( + multi_source_access, + user_api_key_auth, ) # Get allowed servers from key and team @@ -1861,7 +1904,10 @@ class MCPRequestHandler: ): return MCPServerAccess(server_ids=(), scope="scoped") - allowed_mcp_servers_for_team = await MCPRequestHandler._get_allowed_mcp_servers_for_team(user_api_key_auth) + allowed_mcp_servers_for_team = await MCPRequestHandler._get_allowed_mcp_servers_for_policy( + user_api_key_auth, + require_readable_policy=require_readable_team_policy, + ) key_access_group_grants = await MCPRequestHandler._get_key_access_group_mcp_server_extras(user_api_key_auth) @@ -1993,15 +2039,10 @@ class MCPRequestHandler: ), ) except Exception as e: - if isinstance(e, UnloadableEntitlementError): - # A ceiling we KNOW exists and cannot read. Denying is the only answer that does not - # widen this caller past what an operator configured, for both caller shapes. - verbose_logger.warning("Denying MCP access, entitlement unreadable: %s", e) - else: - verbose_logger.warning("Failed to get allowed MCP servers: %s", e) - return MCPServerAccess( - server_ids=(), - scope="scoped" if getattr(key_object_permission, "mcp_servers", None) is not None else "unresolved", + return _mcp_server_access_after_error( + e, + key_object_permission, + require_readable_team_policy=require_readable_team_policy, ) @staticmethod @@ -2144,8 +2185,6 @@ class MCPRequestHandler: # whole union to deny-all over one momentarily-unreadable row. verbose_logger.warning("MCP admitted-subject source team %r unresolvable, skipping: %s", team_id, e) return None - if team_obj is None: - return None member_user_ids: Final = {getattr(m, "user_id", None) for m in (team_obj.members_with_roles or [])} - {None} if auth.user_id not in member_user_ids: return None @@ -2240,35 +2279,156 @@ class MCPRequestHandler: return list(reachable) @staticmethod - async def billing_auth_for_tool_call(auth: UserAPIKeyAuth, tool_name: str) -> UserAPIKeyAuth: + async def billing_auth_for_tool_call( + auth: UserAPIKeyAuth, + tool_name: str, + *, + mcp_servers: Sequence[str] | None = None, + scoped_server: MCPServer | None = None, + native_tool_name: str | None = None, + allowed_mcp_servers: Sequence[MCPServer] | None = None, + ) -> UserAPIKeyAuth: """The auth object a tool call's SPEND should be recorded against. - ``auth`` unchanged for any non-admitted caller (key/JWT billing byte-identical). For an admitted - subject whose call is reached through a team's grant, a copy carrying that team's ``team_id`` and - owning ``org_id`` so the team's budget accumulates and the right org is charged. Falls back to - user-level attribution (rather than guessing a team) when the tool name does not resolve to a - server, reusing the manager's own tool-name lookup.""" - if not is_mcp_admitted_user_subject(auth): + ``auth`` unchanged for callers without inherited team grants. For an admitted subject or personal + key whose call is reached through a team's grant, a copy carrying that team's ``team_id`` and + owning ``org_id`` so the team's budget accumulates and the right org is charged.""" + from litellm.proxy.auth.auth_checks import personal_key_inherits_team_grants + + if not is_mcp_admitted_user_subject(auth) and not personal_key_inherits_team_grants(auth): return auth try: - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.operations import ( + _get_allowed_mcp_servers, + _resolve_display_name_to_original, + resolve_scoped_mcp_server, ) - server: Final = global_mcp_server_manager.get_mcp_server_from_tool_name(tool_name) + servers_for_name_resolution: Final = ( + allowed_mcp_servers + if allowed_mcp_servers is not None + else await _get_allowed_mcp_servers(auth, mcp_servers) + ) + resolved_tool_name: Final = _resolve_display_name_to_original(tool_name, servers_for_name_resolution) + server: Final = ( + scoped_server + or (await resolve_scoped_mcp_server(auth, mcp_servers) if mcp_servers is not None else None) + or global_mcp_server_manager.get_mcp_server_from_tool_name(resolved_tool_name) + or global_mcp_server_manager.server_owning_tool_name_prefix(resolved_tool_name) + ) if server is None: return auth - source: Final = await MCPRequestHandler.attributing_source_for_server(auth, server.server_id) + source: Final = await ( + MCPRequestHandler.attributing_source_for_server(auth, server.server_id) + if is_mcp_admitted_user_subject(auth) + else MCPRequestHandler.personal_key_attributing_source( + auth, + server.server_id, + native_tool_name + if native_tool_name is not None + else strip_known_server_prefix(resolved_tool_name, server), + ) + ) if source is None or not source.team_id: return auth - billed: Final = auth.model_copy() - billed.team_id = source.team_id - billed.org_id = source.org_id - return billed + if is_mcp_admitted_user_subject(auth): + return auth.model_copy( + update=MappingProxyType( + { + "team_id": source.team_id, + "org_id": source.org_id, + "team_metadata": source.team_metadata, + } + ) + ) + source_auth_update: Final = MappingProxyType( + { + "team_id": source.team_id, + "org_id": source.org_id, + "team_metadata": source.team_metadata, + } + ) + try: + team_auth_context: Final = await MCPRequestHandler._personal_key_team_auth_context(auth, source) + except Exception as e: # noqa: BLE001 # keep the selected team's attribution on follow-up read failures + verbose_logger.warning("MCP billing context lookup failed for team %r: %s", source.team_id, e) + return auth.model_copy(update=source_auth_update) + if team_auth_context is None: + return auth.model_copy(update=source_auth_update) + team_object, team_membership = team_auth_context + from litellm.proxy.auth.team_grants import team_grant_auth_update, team_grant_billing_auth_update + + try: + billed_update: Final = await team_grant_billing_auth_update(auth, team_object, team_membership) + except Exception as e: # noqa: BLE001 # retain team controls when organization attribution fails + verbose_logger.warning("MCP billing controls lookup failed for team %r: %s", team_object.team_id, e) + return auth.model_copy( + update=team_grant_auth_update( + team_object, + team_membership, + auth.user_id, + key_org_id=auth.org_id, + ) + ) + return auth.model_copy(update=billed_update) except Exception as e: # noqa: BLE001 # attribution must never fail an authorized call verbose_logger.warning("MCP billing attribution failed for %r, billing the user: %s", tool_name, e) return auth + @staticmethod + async def guardrail_auth_for_call( + guardrail_auth: UserAPIKeyAuth | None, + user_api_key_auth: UserAPIKeyAuth | None, + server: MCPServer | None, + tool_name: str, + allowed_mcp_servers: Sequence[MCPServer] | None = None, + ) -> UserAPIKeyAuth | None: + if guardrail_auth is not None or user_api_key_auth is None or server is None: + return guardrail_auth if guardrail_auth is not None else user_api_key_auth + if not MCPRequestHandler.is_personal_key_inheriting_team_grants(user_api_key_auth): + return user_api_key_auth + return await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, + tool_name, + scoped_server=server, + native_tool_name=tool_name, + allowed_mcp_servers=allowed_mcp_servers, + ) + + @staticmethod + async def _personal_key_team_auth_context( + auth: UserAPIKeyAuth, + source: UserAPIKeyAuth, + ) -> tuple[LiteLLM_TeamTable, LiteLLM_TeamMembership | None] | None: + from litellm.proxy.auth.auth_checks import get_team_membership, get_team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None or source.team_id is None: + return None + team: Final = await get_team_object( + team_id=source.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=auth.requires_fresh_policy, + ) + membership: Final = ( + await get_team_membership( + user_id=auth.user_id, + team_id=source.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=auth.requires_fresh_policy, + ) + if auth.user_id is not None + else None + ) + return team, membership + @staticmethod async def attributing_source_for_server( auth: UserAPIKeyAuth, @@ -2328,6 +2488,201 @@ class MCPRequestHandler: allowed.update(tools) return await MCPRequestHandler._narrow_tools_to_toolset(sorted(allowed), server_id, auth) + @staticmethod + async def _multi_source_server_access(user_api_key_auth: UserAPIKeyAuth | None) -> MCPServerAccess | None: + """Server access for the two caller shapes pinned to no single team, each resolved per grant + source: a keyless admitted subject (its own grants plus its teams') and, under + ``personal_key_model_access_from_teams``, a personal key (its owner's teams). None for every + other caller, which the single-source rules then resolve.""" + from litellm.proxy.auth.auth_checks import personal_key_inherits_team_grants + + if user_api_key_auth is None: + return None + if is_mcp_admitted_user_subject(user_api_key_auth): + return MCPServerAccess( + server_ids=tuple(await MCPRequestHandler.resolve_admitted_subject_servers(user_api_key_auth)), + ) + if personal_key_inherits_team_grants(user_api_key_auth): + return await MCPRequestHandler.resolve_personal_key_team_servers(user_api_key_auth) + return None + + @staticmethod + def is_personal_key_inheriting_team_grants(auth: UserAPIKeyAuth) -> bool: + from litellm.proxy.auth.auth_checks import personal_key_inherits_team_grants + + return personal_key_inherits_team_grants(auth) + + @staticmethod + async def personal_key_team_sources(auth: UserAPIKeyAuth) -> tuple[UserAPIKeyAuth, ...]: + """One source per current team of a personal key's owner, each resolving as the same key pinned + to that team, so key/team inheritance and intersection hold exactly as for a team key. The + pinned ``team_id`` keeps a source out of this branch. A team of another organization grants + nothing to a key issued under its own org, so neither org ceiling is dropped. No team, or an owner that cannot be + loaded, yields no source: the key reaches nothing through its owner.""" + from litellm.proxy.auth.auth_checks import personal_key_owner_teams + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + teams: Final = await personal_key_owner_teams( + valid_token=auth, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=bool(auth.requires_fresh_policy), + ) + return tuple( + auth.model_copy( + update=MappingProxyType( + { + "team_id": team.team_id, + "org_id": auth.org_id or team.organization_id, + "team_metadata": TypeAdapter(Mapping[str, object]).validate_python( + team.model_dump().get("metadata") or {} + ), + } + ) + ) + for team in teams + if MCPRequestHandler._personal_key_team_permission_is_readable(team) + and (auth.org_id is None or team.organization_id in (None, auth.org_id)) + ) + + @staticmethod + def _personal_key_team_permission_is_readable(team: LiteLLM_TeamTable) -> bool: + if team.object_permission_id is None or team.object_permission is not None: + return True + verbose_logger.warning( + "Personal key owner team %s has an unreadable object_permission, granting nothing", + team.team_id, + ) + return False + + @staticmethod + async def personal_key_team_server_grants( + auth: UserAPIKeyAuth, + ) -> tuple[tuple[UserAPIKeyAuth, MCPServerAccess], ...]: + sources: Final = await MCPRequestHandler.personal_key_team_sources(auth) + + async def resolve_source(source: UserAPIKeyAuth) -> tuple[UserAPIKeyAuth, MCPServerAccess]: + try: + return source, await MCPRequestHandler.get_mcp_server_access( + source, + require_readable_team_policy=True, + ) + except Exception as e: # noqa: BLE001 # an unreadable team grant must not widen inherited access + verbose_logger.warning("MCP access lookup failed for owner team %r: %s", source.team_id, e) + return source, MCPServerAccess(server_ids=(), scope="scoped") + + return tuple(await asyncio.gather(*(resolve_source(source) for source in sources))) + + @staticmethod + async def personal_key_attributing_source( + auth: UserAPIKeyAuth, + server_id: str, + tool_name: str, + grants: tuple[tuple[UserAPIKeyAuth, MCPServerAccess], ...] | None = None, + ) -> UserAPIKeyAuth | None: + source_grants: Final = grants or await MCPRequestHandler.personal_key_team_server_grants(auth) + + async def _source_if_tool_allowed(source: UserAPIKeyAuth, access: MCPServerAccess) -> UserAPIKeyAuth | None: + if server_id not in access.server_ids: + return None + try: + allowed_tools: Final = await MCPRequestHandler._single_credential_tools(server_id, source) + except Exception as e: # noqa: BLE001 # attribution metadata must not fail an authorized call + verbose_logger.warning("MCP tool attribution failed for team %r: %s", source.team_id, e) + return None + return source if allowed_tools is None or tool_name in allowed_tools else None + + eligible_sources: Final = tuple( + source + for source in await asyncio.gather( + *(_source_if_tool_allowed(source, access) for source, access in source_grants) + ) + if source is not None + ) + return min(eligible_sources, key=lambda source: source.team_id or "") if eligible_sources else None + + @staticmethod + async def personal_key_team_rpm_limits( + auth: UserAPIKeyAuth, server: MCPServer, tool_name: str + ) -> Mapping[str, Mapping[str, int]] | None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + try: + source: Final = await MCPRequestHandler.personal_key_attributing_source(auth, server.server_id, tool_name) + if source is None or source.team_id is None or prisma_client is None: + return None + team: Final = await get_team_object( + team_id=source.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=bool(auth.requires_fresh_policy), + ) + team_metadata: Final[Mapping[str, object]] = TypeAdapter(Mapping[str, object]).validate_python( + team.model_dump().get("metadata") or {} + ) + team_limit: Final = team_metadata.get("mcp_rpm_limit") + if not isinstance(team_limit, dict) or not team_limit: + return None + validated_limit: Final = TypeAdapter(dict[str, int]).validate_python(team_limit) + applicable: Final = { + name: rpm + for name, rpm in validated_limit.items() + if server.server_id in global_mcp_server_manager.expand_permission_list([name]) + } + except Exception as e: # noqa: BLE001 # throttling metadata must not fail an allowed request + verbose_logger.warning("Failed to resolve per-team MCP rpm limits for personal key: %s", e) + return None + else: + return {source.team_id: applicable} if applicable else None + + @staticmethod + async def resolve_personal_key_team_servers(auth: UserAPIKeyAuth) -> MCPServerAccess: + """Union of what the owner's teams grant the personal key, each answered by the canonical + resolver. Unscoped only when some team leaves the key unscoped, as a team key of it would be.""" + grants: Final = await MCPRequestHandler.personal_key_team_server_grants(auth) + server_ids: Final = tuple(dict.fromkeys(chain.from_iterable(access.server_ids for _source, access in grants))) + unscoped: Final = any(access.scope == "unscoped" for _source, access in grants) + return MCPServerAccess(server_ids=server_ids, scope="unscoped" if unscoped else "scoped") + + @staticmethod + async def resolve_personal_key_team_tools(server_id: str, auth: UserAPIKeyAuth) -> tuple[str, ...] | None: + """Effective tool allowlist on ``server_id`` for a personal key: the union over the owner's teams + that grant the server, where a granting team without a tool rule allows every tool. A server no + team grants is reachable only through an open channel (allow_all_keys, the owner's BYOM, the + admin-view role) and + then carries the key's OWN tool rules, never a team's; otherwise nothing is allowed. Each team + resolves through the raising ``_single_credential_tools`` so one team's fault reaches the + caller's fail-closed handler instead of becoming an allow-all ``None`` in the union.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + grants: Final = await MCPRequestHandler.personal_key_team_server_grants(auth) + granting: Final = tuple(source for source, access in grants if server_id in access.server_ids) + if not granting: + reachable: Final = server_id in await global_mcp_server_manager.operator_open_server_ids( + auth + ) or await MCPRequestHandler.admin_view_unscoped(auth) + if not reachable: + return () + own_tools: Final = await MCPRequestHandler._single_credential_tools(server_id, auth) + return None if own_tools is None else tuple(own_tools) + tool_lists: Final = await asyncio.gather( + *(MCPRequestHandler._single_credential_tools(server_id, source) for source in granting) + ) + if any(tools is None for tools in tool_lists): + return None + return tuple(sorted(frozenset(chain.from_iterable(tools for tools in tool_lists if tools is not None)))) + @staticmethod def _get_key_object_permission( user_api_key_auth: UserAPIKeyAuth | None = None, @@ -2693,6 +3048,8 @@ class MCPRequestHandler: return await managed_agent_tools(server_id, user_api_key_auth) + from litellm.proxy.auth.auth_checks import personal_key_inherits_team_grants + try: # FIRST statement, mirroring get_allowed_mcp_servers: a keyless admitted subject resolves per # source and shares nothing with the single-credential prelude below. Ordering is the invariant: @@ -2700,97 +3057,11 @@ class MCPRequestHandler: if is_mcp_admitted_user_subject(user_api_key_auth): return await MCPRequestHandler.resolve_admitted_subject_tools(server_id, user_api_key_auth) - # Get key and team object permissions (already loaded in main auth flow) - key_obj_perm: Final = MCPRequestHandler._get_key_object_permission(user_api_key_auth) - team_obj_perm: Final = await MCPRequestHandler._get_team_object_permission(user_api_key_auth) + if personal_key_inherits_team_grants(user_api_key_auth): + return _as_list(await MCPRequestHandler.resolve_personal_key_team_tools(server_id, user_api_key_auth)) - # Extract tool permissions for this server. Dict keys may be - # server_ids OR names/aliases; normalize to server_id-keyed form - # before lookup so a name-based key does not silently drop its - # tool restrictions when server_id is the resolved uuid. - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - - key_direct_tools: Final = ( - global_mcp_server_manager.expand_tool_permissions(key_obj_perm.mcp_tool_permissions).get(server_id) - if key_obj_perm - else None - ) - - # Tools granted through the key's toolsets restrict this server exactly - # as direct tool permissions do; union with any direct grants so the - # tool-level check sees the key's full effective tool scope - key_toolset_ids: Final = (key_obj_perm.mcp_toolsets or []) if key_obj_perm else [] - key_toolset_tools: Final = ( - ( - await global_mcp_server_manager.resolve_toolset_tool_permissions( - toolset_ids=key_toolset_ids, - requires_fresh_policy=user_api_key_auth.requires_fresh_policy, - ) - ).get(server_id) - if key_toolset_ids - else None - ) - - key_group_tools: Final = await MCPRequestHandler._key_access_group_tools_for_server( - user_api_key_auth, server_id - ) - key_direct_servers: Final = await MCPRequestHandler._key_direct_servers( - key_obj_perm, requires_fresh_policy=bool(user_api_key_auth.requires_fresh_policy) - ) - key_tools: Final = _as_list( - MCPRequestHandler._key_tool_scope( - key_direct_tools, key_toolset_tools, key_group_tools, server_id in key_direct_servers - ) - ) - team_direct_tools: Final = ( - global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id) - if team_obj_perm - else None - ) - - # Tools granted through the team's toolsets restrict this server exactly - # as the team's direct tool permissions do, mirroring the key path above - team_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server( - team_obj_perm, server_id, requires_fresh_policy=user_api_key_auth.requires_fresh_policy - ) - team_group_tools: Final = await MCPRequestHandler._team_access_group_tools_for_server( - user_api_key_auth, server_id - ) - team_tools: Final = MCPRequestHandler._union_tool_grants( - team_direct_tools, team_toolset_tools, team_group_tools - ) - - # Apply same inheritance logic as get_allowed_mcp_servers. ``is not None`` so an explicit - # empty allowlist stays a deny instead of reading as "no restriction". - if team_tools is not None: - if key_tools is not None: - # Both have restrictions → intersection - allowed_tools = list(set(team_tools) & set(key_tools)) - else: - # Only team has restrictions → inherit from team - allowed_tools = team_tools - else: - # No team restrictions → use key restrictions - allowed_tools = cast(list[str], key_tools) - - allowed_tools = _as_list( - await MCPRequestHandler._apply_end_user_tool_ceiling(allowed_tools, server_id, user_api_key_auth) - ) - - allowed_tools = _as_list( - await MCPRequestHandler._apply_user_tool_ceiling( - allowed_tools, server_id, user_api_key_auth, keyless_source=keyless_source - ) - ) - - allowed_tools = _as_list( - await MCPRequestHandler.apply_agent_caller_tool_ceiling(allowed_tools, server_id, user_api_key_auth) - ) - - return await MCPRequestHandler._apply_agent_and_org_tool_ceilings( - allowed_tools, server_id, user_api_key_auth, keyless_source=keyless_source + return await MCPRequestHandler._single_credential_tools( + server_id, user_api_key_auth, keyless_source=keyless_source ) except Exception as e: @@ -2806,10 +3077,108 @@ class MCPRequestHandler: # keyless_source AND the marker are needed: each source resolves through an UNMARKED auth, so # without keyless_source a fault under a source returns None and wins the union as allow-all. deny_all: Final = ( - unreadable_entitlement or keyless_source or is_mcp_admitted_user_subject(user_api_key_auth) + unreadable_entitlement + or keyless_source + or is_mcp_admitted_user_subject(user_api_key_auth) + or personal_key_inherits_team_grants(user_api_key_auth) ) return [] if deny_all else None + @staticmethod + async def _single_credential_tools( + server_id: str, + user_api_key_auth: UserAPIKeyAuth, + *, + keyless_source: bool = False, + ) -> list[str] | None: # mutable-ok: resolver returns a list + """Tool allowlist on ``server_id`` for one credential pinned to at most one team: the key's own + and its team's tool rules under the end-user, user, agent-caller, agent and org ceilings.""" + # Get key and team object permissions (already loaded in main auth flow) + key_obj_perm: Final = MCPRequestHandler._get_key_object_permission(user_api_key_auth) + team_obj_perm: Final = await MCPRequestHandler._get_team_object_permission(user_api_key_auth) + + # Extract tool permissions for this server. Dict keys may be + # server_ids OR names/aliases; normalize to server_id-keyed form + # before lookup so a name-based key does not silently drop its + # tool restrictions when server_id is the resolved uuid. + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + key_direct_tools: Final = ( + global_mcp_server_manager.expand_tool_permissions(key_obj_perm.mcp_tool_permissions).get(server_id) + if key_obj_perm + else None + ) + + # Tools granted through the key's toolsets restrict this server exactly + # as direct tool permissions do; union with any direct grants so the + # tool-level check sees the key's full effective tool scope + key_toolset_ids: Final = (key_obj_perm.mcp_toolsets or []) if key_obj_perm else [] + key_toolset_tools: Final = ( + ( + await global_mcp_server_manager.resolve_toolset_tool_permissions( + toolset_ids=key_toolset_ids, + requires_fresh_policy=user_api_key_auth.requires_fresh_policy, + ) + ).get(server_id) + if key_toolset_ids + else None + ) + + key_group_tools: Final = await MCPRequestHandler._key_access_group_tools_for_server( + user_api_key_auth, server_id + ) + key_direct_servers: Final = await MCPRequestHandler._key_direct_servers( + key_obj_perm, requires_fresh_policy=bool(user_api_key_auth.requires_fresh_policy) + ) + key_tools: Final = _as_list( + MCPRequestHandler._key_tool_scope( + key_direct_tools, key_toolset_tools, key_group_tools, server_id in key_direct_servers + ) + ) + team_direct_tools: Final = ( + global_mcp_server_manager.expand_tool_permissions(team_obj_perm.mcp_tool_permissions).get(server_id) + if team_obj_perm + else None + ) + + # Tools granted through the team's toolsets restrict this server exactly + # as the team's direct tool permissions do, mirroring the key path above + team_toolset_tools: Final = await MCPRequestHandler._toolset_tools_for_server( + team_obj_perm, server_id, requires_fresh_policy=user_api_key_auth.requires_fresh_policy + ) + team_group_tools: Final = await MCPRequestHandler._team_access_group_tools_for_server( + user_api_key_auth, server_id + ) + team_tools: Final = MCPRequestHandler._union_tool_grants( + team_direct_tools, team_toolset_tools, team_group_tools + ) + + # Apply same inheritance logic as get_allowed_mcp_servers. ``is not None`` so an explicit + # empty allowlist stays a deny instead of reading as "no restriction". + initial_allowed_tools: Final = ( + list(set(team_tools) & set(key_tools)) + if team_tools is not None and key_tools is not None + else team_tools + if team_tools is not None + else key_tools + ) + end_user_allowed_tools: Final = _as_list( + await MCPRequestHandler._apply_end_user_tool_ceiling(initial_allowed_tools, server_id, user_api_key_auth) + ) + user_allowed_tools: Final = _as_list( + await MCPRequestHandler._apply_user_tool_ceiling( + end_user_allowed_tools, server_id, user_api_key_auth, keyless_source=keyless_source + ) + ) + caller_allowed_tools: Final = _as_list( + await MCPRequestHandler.apply_agent_caller_tool_ceiling(user_allowed_tools, server_id, user_api_key_auth) + ) + return await MCPRequestHandler._apply_agent_and_org_tool_ceilings( + caller_allowed_tools, server_id, user_api_key_auth, keyless_source=keyless_source + ) + @staticmethod async def _apply_agent_and_org_tool_ceilings( allowed_tools: list[str] | None, @@ -3069,6 +3438,8 @@ class MCPRequestHandler: @staticmethod async def _get_allowed_mcp_servers_for_team( user_api_key_auth: UserAPIKeyAuth | None = None, + *, + require_readable_policy: bool = False, ) -> list[str]: """Get allowed MCP servers a caller inherits from the team it is pinned to. @@ -3080,7 +3451,24 @@ class MCPRequestHandler: team_ids: Final = await MCPRequestHandler._team_ids_for_mcp_grant(user_api_key_auth) if not team_ids: return [] - return await MCPRequestHandler._allowed_mcp_servers_for_single_team(team_ids[0], user_api_key_auth) + return await MCPRequestHandler._allowed_mcp_servers_for_single_team( + team_ids[0], + user_api_key_auth, + require_readable_policy=require_readable_policy, + ) + + @staticmethod + async def _get_allowed_mcp_servers_for_policy( + user_api_key_auth: UserAPIKeyAuth | None, + *, + require_readable_policy: bool, + ) -> list[str]: + if require_readable_policy: + return await MCPRequestHandler._get_allowed_mcp_servers_for_team( + user_api_key_auth, + require_readable_policy=True, + ) + return await MCPRequestHandler._get_allowed_mcp_servers_for_team(user_api_key_auth) @staticmethod async def _team_ids_for_mcp_grant(user_api_key_auth: UserAPIKeyAuth | None) -> list[str]: @@ -3134,6 +3522,7 @@ class MCPRequestHandler: team_access_group_servers: list[str], *, requires_fresh_policy: bool = False, + require_readable_policy: bool = False, ) -> AbstractSet[str]: """The raw MCP-server set a team grants (before any org ceiling): its object_permission (direct ``mcp_servers``, the ``all_proxy_servers`` sentinel → the full registry, legacy access groups, @@ -3150,10 +3539,11 @@ class MCPRequestHandler: return set(global_mcp_server_manager.get_registry().keys()) legacy_access_group_servers: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups( object_permissions.mcp_access_groups or [], - requires_fresh_policy=requires_fresh_policy, + requires_fresh_policy=requires_fresh_policy or require_readable_policy, ) toolset_grants: Final = await MCPRequestHandler._toolset_tool_permissions( - object_permissions, requires_fresh_policy=requires_fresh_policy + object_permissions, + requires_fresh_policy=requires_fresh_policy or require_readable_policy, ) servers: Final = ( set(global_mcp_server_manager.expand_permission_list(object_permissions.mcp_servers or [])) @@ -3168,6 +3558,8 @@ class MCPRequestHandler: async def _allowed_mcp_servers_for_single_team( team_id: str, user_api_key_auth: UserAPIKeyAuth | None, + *, + require_readable_policy: bool = False, ) -> list[str]: """Allowed MCP servers granted by ONE team (its raw grant, then capped by the team's own org for a keyless admitted subject). @@ -3216,17 +3608,19 @@ class MCPRequestHandler: user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, check_db_only=bool(user_api_key_auth and user_api_key_auth.requires_fresh_policy), + strict_errors=require_readable_policy, ) servers: Final = await MCPRequestHandler._team_granted_servers( team_obj, team_access_group_servers, requires_fresh_policy=bool(user_api_key_auth and user_api_key_auth.requires_fresh_policy), + require_readable_policy=require_readable_policy, ) return list(servers) except Exception as e: if isinstance(e, UnloadableEntitlementError) or ( - user_api_key_auth is not None and user_api_key_auth.requires_fresh_policy + require_readable_policy or (user_api_key_auth is not None and user_api_key_auth.requires_fresh_policy) ): raise verbose_logger.warning("Failed to get allowed MCP servers for team: %s", e) diff --git a/litellm/proxy/_experimental/mcp_server/contracts.py b/litellm/proxy/_experimental/mcp_server/contracts.py index 5c3fbde67a6..84cd2751444 100644 --- a/litellm/proxy/_experimental/mcp_server/contracts.py +++ b/litellm/proxy/_experimental/mcp_server/contracts.py @@ -138,5 +138,6 @@ class AuthorizedToolCall: host_progress_callback: ProgressCallback | None guardrail_context: Mapping[str, object] | None logging_data: Mapping[str, object] + guardrail_auth: UserAPIKeyAuth | None = None input_responses: InputResponses | None = None request_state: str | None = None diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index aba5161d56d..f65310a77e3 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -3580,7 +3580,7 @@ class MCPServerManager: *, allow_all_server_ids: list[str] | None = None, submitted_server_ids: list[str] | None = None, - ) -> set: + ) -> frozenset[str]: """Servers reachable through OPEN channels rather than a grant: operator-opened ``allow_all_keys`` servers, plus the caller's own active BYOM submissions when the caller carries no explicit ``mcp_servers`` scope. @@ -3601,10 +3601,10 @@ class MCPServerManager: if user_api_key_auth is not None and ( user_api_key_auth.mcp_toolset_id is not None or user_api_key_auth.mcp_explicit_grants_only ): - return set() - if allow_all_server_ids is None: - allow_all_server_ids = self.get_allow_all_keys_server_ids() - open_ids: Final = set(allow_all_server_ids) + return frozenset() + open_ids: Final = frozenset( + allow_all_server_ids if allow_all_server_ids is not None else self.get_allow_all_keys_server_ids() + ) key_object_permission: Final = user_api_key_auth.object_permission if user_api_key_auth else None # "Explicitly scoped, so do not widen with BYOM" is a rule about a CREDENTIAL that carries # its own mcp_servers list. It does not describe a keyless admitted subject: its @@ -3616,11 +3616,14 @@ class MCPServerManager: and key_object_permission is not None and (key_object_permission.mcp_servers is not None) ) - if not has_explicit_object_permission: - if submitted_server_ids is None: - submitted_server_ids = await self._get_active_submitted_mcp_server_ids_for_user(user_api_key_auth) - open_ids.update(submitted_server_ids) - return open_ids + if has_explicit_object_permission: + return open_ids + submitted_ids: Final = ( + submitted_server_ids + if submitted_server_ids is not None + else await self._get_active_submitted_mcp_server_ids_for_user(user_api_key_auth) + ) + return open_ids | frozenset(submitted_ids) @staticmethod def _admitted_session_resource_scope(user_api_key_auth: UserAPIKeyAuth | None) -> frozenset[str] | None: @@ -6051,6 +6054,7 @@ class MCPServerManager: guardrail_context: Mapping[str, object] | None = None, tool: MCPTool | None = None, incoming_bearer_token: SecretStr | None | EllipsisType = ..., + guardrail_auth: UserAPIKeyAuth | None = None, ) -> dict[str, Any]: """ Run pre-call checks and guardrail hooks for an MCP tool call. @@ -6107,6 +6111,27 @@ class MCPServerManager: if proxy_logging_obj is None: return hook_result + base_guardrail_auth: Final = await MCPRequestHandler.guardrail_auth_for_call( + guardrail_auth=guardrail_auth, + user_api_key_auth=user_api_key_auth, + server=server, + tool_name=name, + ) + checked_auth: Final = ( + base_guardrail_auth.model_copy( + update={ + "mcp_source_team_rpm_limits": await MCPRequestHandler.personal_key_team_rpm_limits( + user_api_key_auth, server, name + ) + } + ) + if base_guardrail_auth is not None + and user_api_key_auth is not None + and MCPRequestHandler.is_personal_key_inheriting_team_grants(user_api_key_auth) + and base_guardrail_auth.mcp_source_team_rpm_limits is None + else base_guardrail_auth + ) + caller_bearer: Final = ( MCPRequestHandler.get_incoming_bearer_token(raw_headers or {}) if incoming_bearer_token is ... @@ -6119,13 +6144,11 @@ class MCPServerManager: "arguments": arguments, "server_name": server_name, "mcp_rate_limit_server_name": server.alias or server.server_name or server.name, - "user_api_key_auth": user_api_key_auth, - "user_api_key_user_id": (getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None), - "user_api_key_team_id": (getattr(user_api_key_auth, "team_id", None) if user_api_key_auth else None), - "user_api_key_end_user_id": ( - getattr(user_api_key_auth, "end_user_id", None) if user_api_key_auth else None - ), - "user_api_key_hash": (getattr(user_api_key_auth, "api_key_hash", None) if user_api_key_auth else None), + "user_api_key_auth": checked_auth, + "user_api_key_user_id": (getattr(checked_auth, "user_id", None) if checked_auth else None), + "user_api_key_team_id": (getattr(checked_auth, "team_id", None) if checked_auth else None), + "user_api_key_end_user_id": (getattr(checked_auth, "end_user_id", None) if checked_auth else None), + "user_api_key_hash": (getattr(checked_auth, "api_key_hash", None) if checked_auth else None), "incoming_bearer_token": caller_bearer.get_secret_value() if caller_bearer is not None else None, "headers": logging_safe_mcp_headers(raw_headers), "tool_description": tool.description if tool is not None else None, @@ -6146,11 +6169,11 @@ class MCPServerManager: try: # Use standard pre_call_hook modified_data: Final = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_auth, + user_api_key_dict=checked_auth, data=synthetic_llm_data, call_type=CallTypes.call_mcp_tool.value, ) - await proxy_logging_obj.enforce_mcp_server_rate_limits(user_api_key_auth, server) + await proxy_logging_obj.enforce_mcp_server_rate_limits(checked_auth, server) if modified_data: # Convert response back to MCP format and apply modifications modified_kwargs: Final = proxy_logging_obj.convert_mcp_hook_response_to_kwargs( @@ -6911,6 +6934,7 @@ class MCPServerManager: input_responses: InputResponses | None = None, request_state: str | None = None, *, + guardrail_auth: UserAPIKeyAuth | None = None, catalog_auth_header: str | None | EllipsisType = ..., listed_tool: MCPTool | None | EllipsisType = ..., incoming_bearer_token: SecretStr | None | EllipsisType = ..., @@ -6938,6 +6962,12 @@ class MCPServerManager: """ start_time: Final = datetime.datetime.now() mcp_server: Final = self._resolve_mcp_server_for_tool_call(server_name, name) + effective_guardrail_auth: Final = await MCPRequestHandler.guardrail_auth_for_call( + guardrail_auth=guardrail_auth, + user_api_key_auth=user_api_key_auth, + server=mcp_server, + tool_name=name, + ) client_auth_header: Final = _catalog_auth_header(mcp_auth_header, catalog_auth_header) # Resolved before any hook runs so a missing BYOK credential (401) never @@ -6969,6 +6999,7 @@ class MCPServerManager: guardrail_context=guardrail_context, tool=self.get_listed_tool(mcp_server, name, listed_caller) if listed_tool is ... else listed_tool, incoming_bearer_token=incoming_bearer_token, + guardrail_auth=effective_guardrail_auth, ) if "arguments" in hook_result: arguments = hook_result["arguments"] @@ -6980,7 +7011,7 @@ class MCPServerManager: name=name, arguments=arguments, server_name_from_prefix=server_name, - user_api_key_auth=user_api_key_auth, + user_api_key_auth=effective_guardrail_auth, proxy_logging_obj=proxy_logging_obj, start_time=start_time, litellm_logging_obj=litellm_logging_obj, diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py index c45faf38f51..39e882fcab0 100644 --- a/litellm/proxy/_experimental/mcp_server/operations.py +++ b/litellm/proxy/_experimental/mcp_server/operations.py @@ -5,11 +5,11 @@ import time import traceback import types import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from contextvars import ContextVar from dataclasses import dataclass, replace from datetime import datetime -from functools import partial +from functools import partial, reduce from types import EllipsisType from typing import Any, Final, NoReturn, TypeAlias, overload @@ -47,7 +47,7 @@ from mcp.types import ( ToolsCapability, ) from mcp.types import Tool as MCPTool -from pydantic import AnyUrl, ConfigDict, Field, SecretStr, TypeAdapter +from pydantic import AnyUrl, ConfigDict, Field, JsonValue, SecretStr, TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_logger @@ -317,6 +317,7 @@ async def _build_virtual_call_logging_obj( name: str, arguments: dict[str, object], user_api_key_auth: UserAPIKeyAuth, + logging_auth: UserAPIKeyAuth | None = None, raw_headers: Mapping[str, str] | None = None, client_ip: str | None = None, ) -> LiteLLMLoggingObj | None: @@ -336,7 +337,7 @@ async def _build_virtual_call_logging_obj( raw_headers=raw_headers, client_ip=client_ip, ) - _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing( + pre_call_result: Final = await ProxyBaseLLMRequestProcessing( data={"name": name, "arguments": arguments} ).common_processing_pre_call_logic( request=request, @@ -345,10 +346,67 @@ async def _build_virtual_call_logging_obj( route_type=CallTypes.call_mcp_tool.value, proxy_logging_obj=proxy_logging_obj, general_settings=general_settings, + logging_auth=logging_auth, ) + _, logging_candidate = TypeAdapter(tuple[dict[str, object], object]).validate_python(pre_call_result) + if not isinstance(logging_candidate, LiteLLMLoggingObj): + verbose_logger.warning("MCP request processing did not return a logging object") + return None + virtual_logging_obj: Final = logging_candidate return virtual_logging_obj +async def _log_virtual_mcp_tool_failure( + *, + logging_obj: LiteLLMLoggingObj | None, + exc: Exception, + call_start: datetime, + name: str, + arguments: dict[str, object] | None, + user_api_key_auth: UserAPIKeyAuth, + raw_headers: Mapping[str, str] | None, + client_ip: str | None, +) -> None: + try: + failure_logging_obj: Final = ( + logging_obj + if logging_obj is not None + else await _build_virtual_call_logging_obj( + name=name, + arguments=arguments or {}, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + ) + if failure_logging_obj is None: + return + from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj + + failure_end: Final = datetime.now() # noqa: DTZ005 # matches the logging pipeline start time + failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) + failure_logging_obj.failure_handler(exc, failure_traceback, call_start, failure_end) + await failure_logging_obj.async_failure_handler(exc, failure_traceback, call_start, failure_end) + if isinstance(exc, MCPUpstreamAuthError): + return + failure_hook: Final = TypeAdapter(Callable[..., Awaitable[HTTPException | None]]).validate_python( + request_logging_obj.post_call_failure_hook + ) + await failure_hook( + request_data={ + "name": name, + "arguments": arguments, + "litellm_logging_obj": failure_logging_obj, + }, + original_exception=exc, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + traceback_str=failure_traceback, + ) + except Exception: # noqa: BLE001 # failure logging must not mask the tool call error + verbose_logger.exception("Error logging failed MCP proxy tool call") + + async def _dispatch_virtual_mcp_tool( name: str, arguments: dict[str, object] | None, @@ -393,11 +451,51 @@ async def _dispatch_virtual_mcp_tool( if mcp_proxy_mode and name in MCP_PROXY_TOOL_NAMES: assert user_api_key_auth is not None proxy_call_start: Final = datetime.now() # noqa: DTZ005 # logging pipeline uses naive datetimes + proxy_arguments: Final = arguments or {} + proxy_arguments_json: Final = TypeAdapter(dict[str, JsonValue]).validate_python(proxy_arguments) + proxy_tool_id: Final = proxy_arguments.get("tool_id") + try: + proxy_target_tool: Final = ( + await find_mcp_proxy_tool( + tool_id=proxy_tool_id, + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + if name == MCP_PROXY_CALL_TOOL_NAME and isinstance(proxy_tool_id, str) + else None + ) + proxy_guardrail_auth: Final = ( + await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, + tool_name=proxy_target_tool.name, + mcp_servers=mcp_servers, + ) + if proxy_target_tool is not None + else user_api_key_auth + ) + except Exception as exc: + await _log_virtual_mcp_tool_failure( + logging_obj=None, + exc=exc, + call_start=proxy_call_start, + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + raise proxy_logging_obj: Final = ( await _build_virtual_call_logging_obj( name=name, - arguments=arguments or {}, + arguments=proxy_arguments, user_api_key_auth=user_api_key_auth, + logging_auth=proxy_guardrail_auth, raw_headers=raw_headers, client_ip=client_ip, ) @@ -407,7 +505,7 @@ async def _dispatch_virtual_mcp_tool( try: proxy_result: Final = await handle_mcp_proxy_tool( name=name, - arguments=arguments or {}, + arguments=proxy_arguments_json, user_api_key_dict=user_api_key_auth, client_ip=client_ip, mcp_servers=mcp_servers, @@ -417,30 +515,21 @@ async def _dispatch_virtual_mcp_tool( raw_headers=raw_headers, litellm_logging_obj=proxy_logging_obj, incoming_bearer_token=incoming_bearer_token, + guardrail_auth=proxy_guardrail_auth, + resolved_tool=proxy_target_tool if name == MCP_PROXY_CALL_TOOL_NAME else ..., ) except Exception as exc: if proxy_logging_obj is not None: - from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj - - failure_end: Final = datetime.now() # noqa: DTZ005 # matches the logging pipeline start time - failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) - try: - proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end) - await proxy_logging_obj.async_failure_handler(exc, failure_traceback, proxy_call_start, failure_end) - if not isinstance(exc, MCPUpstreamAuthError): - await request_logging_obj.post_call_failure_hook( - request_data={ - "name": name, - "arguments": arguments, - "litellm_logging_obj": proxy_logging_obj, - }, - original_exception=exc, - user_api_key_dict=user_api_key_auth, - route="/mcp/call_tool", - traceback_str=failure_traceback, - ) - except Exception: # noqa: BLE001 # a failing failure hook must not mask the tool call's own error - verbose_logger.exception("Error logging failed MCP proxy tool call") + await _log_virtual_mcp_tool_failure( + logging_obj=proxy_logging_obj, + exc=exc, + call_start=proxy_call_start, + name=name, + arguments=arguments, + user_api_key_auth=proxy_guardrail_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) raise if proxy_logging_obj is not None: return await _fire_mcp_tool_call_logging( @@ -448,8 +537,8 @@ async def _dispatch_virtual_mcp_tool( result=proxy_result, start_time=proxy_call_start, end_time=datetime.now(), # noqa: DTZ005 # matches the logging pipeline start time - user_api_key_auth=user_api_key_auth, - request_data=types.MappingProxyType({"name": name, "arguments": arguments}), + user_api_key_auth=proxy_guardrail_auth, + request_data=types.MappingProxyType({"name": name, "arguments": proxy_arguments}), ) return proxy_result @@ -498,16 +587,22 @@ async def _dispatch_virtual_mcp_tool( top_k=coerce_top_k(args.get("top_k", DEFAULT_SKILL_SEARCH_TOP_K), default=DEFAULT_SKILL_SEARCH_TOP_K), user_api_key_dict=user_api_key_auth, ) + tool_request: Final = TypeAdapter(CallToolRequestParams).validate_python( + {"name": args.get("tool_name", ""), "arguments": args.get("arguments") or {}} + ) + tool_guardrail_auth: Final = await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, + tool_name=tool_request.name, + mcp_servers=mcp_servers, + ) virtual_logging_obj: Final = await _build_virtual_call_logging_obj( - name=name, - arguments=args, + name=tool_request.name, + arguments=tool_request.arguments or {}, user_api_key_auth=user_api_key_auth, + logging_auth=tool_guardrail_auth, raw_headers=raw_headers, client_ip=client_ip, ) - tool_request: Final = CallToolRequestParams.model_validate( - types.MappingProxyType({"name": args.get("tool_name", ""), "arguments": args.get("arguments") or {}}) - ) return await handle_mcp_tool_call( tool_name=tool_request.name, arguments=tool_request.arguments or {}, @@ -520,6 +615,7 @@ async def _dispatch_virtual_mcp_tool( raw_headers=raw_headers, litellm_logging_obj=virtual_logging_obj, incoming_bearer_token=incoming_bearer_token, + guardrail_auth=tool_guardrail_auth, ) @@ -1775,7 +1871,7 @@ async def _list_mcp_resource_templates( def _resolve_display_name_to_original( name: str, - allowed_mcp_servers: list[MCPServer], + allowed_mcp_servers: Sequence[MCPServer], ) -> str: """Translate a display-name override back to the original prefixed tool name. @@ -1792,6 +1888,15 @@ def _resolve_display_name_to_original( return name +def resolve_native_tool_name_for_server( + name: str, + server: MCPServer, + allowed_mcp_servers: Sequence[MCPServer], +) -> str: + display_name_resolved: Final = _resolve_display_name_to_original(name, allowed_mcp_servers) + return strip_known_server_prefix(display_name_resolved, server) + + async def _get_byok_credential( mcp_server: MCPServer, user_api_key_auth: UserAPIKeyAuth | None, @@ -1988,6 +2093,7 @@ async def execute_mcp_tool( raw_headers: dict[str, str] | None = None, host_progress_callback: ProgressCallback | None = None, guardrail_context: Mapping[str, object] | None = None, + guardrail_auth: UserAPIKeyAuth | None = None, client_ip: str | None = None, wire_compat: WireCompat = WireCompat.LEGACY, input_responses: InputResponses | None = None, @@ -2012,6 +2118,7 @@ async def execute_mcp_tool( start_time=start_time, host_progress_callback=host_progress_callback, guardrail_context=guardrail_context, + guardrail_auth=guardrail_auth, logging_data=types.MappingProxyType(kwargs), input_responses=input_responses, request_state=request_state, @@ -2031,6 +2138,7 @@ async def _execute_mcp_tool( raw_headers: dict[str, str] | None = None, host_progress_callback: ProgressCallback | None = None, guardrail_context: Mapping[str, object] | None = None, + guardrail_auth: UserAPIKeyAuth | None = None, client_ip: str | None = None, wire_compat: WireCompat = WireCompat.LEGACY, input_responses: InputResponses | None = None, @@ -2138,7 +2246,7 @@ async def _execute_mcp_tool( break if mcp_server is not None: server_name = mcp_server.name - original_tool_name = strip_known_server_prefix(name, mcp_server) + original_tool_name = resolve_native_tool_name_for_server(name, mcp_server, allowed_mcp_servers) if requested_server is not None: if mcp_server is not None and mcp_server.server_id != requested_server.server_id: @@ -2156,7 +2264,7 @@ async def _execute_mcp_tool( if mcp_server is None: mcp_server = requested_server server_name = requested_server.name - original_tool_name = strip_known_server_prefix(name, requested_server) + original_tool_name = resolve_native_tool_name_for_server(name, requested_server, allowed_mcp_servers) # Only enforce server-level permissions when we can resolve a server if server_name: @@ -2187,6 +2295,14 @@ async def _execute_mcp_tool( name ) + effective_guardrail_auth: Final = await MCPRequestHandler.guardrail_auth_for_call( + guardrail_auth=guardrail_auth, + user_api_key_auth=user_api_key_auth, + server=mcp_server, + tool_name=original_tool_name, + allowed_mcp_servers=allowed_mcp_servers, + ) + client_auth_header: Final = mcp_auth_header if mcp_server: standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get("mcp_server_cost_info") @@ -2274,6 +2390,7 @@ async def _execute_mcp_tool( ), ), incoming_bearer_token=incoming_bearer_token, + guardrail_auth=effective_guardrail_auth, ) # `pre_call_tool_check` may return guardrail-modified # arguments; honor them on the local path too. @@ -2322,6 +2439,7 @@ async def _execute_mcp_tool( name=original_tool_name, arguments=arguments, user_api_key_auth=user_api_key_auth, + guardrail_auth=effective_guardrail_auth, mcp_auth_header=mcp_auth_header, catalog_auth_header=client_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, @@ -2331,7 +2449,11 @@ async def _execute_mcp_tool( litellm_logging_obj=litellm_logging_obj, guardrail_context=guardrail_context, host_progress_callback=_guarded_progress_callback( - host_progress_callback, litellm_logging_obj, user_api_key_auth, kwargs + callback=host_progress_callback, + logging_obj=litellm_logging_obj, + original_auth=user_api_key_auth, + guardrail_auth=effective_guardrail_auth, + request_data=kwargs, ), wire_compat=wire_compat, input_responses=input_responses, @@ -2410,6 +2532,7 @@ async def _execute_mcp_tool( ), ), incoming_bearer_token=incoming_bearer_token, + guardrail_auth=effective_guardrail_auth, ) if "arguments" in hook_result: arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args @@ -2422,7 +2545,8 @@ async def _execute_mcp_tool( return await _run_post_mcp_call_guardrails( result=converted, litellm_logging_obj=litellm_logging_obj, - user_api_key_auth=user_api_key_auth, + original_auth=user_api_key_auth, + guardrail_auth=effective_guardrail_auth, request_data=kwargs, ) @@ -2430,7 +2554,8 @@ async def _execute_mcp_tool( def _guarded_progress_callback( callback: ProgressCallback | None, logging_obj: LiteLLMLoggingObj | None, - caller: UserAPIKeyAuth | None, + original_auth: UserAPIKeyAuth | None, + guardrail_auth: UserAPIKeyAuth | None, request_data: Mapping[str, object], ) -> ProgressCallback | None: if callback is None: @@ -2442,7 +2567,11 @@ def _guarded_progress_callback( return try: checked: Final = await _run_post_mcp_call_guardrails( - CallToolResult(content=[TextContent(type="text", text=message)]), logging_obj, caller, request_data + CallToolResult(content=[TextContent(type="text", text=message)]), + litellm_logging_obj=logging_obj, + original_auth=original_auth, + guardrail_auth=guardrail_auth, + request_data=request_data, ) filtered: Final = TextContent.model_validate(checked.content[0]).text except Exception: # noqa: BLE001 # any guardrail failure must withhold unverified progress text @@ -2457,7 +2586,8 @@ def _guarded_progress_callback( async def _run_post_mcp_call_guardrails( result: CallToolResult, litellm_logging_obj: LiteLLMLoggingObj | None, - user_api_key_auth: UserAPIKeyAuth | None, + original_auth: UserAPIKeyAuth | None, + guardrail_auth: UserAPIKeyAuth | None, request_data: Mapping[str, object], ) -> CallToolResult: """Run ``post_mcp_call`` guardrails over an executed tool result. @@ -2472,15 +2602,139 @@ async def _run_post_mcp_call_guardrails( if proxy_logging_obj is None: return result + post_auth: Final = guardrail_auth if guardrail_auth is not None else original_auth + if post_auth is original_auth: + request_data_for_hook: Final = ( + litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data) + ) + return await proxy_logging_obj.post_mcp_call_hook( + response=result, + request_data=request_data_for_hook, + user_api_key_dict=post_auth, + ) + from litellm.proxy.litellm_pre_call_utils import add_guardrails_from_auth_metadata + + base_request_data: Final[Mapping[str, object]] = ( + litellm_logging_obj.model_call_details if litellm_logging_obj is not None else request_data + ) + request_metadata: Final[object | None] = base_request_data.get("metadata") + metadata: Final[Mapping[str, object]] = ( + TypeAdapter(Mapping[str, object]).validate_python(request_metadata) + if isinstance(request_metadata, Mapping) + else {} + ) + post_auth_metadata: Final[Mapping[str, object]] = ( + TypeAdapter(Mapping[str, object]).validate_python(post_auth.metadata) + if post_auth is not None and post_auth.metadata is not None + else {} + ) + post_auth_team_metadata: Final[Mapping[str, object]] = ( + TypeAdapter(Mapping[str, object]).validate_python(post_auth.team_metadata) + if post_auth is not None and post_auth.team_metadata is not None + else {} + ) + post_auth_project_metadata: Final[Mapping[str, object]] = ( + TypeAdapter(Mapping[str, object]).validate_python(post_auth.project_metadata) + if post_auth is not None and post_auth.project_metadata is not None + else {} + ) + auth_guardrail_data: Final[dict[str, object]] = {"metadata": {}} # mutable-ok: resolver writes metadata + if post_auth is not None: + auth_for_guardrail_resolution: Final = post_auth.model_copy( + update={ + "metadata": _guardrail_metadata_for_legacy_resolver(post_auth_metadata), + "team_metadata": _guardrail_metadata_for_legacy_resolver(post_auth_team_metadata), + "project_metadata": _guardrail_metadata_for_legacy_resolver(post_auth_project_metadata), + } + ) + add_guardrails_from_auth_metadata( + user_api_key_dict=auth_for_guardrail_resolution, + data=auth_guardrail_data, + metadata_variable_name="metadata", + ) + auth_metadata: Final = auth_guardrail_data.get("metadata") + resolved_auth_metadata: Final[Mapping[str, object]] = ( + TypeAdapter(Mapping[str, object]).validate_python(auth_metadata) if isinstance(auth_metadata, Mapping) else {} + ) + request_guardrails: Final = _guardrails_from_metadata(metadata) + request_guardrail_names: Final = frozenset[str]().union( + *(_guardrail_names_from_entry(entry) for entry in request_guardrails) + ) + auth_guardrails: Final = _deduplicate_auth_guardrails( + ( + *_auth_guardrails_from_metadata(post_auth_metadata), + *_auth_guardrails_from_metadata(post_auth_team_metadata), + *_auth_guardrails_from_metadata(post_auth_project_metadata), + *_auth_guardrails_from_metadata(resolved_auth_metadata), + ), + request_guardrail_names, + ) + merged_guardrails: Final = (*request_guardrails, *auth_guardrails) + guardrail_request_data: Final = { + **base_request_data, + "metadata": { + **metadata, + **resolved_auth_metadata, + "guardrails": list(merged_guardrails), + }, + } return await proxy_logging_obj.post_mcp_call_hook( response=result, - request_data=( - litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data) - ), - user_api_key_dict=user_api_key_auth, + request_data=guardrail_request_data, + user_api_key_dict=post_auth, ) +def _guardrails_from_metadata(metadata: Mapping[str, object] | None) -> tuple[object, ...]: + if metadata is None: + return () + guardrails_value: Final = metadata.get("guardrails") + if not isinstance(guardrails_value, list): + return () + return TypeAdapter(tuple[object, ...]).validate_python(guardrails_value) + + +def _guardrail_metadata_for_legacy_resolver(metadata: Mapping[str, object]) -> dict[str, object]: + return { + **metadata, + "guardrails": [entry for entry in _guardrails_from_metadata(metadata) if isinstance(entry, str)], + } + + +def _auth_guardrails_from_metadata(metadata: Mapping[str, object]) -> tuple[object, ...]: + return tuple( + TypeAdapter(Mapping[object, object]).validate_python(guardrail) if isinstance(guardrail, Mapping) else guardrail + for guardrail in _guardrails_from_metadata(metadata) + if isinstance(guardrail, (str, Mapping)) + ) + + +def _guardrail_names_from_entry(entry: object) -> frozenset[str]: + if isinstance(entry, str): + return frozenset({entry}) + if not isinstance(entry, Mapping): + return frozenset() + typed_entry: Final[Mapping[object, object]] = TypeAdapter(Mapping[object, object]).validate_python(entry) + return frozenset(name for name in typed_entry if isinstance(name, str)) + + +def _deduplicate_auth_guardrails( + guardrails: tuple[object, ...], + request_guardrail_names: frozenset[str], +) -> tuple[object, ...]: + def keep_distinct( + accumulated: tuple[tuple[object, ...], frozenset[str]], + entry: object, + ) -> tuple[tuple[object, ...], frozenset[str]]: + kept_entries, kept_names = accumulated + entry_names: Final = _guardrail_names_from_entry(entry) + if entry_names & (request_guardrail_names | kept_names): + return accumulated + return (*kept_entries, entry), kept_names | entry_names + + return reduce(keep_distinct, guardrails, ((), frozenset()))[0] + + def suppress_completed_success_logging(logging_obj: LiteLLMLoggingObj) -> None: """An interim ``InputRequiredResult`` is not a completed call, so the ``@client`` wrapper on ``call_mcp_tool`` must not run the success handlers for it when the coroutine returns.""" @@ -2607,6 +2861,14 @@ def _single_scoped_server( return allowed_mcp_servers[0] +async def resolve_scoped_mcp_server( + user_api_key_auth: UserAPIKeyAuth, + mcp_servers: Sequence[str], +) -> MCPServer | None: + allowed_mcp_servers: Final = await _get_allowed_mcp_servers(user_api_key_auth, mcp_servers) + return _single_scoped_server(mcp_servers, allowed_mcp_servers) + + @client @catalog_operation(lambda: global_mcp_server_manager) async def call_mcp_tool( @@ -2623,6 +2885,7 @@ async def call_mcp_tool( input_responses: InputResponses | None = None, request_state: str | None = None, incoming_bearer_token: SecretStr | None | EllipsisType = ..., + guardrail_auth: UserAPIKeyAuth | None = None, **kwargs: Any, ) -> CallToolResult | InputRequiredResult: """ @@ -2686,6 +2949,7 @@ async def call_mcp_tool( request_state=request_state, incoming_bearer_token=incoming_bearer_token, requested_server_id=scoped_server.server_id if scoped_server is not None else None, + guardrail_auth=guardrail_auth, **kwargs, ) except Exception as e: @@ -2874,6 +3138,7 @@ async def _handle_managed_mcp_tool( litellm_logging_obj: LiteLLMLoggingObj | None = None, host_progress_callback: ProgressCallback | None = None, guardrail_context: Mapping[str, object] | None = None, + guardrail_auth: UserAPIKeyAuth | None = None, client_ip: str | None = None, wire_compat: WireCompat = WireCompat.LEGACY, input_responses: InputResponses | None = None, @@ -2903,6 +3168,7 @@ async def _handle_managed_mcp_tool( host_progress_callback=host_progress_callback, litellm_logging_obj=litellm_logging_obj, guardrail_context=guardrail_context, + guardrail_auth=guardrail_auth, wire_compat=wire_compat, input_responses=input_responses, request_state=request_state, @@ -3112,7 +3378,16 @@ async def _execute_mcp_server_tool_call( raw_headers=raw_headers, client_ip=_client_ip, ) - if user_api_key_auth is not None: + guardrail_auth: Final = ( + await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, + tool_name=params.name, + mcp_servers=mcp_servers, + ) + if user_api_key_auth is not None + else None + ) + if guardrail_auth is not None: data = await add_litellm_data_to_request( data=body_data, request=request, @@ -3124,9 +3399,7 @@ async def _execute_mcp_server_tool_call( # half (an already-over-budget team stops granting) lives in the source gate. # Authorization is unaffected: it ran before this, and the union is resolved # from the untouched auth object passed to call_mcp_tool below. - user_api_key_dict=await MCPRequestHandler.billing_auth_for_tool_call( - user_api_key_auth, tool_name=params.name - ), + user_api_key_dict=guardrail_auth, proxy_config=proxy_config, ) else: @@ -3145,6 +3418,7 @@ async def _execute_mcp_server_tool_call( input_responses=params.input_responses, request_state=params.request_state, incoming_bearer_token=context.incoming_bearer_token, + guardrail_auth=guardrail_auth, **data, # for logging ) except MCPMissingUserEnvVarsError as e: @@ -3599,6 +3873,7 @@ class GatewayOperations: client_ip=_client_ip, host_progress_callback=operation.host_progress_callback, guardrail_context=operation.guardrail_context, + guardrail_auth=operation.guardrail_auth, wire_compat=context.wire_compat, input_responses=operation.input_responses, request_state=operation.request_state, diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index b96b6745951..ee063b81ae4 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,5 +1,6 @@ import asyncio import importlib +import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime @@ -12,7 +13,7 @@ import anyio import httpx import httpx2 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from starlette.datastructures import Headers from typing_extensions import ReadOnly @@ -48,6 +49,7 @@ from litellm.proxy._experimental.mcp_server.ui_session_utils import ( ) from litellm.proxy._experimental.mcp_server.utils import ( MCPMissingUserEnvVarsError, + add_server_prefix_to_name, get_server_prefix, merge_mcp_headers, ) @@ -62,7 +64,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro from litellm.responses.mcp.request_context import MCPRequestContext if TYPE_CHECKING: - from mcp.types import CallToolResult + from mcp.types import CallToolResult, InputRequiredResult from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload @@ -226,6 +228,7 @@ if MCP_AVAILABLE: from mcp.types import Tool as MCPTool from litellm.experimental_mcp_client.client import MCPClient, as_mcp_read_timeout + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.litellm_proxy.skills.skill_search import ( DEFAULT_SKILL_SEARCH_TOP_K, ) @@ -249,6 +252,7 @@ if MCP_AVAILABLE: filter_tools_by_allowed_tools, filter_tools_by_key_team_permissions, fire_mcp_tool_call_failure_logging, + resolve_native_tool_name_for_server, ) from litellm.proxy._experimental.mcp_server.server import ( # noqa: F401 # legacy module exports _apply_toolset_scope, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -1215,6 +1219,102 @@ if MCP_AVAILABLE: "message": f"An unexpected error occurred: {e}", } + async def _execute_rest_mcp_tool( + request: Request, + tool_name: str, + tool_arguments: Mapping[str, object], + allowed_mcp_servers: Sequence[MCPServer], + canonical_server_id: str, + target_server: MCPServer | None, + user_api_key_dict: UserAPIKeyAuth, + data: Mapping[str, object], + logging_obj: "LiteLLMLoggingObj | None", + guardrail_auth: UserAPIKeyAuth, + ) -> "tuple[CallToolResult | InputRequiredResult, datetime]": + ( + mcp_auth_header, + mcp_server_auth_headers, + raw_headers_from_request, + caller_oauth2_headers_from_request, + ) = _extract_mcp_headers_from_request(request, user_api_key_dict) + typed_data: Final = TypeAdapter(dict[str, object]).validate_python(data) + request_metadata: Final = TypeAdapter(dict[str, object]).validate_python(typed_data.get("metadata") or {}) + user_api_key_auth_from_metadata: Final = request_metadata.get("user_api_key_auth") + request_auth: Final = ( + {"user_api_key_auth": TypeAdapter(UserAPIKeyAuth).validate_python(user_api_key_auth_from_metadata)} + if user_api_key_auth_from_metadata is not None + else {} + ) + request_headers: Final = { + **({"mcp_auth_header": mcp_auth_header} if mcp_auth_header else {}), + **({"mcp_server_auth_headers": mcp_server_auth_headers} if mcp_server_auth_headers else {}), + "raw_headers": raw_headers_from_request, + } + execution_data: Final = MappingProxyType({**typed_data, **request_auth, **request_headers}) + _stamp_mcp_tool_metadata(logging_obj, canonical_server_id, tool_name) + user_oauth_extra_headers: Final = ( + await _get_user_oauth_extra_headers(target_server, user_api_key_dict) if target_server is not None else None + ) + caller_oauth2_headers: Final = ( + caller_oauth2_headers_from_request + if target_server is not None and target_server.auth_type in _CLIENT_FORWARDED_TOKEN_AUTH_TYPES + else None + ) + tool_start_time: Final = datetime.now() + executed: Final = await execute_mcp_tool( + name=tool_name, + arguments=TypeAdapter(dict[str, object]).validate_python(tool_arguments), + allowed_mcp_servers=list(allowed_mcp_servers), + start_time=tool_start_time, + user_api_key_auth=TypeAdapter(UserAPIKeyAuth | None).validate_python( + execution_data.get("user_api_key_auth") + ), + mcp_auth_header=TypeAdapter(str | None).validate_python(execution_data.get("mcp_auth_header")), + mcp_server_auth_headers=TypeAdapter(dict[str, dict[str, str]] | None).validate_python( + execution_data.get("mcp_server_auth_headers") + ), + oauth2_headers=user_oauth_extra_headers or caller_oauth2_headers, + raw_headers=TypeAdapter(dict[str, str] | None).validate_python(execution_data.get("raw_headers")), + client_ip=IPAddressUtils.get_mcp_client_ip(request), + litellm_logging_obj=logging_obj, + guardrail_context=MCPRequestContext.resolve_guardrail_context(execution_data), + requested_server_id=canonical_server_id, + incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token(request.headers, user_api_key_dict), + guardrail_auth=guardrail_auth, + ) + return executed, tool_start_time + + async def _rest_tool_call_server_and_auth( + user_api_key_auth: UserAPIKeyAuth, + allowed_mcp_servers: Sequence[MCPServer], + canonical_server_id: str, + tool_name: str, + ) -> tuple[MCPServer | None, UserAPIKeyAuth]: + + target_server: Final = next( + (server for server in allowed_mcp_servers if server.server_id == canonical_server_id), + None, + ) + if target_server is None: + return None, user_api_key_auth + native_tool_name: Final = resolve_native_tool_name_for_server( + tool_name, + target_server, + allowed_mcp_servers, + ) + billing_tool_name: Final = add_server_prefix_to_name( + native_tool_name, + get_server_prefix(target_server), + ) + guardrail_auth: Final = await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, + tool_name=billing_tool_name, + scoped_server=target_server, + native_tool_name=native_tool_name, + allowed_mcp_servers=allowed_mcp_servers, + ) + return target_server, guardrail_auth + @router.post("/tools/call", dependencies=[Depends(user_api_key_auth)]) @catalog_operation(global_manager) async def call_tool_rest_api( @@ -1279,9 +1379,48 @@ if MCP_AVAILABLE: _request_start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the tool start time below try: ( - data, - logging_obj, - ) = await proxy_base_llm_response_processor.common_processing_pre_call_logic( + allowed_mcp_servers, + canonical_server_id, + ) = await _resolve_allowed_mcp_servers_with_ip_filter(request, user_api_key_dict, server_id) + except Exception as access_error: + try: + failure_pre_call_result: Final = ( + await proxy_base_llm_response_processor.common_processing_pre_call_logic( + request=request, + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_type=CallTypes.call_mcp_tool.value, + proxy_logging_obj=proxy_logging_obj, + general_settings=general_settings, + skip_guardrails=True, + logging_auth=user_api_key_dict, + ) + ) + failure_data, failure_logging_candidate = TypeAdapter( + tuple[dict[str, object], object] + ).validate_python(failure_pre_call_result) + if isinstance(failure_logging_candidate, LiteLLMLoggingObj): + _stamp_mcp_tool_metadata(failure_logging_candidate, server_id, tool_name) + await _safe_fire_mcp_tool_call_failure_logging( + failure_logging_candidate, + access_error, + _request_start_time, + user_api_key_dict, + failure_data, + ) + else: + verbose_logger.warning("MCP request processing did not return a logging object") + except Exception as logging_error: # noqa: BLE001 # logging failures must not mask the access error + verbose_logger.warning("MCP tool call access-failure logging failed: %s", logging_error) + raise + target_server, guardrail_auth = await _rest_tool_call_server_and_auth( + user_api_key_dict, + allowed_mcp_servers, + canonical_server_id, + tool_name, + ) + try: + pre_call_result: Final = await proxy_base_llm_response_processor.common_processing_pre_call_logic( request=request, user_api_key_dict=user_api_key_dict, proxy_config=proxy_config, @@ -1289,81 +1428,53 @@ if MCP_AVAILABLE: proxy_logging_obj=proxy_logging_obj, general_settings=general_settings, skip_guardrails=True, + logging_auth=guardrail_auth, ) - - # Extract MCP auth headers from request and add to data dict - ( - mcp_auth_header, - mcp_server_auth_headers, - raw_headers_from_request, - caller_oauth2_headers_from_request, - ) = _extract_mcp_headers_from_request(request, user_api_key_dict) - if mcp_auth_header: - data["mcp_auth_header"] = mcp_auth_header - if mcp_server_auth_headers: - data["mcp_server_auth_headers"] = mcp_server_auth_headers - data["raw_headers"] = raw_headers_from_request - - # Extract user_api_key_auth from metadata and add to top level - # call_mcp_tool expects user_api_key_auth as a top-level parameter - if "metadata" in data and "user_api_key_auth" in data["metadata"]: - data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"] - - _stamp_mcp_tool_metadata(logging_obj, server_id, tool_name) - - # Resolve allowed MCP servers with IP filtering - ( - allowed_mcp_servers, - canonical_server_id, - ) = await _resolve_allowed_mcp_servers_with_ip_filter(request, user_api_key_dict, server_id) - - # Look up per-user OAuth headers for this server (mirrors list_tool_rest_api). - user_oauth_extra_headers: dict[str, str] | None = None - target_server: Final = next( - (s for s in allowed_mcp_servers if s.server_id == canonical_server_id), - None, - ) - if target_server is not None: - user_oauth_extra_headers = await _get_user_oauth_extra_headers(target_server, user_api_key_dict) - caller_oauth2_headers: Final = ( - caller_oauth2_headers_from_request - if target_server is not None and target_server.auth_type in _CLIENT_FORWARDED_TOKEN_AUTH_TYPES - else None + data, logging_candidate = TypeAdapter(tuple[dict[str, object], object]).validate_python(pre_call_result) + logging_obj: Final[LiteLLMLoggingObj | None] = ( + logging_candidate if isinstance(logging_candidate, LiteLLMLoggingObj) else None ) + if logging_obj is None: + verbose_logger.warning("MCP request processing did not return a logging object") - # Call execute_mcp_tool directly (permission checks already done) - _tool_start_time: Final = datetime.now() - executed: Final = await execute_mcp_tool( - name=tool_name, - arguments=tool_arguments, + executed, _tool_start_time = await _execute_rest_mcp_tool( + request=request, + tool_name=tool_name, + tool_arguments=tool_arguments, allowed_mcp_servers=allowed_mcp_servers, - start_time=_tool_start_time, - user_api_key_auth=data.get("user_api_key_auth"), - mcp_auth_header=data.get("mcp_auth_header"), - mcp_server_auth_headers=data.get("mcp_server_auth_headers"), - oauth2_headers=user_oauth_extra_headers or caller_oauth2_headers, - raw_headers=data.get("raw_headers"), - client_ip=IPAddressUtils.get_mcp_client_ip(request), - litellm_logging_obj=data.get("litellm_logging_obj"), - guardrail_context=MCPRequestContext.resolve_guardrail_context(data), - requested_server_id=canonical_server_id, - incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token( - request.headers, user_api_key_dict - ), + canonical_server_id=canonical_server_id, + target_server=target_server, + user_api_key_dict=user_api_key_dict, + data=data, + logging_obj=logging_obj, + guardrail_auth=guardrail_auth, ) result: Final = complete_call_tool_result(executed, WireCompat.LEGACY) - except Exception as e: - request_data: Final = proxy_base_llm_response_processor.data - await _safe_fire_mcp_tool_call_failure_logging( - request_data.get("litellm_logging_obj"), e, _request_start_time, user_api_key_dict, request_data - ) - raise + finally: + failure: Final = sys.exc_info()[1] + if isinstance(failure, Exception): + request_data: Final[Mapping[str, object]] = TypeAdapter(Mapping[str, object]).validate_python( + proxy_base_llm_response_processor.data + ) + request_logging_obj_candidate: Final = request_data.get("litellm_logging_obj") + request_logging_obj: Final[LiteLLMLoggingObj | None] = ( + request_logging_obj_candidate + if isinstance(request_logging_obj_candidate, LiteLLMLoggingObj) + else None + ) + await _safe_fire_mcp_tool_call_failure_logging( + request_logging_obj, + failure, + _request_start_time, + guardrail_auth, + request_data, + ) return await _safe_fire_mcp_tool_call_logging( logging_obj, result, _tool_start_time, datetime.now(), - user_api_key_auth=user_api_key_dict, + user_api_key_auth=guardrail_auth, request_data=data, ) except MCPMissingUserEnvVarsError as e: diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index 18642a9162c..371a1ff6fc9 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -600,6 +600,8 @@ async def handle_mcp_proxy_tool( raw_headers: dict[str, str] | None = None, # mutable-ok: preserve request headers litellm_logging_obj: LiteLLMLoggingObj | None = None, incoming_bearer_token: SecretStr | None | EllipsisType = ..., + guardrail_auth: UserAPIKeyAuth | None = None, + resolved_tool: Tool | None | EllipsisType = ..., ) -> CallToolResult: from fastapi import HTTPException @@ -646,18 +648,22 @@ async def handle_mcp_proxy_tool( tool_id: Final = arguments.get("tool_id") tool: Final = ( - await find_mcp_proxy_tool( - tool_id, - user_api_key_auth=user_api_key_dict, - mcp_servers=mcp_servers, - client_ip=client_ip, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, + resolved_tool + if resolved_tool is not ... + else ( + await find_mcp_proxy_tool( + tool_id, + user_api_key_auth=user_api_key_dict, + mcp_servers=mcp_servers, + client_ip=client_ip, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + if isinstance(tool_id, str) + else None ) - if isinstance(tool_id, str) - else None ) if tool is None: return _text_tool_result("Unknown or unauthorized tool_id", is_error=True) @@ -687,6 +693,7 @@ async def handle_mcp_proxy_tool( raw_headers=raw_headers, litellm_logging_obj=litellm_logging_obj, incoming_bearer_token=incoming_bearer_token, + guardrail_auth=guardrail_auth, ) @@ -704,6 +711,7 @@ async def handle_mcp_tool_call( requested_server_id: str | None = None, guardrail_context: Mapping[str, object] | None = None, incoming_bearer_token: SecretStr | None | EllipsisType = ..., + guardrail_auth: UserAPIKeyAuth | None = None, ) -> CallToolResult: from litellm.proxy._experimental.mcp_server.operations import ( _get_allowed_mcp_servers, @@ -745,6 +753,7 @@ async def handle_mcp_tool_call( litellm_logging_obj=litellm_logging_obj, requested_server_id=requested_server_id, guardrail_context=guardrail_context, + guardrail_auth=guardrail_auth, incoming_bearer_token=incoming_bearer_token, ) return complete_call_tool_result(result, WireCompat.LEGACY) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 5c8849e50ca..c11d296837e 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -731,6 +731,10 @@ async def invoke_agent_a2a( ) user_api_key_dict.invoked_agent_id = agent.agent_id + strict_policy: Final = agent.identity_managed or bool(user_api_key_dict.requires_fresh_policy) + user_api_key_auth_for_call: Final = await AgentRequestHandler.personal_key_agent_billing_auth( + user_api_key_dict, agent.agent_id, strict=strict_policy + ) _enforce_inbound_trace_id(agent, request) @@ -790,7 +794,7 @@ async def invoke_agent_a2a( data, logging_obj = await processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_for_call, proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, route_type="asend_message", @@ -889,7 +893,7 @@ async def invoke_agent_a2a( try: response = await proxy_logging_obj.post_call_success_hook( - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_for_call, data=data, response=response, ) @@ -924,7 +928,7 @@ async def invoke_agent_a2a( metadata=data.get("metadata", {}), proxy_server_request=data.get("proxy_server_request"), agent_extra_headers=agent_extra_headers, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_for_call, request_data=data, proxy_logging_obj=proxy_logging_obj, served_version=served_version, @@ -990,7 +994,7 @@ async def invoke_agent_a2a( response = LiteLLMSendMessageResponse.from_dict(result, request_id=request_id) response = await proxy_logging_obj.post_call_success_hook( - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_for_call, data=data, response=response, ) @@ -1017,7 +1021,7 @@ async def invoke_agent_a2a( request_id=request_id, extra_headers=agent_extra_headers, proxy_logging_obj=proxy_logging_obj, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=user_api_key_auth_for_call, request_data=data, served_version=served_version, ) diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 48213625c4f..dd002a2763e 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -8,6 +8,7 @@ Follows the same pattern as MCP permission handling. import asyncio from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass +from itertools import chain from types import MappingProxyType from typing import Final, Protocol, TypeAlias @@ -45,6 +46,11 @@ class RestrictedAgentAccess: agent_ids: frozenset[str] +@dataclass(frozen=True, slots=True) +class AgentPolicyUnavailable: + team_id: str + + AgentAccess: TypeAlias = UnrestrictedAgentAccess | RestrictedAgentAccess @@ -83,6 +89,16 @@ def _restricted_ids(access: AgentAccess) -> frozenset[str] | None: return _to_stable_ids(access.agent_ids) +def _union_agent_access(accesses: Sequence[AgentAccess]) -> AgentAccess: + if accesses and all(isinstance(access, UnrestrictedAgentAccess) for access in accesses): + return UnrestrictedAgentAccess() + return RestrictedAgentAccess( + frozenset( + chain.from_iterable(access.agent_ids for access in accesses if isinstance(access, RestrictedAgentAccess)) + ) + ) + + def _intersect_agent_access(key_access: AgentAccess, team_access: AgentAccess) -> AgentAccess: key_ids: Final = _restricted_ids(key_access) team_ids: Final = _restricted_ids(team_access) @@ -251,7 +267,11 @@ class AgentRequestHandler: ) return target.agent_id in explicit - match await AgentRequestHandler.resolve_agent_access(user_api_key_auth, resolve_ceiling): + match await AgentRequestHandler.resolve_agent_access( + user_api_key_auth, + resolve_ceiling, + strict=bool(user_api_key_auth is not None and user_api_key_auth.requires_fresh_policy), + ): case UnrestrictedAgentAccess(): return True case RestrictedAgentAccess(allowed_agent_ids): @@ -370,6 +390,200 @@ class AgentRequestHandler: verbose_logger.warning("Failed to get allowed agents for key: %s", e) return UnrestrictedAgentAccess() + @staticmethod + async def _personal_key_owner_team_agent_access(user_api_key_auth: UserAPIKeyAuth, *, strict: bool) -> AgentAccess: + """Union owner-team access; teams without a rule add nothing once any team declares one.""" + from litellm.proxy.auth.auth_checks import personal_key_owner_teams_with_status + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + try: + owner_teams: Final = await personal_key_owner_teams_with_status( + valid_token=user_api_key_auth, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=strict, + ) + except Exception as e: # noqa: BLE001 # fail closed: an owner whose teams cannot be read grants nothing + if strict: + raise HTTPException(503, "Agent invocation policy is unavailable") from e + verbose_logger.warning("Failed to load owner teams for personal key agent access: %s", e) + return RestrictedAgentAccess(frozenset()) + team_results: Final = await asyncio.gather( + *(AgentRequestHandler._owner_team_agent_grants(team, strict=strict) for team in owner_teams.teams) + ) + withheld_results: Final = await asyncio.gather( + *(AgentRequestHandler._owner_team_agent_grants(team, strict=strict) for team in owner_teams.withheld) + ) + if any(isinstance(access, AgentPolicyUnavailable) for access in team_results): + raise HTTPException(503, "Agent invocation policy is unavailable") + team_accesses: Final = tuple( + access for access in team_results if not isinstance(access, AgentPolicyUnavailable) + ) + withheld_restrictions: Final = tuple( + RestrictedAgentAccess(frozenset()) + for access in withheld_results + if isinstance(access, (AgentPolicyUnavailable, RestrictedAgentAccess)) + ) + accesses: Final = ( + (*team_accesses, *withheld_restrictions, RestrictedAgentAccess(frozenset())) + if owner_teams.unreadable + else (*team_accesses, *withheld_restrictions) + ) + return _union_agent_access(accesses) + + @staticmethod + async def personal_key_agent_billing_auth( + auth: UserAPIKeyAuth, + agent_id: str, + *, + strict: bool = False, + ) -> UserAPIKeyAuth: + try: + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + from litellm.proxy.auth.auth_checks import ( + personal_key_inherits_team_grants, + personal_key_owner_teams, + ) + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + if not personal_key_inherits_team_grants(auth): + return auth + teams: Final = await personal_key_owner_teams( + valid_token=auth, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=strict, + ) + accesses: Final = await asyncio.gather( + *(AgentRequestHandler._owner_team_agent_grants(team, strict=strict) for team in teams) + ) + stable_agent_id: Final = global_agent_registry.stable_agent_id(agent_id) + + restricted_sources: Final = tuple( + (team, access) + for team, access in zip(teams, accesses, strict=True) + if isinstance(access, RestrictedAgentAccess) and stable_agent_id in _to_stable_ids(access.agent_ids) + ) + unrestricted_sources: Final = ( + tuple(zip(teams, accesses, strict=True)) + if not restricted_sources + and teams + and all(isinstance(access, UnrestrictedAgentAccess) for access in accesses) + else () + ) + sources: Final = restricted_sources or unrestricted_sources + if not sources: + verbose_logger.warning("No owner team grants agent %s; billing against the personal key", agent_id) + return auth + team: Final = min((source[0] for source in sources), key=lambda source: source.team_id) + except Exception as e: # noqa: BLE001 # attribution must never fail an authorized call + verbose_logger.warning("Failed to attribute agent %s spend to an owner team: %s", agent_id, e) + return auth + return await AgentRequestHandler._personal_key_agent_billing_auth_for_team( + auth, + agent_id, + team, + strict=strict, + ) + + @staticmethod + async def _personal_key_agent_billing_auth_for_team( + auth: UserAPIKeyAuth, + agent_id: str, + team: LiteLLM_TeamTable, + *, + strict: bool, + ) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_checks import get_team_membership + from litellm.proxy.auth.team_grants import team_grant_auth_update, team_grant_billing_auth_update + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + try: + membership: Final = ( + await get_team_membership( + user_id=auth.user_id, + team_id=team.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=strict, + ) + if auth.user_id and prisma_client is not None + else None + ) + team_auth_update: Final = await team_grant_billing_auth_update(auth, team, membership) + except Exception as e: # noqa: BLE001 # a chosen team's controls must survive billing lookup failures + verbose_logger.warning( + "Failed to load billing controls for agent %s from owner team %s: %s", + agent_id, + team.team_id, + e, + ) + return auth.model_copy(update=team_grant_auth_update(team, None, auth.user_id, key_org_id=auth.org_id)) + return auth.model_copy(update=team_auth_update) + + @staticmethod + async def _owner_team_agent_grants( + team: LiteLLM_TeamTable, + *, + strict: bool, + ) -> AgentAccess | AgentPolicyUnavailable: + if team.object_permission_id is not None and team.object_permission is None: + if strict: + return AgentPolicyUnavailable(team_id=team.team_id) + verbose_logger.warning("Owner team %s has an unreadable object_permission, granting nothing", team.team_id) + return RestrictedAgentAccess(frozenset()) + try: + return await AgentRequestHandler._team_agent_grants(team, strict=strict) + except Exception as e: # noqa: BLE001 # per-team isolation: a team whose grants cannot be read grants nothing + if strict: + return AgentPolicyUnavailable(team_id=team.team_id) + verbose_logger.warning("Failed to get allowed agents for owner team %s: %s", team.team_id, e) + return RestrictedAgentAccess(frozenset()) + + @staticmethod + async def _team_agent_grants(team: LiteLLM_TeamTable, *, strict: bool) -> AgentAccess: + """The agents one loaded team grants: its object_permission agents plus its access groups. + Declared-but-empty grants stay restricted; a team with no grant at all does not restrict.""" + # 1. Get agents from object_permission (native permissions) + object_permissions: Final = team.object_permission + direct_agents: Final = tuple(object_permissions.agents or () if object_permissions is not None else ()) + declared_access_groups: Final = tuple( + object_permissions.agent_access_groups or () if object_permissions is not None else () + ) + # 2. Also include agents from team's access_group_ids (unified access groups) + team_access_group_ids: Final = tuple(team.access_group_ids or ()) + + if not direct_agents and not declared_access_groups and not team_access_group_ids: + return UnrestrictedAgentAccess() + + access_group_agents: Final = ( + tuple( + await AgentRequestHandler._get_agents_from_access_groups(declared_access_groups, check_db_only=strict) + ) + if declared_access_groups + else () + ) + unified_agents: Final = ( + tuple( + await AgentRequestHandler._get_unified_access_group_agents(team_access_group_ids, check_db_only=strict) + ) + if team_access_group_ids + else () + ) + + return RestrictedAgentAccess(frozenset(direct_agents + access_group_agents + unified_agents)) + @staticmethod async def _get_allowed_agents_for_team( user_api_key_auth: UserAPIKeyAuth | None = None, @@ -389,7 +603,11 @@ class AgentRequestHandler: return UnrestrictedAgentAccess() if user_api_key_auth.team_id is None: - return UnrestrictedAgentAccess() + from litellm.proxy.auth.auth_checks import personal_key_inherits_team_grants + + if not personal_key_inherits_team_grants(user_api_key_auth): + return UnrestrictedAgentAccess() + return await AgentRequestHandler._personal_key_owner_team_agent_access(user_api_key_auth, strict=strict) try: from litellm.proxy.auth.auth_checks import get_team_object @@ -415,38 +633,7 @@ class AgentRequestHandler: if team_obj is None: return RestrictedAgentAccess(frozenset()) if strict else UnrestrictedAgentAccess() - # 1. Get agents from object_permission (native permissions) - object_permissions: Final = team_obj.object_permission - direct_agents: Final = tuple(object_permissions.agents or () if object_permissions is not None else ()) - declared_access_groups: Final = tuple( - object_permissions.agent_access_groups or () if object_permissions is not None else () - ) - # 2. Also include agents from team's access_group_ids (unified access groups) - team_access_group_ids: Final = tuple(team_obj.access_group_ids or ()) - - if not direct_agents and not declared_access_groups and not team_access_group_ids: - return UnrestrictedAgentAccess() - - access_group_agents: Final = ( - tuple( - await AgentRequestHandler._get_agents_from_access_groups( - declared_access_groups, check_db_only=strict - ) - ) - if declared_access_groups - else () - ) - unified_agents: Final = ( - tuple( - await AgentRequestHandler._get_unified_access_group_agents( - team_access_group_ids, check_db_only=strict - ) - ) - if team_access_group_ids - else () - ) - - return RestrictedAgentAccess(frozenset(direct_agents + access_group_agents + unified_agents)) + return await AgentRequestHandler._team_agent_grants(team_obj, strict=strict) except Exception as e: if strict: raise HTTPException(503, "Agent invocation policy is unavailable") from e diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index a04fba2d905..0f94d2952ec 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -18,7 +18,7 @@ from dataclasses import dataclass from functools import partial from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, Protocol, TypeAlias +from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, Protocol, TypeAlias from fastapi import HTTPException, Request, status from pydantic import BaseModel, TypeAdapter, ValidationError @@ -2585,6 +2585,10 @@ def _membership_from_cached_payload( return cached_membership if cached_membership is not None else _TEAM_MEMBERSHIP_CACHE_MISS +def _raise_no_db_connected() -> NoReturn: + raise Exception("No db connected") + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def _fetch_team_membership_from_db( @@ -2594,9 +2598,13 @@ async def _fetch_team_membership_from_db( user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None = None, proxy_logging_obj: ProxyLogging | None = None, + *, + use_writer: bool = False, ) -> LiteLLM_TeamMembership | None: _ = parent_otel_span, proxy_logging_obj - response: Final = await _dictable_table(TeamMembershipRepository(prisma_client), "team_membership").find_unique( + response: Final = await _dictable_table( + TeamMembershipRepository(prisma_client, use_writer=use_writer), "team_membership" + ).find_unique( where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}}, include={"litellm_budget_table": True}, ) @@ -2650,6 +2658,8 @@ async def get_team_membership( user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None = None, proxy_logging_obj: ProxyLogging | None = None, + *, + check_db_only: bool = False, ) -> Optional["LiteLLM_TeamMembership"]: """ Returns team membership object if user is member of team. @@ -2661,6 +2671,19 @@ async def get_team_membership( _key: Final = team_membership_reservation_cache_key(user_id=user_id, team_id=team_id) + if check_db_only: + if prisma_client is None: + _raise_no_db_connected() + return await _fetch_team_membership_from_db( + user_id=user_id, + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + use_writer=True, + ) + l1_cached: Final[object] = await user_api_key_cache.async_get_cache(key=_key, local_only=True) l1_membership: Final = _membership_from_cached_payload(l1_cached) if not isinstance(l1_membership, _TeamMembershipCacheMiss): @@ -2671,7 +2694,7 @@ async def get_team_membership( return _membership_from_shared_load(await asyncio.shield(inflight)) if prisma_client is None: - raise Exception("No db connected") + _raise_no_db_connected() task: Final = asyncio.ensure_future( _load_team_membership_on_cache_miss( @@ -4674,6 +4697,8 @@ async def get_mcp_server_ids_from_access_groups( user_api_key_cache: UserApiKeyCache | None = None, proxy_logging_obj: ProxyLogging | None = None, check_db_only: bool = False, + *, + strict_errors: bool = False, ) -> list[str]: """ Collect MCP server IDs from unified access groups. @@ -4686,6 +4711,7 @@ async def get_mcp_server_ids_from_access_groups( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, check_db_only=check_db_only, + strict_errors=strict_errors, ) @@ -5470,6 +5496,8 @@ async def _personal_key_owner( prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, + *, + check_db_only: bool = False, ) -> LiteLLM_UserTable | None: if valid_token.user_id is None: return None @@ -5481,6 +5509,7 @@ async def _personal_key_owner( user_id_upsert=False, parent_otel_span=valid_token.parent_otel_span, proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, ) except Exception as e: # noqa: BLE001 # fail closed: an owner that cannot be loaded grants nothing verbose_proxy_logger.warning( @@ -5870,13 +5899,32 @@ def personal_key_team_model_access_applies(valid_token: UserAPIKeyAuth | None) - ) -async def _load_personal_key_team( +def personal_key_inherits_team_grants(valid_token: UserAPIKeyAuth | None) -> bool: + """MCP server, MCP tool and agent inheritance is for a virtual key only. A keyless auth with no + team (one grant source of an admitted subject, a JWT user) resolves as the plain caller it is.""" + return ( + personal_key_team_model_access_applies(valid_token) + and valid_token is not None + and valid_token.api_key is not None + ) + + +@dataclass(frozen=True, slots=True) +class _PersonalKeyTeamLoad: + team: LiteLLM_TeamTableCachedObj | None + unreadable: bool + withheld_team: LiteLLM_TeamTableCachedObj | None + + +async def _load_personal_key_team_with_status( team_id: str, valid_token: UserAPIKeyAuth, prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, -) -> LiteLLM_TeamTableCachedObj | None: + *, + check_db_only: bool = False, +) -> _PersonalKeyTeamLoad: try: team_object: Final = await get_team_object( team_id=team_id, @@ -5884,13 +5932,36 @@ async def _load_personal_key_team( user_api_key_cache=user_api_key_cache, parent_otel_span=valid_token.parent_otel_span, proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, ) except Exception as e: # noqa: BLE001 # fail closed: a team that cannot be loaded grants nothing verbose_proxy_logger.warning( "Personal key team model access: team=%s lookup failed, granting nothing: %s", team_id, e ) - return None - return None if team_object.blocked is True else team_object + return _PersonalKeyTeamLoad(team=None, unreadable=True, withheld_team=None) + if team_object.blocked is True: + return _PersonalKeyTeamLoad(team=None, unreadable=False, withheld_team=team_object) + return _PersonalKeyTeamLoad(team=team_object, unreadable=False, withheld_team=None) + + +async def _load_personal_key_team( + team_id: str, + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, + *, + check_db_only: bool = False, +) -> LiteLLM_TeamTableCachedObj | None: + loaded: Final = await _load_personal_key_team_with_status( + team_id=team_id, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + ) + return loaded.team async def _load_personal_key_teams( @@ -5899,6 +5970,8 @@ async def _load_personal_key_teams( prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, + *, + check_db_only: bool = False, ) -> tuple[LiteLLM_TeamTableCachedObj | None, ...]: return tuple( await asyncio.gather( @@ -5909,6 +5982,7 @@ async def _load_personal_key_teams( prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, ) for team_id in dict.fromkeys(user_object.teams) ) @@ -5988,6 +6062,8 @@ async def _personal_key_listing_teams( prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache | None, proxy_logging_obj: ProxyLogging | None, + *, + check_db_only: bool = False, ) -> tuple[LiteLLM_TeamTableCachedObj | None, ...]: if ( user_api_key_dict.user_id is None @@ -6001,6 +6077,7 @@ async def _personal_key_listing_teams( prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, ) if owner is None: return () @@ -6010,9 +6087,177 @@ async def _personal_key_listing_teams( prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, ) +async def _personal_key_team_within_budget( + team: LiteLLM_TeamTableCachedObj, + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, + *, + check_db_only: bool = False, +) -> bool: + source: Final = valid_token.model_copy( + update={"team_id": team.team_id, "org_id": valid_token.org_id or team.organization_id} + ) + try: + await team_max_budget_check( + team_object=team, + valid_token=source, + proxy_logging_obj=proxy_logging_obj, + ) + await _team_multi_budget_check(team_object=team) + if check_db_only: + team_member_user_id: Final = valid_token.user_id + if team_member_user_id is None: + return False + team_membership: Final = await get_team_membership( + user_id=team_member_user_id, + team_id=team.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=True, + ) + await _check_team_member_budget( + team_object=team, + user_object=None, + valid_token=source, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + team_membership=team_membership, + team_membership_loaded=True, + ) + else: + await _check_team_member_budget( + team_object=team, + user_object=None, + valid_token=source, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await organization_max_budget_check( + valid_token=source, + team_object=team, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except litellm.BudgetExceededError: + return False + except Exception as e: # noqa: BLE001 # fail closed: a team whose budget cannot be checked grants nothing + verbose_proxy_logger.warning("Failed to check budget for personal key team %s, skipping: %s", team.team_id, e) + return False + return True + + +@dataclass(frozen=True, slots=True) +class PersonalKeyOwnerTeams: + teams: tuple[LiteLLM_TeamTableCachedObj, ...] + unreadable: bool + withheld: tuple[LiteLLM_TeamTableCachedObj, ...] + + +async def personal_key_owner_teams_with_status( + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache | None, + proxy_logging_obj: ProxyLogging | None, + *, + check_db_only: bool = False, +) -> PersonalKeyOwnerTeams: + """The owner's current, loadable, unblocked, within-budget teams: the grant sources a personal key inherits + MCP server, MCP tool and agent access from under ``personal_key_model_access_from_teams``.""" + if valid_token.user_id is None or prisma_client is None or user_api_key_cache is None or proxy_logging_obj is None: + return PersonalKeyOwnerTeams(teams=(), unreadable=False, withheld=()) + owner: Final = await _personal_key_owner( + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + ) + if owner is None: + return PersonalKeyOwnerTeams(teams=(), unreadable=False, withheld=()) + team_ids: Final = tuple(team_id for team_id in dict.fromkeys(owner.teams) if team_id and team_id != UI_TEAM_ID) + team_loads: Final = await asyncio.gather( + *( + _load_personal_key_team_with_status( + team_id=team_id, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + ) + for team_id in team_ids + ) + ) + blocked_teams: Final = tuple( + load.withheld_team + for load in team_loads + if load.withheld_team is not None + and (valid_token.org_id is None or load.withheld_team.organization_id in (None, valid_token.org_id)) + ) + loaded_teams: Final = tuple(load.team for load in team_loads if load.team is not None) + eligible: Final = tuple( + team + for team in loaded_teams + if valid_token.org_id is None or team.organization_id in (None, valid_token.org_id) + ) + if not eligible: + return PersonalKeyOwnerTeams( + teams=(), + unreadable=any(load.unreadable for load in team_loads), + withheld=blocked_teams, + ) + within_budget: Final = await asyncio.gather( + *( + _personal_key_team_within_budget( + team=team, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + ) + for team in eligible + ) + ) + return PersonalKeyOwnerTeams( + teams=tuple(team for team, allowed in zip(eligible, within_budget, strict=True) if allowed), + unreadable=any(load.unreadable for load in team_loads), + withheld=( + *blocked_teams, + *(team for team, allowed in zip(eligible, within_budget, strict=True) if not allowed), + ), + ) + + +async def personal_key_owner_teams( + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache | None, + proxy_logging_obj: ProxyLogging | None, + *, + check_db_only: bool = False, +) -> tuple[LiteLLM_TeamTableCachedObj, ...]: + return ( + await personal_key_owner_teams_with_status( + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + check_db_only=check_db_only, + ) + ).teams + + async def personal_key_team_visible_models( models: Sequence[str], user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/auth/team_grants.py b/litellm/proxy/auth/team_grants.py index 450a3ce456e..a48bb12cfb7 100644 --- a/litellm/proxy/auth/team_grants.py +++ b/litellm/proxy/auth/team_grants.py @@ -19,6 +19,7 @@ from litellm.proxy._types import ( LiteLLM_TeamMembership, LiteLLM_TeamTable, Member, + UserAPIKeyAuth, ) from litellm.types.llms.base import LiteLLMBaseModel @@ -74,6 +75,91 @@ class TeamGrants(TypedDict, total=False): team_member_rpm_limit: ReadOnly[int | None] +_TEAM_GRANT_AUTH_FIELDS: Final[frozenset[str]] = frozenset( + { + "team_alias", + "team_tpm_limit", + "team_rpm_limit", + "team_tpd_limit", + "team_max_budget", + "team_soft_budget", + "team_spend", + "team_metadata", + "team_member", + "team_member_spend", + "team_member_tpm_limit", + "team_member_rpm_limit", + } +) + + +def team_grant_auth_update( + team_object: LiteLLM_TeamTable, + team_membership: LiteLLM_TeamMembership | None, + user_id: str | None, + *, + key_org_id: str | None, +) -> Mapping[str, object]: + grant_values: Final = team_grants(team_object, team_membership, user_id) + return MappingProxyType( + { + "team_id": team_object.team_id, + "org_id": key_org_id or team_object.organization_id, + **{field: value for field, value in grant_values.items() if field in _TEAM_GRANT_AUTH_FIELDS}, + } + ) + + +async def team_grant_billing_auth_update( + auth: UserAPIKeyAuth, + team_object: LiteLLM_TeamTable, + team_membership: LiteLLM_TeamMembership | None, +) -> Mapping[str, object]: + team_auth_update: Final = team_grant_auth_update( + team_object, + team_membership, + auth.user_id, + key_org_id=auth.org_id, + ) + if auth.org_id is not None or team_object.organization_id is None: + return team_auth_update + + organization_auth_fields: Final[Mapping[str, object]] = MappingProxyType( + { + "organization_alias": None, + "organization_metadata": None, + "organization_max_budget": None, + "organization_tpm_limit": None, + "organization_rpm_limit": None, + } + ) + from litellm.proxy.auth.auth_checks import get_org_object_for_request + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None: + return MappingProxyType({**team_auth_update, **organization_auth_fields}) + org_object: Final = await get_org_object_for_request( + org_id=team_object.organization_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if org_object is None: + return MappingProxyType({**team_auth_update, **organization_auth_fields}) + budget: Final = org_object.litellm_budget_table + return MappingProxyType( + { + **team_auth_update, + "organization_alias": org_object.organization_alias, + "organization_metadata": org_object.metadata, + "organization_max_budget": budget.max_budget if budget is not None else None, + "organization_tpm_limit": budget.tpm_limit if budget is not None else None, + "organization_rpm_limit": budget.rpm_limit if budget is not None else None, + } + ) + + def _json_columns(team_object: LiteLLM_TeamTable) -> _TeamJsonColumns: try: return _TeamJsonColumns.model_validate(team_object.model_dump(include=_JSON_COLUMNS)) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index b443711752a..24a7b46da4b 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -253,7 +253,9 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, add_litellm_data_to_request, + get_metadata_variable_name, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, sanitize_for_log, @@ -2244,6 +2246,7 @@ class ProxyBaseLLMRequestProcessing: llm_router: Router | None = None, rate_limited_model: str | None = None, skip_guardrails: bool = False, + logging_auth: UserAPIKeyAuth | None = None, ) -> tuple[dict, LiteLLMLoggingObj]: start_time: Final = datetime.now() # start before calling guardrail hooks @@ -2263,6 +2266,14 @@ class ProxyBaseLLMRequestProcessing: version=version, proxy_config=proxy_config, ) + if logging_auth is not None: + metadata_variable_name: Final = get_metadata_variable_name(request) + request_metadata: Final = self.data.get(metadata_variable_name) + if isinstance(request_metadata, dict): + self.data[metadata_variable_name] = { + **TypeAdapter(dict[str, object]).validate_python(request_metadata), + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(logging_auth), + } if not general_settings.get("expose_fallback_errors_to_caller"): self.data.pop("include_fallback_errors", None) if route_type in {"aresponses", "_aresponses_websocket"}: @@ -2285,8 +2296,6 @@ class ProxyBaseLLMRequestProcessing: # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: - from litellm.proxy.litellm_pre_call_utils import get_metadata_variable_name - _metadata_variable_name: Final = get_metadata_variable_name(request) if _metadata_variable_name not in self.data: self.data[_metadata_variable_name] = {} diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 81e77283ed8..d858eff3348 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -105,6 +105,7 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +_MCP_KEY_RPM_LIMITS: Final[TypeAdapter[dict[str, int] | None]] = TypeAdapter(dict[str, int] | None) @dataclass(frozen=True, slots=True) @@ -3000,12 +3001,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): MCP tool calls have no token usage, so only requests_per_unit is set; tokens_per_unit stays None so the TPM reservation path is never engaged. """ - from litellm.proxy.auth.auth_utils import get_key_mcp_rpm_limit - if not mcp_server_name or not user_api_key_dict.api_key: return - mcp_rpm_limit: Final = get_key_mcp_rpm_limit(user_api_key_dict) + metadata: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(user_api_key_dict.metadata or {}) + mcp_rpm_limit: Final = _MCP_KEY_RPM_LIMITS.validate_python(metadata.get("mcp_rpm_limit")) if not mcp_rpm_limit: return @@ -3046,11 +3046,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # team's mcp_rpm_limit. Every applicable team is charged rather than one being picked: the # limiter enforces all descriptors, so each team's own ceiling binds on a call made through # its grant, and there is no arbitrary attribution when several teams grant the same server. - team_limits: Final[list[tuple[str | None, dict[str, int] | None]]] = [] - if user_api_key_dict.team_id: - team_limits.append((user_api_key_dict.team_id, get_team_mcp_rpm_limit(user_api_key_dict))) - for source_team_id, source_limit in (user_api_key_dict.mcp_source_team_rpm_limits or {}).items(): - team_limits.append((source_team_id, source_limit)) + source_team_limits: Final = user_api_key_dict.mcp_source_team_rpm_limits or {} + primary_team_ids: Final = (user_api_key_dict.team_id,) if user_api_key_dict.team_id else () + team_ids: Final = tuple(dict.fromkeys((*primary_team_ids, *source_team_limits))) + team_limits: Final = tuple( + ( + team_id, + get_team_mcp_rpm_limit(user_api_key_dict) or source_team_limits.get(team_id) + if team_id == user_api_key_dict.team_id + else source_team_limits[team_id], + ) + for team_id in team_ids + ) for team_id, mcp_rpm_limit in team_limits: if not team_id or not mcp_rpm_limit: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 06033f092dd..dbeda4e27e3 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3718,7 +3718,7 @@ def _add_guardrails_from_policies_in_metadata( def add_guardrails_from_auth_metadata( user_api_key_dict: UserAPIKeyAuth, - data: dict, # mutable-ok: writes guardrails into the live request dict, same contract as the helpers it wraps + data: dict[str, object], # mutable-ok: writes guardrails into the live request dict, same contract as helpers metadata_variable_name: str, ) -> None: """Resolve key, team, and project guardrails, direct and via policies, onto the request metadata.""" @@ -3776,7 +3776,7 @@ async def move_guardrails_to_metadata( add_guardrails_from_auth_metadata( user_api_key_dict=user_api_key_dict, - data=data, + data=data, # pyright: ignore[reportUnknownArgumentType] # request data retains a legacy dynamic shape metadata_variable_name=_metadata_variable_name, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4ea3100f869..5030b1512a6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -20023,9 +20023,9 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie "personal_key_model_access_from_teams": { "type": "Boolean", "description": ( - "Caps each personal key (a key with no team) to the models its owner's current teams can call, " - "re-evaluated on every request. A user in no team gets no model access. A user in several teams " - "can call any model one of those teams can call. Team keys are unaffected." + "Caps each personal key (a key with no team) to the models, MCP servers, MCP tools and agents its " + "owner's current teams can reach, re-evaluated on every request. A user in no team gets none of them. " + "A user in several teams gets the union of what those teams grant. Team keys are unaffected." ), }, "max_ui_session_budget": { diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index c4414003ca5..f73a8547158 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -23,6 +23,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( is_mcp_admitted_user_subject, ) from litellm.proxy._types import ( + UI_TEAM_ID, LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -8649,7 +8650,11 @@ class TestUserSubjectTeamUnion: "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.get_mcp_server_from_tool_name", MagicMock(return_value=server), ): - billed = await MCPRequestHandler.billing_auth_for_tool_call(auth, tool_name="t-grant/tool_a") + billed = await MCPRequestHandler.billing_auth_for_tool_call( + auth, + tool_name="t-grant/tool_a", + allowed_mcp_servers=(), + ) assert (billed.team_id, billed.org_id) == ("t-grant", "org-team") assert (auth.team_id, auth.org_id) == (None, "org-user-primary"), "authz object must be untouched" @@ -11666,3 +11671,1016 @@ async def test_token_minted_credential_lists_tools_for_every_scope_shape(monkeyp assert database.writer_db.litellm_usertable.find_unique.await_count >= 1 database.writer_db.litellm_verificationtoken.find_unique.assert_not_awaited() assert credential.team_id == "team-eng" and credential.allowed_routes == list(granted.allowed_routes or ()) + + +_PERSONAL_KEY_TEAMS: Final = { + "team-a": ("srv-a", {"srv-a": ["add"]}), + "team-b": ("srv-b", None), +} + + +def _personal_key_team(team_id: str) -> LiteLLM_TeamTable: + server_id, tool_permissions = _PERSONAL_KEY_TEAMS[team_id] + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=[server_id], + mcp_tool_permissions=tool_permissions, + ), + ) + + +def _personal_key_team_granting_tools(team_id: str, tools: tuple[str, ...]) -> LiteLLM_TeamTable: + return _personal_key_team(team_id).model_copy( + update={ + "object_permission": LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=["srv-b"], + mcp_tool_permissions={"srv-b": list(tools)}, + ) + } + ) + + +@pytest.fixture +def personal_key_owner_teams(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest): + """A personal key whose owner is on ``request.param`` teams, resolved against a registry of + srv-a (team-a, tool ``add`` only), srv-b (team-b, every tool), srv-c (granted to no team) and + srv-open (``allow_all_keys``, granted to no team).""" + import litellm + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.auth import auth_checks + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id) + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=list(request.param))), + ) + for server_id in ("srv-a", "srv-b", "srv-c", "srv-open"): + global_mcp_server_manager.registry[server_id] = MCPServer( + server_id=server_id, + name=server_id, + server_name=server_id, + url=f"https://{server_id}.example.com", + transport=MCPTransport.http, + allow_all_keys=server_id == "srv-open", + ) + yield UserAPIKeyAuth(api_key="sk-personal", user_id="alice", team_id=None) + for server_id in ("srv-a", "srv-b", "srv-c", "srv-open"): + global_mcp_server_manager.registry.pop(server_id, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "personal_key_owner_teams, expected_servers", + [(("team-a", "team-b"), {"srv-a", "srv-b"}), (("team-b",), {"srv-b"}), ((), set())], + indirect=["personal_key_owner_teams"], +) +async def test_personal_key_mcp_servers_are_the_union_of_its_owner_team_grants( + personal_key_owner_teams: UserAPIKeyAuth, expected_servers: set[str] +) -> None: + assert set(await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams)) == expected_servers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_mcp_listing_applies_agent_caller_ceiling( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers + + auth: Final = personal_key_owner_teams.model_copy(update={"agent_id": "agent-1"}) + caller: Final = agent_caller_from_headers( + {"x-litellm-team-id": "team-b"}, + auth, + ) + auth_with_caller: Final = auth.model_copy(update={"agent_caller": caller}) + + capped: Final = await MCPRequestHandler.get_mcp_server_access(auth_with_caller) + uncapped: Final = await MCPRequestHandler.get_mcp_server_access(auth) + + assert capped == MCPServerAccess(server_ids=("srv-b",), scope="scoped") + assert uncapped == MCPServerAccess(server_ids=("srv-a", "srv-b"), scope="scoped") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", UI_TEAM_ID)], indirect=True) +async def test_personal_key_ui_team_does_not_expand_mcp_access( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == UI_TEAM_ID: + return LiteLLM_TeamTable(team_id=team_id) + return _personal_key_team(team_id) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == ["srv-a"] + assert (await MCPRequestHandler.get_mcp_server_access(personal_key_owner_teams)).scope == "scoped" + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-c", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_tool_call_bills_granting_team( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix + from litellm.proxy.auth import auth_checks + + team_membership: Final = LiteLLM_TeamMembership( + user_id=personal_key_owner_teams.user_id, + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=3), + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team: Final = _personal_key_team_granting_tools(team_id, ("divide",) if team_id == "team-a" else ("multiply",)) + return team.model_copy( + update={ + "metadata": {"guardrails": ["team-guardrail"]} if team_id == "team-b" else team.metadata, + "rpm_limit": 7 if team_id == "team-a" else team.rpm_limit, + } + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=team_membership)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + server_b: Final = global_mcp_server_manager.registry["srv-b"] + server_open: Final = global_mcp_server_manager.registry["srv-open"] + tool_name: Final = f"{get_server_prefix(server_b)}{MCP_TOOL_PREFIX_SEPARATOR}multiply" + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server_b): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call(personal_key_owner_teams, tool_name) + assert billed is not personal_key_owner_teams + assert (billed.team_id, billed.org_id) == ("team-b", "org-team-b") + assert billed.team_metadata == {"guardrails": ["team-guardrail"]} + assert personal_key_owner_teams.team_id is None + + team_a_tool_name: Final = f"{get_server_prefix(server_b)}{MCP_TOOL_PREFIX_SEPARATOR}divide" + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server_b): + team_a_billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, team_a_tool_name + ) + assert team_a_billed.team_id == "team-a" + assert team_a_billed.team_rpm_limit == 7 + assert team_a_billed.team_member_rpm_limit == 3 + + open_tool_name: Final = f"{get_server_prefix(server_open)}{MCP_TOOL_PREFIX_SEPARATOR}tool" + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server_open): + unbilled: Final = await MCPRequestHandler.billing_auth_for_tool_call(personal_key_owner_teams, open_tool_name) + assert unbilled is personal_key_owner_teams + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_org_id,team_org_id,expected_org_id", + [("org-1", None, "org-1"), (None, "org-2", "org-2")], +) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_preserves_key_organization( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + key_org_id: str | None, + team_org_id: str | None, + expected_org_id: str, +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=team_org_id, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=["srv-a"], + mcp_tool_permissions={"srv-a": ["add"]}, + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": key_org_id}) + server: Final = global_mcp_server_manager.registry["srv-a"] + tool_name: Final = f"{get_server_prefix(server)}{MCP_TOOL_PREFIX_SEPARATOR}add" + + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call(auth, tool_name) + + assert billed.team_id == "team-a" + assert billed.org_id == expected_org_id + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_display_name_billing_uses_original_tool_grant( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + server: Final = global_mcp_server_manager.registry["srv-a"].model_copy( + update={ + "name": "math_a", + "server_name": "math_a", + "tool_name_to_display_name": {"add": "Add Numbers"}, + } + ) + monkeypatch.setitem(global_mcp_server_manager.registry, "srv-a", server) + monkeypatch.setitem(global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, "math_a-add", "math_a") + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="Add Numbers", + ) + + assert billed.team_id == "team-a" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_display_name_billing_uses_allowed_servers( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + hidden_server: Final = MCPServer( + server_id="srv_hidden", + name="srv_hidden", + server_name="srv_hidden", + transport=MCPTransport.http, + tool_name_to_display_name={"foo": "Shared"}, + ) + allowed_server: Final = MCPServer( + server_id="srv_a", + name="srv_a", + server_name="srv_a", + transport=MCPTransport.http, + tool_name_to_display_name={"bar": "Shared"}, + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id).model_copy( + update={ + "object_permission": LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=["srv_a"], + mcp_tool_permissions={"srv_a": ["bar"]}, + ) + } + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setitem(global_mcp_server_manager.registry, hidden_server.server_id, hidden_server) + monkeypatch.setitem(global_mcp_server_manager.registry, allowed_server.server_id, allowed_server) + + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="Shared", + ) + + assert billed.team_id == "team-a" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_tool_call_bills_lowest_team_when_both_allow( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR, get_server_prefix + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools(team_id, ("multiply",)) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + server_b: Final = global_mcp_server_manager.registry["srv-b"] + tool_name: Final = f"{get_server_prefix(server_b)}{MCP_TOOL_PREFIX_SEPARATOR}multiply" + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server_b): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call(personal_key_owner_teams, tool_name) + assert (billed.team_id, billed.org_id) == ("team-a", "org-team-a") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_attribution_preserves_hyphenated_native_tool_names( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools( + team_id, ("math-divide",) if team_id == "team-a" else ("math-multiply",) + ).model_copy(update={"metadata": {"mcp_rpm_limit": {"srv-b": 3 if team_id == "team-b" else 9}}}) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + server: Final = global_mcp_server_manager.registry["srv-b"].model_copy(update={"alias": "math"}) + with ( + patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server), + patch.object(global_mcp_server_manager, "get_mcp_server_by_id", return_value=server), + ): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, "math-math-multiply" + ) + assert (billed.team_id, billed.org_id) == ("team-b", "org-team-b") + assert personal_key_owner_teams.team_id is None + assert await MCPRequestHandler.personal_key_team_rpm_limits(personal_key_owner_teams, server, "math-multiply") == { + "team-b": {"srv-b": 3} + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_guardrail_auth_preserves_native_tool_name_matching_server_alias( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools( + team_id, ("math-multiply",) if team_id == "team-b" else ("other",) + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + server: Final = global_mcp_server_manager.registry["srv-b"].model_copy(update={"alias": "math"}) + + billed: Final = await MCPRequestHandler.guardrail_auth_for_call( + None, + personal_key_owner_teams, + server, + "math-multiply", + allowed_mcp_servers=[server], + ) + + assert billed is not None + assert billed.team_id == "team-b" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_inherits_adopted_organization_limits( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_OrganizationTable + + organization: Final = LiteLLM_OrganizationTable( + organization_id="org-1", + organization_alias="org-one", + budget_id="budget-1", + created_by="test", + updated_by="test", + metadata={"k": "v"}, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=100, rpm_limit=5, tpm_limit=50), + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( + update={"organization_id": "org-1"} + ) + + org_loader: Final = AsyncMock(return_value=organization) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) + server: Final = global_mcp_server_manager.registry["srv-b"] + tool_name: Final = "srv-b-divide" + + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call(personal_key_owner_teams, tool_name) + + assert ( + billed.org_id, + billed.organization_rpm_limit, + billed.organization_tpm_limit, + billed.organization_metadata, + ) == ("org-1", 5, 50, {"k": "v"}) + assert billed.organization_max_budget == 100 + assert ( + personal_key_owner_teams.org_id, + personal_key_owner_teams.organization_rpm_limit, + personal_key_owner_teams.organization_tpm_limit, + personal_key_owner_teams.organization_metadata, + ) == (None, None, None, None) + org_loader.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_keeps_key_organization_limits( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_OrganizationTable + + organization: Final = LiteLLM_OrganizationTable( + organization_id="org-1", + budget_id="budget-1", + created_by="test", + updated_by="test", + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=5), + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( + update={"organization_id": "org-1"} + ) + + org_loader: Final = AsyncMock(return_value=organization) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) + auth: Final = personal_key_owner_teams.model_copy( + update={"org_id": "org-k", "organization_rpm_limit": 9} + ) + server: Final = global_mcp_server_manager.registry["srv-b"] + + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call(auth, "srv-b-divide") + + assert (billed.org_id, billed.organization_rpm_limit) == ("org-k", 9) + org_loader.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_keeps_team_controls_when_org_lookup_fails( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy import proxy_server + from litellm.proxy.auth import auth_checks + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( + update={ + "organization_id": "org-1", + "rpm_limit": 5, + "max_budget": 40, + "metadata": {"team_private": True}, + } + ) + + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + org_loader: Final = AsyncMock(side_effect=RuntimeError("org lookup failed")) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) + server: Final = global_mcp_server_manager.registry["srv-b"] + + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + "srv-b-divide", + scoped_server=server, + native_tool_name="divide", + allowed_mcp_servers=[server], + ) + + assert billed is not personal_key_owner_teams + assert billed.team_id == "team-a" + assert billed.team_rpm_limit == 5 + assert billed.team_metadata == {"team_private": True} + assert billed.team_max_budget == 40 + assert billed.org_id == "org-1" + assert personal_key_owner_teams.team_id is None + assert personal_key_owner_teams.team_rpm_limit is None + org_loader.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_keeps_team_id_when_membership_lookup_fails( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy import proxy_server + from litellm.proxy.auth import auth_checks + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team_granting_tools(team_id, ("divide",)) + + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + membership_loader: Final = AsyncMock(side_effect=RuntimeError("membership lookup failed")) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr( + auth_checks, + "get_team_membership", + membership_loader, + ) + server: Final = global_mcp_server_manager.registry["srv-b"] + + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + "srv-b-divide", + scoped_server=server, + native_tool_name="divide", + allowed_mcp_servers=[server], + ) + + membership_loader.assert_awaited_once() + assert billed is not personal_key_owner_teams + assert billed.team_id == "team-a" + assert billed.org_id == "org-team-a" + assert personal_key_owner_teams.team_id is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requires_fresh_policy, expected_team_rpm, expected_member_rpm, expected_metadata", + [ + pytest.param(False, 2, 3, {"policy_source": "cache"}, id="default-uses-cache"), + pytest.param(True, 7, 8, {"policy_source": "database"}, id="strict-uses-database"), + ], +) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_mcp_billing_uses_fresh_team_and_membership_for_strict_policy( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + requires_fresh_policy: bool, + expected_team_rpm: int, + expected_member_rpm: int, + expected_metadata: dict[str, str], +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + async def _get_team_object(team_id: str, **kwargs: object) -> LiteLLM_TeamTable: + database_value: Final = bool(kwargs.get("check_db_only")) + return _personal_key_team_granting_tools(team_id, ("divide",)).model_copy( + update={ + "organization_id": None, + "rpm_limit": 7 if database_value else 2, + "metadata": {"policy_source": "database" if database_value else "cache"}, + } + ) + + async def _get_team_membership( + user_id: str, team_id: str, **kwargs: object + ) -> LiteLLM_TeamMembership: + database_value: Final = bool(kwargs.get("check_db_only")) + return LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=8 if database_value else 3), + ) + + team_loader: Final = AsyncMock(side_effect=_get_team_object) + membership_loader: Final = AsyncMock(side_effect=_get_team_membership) + monkeypatch.setattr(auth_checks, "get_team_object", team_loader) + monkeypatch.setattr(auth_checks, "get_team_membership", membership_loader) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) + auth: Final = personal_key_owner_teams.model_copy(update={"requires_fresh_policy": requires_fresh_policy}) + server: Final = global_mcp_server_manager.registry["srv-b"] + + with patch.object(global_mcp_server_manager, "get_mcp_server_from_tool_name", return_value=server): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + auth, + "srv-b-divide", + scoped_server=server, + native_tool_name="divide", + allowed_mcp_servers=[server], + ) + + assert (billed.team_rpm_limit, billed.team_member_rpm_limit, billed.team_metadata) == ( + expected_team_rpm, + expected_member_rpm, + expected_metadata, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_team_mcp_rpm_limit_charges_granting_team( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + PROXY_MaxParallelRequestsHandler_v3, + RateLimitDescriptor, + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team: Final = _personal_key_team_granting_tools(team_id, ("divide",) if team_id == "team-a" else ("multiply",)) + return team.model_copy( + update={ + "metadata": ( + {"mcp_rpm_limit": {"srv-b": 3, "srv-a": 9}} + if team_id == "team-b" + else {"mcp_rpm_limit": {"srv-b": 9}} + ) + } + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + server_b: Final = global_mcp_server_manager.registry["srv-b"] + limits: Final = await MCPRequestHandler.personal_key_team_rpm_limits(personal_key_owner_teams, server_b, "multiply") + assert limits == {"team-b": {"srv-b": 3}} + + stamped: Final = personal_key_owner_teams.model_copy(update={"mcp_source_team_rpm_limits": limits}) + descriptors: list[RateLimitDescriptor] = [] + limiter: Final = PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=MagicMock()) + limiter._add_mcp_per_team_rate_limit_descriptor(stamped, server_b.name, descriptors) + assert len(descriptors) == 1 + assert descriptors[0]["key"] == "mcp_per_team" + assert descriptors[0]["value"] == f"team-b:{server_b.name}" + assert descriptors[0]["rate_limit"]["requests_per_unit"] == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_over_budget_team_grants_no_mcp_access( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team: Final = _personal_key_team(team_id) + return team.model_copy(update={"max_budget": 1, "spend": 5}) if team_id == "team-b" else team + + async def _spend_from_fallback(*, fallback_spend: float, **_: object) -> float: + return fallback_spend + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "get_current_spend", _spend_from_fallback) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "budget_alerts", AsyncMock()) + assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == ["srv-a"] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-b", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_mcp_tools_follow_the_rule_of_the_team_granting_the_server( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == ["add"] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-b", personal_key_owner_teams) is None + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-c", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [()], indirect=True) +async def test_personal_key_without_teams_reaches_no_mcp_tool(personal_key_owner_teams: UserAPIKeyAuth) -> None: + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_mcp_access_is_unchanged_when_flag_off( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + import litellm + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", False) + assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == [] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_team_key_mcp_access_ignores_its_owner_other_teams(personal_key_owner_teams: UserAPIKeyAuth) -> None: + from litellm.proxy.auth import auth_checks + + team_key: Final = personal_key_owner_teams.model_copy(update={"team_id": "team-b"}) + assert await MCPRequestHandler.get_allowed_mcp_servers(team_key) == ["srv-b"] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", team_key) is None + auth_checks.get_user_object.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [(), ("team-a",)], indirect=True) +async def test_personal_key_keeps_its_own_tool_rule_on_an_allow_all_server( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) is None + scoped_key: Final = personal_key_owner_teams.model_copy( + update={ + "object_permission": LiteLLM_ObjectPermissionTable( + object_permission_id="perm-key", mcp_tool_permissions={"srv-open": ["add"]} + ) + } + ) + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", scoped_key) == ["add"] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-c", scoped_key) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_mcp_tools_are_denied_when_a_granting_team_cannot_be_read( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + """team-b grants srv-b, but the tool-rule read behind the source pinned to team-b fails. The fault + must deny the server's tools, not become the allow-all ``None`` of a team without a tool rule.""" + + from litellm.proxy.auth import auth_checks + + team_object_reads: Final = AsyncMock() + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team_b_reads: Final = tuple( + call for call in team_object_reads.await_args_list if call.kwargs.get("team_id") == "team-b" + ) + if team_id == "team-b" and len(team_b_reads) == 3: + raise RuntimeError("connection reset by peer") + return _personal_key_team(team_id) + + team_object_reads.side_effect = _get_team_object + monkeypatch.setattr(auth_checks, "get_team_object", team_object_reads) + srv_b_tools: Final = await MCPRequestHandler.get_allowed_tools_for_server("srv-b", personal_key_owner_teams) + team_b_reads: Final = tuple( + call for call in team_object_reads.await_args_list if call.kwargs.get("team_id") == "team-b" + ) + assert len(team_b_reads) == 3 + assert srv_b_tools == [] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == ["add"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "personal_key_owner_teams, key_org, expected_orgs", + [ + (("team-a", "team-b"), None, ("org-team-a", "org-team-b")), + (("team-a", "team-b"), "org-team-a", ("org-team-a",)), + (("team-a", "team-b"), "org-key", ()), + ], + indirect=["personal_key_owner_teams"], +) +async def test_personal_key_sources_keep_the_key_own_org_scope( + personal_key_owner_teams: UserAPIKeyAuth, key_org: str | None, expected_orgs: tuple[str, ...] +) -> None: + """A key without an org takes each team's org. A key issued under an org keeps it and only its own org's + teams become sources, a team of another org grants nothing, so neither org ceiling is dropped.""" + key: Final = personal_key_owner_teams.model_copy(update={"org_id": key_org}) + sources: Final = await MCPRequestHandler.personal_key_team_sources(key) + assert tuple(source.org_id for source in sources) == expected_orgs + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_unreadable_team_permission_is_not_a_server_or_billing_source( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.auth import auth_checks + + async def _unreadable_team(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + organization_id="org-team-a", + object_permission_id="permission-that-cannot-be-loaded", + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _unreadable_team) + + assert await MCPRequestHandler.personal_key_team_sources(personal_key_owner_teams) == () + assert await MCPRequestHandler.resolve_personal_key_team_servers(personal_key_owner_teams) == MCPServerAccess( + server_ids=(), scope="scoped" + ) + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == [] + assert await MCPRequestHandler.personal_key_attributing_source(personal_key_owner_teams, "srv-a", "add") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_team_server_access_group_read_fault_stays_scoped( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPServerAccess + from litellm.proxy.auth import auth_checks + + async def _team_with_access_group(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=team_id, access_group_ids=["unreadable-group"]) + + monkeypatch.setattr(auth_checks, "get_team_object", _team_with_access_group) + get_access_object: Final = AsyncMock(side_effect=RuntimeError("access group read failed")) + monkeypatch.setattr(auth_checks, "get_access_object", get_access_object) + + team_grants: Final = await MCPRequestHandler.personal_key_team_server_grants(personal_key_owner_teams) + assert len(team_grants) == 1 + assert team_grants[0][1] == MCPServerAccess(server_ids=(), scope="scoped") + assert await MCPRequestHandler.resolve_personal_key_team_servers(personal_key_owner_teams) == MCPServerAccess( + server_ids=(), scope="scoped" + ) + get_access_object.assert_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_scoped_bare_tool_name_bills_granting_team( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy import proxy_server + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id).model_copy( + update={ + "object_permission": LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=["srv-a"], + mcp_tool_permissions={"srv-a": ["add"]}, + ) + } + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + + async with global_mcp_server_manager.catalog.operation(): + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="add", + mcp_servers=("srv-a",), + ) + unscoped: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="add", + ) + + assert billed.team_id == "team-a" + assert unscoped is personal_key_owner_teams + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_prefixed_tool_billing_resolves_server_without_loaded_tools( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server: Final = MCPServer( + server_id="math_a", + name="math_a", + server_name="math_a", + transport=MCPTransport.http, + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return _personal_key_team(team_id).model_copy( + update={ + "object_permission": LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=["math_a"], + mcp_tool_permissions={"math_a": ["add"]}, + ) + } + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + + assert global_mcp_server_manager.tool_name_to_mcp_server_name_mapping == {} + + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="math_a-add", + ) + unknown_prefix: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="unknown-add", + ) + + assert billed.team_id == "team-a" + assert unknown_prefix is personal_key_owner_teams + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_scoped_tool_billing_prefers_scoped_server( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + server_id: Final = "srv-a" if team_id == "team-a" else "srv-b" + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + mcp_servers=[server_id], + mcp_tool_permissions={server_id: ["search"]}, + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setitem(global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, "search", "srv-a") + + billed: Final = await MCPRequestHandler.billing_auth_for_tool_call( + personal_key_owner_teams, + tool_name="search", + mcp_servers=("srv-b",), + ) + + assert billed.team_id == "team-b" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [()], indirect=True) +async def test_admin_view_personal_key_keeps_its_own_tools_on_every_server( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + """The admin-view role lists every registry server for a personal key, so its tools resolve from the + key's own rules (none here, so unrestricted) instead of being denied for lack of a team grant.""" + from litellm.proxy._types import LitellmUserRoles + + viewer: Final = personal_key_owner_teams.model_copy(update={"user_role": LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY}) + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-c", viewer) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_keyless_grant_source_of_the_owner_does_not_inherit_its_teams( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + """A source auth of an admitted subject has the owner's user_id, no team and no api_key. It must + resolve as the plain caller it is, or a token scoped to one team reaches every team's servers.""" + source: Final = UserAPIKeyAuth(user_id=personal_key_owner_teams.user_id, team_id=None) + assert await MCPRequestHandler.get_allowed_mcp_servers(source) == [] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", source) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_mcp_access_is_denied_when_owner_teams_cannot_be_read( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy.auth import auth_checks + + monkeypatch.setattr(auth_checks, "personal_key_owner_teams", AsyncMock(side_effect=RuntimeError("db down"))) + assert await MCPRequestHandler.get_allowed_mcp_servers(personal_key_owner_teams) == [] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-a", personal_key_owner_teams) == [] + assert await MCPRequestHandler.get_allowed_tools_for_server("srv-open", personal_key_owner_teams) == [] diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index d314291ef74..4896c2f1706 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -56,6 +56,7 @@ from litellm.experimental_mcp_client.client import MCPClient, PersistentMCPSessi from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( ListedToolsCaller, MCPServerManager, + global_mcp_server_manager, _deserialize_json_dict, _flow_endpoints_missing, _mcp_oauth_discovery_on_startup_enabled, @@ -67,9 +68,13 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( should_strip_caller_authorization, listed_tools_caller_for, ) +from litellm.proxy import proxy_server +from litellm.proxy.auth import auth_checks from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, MCPApprovalStatus, MCPEnvVar, @@ -167,7 +172,13 @@ async def test_elicitation_callback_keeps_initiating_session(): from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback from types import SimpleNamespace - from mcp.types import ClientCapabilities, ElicitationCapability, FormElicitationCapability, ElicitRequestFormParams, ElicitResult + from mcp.types import ( + ClientCapabilities, + ElicitationCapability, + FormElicitationCapability, + ElicitRequestFormParams, + ElicitResult, + ) from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var capabilities = ClientCapabilities(elicitation=ElicitationCapability(form=FormElicitationCapability())) @@ -183,7 +194,7 @@ async def test_elicitation_callback_keeps_initiating_session(): legacy_server.active_mcp_session_var.set(SimpleNamespace()) active_mcp_request_ctx_var.set(None) capabilities.elicitation = None - result = await callback(None, ElicitRequestFormParams(message="Confirm", requested_schema={"type":"object"})) + result = await callback(None, ElicitRequestFormParams(message="Confirm", requested_schema={"type": "object"})) assert result is accepted assert request.await_args.kwargs["related_request_id"] == "initiating-call" finally: @@ -1289,7 +1300,9 @@ class TestMCPServerManager: raw_row = MagicMock() raw_row.model_dump.return_value = row.model_dump() repository = MagicMock() - other_row: Final = row.model_copy(update={"server_id": "changed-other", "server_name": "changed_other", "auth_type": MCPAuth.none}) + other_row: Final = row.model_copy( + update={"server_id": "changed-other", "server_name": "changed_other", "auth_type": MCPAuth.none} + ) repository.table.find_many = AsyncMock(return_value=[raw_row, other_row]) async def publish_while_staged(*_args, **_kwargs): @@ -1367,7 +1380,11 @@ class TestMCPServerManager: table = SimpleNamespace( find_many=AsyncMock(return_value=[_row(cached.server_id, corrupted), _row("healthy-sibling", stored)]) ) - prisma = SimpleNamespace(db=SimpleNamespace(litellm_config=SimpleNamespace(find_unique=AsyncMock(return_value=None)), litellm_mcpservertable=table)) + prisma = SimpleNamespace( + db=SimpleNamespace( + litellm_config=SimpleNamespace(find_unique=AsyncMock(return_value=None)), litellm_mcpservertable=table + ) + ) prisma.writer_db = prisma.db monkeypatch.setattr(proxy_server, "prisma_client", prisma) @@ -2301,12 +2318,17 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_managed_scope_challenge_requires_reauthorization_without_retry(self): server = MCPServer( - server_id="managed-scope", name="managed-scope", alias="managed-scope", - url="https://up.example.com/mcp", transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", + server_id="managed-scope", + name="managed-scope", + alias="managed-scope", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + oauth2_flow="authorization_code", ) upstream_error = self._upstream_status_error( - 403, 'Bearer error="insufficient_scope", scope="tools.write", resource_metadata="https://untrusted.test/metadata"' + 403, + 'Bearer error="insufficient_scope", scope="tools.write", resource_metadata="https://untrusted.test/metadata"', ) manager = MCPServerManager() client = AsyncMock() @@ -7390,12 +7412,21 @@ class TestMCPServerManager: caller=ListedToolsCaller(user_api_key_auth=auth), ) if managed: - manager.registry["test-server"] = manager.registry["test-server"].model_copy(update={ - "auth_type": MCPAuth.oauth2, "oauth2_flow": "authorization_code", - }) + manager.registry["test-server"] = manager.registry["test-server"].model_copy( + update={ + "auth_type": MCPAuth.oauth2, + "oauth2_flow": "authorization_code", + } + ) send = AsyncMock() gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/test-server") - context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) + context = ServerRequestContext( + session=MagicMock(), + lifespan_context={}, + protocol_version="2025-11-25", + method="tools/call", + request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate}), + ) token = active_mcp_request_ctx_var.set(context) await gate({"type": "http.response.start", "status": 200, "headers": []}) progress = asyncio.create_task(gate({"type": "http.response.body", "body": b"progress", "more_body": True})) @@ -7407,7 +7438,13 @@ class TestMCPServerManager: manager.create_mcp_client.return_value.call_tool.side_effect = upstream_call try: - result = await manager.call_tool(server_name="test-server", name="test_tool", arguments={}, user_api_key_auth=auth, proxy_logging_obj=logging) + result = await manager.call_tool( + server_name="test-server", + name="test_tool", + arguments={}, + user_api_key_auth=auth, + proxy_logging_obj=logging, + ) assert not result.is_error finally: gate.allow() @@ -8040,9 +8077,7 @@ class TestMCPServerManager: "litellm.proxy.guardrails.guardrail_hooks.mcp_jwt_signer.mcp_jwt_signer.get_mcp_jwt_signer", return_value=signer, ): - manager.record_listed_tools( - server, [MCPTool(name="turn", description="alice view", inputSchema={})], alice - ) + manager.record_listed_tools(server, [MCPTool(name="turn", description="alice view", inputSchema={})], alice) assert manager.get_listed_tool(server, "turn", bob) is None for_alice = manager.get_listed_tool(server, "turn", alice) @@ -9141,9 +9176,7 @@ class TestMCPServerTimestamps: registration_url=None, ) assert oauth_endpoints_unresolved(relay_arm) is True - assert ( - oauth_endpoints_unresolved(relay_arm.model_copy(update={"registration_url": "https://idp/reg"})) is False - ) + assert oauth_endpoints_unresolved(relay_arm.model_copy(update={"registration_url": "https://idp/reg"})) is False assert oauth_endpoints_unresolved(relay_arm.model_copy(update={"client_id": "admin-client"})) is False def test_entra_obo_without_scopes_is_unresolved(self): @@ -9405,11 +9438,13 @@ class TestMCPServerTimestamps: assert blipped.issuer == "https://idp.example.com" assert blipped.authorization_response_iss_parameter_supported is True fallback = MCPServerManager._merge_discovered_oauth_metadata( - blipped, MCPOAuthMetadata(from_origin_fallback=True), + blipped, + MCPOAuthMetadata(from_origin_fallback=True), ) assert fallback.authorization_response_iss_parameter_supported is True refreshed = MCPServerManager._merge_discovered_oauth_metadata( - fallback, MCPOAuthMetadata(discovered_issuer="https://idp.example.com"), + fallback, + MCPOAuthMetadata(discovered_issuer="https://idp.example.com"), ) assert refreshed.authorization_response_iss_parameter_supported is False @@ -9764,7 +9799,7 @@ class TestMCPServerTimestamps: "unlimited_server": { "url": "https://example.com/other-mcp", "transport": MCPTransport.http, - } + }, } await manager.load_servers_from_config(config) servers = list(manager.config_mcp_servers.values()) @@ -15213,6 +15248,457 @@ async def test_pre_call_tool_check_honors_guardrail_attached_to_key(monkeypatch, assert exc_info.value.status_code == 400 +@pytest.mark.asyncio +async def test_pre_call_tool_check_stamps_personal_key_team_rpm_limits(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + manager: Final = MCPServerManager() + server: Final = MCPServer( + server_id="srv-b", + name="srv-b", + server_name="srv-b", + transport=MCPTransport.stdio, + ) + user_api_key_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") + expected: Final = {"team-b": {"srv-b": 3}} + proxy_logging_obj: Final = _permissive_proxy_logging() + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(manager, "check_tool_permission_for_key_team", AsyncMock(return_value=None)) + monkeypatch.setattr(MCPRequestHandler, "personal_key_team_rpm_limits", AsyncMock(return_value=expected)) + await manager.pre_call_tool_check( + name="multiply", + arguments={}, + server_name=server.name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=server, + ) + + hook_auth: Final = proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] + assert hook_auth is not user_api_key_auth + assert hook_auth.mcp_source_team_rpm_limits == expected + assert user_api_key_auth.mcp_source_team_rpm_limits is None + assert proxy_logging_obj.enforce_mcp_server_rate_limits.await_args.args[0] is hook_auth + assert MCPRequestHandler.personal_key_team_rpm_limits.await_args.args == ( + user_api_key_auth, + server, + "multiply", + ) + + +@pytest.mark.asyncio +async def test_pre_call_tool_check_uses_inherited_team_auth_for_guardrails(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + server: Final = MCPServer( + server_id="srv-guardrails", + name="math_a", + server_name="math_a", + transport=MCPTransport.stdio, + ) + team: Final = LiteLLM_TeamTable( + team_id="team-a", + object_permission_id="perm-team-a", + metadata={"guardrails": ["team-guardrail"]}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-a", + mcp_servers=[server.server_id], + mcp_tool_permissions={server.server_id: ["multiply"]}, + ), + ) + user_api_key_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") + manager: Final = MCPServerManager() + proxy_logging_obj: Final = _permissive_proxy_logging() + user_api_key_cache: Final = MagicMock() + user_api_key_cache.async_get_cache = AsyncMock(return_value=None) + user_api_key_cache.async_set_cache = AsyncMock() + + async def get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + assert team_id == team.team_id + return team + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=["team-a"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_personal_key_team_within_budget", AsyncMock(return_value=True)) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + monkeypatch.setattr(manager, "check_tool_permission_for_key_team", AsyncMock(return_value=None)) + + team_grants: Final = await MCPRequestHandler.personal_key_team_server_grants(user_api_key_auth) + assert team_grants[0][0].team_id == "team-a" + assert server.server_id in team_grants[0][1].server_ids + + await manager.pre_call_tool_check( + name="multiply", + arguments={}, + server_name=server.name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=server, + guardrail_auth=None, + ) + + hook_auth: Final = proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] + assert hook_auth.team_id == "team-a" + assert hook_auth.team_metadata == {"guardrails": ["team-guardrail"]} + + +@pytest.mark.asyncio +async def test_pre_call_tool_check_keeps_native_name_for_team_attribution(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + + server: Final = MCPServer( + server_id="math-a", + name="math_a", + alias="math_a", + server_name="math_a", + transport=MCPTransport.stdio, + ) + team_a: Final = LiteLLM_TeamTable( + team_id="team-a", + object_permission_id="perm-team-a", + metadata={"mcp_rpm_limit": {"math-a": 7}}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-a", + mcp_servers=["math-a"], + mcp_tool_permissions={"math-a": ["math_a-foo"]}, + ), + ) + team_b: Final = LiteLLM_TeamTable( + team_id="team-b", + object_permission_id="perm-team-b", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-b", + mcp_servers=["math-a"], + mcp_tool_permissions={"math-a": ["foo"]}, + ), + ) + user_api_key_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") + manager: Final = MCPServerManager() + proxy_logging_obj: Final = _permissive_proxy_logging() + user_api_key_cache: Final = MagicMock() + user_api_key_cache.async_get_cache = AsyncMock(return_value=None) + user_api_key_cache.async_set_cache = AsyncMock() + + async def get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return team_a if team_id == team_a.team_id else team_b + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=["team-a", "team-b"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_personal_key_team_within_budget", AsyncMock(return_value=True)) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + + await manager.pre_call_tool_check( + name="math_a-foo", + arguments={}, + server_name=server.name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=server, + guardrail_auth=None, + ) + + hook_auth: Final = proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] + assert hook_auth.team_id == "team-a" + assert hook_auth.mcp_source_team_rpm_limits == {"team-a": {"math-a": 7}} + + +@pytest.mark.asyncio +async def test_pre_call_tool_check_keeps_original_auth_for_team_key(monkeypatch: pytest.MonkeyPatch) -> None: + server: Final = MCPServer( + server_id="srv-team-key", + name="team-server", + server_name="team-server", + transport=MCPTransport.stdio, + ) + user_api_key_auth: Final = UserAPIKeyAuth( + api_key="sk-team", + team_id="team-a", + team_metadata={"guardrails": ["team-guardrail"]}, + ) + manager: Final = MCPServerManager() + proxy_logging_obj: Final = _permissive_proxy_logging() + monkeypatch.setattr(manager, "check_tool_permission_for_key_team", AsyncMock(return_value=None)) + + await manager.pre_call_tool_check( + name="multiply", + arguments={}, + server_name=server.name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=server, + guardrail_auth=None, + ) + + assert proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] is user_api_key_auth + + +@pytest.mark.asyncio +async def test_call_tool_passes_attributed_auth_to_guardrail_hooks(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.operations import _run_post_mcp_call_guardrails + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + + original_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") + team: Final = LiteLLM_TeamTable( + team_id="team-b", + organization_id="org-team-b", + object_permission_id="perm-team-b", + metadata={"guardrails": ["team-guardrail"]}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-b", + mcp_servers=["test-server"], + mcp_tool_permissions={"test-server": ["test_tool"]}, + ), + ) + manager: Final = TestMCPServerManager._manager_ready_for_call_tool( + [MCPTool(name="test_tool", inputSchema={"type": "object"})], + caller=ListedToolsCaller(user_api_key_auth=original_auth), + )[0] + server: Final = manager.registry["test-server"] + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + proxy_logging_obj.enforce_mcp_server_rate_limits = AsyncMock() + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr("litellm.proxy.utils.premium_user_check", MagicMock()) + monkeypatch.setattr(manager, "check_tool_permission_for_key_team", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging_obj) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=["team-b"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=team)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_personal_key_team_within_budget", AsyncMock(return_value=True)) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + + await manager.call_tool( + server_name="test-server", + name="test_tool", + arguments={}, + user_api_key_auth=original_auth, + proxy_logging_obj=proxy_logging_obj, + ) + + pre_auth: Final = proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] + during_auth: Final = proxy_logging_obj.during_call_hook.await_args.kwargs["user_api_key_dict"] + assert pre_auth.team_id == "team-b" + assert pre_auth.team_metadata == {"guardrails": ["team-guardrail"]} + assert during_auth.team_id == "team-b" + assert during_auth.team_metadata == {"guardrails": ["team-guardrail"]} + assert proxy_logging_obj.pre_call_hook.await_args.kwargs["data"]["metadata"]["guardrails"] == ["team-guardrail"] + assert proxy_logging_obj.during_call_hook.await_args.kwargs["data"]["metadata"]["guardrails"] == ["team-guardrail"] + assert original_auth.team_id is None + assert original_auth.team_metadata is None + + non_inherited_auth: Final = UserAPIKeyAuth( + api_key="sk-team", + team_id="team-a", + team_metadata={"guardrails": ["team-guardrail"]}, + ) + await manager.call_tool( + server_name="test-server", + name="test_tool", + arguments={}, + user_api_key_auth=non_inherited_auth, + proxy_logging_obj=proxy_logging_obj, + ) + assert proxy_logging_obj.pre_call_hook.await_args_list[-1].kwargs["user_api_key_dict"] is non_inherited_auth + assert proxy_logging_obj.during_call_hook.await_args_list[-1].kwargs["user_api_key_dict"] is non_inherited_auth + + from mcp.types import CallToolResult + + result: Final = CallToolResult(content=[]) + non_inherited_metadata: Final = {"existing": "value"} + model_call_details: Final = {"metadata": non_inherited_metadata} + post_guardrail_auth: Final = original_auth.model_copy(update={"team_metadata": {"guardrails": ["g1"]}}) + inherited_request_data: Final = {"metadata": {"guardrails": [{"req-g": {"x": 1}}]}} + await _run_post_mcp_call_guardrails( + result=result, + litellm_logging_obj=None, + original_auth=original_auth, + guardrail_auth=post_guardrail_auth, + request_data=inherited_request_data, + ) + post_kwargs: Final = proxy_logging_obj.post_mcp_call_hook.await_args.kwargs + assert post_kwargs["user_api_key_dict"] is post_guardrail_auth + assert post_kwargs["request_data"]["metadata"]["guardrails"] == [{"req-g": {"x": 1}}, "g1"] + assert inherited_request_data["metadata"]["guardrails"] == [{"req-g": {"x": 1}}] + await _run_post_mcp_call_guardrails( + result=result, + litellm_logging_obj=MagicMock(model_call_details=model_call_details), + original_auth=non_inherited_auth, + guardrail_auth=non_inherited_auth, + request_data={"metadata": {}}, + ) + non_inherited_post_kwargs: Final = proxy_logging_obj.post_mcp_call_hook.await_args.kwargs + assert non_inherited_post_kwargs["user_api_key_dict"] is non_inherited_auth + assert non_inherited_post_kwargs["request_data"] is model_call_details + assert non_inherited_post_kwargs["request_data"]["metadata"] == {"existing": "value"} + assert model_call_details["metadata"] is non_inherited_metadata + + +@pytest.mark.asyncio +async def test_virtual_mcp_calls_use_attributed_auth_for_logging_and_hooks(monkeypatch: pytest.MonkeyPatch) -> None: + from mcp.types import CallToolResult, Tool + + from litellm.proxy._experimental.mcp_server import operations, tool_search + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + from litellm.proxy import proxy_server + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.types.mcp_server.tool_registry import MCPTool + + original_auth: Final = UserAPIKeyAuth( + api_key="sk-personal", + user_id="owner", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="virtual-tools", + mcp_tool_search_enabled=True, + ), + ) + team: Final = LiteLLM_TeamTable( + team_id="team-a", + organization_id="org-team-a", + object_permission_id="perm-team-a", + metadata={"guardrails": ["g1"]}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-a", + mcp_servers=["math-a"], + mcp_tool_permissions={"math-a": ["foo"]}, + ), + ) + server: Final = MCPServer( + server_id="math-a", + name="math_a", + server_name="math_a", + transport=MCPTransport.http, + ) + registered_tool: Final = MCPTool( + name="math_a-foo", + description="Foo", + input_schema={"type": "object"}, + handler=AsyncMock(return_value=None), + server_id="math-a", + ) + result: Final = CallToolResult(content=[]) + target_tool: Final = Tool(name="math_a-foo", inputSchema={"type": "object"}) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + proxy_logging_obj.enforce_mcp_server_rate_limits = AsyncMock() + + async def build_logging( + name: str, + arguments: dict[str, object], + user_api_key_auth: UserAPIKeyAuth, + logging_auth: UserAPIKeyAuth | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, + ) -> MagicMock: + logger_auth: Final = logging_auth or user_api_key_auth + return MagicMock(model_call_details={"metadata": {"user_api_key_team_id": logger_auth.team_id}}) + + logging_builder: Final = AsyncMock(side_effect=build_logging) + tool_call_handler: Final = AsyncMock(return_value=result) + proxy_handler: Final = AsyncMock(return_value=result) + + async def get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + assert team_id == "team-a" + return team + + async def get_user_object(user_id: str, **_: object) -> LiteLLM_UserTable: + return LiteLLM_UserTable(user_id=user_id, teams=["team-a"]) + + monkeypatch.setattr(auth_checks, "get_team_object", get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_user_object", get_user_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging_obj) + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setitem(global_mcp_server_manager.registry, "math-a", server) + monkeypatch.setitem(global_mcp_tool_registry.tools, "math_a-foo", registered_tool) + monkeypatch.setattr(operations, "_build_virtual_call_logging_obj", logging_builder) + monkeypatch.setattr(tool_search, "handle_mcp_tool_call", tool_call_handler) + monkeypatch.setattr(tool_search, "handle_mcp_proxy_tool", proxy_handler) + monkeypatch.setattr(operations, "find_mcp_proxy_tool", AsyncMock(return_value=target_tool)) + monkeypatch.setattr(operations, "_fire_mcp_tool_call_logging", AsyncMock(return_value=result)) + + await operations._dispatch_virtual_mcp_tool( + name="mcp_tool_call", + arguments={"tool_name": "math_a-foo", "arguments": {}}, + user_api_key_auth=original_auth, + client_ip=None, + ) + + assert logging_builder.await_args.kwargs["user_api_key_auth"] is original_auth + assert logging_builder.await_args.kwargs["logging_auth"].team_id == "team-a" + assert logging_builder.await_args.kwargs["logging_auth"].team_metadata == {"guardrails": ["g1"]} + assert tool_call_handler.await_args.kwargs["user_api_key_dict"] is original_auth + assert tool_call_handler.await_args.kwargs["guardrail_auth"].team_id == "team-a" + assert tool_call_handler.await_args.kwargs["guardrail_auth"].team_metadata == {"guardrails": ["g1"]} + direct_logger: Final = tool_call_handler.await_args.kwargs["litellm_logging_obj"] + assert direct_logger.model_call_details["metadata"]["user_api_key_team_id"] == "team-a" + + await operations._dispatch_virtual_mcp_tool( + name="call_tool", + arguments={"tool_id": "opaque-tool-id", "arguments": {}}, + user_api_key_auth=original_auth, + client_ip=None, + mcp_proxy_mode=True, + ) + + assert logging_builder.await_args_list[1].kwargs["user_api_key_auth"] is original_auth + assert logging_builder.await_args_list[1].kwargs["logging_auth"].team_id == "team-a" + assert logging_builder.await_args_list[1].kwargs["logging_auth"].team_metadata == {"guardrails": ["g1"]} + assert proxy_handler.await_args.kwargs["user_api_key_dict"] is original_auth + assert proxy_handler.await_args.kwargs["guardrail_auth"].team_id == "team-a" + assert proxy_handler.await_args.kwargs["guardrail_auth"].team_metadata == {"guardrails": ["g1"]} + assert proxy_handler.await_args.kwargs["resolved_tool"] is target_tool + proxy_logger: Final = operations._fire_mcp_tool_call_logging.await_args.kwargs["logging_obj"] + assert proxy_logger.model_call_details["metadata"]["user_api_key_team_id"] == "team-a" + assert original_auth.team_id is None + assert original_auth.team_metadata is None + + @pytest.mark.asyncio @pytest.mark.parametrize( ("config", "extra_headers", "expected_source", "expected_authorization"), @@ -15917,9 +16403,7 @@ async def test_discovery_cache_isolates_forwarded_credentials_and_static_auth_ca @pytest.mark.parametrize("kind", ("prompts", "resources", "templates")) @pytest.mark.parametrize("header", ("Authorization", "X-LiteLLM-API-Key")) @pytest.mark.parametrize("identified", (False, True)) -async def test_discovery_cache_isolates_keyless_admission_credentials( - kind: str, header: str, identified: bool -) -> None: +async def test_discovery_cache_isolates_keyless_admission_credentials(kind: str, header: str, identified: bool) -> None: manager: Final = MCPServerManager() upstream: Final = _DiscoveryUpstream() server: Final = _discovery_server().model_copy(update={"static_headers": {"Authorization": "Bearer upstream"}}) @@ -17283,6 +17767,7 @@ async def test_catalog_config_only_snapshot_cleans_up_after_error(monkeypatch): manager = MCPServerManager() configured = MCPServer(server_id="configured", name="configured", transport="stdio", command="echo") manager.config_mcp_servers = {configured.server_id: configured} + async def fail_operation(): async with manager.catalog.operation(): assert await manager.catalog.resolve("configured") == configured @@ -17331,14 +17816,26 @@ async def test_catalog_publishes_rediscovered_routes_despite_concurrent_owner_ch @pytest.mark.parametrize("anchored", [False, True]) async def test_catalog_reload_preserves_concurrent_config_discovery_and_routes(anchored): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="config-race", name="config_race", transport=MCPTransport.http, - url="https://upstream.example/mcp", auth_type=MCPAuth.oauth2, - issuer="https://issuer.example" if anchored else None, issuer_is_anchored=anchored) + server: Final = MCPServer( + server_id="config-race", + name="config_race", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + auth_type=MCPAuth.oauth2, + issuer="https://issuer.example" if anchored else None, + issuer_is_anchored=anchored, + ) manager.config_mcp_servers = {server.server_id: server} manager._set_oauth_discovery_deferred(server.server_id, True) generation: Final = manager.oauth_discovery_slot(server.server_id).generation - resolved: Final = server.model_copy(update={"authorization_url": "https://issuer.example/authorize", - "token_url": "https://issuer.example/token", "scopes": ["read"], "issuer": "https://issuer.example"}) + resolved: Final = server.model_copy( + update={ + "authorization_url": "https://issuer.example/authorize", + "token_url": "https://issuer.example/token", + "scopes": ["read"], + "issuer": "https://issuer.example", + } + ) async def hydrate(target: MCPServer) -> bool: target.client_id = "persisted-client" @@ -17355,7 +17852,10 @@ async def test_catalog_reload_preserves_concurrent_config_discovery_and_routes(a prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), - patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", side_effect=hydrate), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", + side_effect=hydrate, + ), ): await manager.reload_servers_from_database() current: Final = manager.get_mcp_server_by_id(server.server_id) @@ -17372,14 +17872,16 @@ async def test_catalog_reload_preserves_concurrent_config_discovery_and_routes(a @pytest.mark.parametrize("change", ["credentials", "delete"]) async def test_catalog_reload_does_not_restore_replaced_config_credentials_or_routes(change): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="config-replaced", name="config_replaced", transport=MCPTransport.http, - client_id="previous-client") + server: Final = MCPServer( + server_id="config-replaced", name="config_replaced", transport=MCPTransport.http, client_id="previous-client" + ) manager.config_mcp_servers = {server.server_id: server} manager.published_tool_routes["config_replaced-search"] = "config_replaced" async def read_rows(**_kwargs): - manager.config_mcp_servers = {} if change == "delete" else { - server.server_id: server.model_copy(update={"client_id": "new-client"})} + manager.config_mcp_servers = ( + {} if change == "delete" else {server.server_id: server.model_copy(update={"client_id": "new-client"})} + ) return [] prisma: Final = MagicMock() @@ -17402,8 +17904,13 @@ async def test_catalog_operation_retains_routes_only_for_same_configured_target( from mcp.types import Tool manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="route-race", name="route_race", transport=MCPTransport.http, - url="https://upstream.example/mcp", auth_type=MCPAuth.oauth2) + server: Final = MCPServer( + server_id="route-race", + name="route_race", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + auth_type=MCPAuth.oauth2, + ) manager.registry = {server.server_id: server} with patch("litellm.proxy.proxy_server.prisma_client", None): async with manager.catalog.operation(): @@ -17411,9 +17918,16 @@ async def test_catalog_operation_retains_routes_only_for_same_configured_target( if change == "delete": manager.registry = {} else: - manager.registry[server.server_id] = server.model_copy(update=( - {"authorization_url": "https://issuer.example/authorize", "token_url": "https://issuer.example/token"} - if change == "discovery" else {"url": "https://changed.example/mcp"})) + manager.registry[server.server_id] = server.model_copy( + update=( + { + "authorization_url": "https://issuer.example/authorize", + "token_url": "https://issuer.example/token", + } + if change == "discovery" + else {"url": "https://changed.example/mcp"} + ) + ) assert (tools[0].name in manager.published_tool_routes) is (change == "discovery") @@ -17426,8 +17940,10 @@ async def test_catalog_reload_keeps_new_route_owner_over_earlier_route(change, m manager: Final = MCPServerManager() registry: Final = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) - servers: Final = {name: MCPServer(server_id=name, name=name, transport=MCPTransport.http, - client_id="configured-client") for name in ("old_owner", "new_owner", "concurrent_owner")} + servers: Final = { + name: MCPServer(server_id=name, name=name, transport=MCPTransport.http, client_id="configured-client") + for name in ("old_owner", "new_owner", "concurrent_owner") + } manager.config_mcp_servers = servers manager.published_tool_routes = {"shared-search": "old_owner"} if mapped else {} @@ -17453,7 +17969,9 @@ async def test_catalog_reload_keeps_new_route_owner_over_earlier_route(change, m registry.published_tools.clear() elif change == "staged_with_concurrent": manager.published_tool_routes["shared-search"] = "concurrent_owner" - registry.published_tools["shared-search"] = original_tool.model_copy(update={"handler": concurrent_handler}) + registry.published_tools["shared-search"] = original_tool.model_copy( + update={"handler": concurrent_handler} + ) manager.tool_name_to_mcp_server_name_mapping["shared-search"] = "new_owner" registry.register_tool("shared-search", "Search", {}, new_handler) else: @@ -17486,10 +18004,16 @@ async def test_catalog_observes_committed_update_and_delete_without_background_r timestamp: Final = datetime.now() row: Final = LiteLLM_MCPServerTable( - server_id="catalog-server", server_name="catalog_server", alias="catalog_server", - transport=MCPTransport.http, url="https://first.example.com/mcp", updated_at=timestamp, + server_id="catalog-server", + server_name="catalog_server", + alias="catalog_server", + transport=MCPTransport.http, + url="https://first.example.com/mcp", + updated_at=timestamp, + ) + updated: Final = row.model_copy( + update={"url": "https://second.example.com/mcp", "updated_at": timestamp + timedelta(seconds=1)} ) - updated: Final = row.model_copy(update={"url": "https://second.example.com/mcp", "updated_at": timestamp + timedelta(seconds=1)}) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(side_effect=([row], [updated], [])) @@ -17515,8 +18039,12 @@ async def test_catalog_observes_committed_update_and_delete_without_background_r async def test_catalog_rebuilt_unchanged_server_keeps_discovered_tool_routes(): from mcp.types import Tool - row: Final = LiteLLM_MCPServerTable(server_id="unchanged-routes", alias="unchanged_routes", - url="https://upstream.example/mcp", transport=MCPTransport.http) + row: Final = LiteLLM_MCPServerTable( + server_id="unchanged-routes", + alias="unchanged_routes", + url="https://upstream.example/mcp", + transport=MCPTransport.http, + ) manager: Final = MCPServerManager() prisma: Final = MagicMock() prisma.writer_db = prisma.db @@ -17539,9 +18067,14 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change from mcp.types import Tool from litellm.proxy._experimental.mcp_server import tool_registry - row: Final = LiteLLM_MCPServerTable(server_id="late-routes", alias="late_routes", - url="https://upstream.example/mcp", transport=MCPTransport.http, updated_at=datetime.now(), - spec_path="late-openapi.json" if openapi else None) + row: Final = LiteLLM_MCPServerTable( + server_id="late-routes", + alias="late_routes", + url="https://upstream.example/mcp", + transport=MCPTransport.http, + updated_at=datetime.now(), + spec_path="late-openapi.json" if openapi else None, + ) manager: Final = MCPServerManager() registry: Final = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) @@ -17565,9 +18098,17 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change await published.wait() if change == "deleted": return [] - return [row if change == "same" else row.model_copy(update={ - "url": "https://updated.example/mcp", "spec_path": None, - "updated_at": row.updated_at + timedelta(seconds=1)})] + return [ + row + if change == "same" + else row.model_copy( + update={ + "url": "https://updated.example/mcp", + "spec_path": None, + "updated_at": row.updated_at + timedelta(seconds=1), + } + ) + ] prisma: Final = MagicMock() prisma.writer_db = prisma.db @@ -17584,7 +18125,9 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change assert manager.get_mcp_server_by_id(row.server_id) is None assert "late_routes-search" not in manager.published_tool_routes else: - assert manager.server_exposes_tool(manager.get_mcp_server_by_id(row.server_id), "late_routes-search") is (change == "same") + assert manager.server_exposes_tool(manager.get_mcp_server_by_id(row.server_id), "late_routes-search") is ( + change == "same" + ) tool: Final = registry.get_tool("late_routes-search") if openapi and change == "same": assert tool is not None @@ -17595,7 +18138,9 @@ async def test_catalog_reload_retains_routes_discovered_for_a_late_server(change @pytest.mark.asyncio @pytest.mark.parametrize("retain_operation", [False, True]) -async def test_catalog_openapi_refresh_does_not_restore_removed_operations(tmp_path, monkeypatch, respx_mock, retain_operation): +async def test_catalog_openapi_refresh_does_not_restore_removed_operations( + tmp_path, monkeypatch, respx_mock, retain_operation +): from litellm.proxy._experimental.mcp_server import tool_registry monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") @@ -17606,8 +18151,13 @@ async def test_catalog_openapi_refresh_does_not_restore_removed_operations(tmp_p paths: Final = {"/removed": {"get": {"operationId": "removed"}}, "/retained": {"get": {"operationId": "retained"}}} spec: Final = {"openapi": "3.0.0", "info": {"title": "Refresh", "version": "1"}, "paths": paths} spec_path.write_text(json.dumps(spec)) - row: Final = LiteLLM_MCPServerTable(server_id="spec-refresh", alias="spec_refresh", - url="https://upstream.example", transport=MCPTransport.http, spec_path=str(spec_path)) + row: Final = LiteLLM_MCPServerTable( + server_id="spec-refresh", + alias="spec_refresh", + url="https://upstream.example", + transport=MCPTransport.http, + spec_path=str(spec_path), + ) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[row]) @@ -17647,10 +18197,15 @@ async def test_catalog_lookup_uses_one_snapshot_until_operation_finishes(): from datetime import timedelta row: Final = LiteLLM_MCPServerTable( - server_id="snapshot-server", alias="snapshot_server", transport=MCPTransport.http, - url="https://first.example.com/mcp", updated_at=datetime.now(), + server_id="snapshot-server", + alias="snapshot_server", + transport=MCPTransport.http, + url="https://first.example.com/mcp", + updated_at=datetime.now(), + ) + updated: Final = row.model_copy( + update={"url": "https://second.example.com/mcp", "updated_at": row.updated_at + timedelta(seconds=1)} ) - updated: Final = row.model_copy(update={"url": "https://second.example.com/mcp", "updated_at": row.updated_at + timedelta(seconds=1)}) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(side_effect=([row], [updated], [updated])) @@ -17746,8 +18301,15 @@ async def test_catalog_cancelled_openapi_refresh_retains_tools_and_discovery(mon manager: Final = MCPServerManager() stamp: Final = datetime.now() - server: Final = MCPServer(server_id="staged", name="staged", transport=MCPTransport.http, - url="https://before.example/mcp", spec_path="before.json", updated_at=stamp, auth_type=MCPAuth.oauth2) + server: Final = MCPServer( + server_id="staged", + name="staged", + transport=MCPTransport.http, + url="https://before.example/mcp", + spec_path="before.json", + updated_at=stamp, + auth_type=MCPAuth.oauth2, + ) manager.registry = {server.server_id: server} manager._set_oauth_discovery_deferred(server.server_id, True) original_slot: Final = manager.oauth_discovery_slot(server.server_id) @@ -17756,8 +18318,15 @@ async def test_catalog_cancelled_openapi_refresh_retains_tools_and_discovery(mon registry: Final = tool_registry.MCPToolRegistry() registry.register_tool("staged-existing", "existing", {}, lambda: "existing") monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) - row: Final = LiteLLM_MCPServerTable(server_id=server.server_id, alias="staged", transport=MCPTransport.http, - url="https://after.example/mcp", spec_path="after.json", updated_at=stamp + timedelta(seconds=1), auth_type=MCPAuth.oauth2) + row: Final = LiteLLM_MCPServerTable( + server_id=server.server_id, + alias="staged", + transport=MCPTransport.http, + url="https://after.example/mcp", + spec_path="after.json", + updated_at=stamp + timedelta(seconds=1), + auth_type=MCPAuth.oauth2, + ) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[row]) @@ -17785,8 +18354,14 @@ async def test_catalog_cancelled_openapi_refresh_retains_tools_and_discovery(mon @pytest.mark.asyncio async def test_catalog_snapshot_identity_is_independent_of_worker_oauth_discovery(): - row: Final = LiteLLM_MCPServerTable(server_id="identity-server", alias="identity_server", transport=MCPTransport.http, - url="https://upstream.example/mcp", auth_type=MCPAuth.oauth2, updated_at=datetime.now()) + row: Final = LiteLLM_MCPServerTable( + server_id="identity-server", + alias="identity_server", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + auth_type=MCPAuth.oauth2, + updated_at=datetime.now(), + ) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[row]) @@ -17799,8 +18374,9 @@ async def test_catalog_snapshot_identity_is_independent_of_worker_oauth_discover ): async with manager.catalog.operation() as unresolved: assert manager.get_mcp_server_by_id(row.server_id).authorization_url is None - resolved: Final = manager.registry[row.server_id].model_copy(update={ - "authorization_url": "https://idp.example/authorize", "token_url": "https://idp.example/token"}) + resolved: Final = manager.registry[row.server_id].model_copy( + update={"authorization_url": "https://idp.example/authorize", "token_url": "https://idp.example/token"} + ) manager.registry[row.server_id] = resolved async with manager.catalog.operation() as discovered: assert manager.get_mcp_server_by_id(row.server_id).authorization_url == resolved.authorization_url @@ -17814,8 +18390,13 @@ async def test_catalog_failed_openapi_row_does_not_publish_partial_handlers(monk manager: Final = MCPServerManager() registry: Final = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) - row: Final = LiteLLM_MCPServerTable(server_id="broken", alias="broken", transport=MCPTransport.http, - url="https://upstream.example/mcp", spec_path="broken.json") + row: Final = LiteLLM_MCPServerTable( + server_id="broken", + alias="broken", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + spec_path="broken.json", + ) prisma: Final = MagicMock() prisma.writer_db = prisma.db prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[row]) @@ -17856,16 +18437,22 @@ async def test_catalog_reuses_openapi_tools_until_configuration_or_background_re registry = tool_registry.MCPToolRegistry() monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) spec_path = tmp_path / "catalog.json" - spec_path.write_text(json.dumps({ - "openapi": "3.0.0", "info": {"title": "Catalog", "version": "1"}, - "paths": {"/echo": {"get": {"operationId": "echo"}}}, - })) + spec_path.write_text( + json.dumps( + { + "openapi": "3.0.0", + "info": {"title": "Catalog", "version": "1"}, + "paths": {"/echo": {"get": {"operationId": "echo"}}}, + } + ) + ) row = _catalog_row().model_copy(update={"spec_path": str(spec_path)}) read_rows = AsyncMock(return_value=[row]) _catalog_database(monkeypatch, read_rows) manager = MCPServerManager() with patch.object( - openapi_to_mcp_generator, "load_openapi_spec_async", + openapi_to_mcp_generator, + "load_openapi_spec_async", wraps=openapi_to_mcp_generator.load_openapi_spec_async, ) as load_spec: await manager.catalog.list() @@ -17912,10 +18499,15 @@ async def test_catalog_changed_openapi_handler_never_dispatches(monkeypatch, tmp monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) monkeypatch.setattr(operations, "global_mcp_tool_registry", registry) spec_path = tmp_path / "catalog-race.json" - spec_path.write_text(json.dumps({ - "openapi": "3.0.0", "info": {"title": "Catalog", "version": "1"}, - "paths": {"/echo": {"get": {"operationId": "echo"}}}, - })) + spec_path.write_text( + json.dumps( + { + "openapi": "3.0.0", + "info": {"title": "Catalog", "version": "1"}, + "paths": {"/echo": {"get": {"operationId": "echo"}}}, + } + ) + ) row = _catalog_row().model_copy(update={"spec_path": str(spec_path)}) read_rows = AsyncMock(return_value=[row]) _catalog_database(monkeypatch, read_rows) @@ -17924,9 +18516,14 @@ async def test_catalog_changed_openapi_handler_never_dispatches(monkeypatch, tmp destination = respx_mock.get("https://changed.example/echo").respond(200, text="must not execute") async with manager.catalog.operation(): admitted = manager.get_mcp_server_by_id("catalog-server") - read_rows.return_value = [row.model_copy(update={ - "url": "https://changed.example", "updated_at": datetime(2026, 1, 2), - })] + read_rows.return_value = [ + row.model_copy( + update={ + "url": "https://changed.example", + "updated_at": datetime(2026, 1, 2), + } + ) + ] await asyncio.create_task(manager.reload_servers_from_database()) with pytest.raises(HTTPException) as exc: if dispatch == "managed": @@ -17962,7 +18559,8 @@ async def test_catalog_deferred_discovery_preserves_admitted_configuration(monke entered = asyncio.Event() release = asyncio.Event() metadata = MCPOAuthMetadata( - authorization_url="https://issuer.example/authorize", token_url="https://issuer.example/token", + authorization_url="https://issuer.example/authorize", + token_url="https://issuer.example/token", ) async def discover(server): @@ -17992,15 +18590,19 @@ async def test_catalog_deferred_discovery_preserves_admitted_configuration(monke async def test_catalog_cancelled_config_hydration_preserves_published_credentials(): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="config-hydration", name="config_hydration", transport=MCPTransport.http, - client_id="previous-client") + server: Final = MCPServer( + server_id="config-hydration", name="config_hydration", transport=MCPTransport.http, client_id="previous-client" + ) manager.config_mcp_servers = {server.server_id: server} async def cancelled_hydration(target: MCPServer) -> bool: target.client_id = "unpublished-client" raise asyncio.CancelledError() - with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", side_effect=cancelled_hydration): + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.hydrate_config_server_dcr_client", + side_effect=cancelled_hydration, + ): with pytest.raises(asyncio.CancelledError): await manager.reload_servers_from_database() assert manager.get_mcp_server_by_id(server.server_id).client_id == "previous-client" @@ -18009,15 +18611,22 @@ async def test_catalog_cancelled_config_hydration_preserves_published_credential @pytest.mark.asyncio async def test_catalog_oauth_resolution_cannot_replace_an_operation_target_after_update(): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="pinned-oauth", name="pinned_oauth", transport=MCPTransport.http, - url="https://before.example/mcp", auth_type=MCPAuth.oauth2, - authorization_url="https://before.example/authorize", token_url="https://before.example/token") + server: Final = MCPServer( + server_id="pinned-oauth", + name="pinned_oauth", + transport=MCPTransport.http, + url="https://before.example/mcp", + auth_type=MCPAuth.oauth2, + authorization_url="https://before.example/authorize", + token_url="https://before.example/token", + ) manager.registry = {server.server_id: server} with patch("litellm.proxy.proxy_server.prisma_client", None): async with manager.catalog.operation(): selected: Final = manager.get_mcp_server_by_id(server.server_id) - manager.registry[server.server_id] = server.model_copy(update={ - "url": "https://after.example/mcp", "authorization_url": "https://after.example/authorize"}) + manager.registry[server.server_id] = server.model_copy( + update={"url": "https://after.example/mcp", "authorization_url": "https://after.example/authorize"} + ) resolved: Final = await manager.ensure_oauth_metadata_discovered(selected) assert resolved.url == "https://before.example/mcp" assert resolved.authorization_url == "https://before.example/authorize" @@ -18027,8 +18636,13 @@ async def test_catalog_oauth_resolution_cannot_replace_an_operation_target_after @pytest.mark.asyncio async def test_catalog_unresolved_oauth_snapshot_fails_closed_if_target_was_replaced(): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="replaced-oauth", name="replaced_oauth", transport=MCPTransport.http, - url="https://before.example/mcp", auth_type=MCPAuth.oauth2) + server: Final = MCPServer( + server_id="replaced-oauth", + name="replaced_oauth", + transport=MCPTransport.http, + url="https://before.example/mcp", + auth_type=MCPAuth.oauth2, + ) manager.registry = {server.server_id: server} with ( patch("litellm.proxy.proxy_server.prisma_client", None), @@ -18036,9 +18650,13 @@ async def test_catalog_unresolved_oauth_snapshot_fails_closed_if_target_was_repl ): async with manager.catalog.operation(): selected: Final = manager.get_mcp_server_by_id(server.server_id) - manager.registry[server.server_id] = server.model_copy(update={ - "url": "https://after.example/mcp", "authorization_url": "https://after.example/authorize", - "token_url": "https://after.example/token"}) + manager.registry[server.server_id] = server.model_copy( + update={ + "url": "https://after.example/mcp", + "authorization_url": "https://after.example/authorize", + "token_url": "https://after.example/token", + } + ) with pytest.raises(HTTPException) as exc: await manager.ensure_oauth_metadata_discovered(selected) assert exc.value.status_code == 503 @@ -18048,11 +18666,21 @@ async def test_catalog_unresolved_oauth_snapshot_fails_closed_if_target_was_repl @pytest.mark.asyncio async def test_catalog_unresolved_oauth_snapshot_accepts_discovery_for_the_same_target(): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="same-oauth", name="same_oauth", transport=MCPTransport.http, - url="https://upstream.example/mcp", auth_type=MCPAuth.oauth2) + server: Final = MCPServer( + server_id="same-oauth", + name="same_oauth", + transport=MCPTransport.http, + url="https://upstream.example/mcp", + auth_type=MCPAuth.oauth2, + ) manager.registry = {server.server_id: server} - discovered: Final = server.model_copy(update={"authorization_url": "https://issuer.example/authorize", - "token_url": "https://issuer.example/token", "scopes": ["read"]}) + discovered: Final = server.model_copy( + update={ + "authorization_url": "https://issuer.example/authorize", + "token_url": "https://issuer.example/token", + "scopes": ["read"], + } + ) with ( patch("litellm.proxy.proxy_server.prisma_client", None), patch.object(manager, "_ensure_oauth_metadata_discovered", return_value=discovered) as discovery, @@ -18068,8 +18696,9 @@ async def test_catalog_unresolved_oauth_snapshot_accepts_discovery_for_the_same_ @pytest.mark.asyncio async def test_catalog_fresh_lookup_does_not_fall_back_to_stale_grants_when_database_fails(): manager: Final = MCPServerManager() - server: Final = MCPServer(server_id="stale-grant", name="stale_grant", transport=MCPTransport.http, - allow_all_keys=True) + server: Final = MCPServer( + server_id="stale-grant", name="stale_grant", transport=MCPTransport.http, allow_all_keys=True + ) manager.registry = {server.server_id: server} prisma: Final = MagicMock() prisma.writer_db = prisma.db @@ -18107,7 +18736,6 @@ async def test_catalog_cancelled_registration_does_not_publish_partial_handlers( assert manager.registry == {} - def _revision_row(revision): from types import SimpleNamespace @@ -18388,9 +19016,7 @@ async def test_release_during_auth_fingerprinting_does_not_open_upstream_session _RetiringSessionStub(), ) manager.track_gateway_session(gateway_session_id) - request_task: Final = asyncio.create_task( - manager._upstream_session_for(client, server, gateway_session_headers) - ) + request_task: Final = asyncio.create_task(manager._upstream_session_for(client, server, gateway_session_headers)) try: await asyncio.wait_for(fingerprint_started.wait(), 5) @@ -18428,7 +19054,9 @@ async def test_upstream_session_is_shared_per_gateway_session_and_released_with_ first: Final = await manager._upstream_session_for(client, server, {"Mcp-Session-Id": "gw-1"}) second: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-1"}) other: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-2"}) - assert first is not None and first is second, "every call of one gateway session must share one upstream session" + assert first is not None and first is second, ( + "every call of one gateway session must share one upstream session" + ) assert other is not None and other is not first, "distinct gateway sessions must not share upstream state" manager.release_upstream_sessions("gw-1") @@ -18488,9 +19116,7 @@ async def test_upstream_sessions_are_partitioned_by_discovery_auth_fingerprint( assert first_session.closed assert len(manager._upstream_sessions) == 1 assert tuple(manager._upstream_sessions.values()) == (second_session,) - first_session_again: Final = await manager._upstream_session_for( - first_client, server, gateway_session_headers - ) + first_session_again: Final = await manager._upstream_session_for(first_client, server, gateway_session_headers) assert first_session_again is not None assert first_session_again is not first_session assert first_session_again is not second_session @@ -18541,9 +19167,7 @@ async def test_upstream_sessions_are_partitioned_by_connection_settings( assert first_session.closed assert len(manager._upstream_sessions) == 1 assert tuple(manager._upstream_sessions.values()) == (changed_session,) - first_session_again: Final = await manager._upstream_session_for( - third_client, server, gateway_session_headers - ) + first_session_again: Final = await manager._upstream_session_for(third_client, server, gateway_session_headers) assert first_session_again is not None assert first_session_again is not first_session @@ -18780,9 +19404,7 @@ async def test_upstream_session_is_not_reused_outside_an_mcp_protocol_request() manager.track_gateway_session("gw-rest") try: - session: Final = await manager._upstream_session_for( - client, server, {"mcp-session-id": "gw-rest"} - ) + session: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-rest"}) assert session is None finally: manager.release_upstream_sessions("gw-rest") @@ -19566,8 +20188,11 @@ async def test_upstream_preparation_honors_case_sensitive_extra_command(monkeypa monkeypatch.setenv("LITELLM_ENABLE_MCP_STDIO", "true") monkeypatch.setattr(upstream, "MCP_STDIO_ALLOWED_COMMANDS", frozenset({"CustomRunner"})) server: Final = MCPServer( - server_id="custom-stdio", name="custom-stdio", transport=MCPTransport.stdio, - command="/opt/tools/CustomRunner", args=[], + server_id="custom-stdio", + name="custom-stdio", + transport=MCPTransport.stdio, + command="/opt/tools/CustomRunner", + args=[], ) client: Final = await MCPServerManager()._create_mcp_client(server) assert client.stdio_config is not None @@ -19759,10 +20384,12 @@ async def test_catalog_observes_writer_changes_while_read_replica_lags(monkeypat litellm_mcpservertable=SimpleNamespace(find_many=AsyncMock(return_value=[original])), litellm_config=SimpleNamespace(find_unique=AsyncMock(return_value=_revision_row(7))), ) - routed: Final = SimpleNamespace(**{ - name: _RoutedActions(getattr(writer, name), getattr(reader, name), lambda: True) - for name in ("litellm_mcpservertable", "litellm_config") - }) + routed: Final = SimpleNamespace( + **{ + name: _RoutedActions(getattr(writer, name), getattr(reader, name), lambda: True) + for name in ("litellm_mcpservertable", "litellm_config") + } + ) monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=routed, writer_db=writer)) manager: Final = MCPServerManager() assert (await manager.catalog.resolve(original.server_id)).name == "initial" @@ -19779,7 +20406,9 @@ async def test_catalog_observes_writer_changes_while_read_replica_lags(monkeypat @pytest.mark.asyncio async def test_catalog_revision_failure_preserves_state_and_recovers(monkeypatch): reader: Final = AsyncMock(side_effect=([_catalog_row()], [_catalog_row("updated")])) - revisions: Final = AsyncMock(side_effect=[_revision_row(7), RuntimeError("private database detail"), _revision_row(8)]) + revisions: Final = AsyncMock( + side_effect=[_revision_row(7), RuntimeError("private database detail"), _revision_row(8)] + ) _catalog_database(monkeypatch, reader, revisions) manager: Final = MCPServerManager() assert (await manager.catalog.resolve("catalog-server")).name == "initial" @@ -19905,15 +20534,20 @@ async def test_catalog_accepts_discovered_issuer_response_support( manager: Final = MCPServerManager() server: Final = MCPServer( - server_id="discovered-issuer", name="discovered_issuer", transport=MCPTransport.http, - url="https://resource.example/mcp", auth_type=MCPAuth.oauth2, - issuer="https://issuer.example" if anchored else None, issuer_is_anchored=anchored, + server_id="discovered-issuer", + name="discovered_issuer", + transport=MCPTransport.http, + url="https://resource.example/mcp", + auth_type=MCPAuth.oauth2, + issuer="https://issuer.example" if anchored else None, + issuer_is_anchored=anchored, ) manager.registry = {server.server_id: server} manager._set_oauth_discovery_deferred(server.server_id, True) metadata: Final = MCPOAuthMetadata( discovered_issuer="https://issuer.example", - authorization_url="https://issuer.example/authorize", token_url="https://issuer.example/token", + authorization_url="https://issuer.example/authorize", + token_url="https://issuer.example/token", authorization_response_iss_parameter_supported=issuer_parameter_supported, ) discovery: Final = AsyncMock(return_value=metadata) @@ -19935,9 +20569,13 @@ async def test_catalog_rejects_a_changed_anchored_issuer_during_discovery(monkey manager: Final = MCPServerManager() server: Final = MCPServer( - server_id="anchored-issuer", name="anchored_issuer", transport=MCPTransport.http, - url="https://resource.example/mcp", auth_type=MCPAuth.oauth2, - issuer="https://original.example", issuer_is_anchored=True, + server_id="anchored-issuer", + name="anchored_issuer", + transport=MCPTransport.http, + url="https://resource.example/mcp", + auth_type=MCPAuth.oauth2, + issuer="https://original.example", + issuer_is_anchored=True, ) manager.registry = {server.server_id: server} discovery: Final = AsyncMock() @@ -20023,13 +20661,24 @@ async def test_paginated_listing_keeps_earlier_tool_metadata_and_caller_isolatio @pytest.mark.asyncio -@pytest.mark.parametrize("request_name,result_field", [("ListPromptsRequest", "prompts"), ("ListResourcesRequest", "resources"), ("ListResourceTemplatesRequest", "resource_templates")]) +@pytest.mark.parametrize( + "request_name,result_field", + [ + ("ListPromptsRequest", "prompts"), + ("ListResourcesRequest", "resources"), + ("ListResourceTemplatesRequest", "resource_templates"), + ], +) @pytest.mark.parametrize("cursor", [None, "next"]) -async def test_disabled_stdio_catalog_is_empty_initially_and_rejects_continuation(monkeypatch, request_name, result_field, cursor): +async def test_disabled_stdio_catalog_is_empty_initially_and_rejects_continuation( + monkeypatch, request_name, result_field, cursor +): from mcp import types monkeypatch.delenv("LITELLM_ENABLE_MCP_STDIO", raising=False) - server = MCPServer(server_id="disabled", name="disabled", transport=MCPTransport.stdio, command="blocked-executable", args=[]) + server = MCPServer( + server_id="disabled", name="disabled", transport=MCPTransport.stdio, command="blocked-executable", args=[] + ) request = getattr(types, request_name)(params=types.PaginatedRequestParams(cursor=cursor)) manager = MCPServerManager() if cursor is not None: @@ -20041,7 +20690,6 @@ async def test_disabled_stdio_catalog_is_empty_initially_and_rejects_continuatio assert page.next_cursor is None - @pytest.mark.asyncio @pytest.mark.parametrize("cursor", [None, "next"]) async def test_disabled_stdio_tools_are_empty_initially_and_reject_continuation(monkeypatch, cursor): @@ -20149,8 +20797,10 @@ async def test_failed_aggregate_continuation_preserves_only_delivered_tool_metad ListToolsResult(tools=[first], next_cursor="next"), ListToolsResult(tools=[unseen], next_cursor="next" if client is clients["beta"] else None), ] + async def create_client(server, **kwargs): return clients[server.server_id] + manager.create_mcp_client = create_client monkeypatch.setattr(operations, "global_mcp_server_manager", manager) caller = UserAPIKeyAuth(api_key="owned-caller", user_id="alice") @@ -20187,7 +20837,9 @@ async def test_aggregate_publishes_complete_bare_routes_only_after_delivering_a_ clients = {server.server_id: AsyncMock() for server in servers} for name, client in clients.items(): client._last_initialize_instructions = None - client.list_tools_page.return_value = ListToolsResult(tools=[tool], next_cursor="next" if name == "beta" else None) + client.list_tools_page.return_value = ListToolsResult( + tools=[tool], next_cursor="next" if name == "beta" else None + ) async def create_client(server, **kwargs): return clients[server.server_id] @@ -20278,27 +20930,55 @@ async def test_manual_oauth_endpoints_discover_client_metadata_once( monkeypatch.setattr(manager_module, "global_mcp_server_manager", manager) configured: Final[MCPServer] if source == "config": - await manager.load_servers_from_config({"manual": { - "url": "https://up.example.com/mcp", "transport": "http", "auth_type": "oauth2", - "oauth2_flow": "authorization_code", "authorization_url": "https://up.example.com/authorize", - "token_url": "https://up.example.com/token", "scopes": ["read"], - }}) + await manager.load_servers_from_config( + { + "manual": { + "url": "https://up.example.com/mcp", + "transport": "http", + "auth_type": "oauth2", + "oauth2_flow": "authorization_code", + "authorization_url": "https://up.example.com/authorize", + "token_url": "https://up.example.com/token", + "scopes": ["read"], + } + } + ) configured = next(iter(manager.config_mcp_servers.values())) else: row: Final = LiteLLM_MCPServerTable( - server_id="manual", alias="manual", url="https://up.example.com/mcp", transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", - authorization_url="https://up.example.com/authorize", token_url="https://up.example.com/token", - credentials={"scopes": ["read"]}, created_at=datetime.now(), updated_at=datetime.now(), + server_id="manual", + alias="manual", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + oauth2_flow="authorization_code", + authorization_url="https://up.example.com/authorize", + token_url="https://up.example.com/token", + credentials={"scopes": ["read"]}, + created_at=datetime.now(), + updated_at=datetime.now(), ) built: Final = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) manager.registry[built.server_id] = built configured = built async with manager.catalog.operation(): response: Final = await discoverable_endpoints.register_client_with_server( - Request({"type": "http", "scheme": "https", "server": ("gateway.example.com", 443), - "path": "/register", "root_path": "", "headers": [], "query_string": b""}), - configured, "Gateway", None, None, None, + Request( + { + "type": "http", + "scheme": "https", + "server": ("gateway.example.com", 443), + "path": "/register", + "root_path": "", + "headers": [], + "query_string": b"", + } + ), + configured, + "Gateway", + None, + None, + None, ) body: Final = json.loads(response.body) if hasattr(response, "body") else response assert body["client_id"] == ( @@ -20326,11 +21006,19 @@ async def test_optional_client_metadata_discovery_failure_preserves_manual_endpo respx_mock.get("https://up.example.com/.well-known/oauth-authorization-server").respond(503) respx_mock.route().respond(404) manager: Final = MCPServerManager() - await manager.load_servers_from_config({"manual": { - "url": "https://up.example.com/mcp", "transport": "http", "auth_type": "oauth2", - "oauth2_flow": "authorization_code", "authorization_url": "https://up.example.com/authorize", - "token_url": "https://up.example.com/token", "scopes": ["read"], - }}) + await manager.load_servers_from_config( + { + "manual": { + "url": "https://up.example.com/mcp", + "transport": "http", + "auth_type": "oauth2", + "oauth2_flow": "authorization_code", + "authorization_url": "https://up.example.com/authorize", + "token_url": "https://up.example.com/token", + "scopes": ["read"], + } + } + ) configured: Final = next(iter(manager.config_mcp_servers.values())) resolved: Final = await manager.ensure_oauth_metadata_discovered(configured) attempts: Final = len(respx_mock.calls) @@ -20350,26 +21038,43 @@ def test_oauth_rebuild_retains_only_corroborated_cimd_capability(capability: boo from litellm.proxy._experimental.mcp_server.mcp_server_manager import carry_forward_resolved_oauth_endpoints previous: Final = MCPServer( - server_id="cimd-rebuild", name="cimd_rebuild", url="https://mcp.example.com/mcp", - transport=MCPTransport.http, auth_type=MCPAuth.oauth2, - authorization_url="https://idp.example.com/authorize", token_url="https://idp.example.com/token", + server_id="cimd-rebuild", + name="cimd_rebuild", + url="https://mcp.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", client_id_metadata_document_supported=capability, - authorization_server_metadata={"issuer": "https://idp.example.com", "response_types_supported": ["code"], "scopes_supported": ["offline_access"]}, + authorization_server_metadata={ + "issuer": "https://idp.example.com", + "response_types_supported": ["code"], + "scopes_supported": ["offline_access"], + }, + ) + rebuilt: Final = previous.model_copy( + update={ + "client_id_metadata_document_supported": not capability if rebuild == "fresh_discovery" else None, + "authorization_url": "https://changed.example.com/authorize" + if rebuild == "repointed" + else previous.authorization_url, + "token_url": None, + "authorization_server_metadata": {"scopes_supported": []} if rebuild == "fresh_discovery" else None, + "issuer": "https://idp.example.com" if rebuild == "anchored" else None, + "issuer_is_anchored": rebuild == "anchored", + } ) - rebuilt: Final = previous.model_copy(update={ - "client_id_metadata_document_supported": not capability if rebuild == "fresh_discovery" else None, - "authorization_url": "https://changed.example.com/authorize" if rebuild == "repointed" else previous.authorization_url, - "token_url": None, - "authorization_server_metadata": {"scopes_supported": []} if rebuild == "fresh_discovery" else None, - "issuer": "https://idp.example.com" if rebuild == "anchored" else None, - "issuer_is_anchored": rebuild == "anchored", - }) carry_forward_resolved_oauth_endpoints(rebuilt, previous) - expected: Final = not capability if rebuild == "fresh_discovery" else None if rebuild in ("repointed", "anchored") else capability + expected: Final = ( + not capability if rebuild == "fresh_discovery" else None if rebuild in ("repointed", "anchored") else capability + ) assert rebuilt.client_id_metadata_document_supported is expected assert rebuilt.authorization_server_metadata == ( - {"scopes_supported": []} if rebuild == "fresh_discovery" - else previous.authorization_server_metadata if rebuild == "same" else None + {"scopes_supported": []} + if rebuild == "fresh_discovery" + else previous.authorization_server_metadata + if rebuild == "same" + else None ) @@ -20378,13 +21083,19 @@ def test_oauth_rebuild_retains_only_corroborated_cimd_capability(capability: boo async def test_repeated_stale_discovery_uses_current_callers_endpoint(endpoint: str) -> None: manager: Final = MCPServerManager() original: Final = MCPServer( - server_id="partial-replacement", name="replacement", url="https://old.example.com/mcp", - transport=MCPTransport.http, auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", + server_id="partial-replacement", + name="replacement", + url="https://old.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + oauth2_flow="authorization_code", ) replacement: Final = original.model_copy(update={endpoint: "https://new.example.com/oauth"}) manager.registry[original.server_id] = replacement resolved: Final = await manager._rejoin_oauth_metadata_discovery( - original, needed_endpoint=lambda server: getattr(server, endpoint), retry_stale=False, + original, + needed_endpoint=lambda server: getattr(server, endpoint), + retry_stale=False, ) assert resolved is replacement @@ -20394,11 +21105,19 @@ def test_openapi_extra_headers_exclude_gateway_admission_key(header_name: str) - caller_key: Final = "Bearer sk-admission-only" raw_headers: Final = {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} server: Final = MCPServer( - server_id="header-boundary", name="header-boundary", transport=MCPTransport.http, - spec_path="/spec.yaml", auth_type=MCPAuth.none, extra_headers=[header_name, "X-Tenant"], + server_id="header-boundary", + name="header-boundary", + transport=MCPTransport.http, + spec_path="/spec.yaml", + auth_type=MCPAuth.none, + extra_headers=[header_name, "X-Tenant"], ) assert resolve_openapi_tool_auth( - server, None, None, raw_headers, UserAPIKeyAuth(api_key="sk-admission-only"), + server, + None, + None, + raw_headers, + UserAPIKeyAuth(api_key="sk-admission-only"), ) == (None, {"X-Tenant": "tenant-control"}, None) assert raw_headers == {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index 66d66271f71..b8e28c3f9b8 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -1335,9 +1335,12 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): - with patch( - "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", - ) as mock_logger, patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger): + with ( + patch( + "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", + ) as mock_logger, + patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger), + ): # Test with server-specific auth headers mcp_server_auth_headers = { "working": {"Authorization": "Bearer working-token"}, @@ -1431,9 +1434,12 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): - with patch( - "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", - ) as mock_logger, patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger): + with ( + patch( + "litellm.proxy._experimental.mcp_server.catalog.verbose_logger", + ) as mock_logger, + patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger), + ): # Test with server-specific auth headers mcp_server_auth_headers = { "failing1": {"Authorization": "Bearer failing1-token"}, @@ -3535,8 +3541,15 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): patch( # test-quality-ok: registry is empty in unit tests; key owns one server "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, - return_value=[MCPServer(server_id="new-server", name="new-server", url="http://upstream/mcp", - transport=MCPTransport.http, auth_type=MCPAuth.none)], + return_value=[ + MCPServer( + server_id="new-server", + name="new-server", + url="http://upstream/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.none, + ) + ], ), patch( "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", @@ -3760,10 +3773,13 @@ async def test_owner_fingerprint_hashes_custom_api_keys(): @pytest.mark.asyncio -@pytest.mark.parametrize("body", [ - b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', - b'{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":0}}', -]) +@pytest.mark.parametrize( + "body", + [ + b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', + b'{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":0}}', + ], +) async def test_stateful_mcp_session_owner_mismatch_returns_403(body: bytes): """ A stateful mcp-session-id is bound to its creator. A different @@ -7328,9 +7344,9 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), mcp_servers=["passthrough_server"], client_ip=None, - oauth2_headers={"Authorization": "Bearer upstream-token"}, - mcp_server_auth_headers=None, - raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, ) assert result is None else: @@ -7340,9 +7356,9 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu user_api_key_auth=UserAPIKeyAuth(user_id="admitted-user"), mcp_servers=["passthrough_server"], client_ip=None, - oauth2_headers={"Authorization": "Bearer upstream-token"}, - mcp_server_auth_headers=None, - raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + mcp_server_auth_headers=None, + raw_headers={"x-litellm-api-key": "sk-litellm-proxy-key", "Authorization": "Bearer upstream-token"}, ) assert exc_info.value.status_code == expected_status if expected_status == 401: @@ -10710,6 +10726,7 @@ async def test_post_mcp_call_guardrails_return_the_rewritten_result(): logging_obj = _mock_mcp_logging_obj() raw_result = _call_tool_result(False, "jane@example.com") masked_result = _call_tool_result(False, "") + auth: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user") proxy_logging_mock = _mock_mcp_proxy_logging() proxy_logging_mock.post_mcp_call_hook = AsyncMock(return_value=masked_result) @@ -10717,7 +10734,8 @@ async def test_post_mcp_call_guardrails_return_the_rewritten_result(): returned = await _run_post_mcp_call_guardrails( result=raw_result, litellm_logging_obj=logging_obj, - user_api_key_auth=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + original_auth=auth, + guardrail_auth=auth, request_data={}, ) @@ -10741,6 +10759,7 @@ async def test_post_mcp_call_guardrails_run_without_a_logging_object(): raw_result = _call_tool_result(False, "jane@example.com") masked_result = _call_tool_result(False, "") + auth: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user") proxy_logging_mock = _mock_mcp_proxy_logging() proxy_logging_mock.post_mcp_call_hook = AsyncMock(return_value=masked_result) @@ -10748,7 +10767,8 @@ async def test_post_mcp_call_guardrails_run_without_a_logging_object(): returned = await _run_post_mcp_call_guardrails( result=raw_result, litellm_logging_obj=None, - user_api_key_auth=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + original_auth=auth, + guardrail_auth=auth, request_data={"name": "fetch_record"}, ) @@ -10775,11 +10795,56 @@ async def test_post_mcp_call_guardrails_propagate_a_block(): await _run_post_mcp_call_guardrails( result=_call_tool_result(False, "jane@example.com"), litellm_logging_obj=_mock_mcp_logging_obj(), - user_api_key_auth=None, + original_auth=None, + guardrail_auth=None, request_data={}, ) +@pytest.mark.asyncio +async def test_post_mcp_call_guardrails_preserve_and_deduplicate_mapping_entries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._experimental.mcp_server.server import _run_post_mcp_call_guardrails + from litellm.proxy import proxy_server + + monkeypatch.setattr("litellm.proxy.utils.premium_user_check", MagicMock()) + monkeypatch.setattr(proxy_server, "premium_user", True, raising=False) + original_auth: Final = UserAPIKeyAuth(api_key="personal-key", user_id="alice") + guardrail_auth: Final = original_auth.model_copy( + update={ + "team_id": "team-a", + "team_metadata": { + "guardrails": [ + {"request-guardrail": {"team": True}}, + {"team-guardrail": {"x": 1}}, + {"team-guardrail": {"x": 2}}, + "request-guardrail", + "string-guardrail", + ] + }, + } + ) + proxy_logging_mock = _mock_mcp_proxy_logging() + proxy_logging_mock.post_mcp_call_hook = AsyncMock(return_value=_call_tool_result(False, "ok")) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): + await _run_post_mcp_call_guardrails( + result=_call_tool_result(False, "ok"), + litellm_logging_obj=None, + original_auth=original_auth, + guardrail_auth=guardrail_auth, + request_data={"metadata": {"guardrails": [{"request-guardrail": {"request": True}}]}}, + ) + + request_data: Final = proxy_logging_mock.post_mcp_call_hook.await_args.kwargs["request_data"] + assert request_data["metadata"]["guardrails"] == [ + {"request-guardrail": {"request": True}}, + {"team-guardrail": {"x": 1}}, + "string-guardrail", + ] + + class TestListFiltersHonorThePrefixBoundary: """The listing filters compare a published (prefixed) tool name against configured entries, so they have to locate the boundary with the server's @@ -11836,8 +11901,7 @@ async def test_passthrough_probes_bind_each_token_to_its_authorized_server(allow ) assert result is None assert tuple( - (str(call.request.url), call.request.headers["Authorization"]) - for call in (*first.calls, *second.calls) + (str(call.request.url), call.request.headers["Authorization"]) for call in (*first.calls, *second.calls) ) == ( (("http://first/mcp", "Bearer first-token"), ("http://second/mcp", "Bearer second-token")) if allowed and header_shape != "non_authorization" @@ -11899,6 +11963,7 @@ async def test_streamable_http_admission_auth_sees_only_peeked_jsonrpc_method() stateless_handle.assert_not_awaited() send.assert_not_awaited() + @pytest.mark.asyncio @pytest.mark.parametrize( "payload, valid", @@ -11991,16 +12056,22 @@ async def test_progress_text_obeys_request_output_guardrails(monkeypatch, action from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks policy = ContentFilterGuardrail( - guardrail_name="progress-policy", event_hook=GuardrailEventHooks.post_mcp_call, + guardrail_name="progress-policy", + event_hook=GuardrailEventHooks.post_mcp_call, blocked_words=[BlockedWord(keyword="halfway", action=ContentFilterAction(action))], - keyword_redaction_tag="[masked]", default_on=False, + keyword_redaction_tag="[masked]", + default_on=False, ) monkeypatch.setattr(litellm, "callbacks", [policy]) monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) delivered = AsyncMock() caller = UserAPIKeyAuth(user_id="progress-user") callback = _guarded_progress_callback( - delivered, None, caller, {"metadata": {"guardrails": ["progress-policy"] if selected else []}} + delivered, + None, + caller, + caller, + {"metadata": {"guardrails": ["progress-policy"] if selected else []}}, ) assert callback is not None await callback(0, None, message) @@ -12014,8 +12085,98 @@ async def test_progress_callback_preserves_absence_and_cancellation(): from litellm.proxy._experimental.mcp_server.operations import _guarded_progress_callback - assert _guarded_progress_callback(None, None, None, {}) is None - callback = _guarded_progress_callback(AsyncMock(side_effect=asyncio.CancelledError), None, None, {}) + assert _guarded_progress_callback(None, None, None, None, {}) is None + callback = _guarded_progress_callback(AsyncMock(side_effect=asyncio.CancelledError), None, None, None, {}) assert callback is not None with pytest.raises(asyncio.CancelledError): await callback(0, 0, None) + + +@pytest.mark.asyncio +async def test_execute_mcp_tool_derives_team_guardrail_auth_for_personal_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.utils import get_server_prefix + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.utils import ProxyLogging + + user_api_key_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="owner") + server: Final = MCPServer( + server_id="team-server", + name="math_a", + server_name="math_a", + transport=MCPTransport.http, + url="http://localhost/mcp", + ) + team: Final = LiteLLM_TeamTable( + team_id="team-a", + organization_id="org-team-a", + object_permission_id="perm-team-a", + metadata={"guardrails": ["team-guardrail"]}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-a", + mcp_servers=["team-server"], + mcp_tool_permissions={"team-server": ["test_tool"]}, + ), + ) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) + + async def fake_handle_managed_mcp_tool( + *, + server_name: str, + name: str, + arguments: dict[str, object], + user_api_key_auth: UserAPIKeyAuth | None, + guardrail_auth: UserAPIKeyAuth | None, + **_: object, + ) -> CallToolResult: + await global_mcp_server_manager.pre_call_tool_check( + name=name, + arguments=arguments, + server_name=server_name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=server, + guardrail_auth=guardrail_auth, + ) + return CallToolResult(content=[], is_error=False) + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging_obj) + monkeypatch.setattr("litellm.proxy.utils.premium_user_check", MagicMock()) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="owner", teams=["team-a"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=team)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_personal_key_team_within_budget", AsyncMock(return_value=True)) + monkeypatch.setattr(global_mcp_server_manager, "check_tool_permission_for_key_team", AsyncMock(return_value=None)) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + monkeypatch.setattr(mcp_operations, "_list_tools_before_first_call", AsyncMock(return_value=None)) + monkeypatch.setattr(mcp_operations, "_handle_managed_mcp_tool", fake_handle_managed_mcp_tool) + + await mcp_operations.execute_mcp_tool( + name=f"{get_server_prefix(server)}-test_tool", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(), + user_api_key_auth=user_api_key_auth, + requested_server_id=server.server_id, + ) + + guardrail_auth: Final = proxy_logging_obj.pre_call_hook.await_args.kwargs["user_api_key_dict"] + assert guardrail_auth.team_id == "team-a" + assert guardrail_auth.team_metadata == {"guardrails": ["team-guardrail"]} diff --git a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py index 2a04c8cda2a..73e248ff52e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1564,7 +1564,13 @@ class TestListToolsRestAPI: monkeypatch, ): """The REST tools/list path should include tools beyond the upstream first page.""" - from mcp.types import Implementation, InitializeResult, ListToolsResult, PaginatedRequestParams, ServerCapabilities + from mcp.types import ( + Implementation, + InitializeResult, + ListToolsResult, + PaginatedRequestParams, + ServerCapabilities, + ) from mcp.types import Tool as MCPTool import litellm.experimental_mcp_client.client as mcp_client_module @@ -1604,11 +1610,13 @@ class TestListToolsRestAPI: mock_session_ctx = AsyncMock() mock_session_instance = AsyncMock() - mock_session_instance.initialize = AsyncMock(return_value=InitializeResult( - protocol_version="2025-11-25", - capabilities=ServerCapabilities(), - server_info=Implementation(name="stub", version="1"), - )) + mock_session_instance.initialize = AsyncMock( + return_value=InitializeResult( + protocol_version="2025-11-25", + capabilities=ServerCapabilities(), + server_info=Implementation(name="stub", version="1"), + ) + ) mock_session_instance.list_tools.side_effect = [ ListToolsResult( tools=[ @@ -2556,10 +2564,23 @@ class TestCallToolRestAPI: assert logged_data["metadata"]["model_group"] == "MCP: demo-tool" logging_obj = failure_log.await_args.args[0] assert logging_obj.model_call_details["mcp_tool_call_metadata"] == { - "name": "demo-tool", "mcp_server_name": "Example server", + "name": "demo-tool", + "mcp_server_name": "Example server", } - async def test_executes_tool_when_allowed(self, monkeypatch): + @pytest.mark.parametrize( + ("tool_name", "native_tool_name"), + ( + ("math_a-add", "add"), + ("add", "add"), + ("math_a-math_a-foo", "math_a-foo"), + ), + ) + async def test_executes_tool_when_allowed(self, monkeypatch, tool_name, native_tool_name): + import litellm + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler, MCPServerAccess + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + async def fake_contexts(user_api_key_auth): return [user_api_key_auth] @@ -2568,8 +2589,9 @@ class TestCallToolRestAPI: class StubServer: server_id = "server-1" - alias = "server-1" - server_name = "server-1" + alias = "math_a" + server_name = "math_a" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -2577,6 +2599,17 @@ class TestCallToolRestAPI: auth_type = None stub_server = StubServer() + original_auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="owner") + source_auth: Final = original_auth.model_copy(update={"team_id": "team-a", "org_id": "org-a"}) + + async def fake_attributing_source( + auth: UserAPIKeyAuth, + server_id: str, + tool_name: str, + grants: tuple[tuple[UserAPIKeyAuth, MCPServerAccess], ...] | None = None, + ) -> UserAPIKeyAuth | None: + captured["attributed_tool_name"] = tool_name + return source_auth if server_id == "server-1" and tool_name == native_tool_name else None async def fake_add_litellm_data_to_request(**kwargs): return kwargs.get("data", {}) @@ -2585,6 +2618,11 @@ class TestCallToolRestAPI: async def fake_execute_mcp_tool(**kwargs): captured.update(kwargs) + captured["executed_tool_name"] = mcp_operations.resolve_native_tool_name_for_server( + kwargs["name"], + stub_server, + kwargs["allowed_mcp_servers"], + ) return _OK_TOOL_RESULT monkeypatch.setattr( @@ -2605,6 +2643,13 @@ class TestCallToolRestAPI: lambda server_id: stub_server if server_id == "server-1" else None, raising=False, ) + monkeypatch.setattr( + global_mcp_server_manager, + "get_mcp_server_from_tool_name", + lambda tool_name: stub_server if tool_name.startswith("math_a-") else None, + ) + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(MCPRequestHandler, "personal_key_attributing_source", fake_attributing_source) monkeypatch.setattr( "litellm.proxy.proxy_server.add_litellm_data_to_request", fake_add_litellm_data_to_request, @@ -2631,7 +2676,7 @@ class TestCallToolRestAPI: request_payload = { "server_id": "server-1", - "name": "demo-tool", + "name": tool_name, "arguments": {"foo": "bar"}, } request = _build_request( @@ -2642,16 +2687,84 @@ class TestCallToolRestAPI: result = await rest_endpoints.call_tool_rest_api( request, - user_api_key_dict=UserAPIKeyAuth(), + user_api_key_dict=original_auth, ) assert result == _OK_TOOL_RESULT - assert captured["name"] == "demo-tool" + assert captured["name"] == tool_name + assert captured["executed_tool_name"] == native_tool_name + assert captured["attributed_tool_name"] == native_tool_name assert captured["arguments"] == {"foo": "bar"} assert captured["allowed_mcp_servers"] == [stub_server] assert captured["oauth2_headers"] is None + assert captured["user_api_key_auth"].team_id is None + assert captured["user_api_key_auth"].org_id is None + assert captured["guardrail_auth"].team_id == "team-a" + assert captured["guardrail_auth"].org_id == "org-a" + assert fire_logging.await_args.kwargs["user_api_key_auth"] is captured["guardrail_auth"] + logging_obj = fire_logging.await_args.args[0] + assert logging_obj.model_call_details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" fire_logging.assert_awaited_once() + @pytest.mark.asyncio + async def test_rest_billing_uses_target_server_when_global_tool_map_is_empty( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + import litellm + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.auth import auth_checks + + target_server: Final = MCPServer( + server_id="server-b", + name="server-b", + server_name="server-b", + transport=MCPTransport.http, + ) + team: Final = LiteLLM_TeamTable( + team_id="team-b", + object_permission_id="perm-team-b", + organization_id="org-team-b", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-b", + mcp_servers=["server-b"], + mcp_tool_permissions={"server-b": ["search"]}, + ), + ) + auth: Final = UserAPIKeyAuth(api_key="sk-personal", user_id="alice") + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + assert team_id == "team-b" + return team + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "user_api_key_cache", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=["team-b"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setitem(global_mcp_server_manager.registry, target_server.server_id, target_server) + monkeypatch.setattr(global_mcp_server_manager, "tool_name_to_mcp_server_name_mapping", {}) + + resolved_server, billed = await rest_endpoints._rest_tool_call_server_and_auth( + auth, + (target_server,), + target_server.server_id, + "search", + ) + + assert resolved_server is target_server + assert billed.team_id == "team-b" + async def test_extract_mcp_headers_scrubs_admitted_caller_credential(self) -> None: caller_key: Final = "sk-rest-caller-admission-key-123" upstream_token: Final = "Bearer unrelated-upstream-token" @@ -2736,6 +2849,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -2813,6 +2927,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -2876,6 +2991,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -2943,6 +3059,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -3036,6 +3153,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -3123,6 +3241,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -3192,6 +3311,7 @@ class TestCallToolRestAPI: server_id = "server-1" alias = "server-1" server_name = "server-1" + tool_name_to_display_name = None name = "stub" allowed_tools = None mcp_info = {"server_name": "stub"} @@ -3291,19 +3411,29 @@ class TestCallToolRestAPI: from litellm.proxy import proxy_server guardrail_error = HTTPException(status_code=400, detail={"error": "Content blocked"}) + server = MagicMock() + server.server_id = "server-1" + server.alias = "server-1" + server.server_name = "server-1" + server.short_prefix = None + server.tool_name_to_display_name = None + server.name = "stub" async def fake_add_litellm_data_to_request(**kwargs): return kwargs.get("data", {}) - async def blocking_pre_call_hook(user_api_key_dict, data, call_type, skip_guardrails=False): - raise guardrail_error - failure_logging = AsyncMock(side_effect=RuntimeError("spend log db down")) + execute_tool = AsyncMock(side_effect=guardrail_error) + monkeypatch.setattr( + rest_endpoints, + "_resolve_allowed_mcp_servers_with_ip_filter", + AsyncMock(return_value=([server], "server-1")), + ) + monkeypatch.setattr(rest_endpoints, "execute_mcp_tool", execute_tool) monkeypatch.setattr( proxy_server, "add_litellm_data_to_request", fake_add_litellm_data_to_request, raising=False ) monkeypatch.setattr(proxy_server, "proxy_config", {}, raising=False) - monkeypatch.setattr(proxy_server.proxy_logging_obj, "pre_call_hook", blocking_pre_call_hook) monkeypatch.setattr(rest_endpoints, "fire_mcp_tool_call_failure_logging", failure_logging, raising=False) request = _build_request( @@ -3318,6 +3448,7 @@ class TestCallToolRestAPI: ) assert exc_info.value is guardrail_error + execute_tool.assert_awaited_once() failure_logging.assert_awaited_once() async def test_success_logging_cancellation_propagates(self, monkeypatch): @@ -3792,7 +3923,11 @@ class TestGetToolsForSingleServer: from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport - mock_tools = [Tool(name="tool1", description="tool1", inputSchema={}), Tool(name="tool2", description="tool2", inputSchema={}), Tool(name="tool3", description="tool3", inputSchema={})] + mock_tools = [ + Tool(name="tool1", description="tool1", inputSchema={}), + Tool(name="tool2", description="tool2", inputSchema={}), + Tool(name="tool3", description="tool3", inputSchema={}), + ] async def fake_get_tools_from_server(**kwargs): return mock_tools @@ -4531,6 +4666,7 @@ class TestToolResponseMcpInfoEnrichment: assert response.model_dump(by_alias=True, exclude={"mcp_info"}) == original assert tool.model_dump(by_alias=True) == original + class TestRestListToolsetFiltering: @pytest.mark.asyncio async def test_rest_list_filters_toolset_only_key_to_toolset_tools(self, monkeypatch): @@ -4799,8 +4935,11 @@ async def test_preview_client_honors_protocol_metadata(revision: MCPUpstreamProt from litellm.experimental_mcp_client.client import MCPClient payload: Final = NewMCPServerRequest( - server_name="preview", url="http://127.0.0.1:9/mcp", transport="http", - auth_type=MCPAuth.none, mcp_info={"protocol_version": revision}, + server_name="preview", + url="http://127.0.0.1:9/mcp", + transport="http", + auth_type=MCPAuth.none, + mcp_info={"protocol_version": revision}, ) async def inspect_client(client: MCPClient) -> dict[str, str]: @@ -4824,8 +4963,10 @@ async def test_preview_client_honors_protocol_metadata(revision: MCPUpstreamProt ), ) async def test_saved_preview_protocol_omission_and_explicit_edits( - monkeypatch: pytest.MonkeyPatch, auth_type: MCPAuth, - metadata: dict[str, str] | None, expected: MCPUpstreamProtocol, + monkeypatch: pytest.MonkeyPatch, + auth_type: MCPAuth, + metadata: dict[str, str] | None, + expected: MCPUpstreamProtocol, ) -> None: from starlette.datastructures import Headers @@ -4834,19 +4975,29 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( from litellm.proxy.management_endpoints import mcp_management_endpoints saved: Final = MCPServer( - server_id="saved-preview", name="preview", url="https://example.com/mcp", - transport="http", auth_type=auth_type, protocol_version="2025-11-25", + server_id="saved-preview", + name="preview", + url="https://example.com/mcp", + transport="http", + auth_type=auth_type, + protocol_version="2025-11-25", authentication_token="stored-token", - authorization_url="https://example.com/authorize", token_url="https://example.com/token", + authorization_url="https://example.com/authorize", + token_url="https://example.com/token", ) manager: Final = MCPServerManager() manager.registry = {saved.server_id: saved} monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) monkeypatch.setattr(mcp_management_endpoints, "global_mcp_server_manager", manager) payload: Final = NewMCPServerRequest( - server_id=saved.server_id, server_name=saved.name, url=saved.url, transport="http", - auth_type=auth_type, mcp_info=metadata, - authorization_url=saved.authorization_url, token_url=saved.token_url, + server_id=saved.server_id, + server_name=saved.name, + url=saved.url, + transport="http", + auth_type=auth_type, + mcp_info=metadata, + authorization_url=saved.authorization_url, + token_url=saved.token_url, ) staged: Final = rest_endpoints._stage_server_test( payload, Headers({"x-litellm-api-key": "sk-admin", "authorization": "Bearer preview-token"}) @@ -4856,8 +5007,10 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( return {"protocol_version": client.protocol_version} result: Final = await rest_endpoints._execute_with_mcp_client( - staged.request, inspect_client, - mcp_auth_header=staged.mcp_auth_header, oauth2_headers=staged.oauth2_headers, + staged.request, + inspect_client, + mcp_auth_header=staged.mcp_auth_header, + oauth2_headers=staged.oauth2_headers, ) assert result == {"protocol_version": expected} assert saved.protocol_version == "2025-11-25" @@ -4866,14 +5019,20 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( @pytest.mark.asyncio @pytest.mark.parametrize("virtual", [False, True]) -@pytest.mark.parametrize("bearer,claims,expected", [ - ("sk-guardrail-admission", None, None), - ("caller.jwt.assertion", {"sub": "caller"}, "caller.jwt.assertion"), - ("upstream-token", None, "upstream-token"), -]) +@pytest.mark.parametrize( + "bearer,claims,expected", + [ + ("sk-guardrail-admission", None, None), + ("caller.jwt.assertion", {"sub": "caller"}, "caller.jwt.assertion"), + ("upstream-token", None, "upstream-token"), + ], +) async def test_rest_guardrail_bearer_excludes_admission_key( - monkeypatch: pytest.MonkeyPatch, virtual: bool, - bearer: str, claims: dict[str, str] | None, expected: str | None, + monkeypatch: pytest.MonkeyPatch, + virtual: bool, + bearer: str, + claims: dict[str, str] | None, + expected: str | None, ) -> None: import litellm from litellm.caching.dual_cache import DualCache @@ -4884,8 +5043,11 @@ async def test_rest_guardrail_bearer_excludes_admission_key( class CaptureBearer(CustomLogger): async def async_pre_call_hook( - self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, - data: dict[str, object], call_type: str, + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], + call_type: str, ) -> None: if call_type == "call_mcp_tool" and "incoming_bearer_token" in data: observed.put_nowait(data["incoming_bearer_token"]) @@ -4894,23 +5056,36 @@ async def test_rest_guardrail_bearer_excludes_admission_key( return "guarded-result" auth: Final = UserAPIKeyAuth( - api_key="sk-guardrail-admission", jwt_claims=claims, - object_permission={"object_permission_id": "guarded", "mcp_servers": ["guarded"], "mcp_tool_search_enabled": virtual}, + api_key="sk-guardrail-admission", + jwt_claims=claims, + object_permission={ + "object_permission_id": "guarded", + "mcp_servers": ["guarded"], + "mcp_tool_search_enabled": virtual, + }, ) managed: Final = MCPServer( - server_id="guarded", name="guarded", transport=MCPTransport.http, - spec_path="/catalog.yaml", allow_all_keys=True, + server_id="guarded", + name="guarded", + transport=MCPTransport.http, + spec_path="/catalog.yaml", + allow_all_keys=True, ) monkeypatch.setitem(rest_endpoints.global_mcp_server_manager.registry, managed.server_id, managed) monkeypatch.setattr(litellm, "callbacks", [CaptureBearer()]) global_mcp_tool_registry.register_tool( - name="guarded-echo", description="Echo", input_schema={"type": "object"}, handler=echo, + name="guarded-echo", + description="Echo", + input_schema={"type": "object"}, + handler=echo, ) - headers: Final = {"authorization": f"Bearer {bearer}", **( - {} if claims else {"x-litellm-api-key": "sk-guardrail-admission"} - )} + headers: Final = { + "authorization": f"Bearer {bearer}", + **({} if claims else {"x-litellm-api-key": "sk-guardrail-admission"}), + } payload: Final = { - "server_id": "guarded", "name": "mcp_tool_call" if virtual else "guarded-echo", + "server_id": "guarded", + "name": "mcp_tool_call" if virtual else "guarded-echo", "arguments": {"tool_name": "guarded-echo", "arguments": {}} if virtual else {}, } try: @@ -4923,7 +5098,8 @@ async def test_rest_guardrail_bearer_excludes_admission_key( assert observed.get_nowait() == expected if claims is None: list_headers: Final = rest_endpoints._extract_mcp_headers_from_request( - _build_request(headers), auth, + _build_request(headers), + auth, )[2] authorization: Final = list_headers.get("authorization", "") assert (authorization.removeprefix("Bearer ") or None) == expected diff --git a/tests/unit/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/unit/proxy/agent_endpoints/auth/test_agent_permission_handler.py index 45d3d7fb22e..4692088e536 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_agent_permission_handler.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_agent_permission_handler.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch import pytest from litellm.constants import UI_SESSION_TOKEN_TEAM_ID -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import UI_TEAM_ID, LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( @@ -76,71 +76,39 @@ class TestAgentRequestHandler: ) # Case 1: Both key and team have agents - intersection - with patch.object( - AgentRequestHandler, "get_allowed_agents_for_key" - ) as mock_key: - with patch.object( - AgentRequestHandler, "_get_allowed_agents_for_team" - ) as mock_team: - mock_key.return_value = RestrictedAgentAccess( - frozenset({"agent1", "agent2", "agent3"}) - ) - mock_team.return_value = RestrictedAgentAccess( - frozenset({"agent2", "agent4"}) - ) + with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: + with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: + mock_key.return_value = RestrictedAgentAccess(frozenset({"agent1", "agent2", "agent3"})) + mock_team.return_value = RestrictedAgentAccess(frozenset({"agent2", "agent4"})) - result = await AgentRequestHandler.resolve_agent_access( - user_api_key_auth=mock_user_auth - ) + result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=mock_user_auth) assert result == RestrictedAgentAccess(frozenset({"agent2"})) # Case 2: Team has agents, key has none - inherit from team - with patch.object( - AgentRequestHandler, "get_allowed_agents_for_key" - ) as mock_key: - with patch.object( - AgentRequestHandler, "_get_allowed_agents_for_team" - ) as mock_team: + with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: + with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: mock_key.return_value = UnrestrictedAgentAccess() - mock_team.return_value = RestrictedAgentAccess( - frozenset({"team_agent1", "team_agent2"}) - ) + mock_team.return_value = RestrictedAgentAccess(frozenset({"team_agent1", "team_agent2"})) - result = await AgentRequestHandler.resolve_agent_access( - user_api_key_auth=mock_user_auth - ) - assert result == RestrictedAgentAccess( - frozenset({"team_agent1", "team_agent2"}) - ) + result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=mock_user_auth) + assert result == RestrictedAgentAccess(frozenset({"team_agent1", "team_agent2"})) # Case 3: Key has agents, team has none - key restrictions stand - with patch.object( - AgentRequestHandler, "get_allowed_agents_for_key" - ) as mock_key: - with patch.object( - AgentRequestHandler, "_get_allowed_agents_for_team" - ) as mock_team: + with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: + with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: mock_key.return_value = RestrictedAgentAccess(frozenset({"key_agent1"})) mock_team.return_value = UnrestrictedAgentAccess() - result = await AgentRequestHandler.resolve_agent_access( - user_api_key_auth=mock_user_auth - ) + result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=mock_user_auth) assert result == RestrictedAgentAccess(frozenset({"key_agent1"})) # Case 4: No grant anywhere - unrestricted (documented open-by-default) - with patch.object( - AgentRequestHandler, "get_allowed_agents_for_key" - ) as mock_key: - with patch.object( - AgentRequestHandler, "_get_allowed_agents_for_team" - ) as mock_team: + with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: + with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: mock_key.return_value = UnrestrictedAgentAccess() mock_team.return_value = UnrestrictedAgentAccess() - result = await AgentRequestHandler.resolve_agent_access( - user_api_key_auth=mock_user_auth - ) + result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=mock_user_auth) assert result == UnrestrictedAgentAccess() async def test_agent_access_default_deny_is_opt_in(self): @@ -317,9 +285,7 @@ class TestAgentRequestHandler: """LIT-5143: a key restricted to one agent inside a team restricted to another must reach nothing. The empty intersection used to read as "no restrictions", so adding the team grant handed the key every agent on the proxy.""" - mock_user_auth: Final = UserAPIKeyAuth( - api_key="test-key", user_id="test-user", team_id="test-team" - ) + mock_user_auth: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", team_id="test-team") with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: @@ -332,9 +298,7 @@ class TestAgentRequestHandler: for agent_id in ("agent-alpha", "agent-beta", "agent-secret"): assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id=agent_id, user_api_key_auth=mock_user_auth - ) + await AgentRequestHandler.is_agent_allowed(agent_id=agent_id, user_api_key_auth=mock_user_auth) is False ), agent_id @@ -496,9 +460,7 @@ class TestAgentRequestHandler: agent_access_groups=["group-with-no-agents"], ) - with patch.object( - AgentRequestHandler, "_get_agents_from_access_groups", new_callable=AsyncMock - ) as mock_groups: + with patch.object(AgentRequestHandler, "_get_agents_from_access_groups", new_callable=AsyncMock) as mock_groups: mock_groups.return_value = [] assert await AgentRequestHandler.get_allowed_agents_for_key( @@ -506,20 +468,17 @@ class TestAgentRequestHandler: ) == RestrictedAgentAccess(frozenset()) assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id="agent-secret", user_api_key_auth=mock_user_auth - ) + await AgentRequestHandler.is_agent_allowed(agent_id="agent-secret", user_api_key_auth=mock_user_auth) is False ) - with patch.object( - AgentRequestHandler, "_get_agents_from_access_groups", new_callable=AsyncMock - ) as mock_groups: + with patch.object(AgentRequestHandler, "_get_agents_from_access_groups", new_callable=AsyncMock) as mock_groups: mock_groups.side_effect = Exception("DB Error") - assert await AgentRequestHandler.get_allowed_agents_for_key( - user_api_key_auth=mock_user_auth - ) == UnrestrictedAgentAccess() + assert ( + await AgentRequestHandler.get_allowed_agents_for_key(user_api_key_auth=mock_user_auth) + == UnrestrictedAgentAccess() + ) async def test_is_agent_allowed_respects_permissions(self): """ @@ -529,54 +488,32 @@ class TestAgentRequestHandler: mock_user_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user") # Agent in allowed list - should be allowed - with patch.object( - AgentRequestHandler, "resolve_agent_access" - ) as mock_get_allowed: - mock_get_allowed.return_value = RestrictedAgentAccess( - frozenset({"agent1", "agent2"}) - ) + with patch.object(AgentRequestHandler, "resolve_agent_access") as mock_get_allowed: + mock_get_allowed.return_value = RestrictedAgentAccess(frozenset({"agent1", "agent2"})) assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id="agent1", user_api_key_auth=mock_user_auth - ) - is True + await AgentRequestHandler.is_agent_allowed(agent_id="agent1", user_api_key_auth=mock_user_auth) is True ) # Agent not in allowed list - should be denied - with patch.object( - AgentRequestHandler, "resolve_agent_access" - ) as mock_get_allowed: - mock_get_allowed.return_value = RestrictedAgentAccess( - frozenset({"agent1", "agent2"}) - ) + with patch.object(AgentRequestHandler, "resolve_agent_access") as mock_get_allowed: + mock_get_allowed.return_value = RestrictedAgentAccess(frozenset({"agent1", "agent2"})) assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id="agent3", user_api_key_auth=mock_user_auth - ) - is False + await AgentRequestHandler.is_agent_allowed(agent_id="agent3", user_api_key_auth=mock_user_auth) is False ) # Restricted to nothing - should deny every agent - with patch.object( - AgentRequestHandler, "resolve_agent_access" - ) as mock_get_allowed: + with patch.object(AgentRequestHandler, "resolve_agent_access") as mock_get_allowed: mock_get_allowed.return_value = RestrictedAgentAccess(frozenset()) assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id="any_agent", user_api_key_auth=mock_user_auth - ) + await AgentRequestHandler.is_agent_allowed(agent_id="any_agent", user_api_key_auth=mock_user_auth) is False ) # Unrestricted - should allow any agent - with patch.object( - AgentRequestHandler, "resolve_agent_access" - ) as mock_get_allowed: + with patch.object(AgentRequestHandler, "resolve_agent_access") as mock_get_allowed: mock_get_allowed.return_value = UnrestrictedAgentAccess() assert ( - await AgentRequestHandler.is_agent_allowed( - agent_id="any_agent", user_api_key_auth=mock_user_auth - ) + await AgentRequestHandler.is_agent_allowed(agent_id="any_agent", user_api_key_auth=mock_user_auth) is True ) @@ -587,9 +524,7 @@ class TestAgentRequestHandler: result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=None) assert result == UnrestrictedAgentAccess() - is_allowed = await AgentRequestHandler.is_agent_allowed( - agent_id="any_agent", user_api_key_auth=None - ) + is_allowed = await AgentRequestHandler.is_agent_allowed(agent_id="any_agent", user_api_key_auth=None) assert is_allowed is True async def test_resolve_agent_access_handles_errors_gracefully(self): @@ -605,18 +540,12 @@ class TestAgentRequestHandler: object_permission_id="test-permission", ) - with patch.object( - AgentRequestHandler, "get_allowed_agents_for_key" - ) as mock_key: - with patch.object( - AgentRequestHandler, "_get_allowed_agents_for_team" - ) as mock_team: + with patch.object(AgentRequestHandler, "get_allowed_agents_for_key") as mock_key: + with patch.object(AgentRequestHandler, "_get_allowed_agents_for_team") as mock_team: mock_key.side_effect = Exception("DB Error") mock_team.return_value = UnrestrictedAgentAccess() - result = await AgentRequestHandler.resolve_agent_access( - user_api_key_auth=mock_user_auth - ) + result = await AgentRequestHandler.resolve_agent_access(user_api_key_auth=mock_user_auth) assert result == UnrestrictedAgentAccess() async def test_accessible_agents_hides_ungranted_agents_from_non_admins(self): @@ -702,20 +631,14 @@ class TestAgentRequestHandler: access_group_ids=["ag-with-agents"], ) - with patch.object( - AgentRequestHandler, "_get_key_object_permission", return_value=None - ): + with patch.object(AgentRequestHandler, "_get_key_object_permission", return_value=None): with patch( "litellm.proxy.auth.auth_checks.get_agent_ids_from_access_groups", new_callable=AsyncMock, return_value=["agent-from-ag-1", "agent-from-ag-2"], ): - result = await AgentRequestHandler.get_allowed_agents_for_key( - user_api_key_auth=mock_user_auth - ) - assert result == RestrictedAgentAccess( - frozenset({"agent-from-ag-1", "agent-from-ag-2"}) - ) + result = await AgentRequestHandler.get_allowed_agents_for_key(user_api_key_auth=mock_user_auth) + assert result == RestrictedAgentAccess(frozenset({"agent-from-ag-1", "agent-from-ag-2"})) async def testget_allowed_agents_for_key_combines_native_and_access_groups(self): """ @@ -742,12 +665,8 @@ class TestAgentRequestHandler: new_callable=AsyncMock, return_value=["agent-from-ag"], ): - result = await AgentRequestHandler.get_allowed_agents_for_key( - user_api_key_auth=mock_user_auth - ) - assert result == RestrictedAgentAccess( - frozenset({"agent-from-ag", "native-agent-1"}) - ) + result = await AgentRequestHandler.get_allowed_agents_for_key(user_api_key_auth=mock_user_auth) + assert result == RestrictedAgentAccess(frozenset({"agent-from-ag", "native-agent-1"})) async def test_is_agent_allowed_accepts_legacy_config_agent_id_grants(self): """LIT-5144: object_permission grants stored under the pre-fix full-entry hash @@ -902,9 +821,17 @@ async def test_managed_agent_invocation_grants_intersect_verified_user_grants( assert access == RestrictedAgentAccess(frozenset({"shared"} if delegated else {"shared", "agent-only"})) target: Final = AgentResponse( - agent_id="shared", agent_name="Shared", agent_card_params={}, identity_managed=True, + agent_id="shared", + agent_name="Shared", + agent_card_params={}, + identity_managed=True, identity=AgentIdentityBinding( - agent_id="shared", provider="microsoft_entra", tenant_id="tenant", client_id="client", issuer="issuer", revision="current" + agent_id="shared", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", ), ) database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=target) @@ -912,7 +839,9 @@ async def test_managed_agent_invocation_grants_intersect_verified_user_grants( @pytest.mark.asyncio -@pytest.mark.parametrize("revoked", ["user", "team-member", "team-grant", "team-permission", "direct-grant", "access-group"]) +@pytest.mark.parametrize( + "revoked", ["user", "team-member", "team-grant", "team-permission", "direct-grant", "access-group"] +) async def test_delegated_grants_revoke_with_warm_user_team_and_permission_caches( monkeypatch: pytest.MonkeyPatch, revoked: str ) -> None: @@ -1003,9 +932,7 @@ async def test_strict_legacy_group_grants_ignore_stale_replica(monkeypatch: pyte frozenset({"revoked"}) ) database.writer_db.litellm_agentstable.find_many.return_value = [] - assert await AgentRequestHandler.get_allowed_agents_for_key(auth, strict=True) == RestrictedAgentAccess( - frozenset() - ) + assert await AgentRequestHandler.get_allowed_agents_for_key(auth, strict=True) == RestrictedAgentAccess(frozenset()) database.db.litellm_agentstable.find_many.assert_not_awaited() @@ -1030,9 +957,9 @@ async def test_strict_invocation_policy_outage_denies_instead_of_allowing_all( monkeypatch.setattr(proxy_server, "prisma_client", database) auth: Final = UserAPIKeyAuth( team_id="team" if team else None, - object_permission=None if team else LiteLLM_ObjectPermissionTable( - object_permission_id="grant", agent_access_groups=["group"] - ), + object_permission=None + if team + else LiteLLM_ObjectPermissionTable(object_permission_id="grant", agent_access_groups=["group"]), ) with pytest.raises(HTTPException, match="policy is unavailable") as denied: await AgentRequestHandler.resolve_key_team_agent_access(auth, strict=True) @@ -1065,9 +992,9 @@ async def test_registered_managed_target_cannot_bypass_missing_or_unavailable_po from litellm.types.agents import AgentResponse registry: Final = AgentRegistry() - registry.register_agent(AgentResponse( - agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True - )) + registry.register_agent( + AgentResponse(agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True) + ) monkeypatch.setattr(agent_registry, "global_agent_registry", registry) database: Final = MagicMock() database.writer_db.litellm_agentstable.find_unique = AsyncMock( @@ -1091,7 +1018,9 @@ async def test_delegation_without_a_verified_human_never_grants_agents(grant: bo auth: Final = UserAPIKeyAuth(agent_id="actor") auth.managed_agent_policy = AgentResponse( - agent_id="actor", agent_name="Actor", agent_card_params={}, + agent_id="actor", + agent_name="Actor", + agent_card_params={}, object_permission={"object_permission_id": "own", "agents": ["target"]} if grant else None, ) auth.managed_agent_context = ManagedAgentContext(agent_id="actor", mode="delegated") @@ -1100,7 +1029,9 @@ async def test_delegation_without_a_verified_human_never_grants_agents(grant: bo @pytest.mark.asyncio -@pytest.mark.parametrize("change", ("grant", "permission_reference", "groups", "team", "blocked", "expired", "deleted", "outage")) +@pytest.mark.parametrize( + "change", ("grant", "permission_reference", "groups", "team", "blocked", "expired", "deleted", "outage") +) async def test_managed_target_rechecks_authoritative_key_after_peer_revocation( monkeypatch: pytest.MonkeyPatch, change: str ) -> None: @@ -1112,10 +1043,22 @@ async def test_managed_target_rechecks_authoritative_key_after_peer_revocation( from litellm.types.proxy.agent_identity import AgentIdentityBinding permission: Final = LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["target"]) - warm: Final = UserAPIKeyAuth(api_key="a" * 64, token="a" * 64, object_permission_id="grant", object_permission=permission) + warm: Final = UserAPIKeyAuth( + api_key="a" * 64, token="a" * 64, object_permission_id="grant", object_permission=permission + ) target: Final = AgentResponse( - agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True, - identity=AgentIdentityBinding(agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", issuer="issuer", revision="current"), + agent_id="target", + agent_name="Target", + agent_card_params={}, + identity_managed=True, + identity=AgentIdentityBinding( + agent_id="target", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", + ), ) client: Final = MagicMock() client.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=target) @@ -1126,26 +1069,38 @@ async def test_managed_target_rechecks_authoritative_key_after_peer_revocation( monkeypatch.setattr(proxy_server, "prisma_client", client) monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) assert await AgentRequestHandler.is_agent_allowed("target", warm) is True - client.get_data.return_value = warm.model_copy(update={ - "object_permission": None, - "object_permission_id": "replacement" if change == "permission_reference" else "grant", - "access_group_ids": [], - "team_id": "new-team" if change == "team" else None, - "blocked": change == "blocked", - "expires": "2000-01-01T00:00:00+00:00" if change == "expired" else None, - }) - client.writer_db.litellm_objectpermissiontable.find_unique.return_value = permission.model_copy(update={"agents": []}) + client.get_data.return_value = warm.model_copy( + update={ + "object_permission": None, + "object_permission_id": "replacement" if change == "permission_reference" else "grant", + "access_group_ids": [], + "team_id": "new-team" if change == "team" else None, + "blocked": change == "blocked", + "expires": "2000-01-01T00:00:00+00:00" if change == "expired" else None, + } + ) + client.writer_db.litellm_objectpermissiontable.find_unique.return_value = permission.model_copy( + update={"agents": []} + ) if change == "team": from litellm.proxy._types import LiteLLM_TeamTable from litellm.proxy.auth import auth_checks + client.writer_db.litellm_objectpermissiontable.find_unique.return_value = permission - monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=LiteLLM_TeamTable( - team_id="new-team", object_permission=permission.model_copy(update={"agents": ["other"]}) - ))) + monkeypatch.setattr( + auth_checks, + "get_team_object", + AsyncMock( + return_value=LiteLLM_TeamTable( + team_id="new-team", object_permission=permission.model_copy(update={"agents": ["other"]}) + ) + ), + ) if change == "groups": warm.object_permission = None warm.access_group_ids = ["old-group"] from litellm.proxy.auth import auth_checks + monkeypatch.setattr(auth_checks, "get_agent_ids_from_access_groups", AsyncMock(return_value=["target"])) if change == "deleted": client.get_data.return_value = None @@ -1174,14 +1129,23 @@ async def test_managed_target_preserves_ordinary_actor_ceilings_after_key_reload from litellm.types.proxy.agent_identity import AgentIdentityBinding target: Final = AgentResponse( - agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True, + agent_id="target", + agent_name="Target", + agent_card_params={}, + identity_managed=True, identity=AgentIdentityBinding( - agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", - issuer="issuer", revision="current", + agent_id="target", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", ), ) actor: Final = AgentResponse( - agent_id="ordinary", agent_name="Ordinary", agent_card_params={}, + agent_id="ordinary", + agent_name="Ordinary", + agent_card_params={}, access_group_ids=["actor-group"] if ceiling != "caller-team" else [], ) registry: Final = AgentRegistry() @@ -1191,18 +1155,24 @@ async def test_managed_target_preserves_ordinary_actor_ceilings_after_key_reload object_permission_id="key-grant", agents=[] if ceiling == "group-without-grant" else ["target"] ) persisted: Final = UserAPIKeyAuth( - api_key="a" * 64, agent_id="ordinary", object_permission_id="key-grant", object_permission=permission, + api_key="a" * 64, + agent_id="ordinary", + object_permission_id="key-grant", + object_permission=permission, ) auth: Final = persisted.model_copy() auth.agent_caller = AgentCaller(team_id="caller-team") if ceiling == "caller-team" else None group: Final = LiteLLM_AccessGroupTable( - access_group_id="actor-group", access_group_name="Actor group", + access_group_id="actor-group", + access_group_name="Actor group", access_agent_ids=["target"] if permitted else ["other"], ) team: Final = LiteLLM_TeamTable( - team_id="caller-team", object_permission_id="caller-grant", + team_id="caller-team", + object_permission_id="caller-grant", object_permission=LiteLLM_ObjectPermissionTable( - object_permission_id="caller-grant", agents=["target"] if permitted else ["other"], + object_permission_id="caller-grant", + agents=["target"] if permitted else ["other"], ), ) database: Final = MagicMock() @@ -1220,7 +1190,9 @@ async def test_managed_target_preserves_ordinary_actor_ceilings_after_key_reload monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) monkeypatch.setattr(agent_registry, "global_agent_registry", registry) - assert await AgentRequestHandler.is_agent_allowed("target", auth) is (permitted and ceiling != "group-without-grant") + assert await AgentRequestHandler.is_agent_allowed("target", auth) is ( + permitted and ceiling != "group-without-grant" + ) database.get_data.assert_awaited_once() assert auth.agent_caller == (AgentCaller(team_id="caller-team") if ceiling == "caller-team" else None) @@ -1319,15 +1291,24 @@ async def test_target_authorization_uses_live_policy_despite_stale_unmanaged_reg registry.register_agent(stale) monkeypatch.setattr(agent_registry, "global_agent_registry", registry) binding: Final = AgentIdentityBinding( - agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", - issuer="issuer", revision="current", + agent_id="target", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + issuer="issuer", + revision="current", + ) + current: Final = stale.model_copy( + update={ + "identity_managed": managed, + "identity": binding if managed else None, + "enabled": enabled, + } ) - current: Final = stale.model_copy(update={ - "identity_managed": managed, "identity": binding if managed else None, "enabled": enabled, - }) database: Final = MagicMock() database.writer_db.litellm_agentstable.find_unique = AsyncMock( - return_value=current, side_effect=ConnectionError("writer unavailable") if outage else None, + return_value=current, + side_effect=ConnectionError("writer unavailable") if outage else None, ) monkeypatch.setattr(proxy_server, "prisma_client", database) permission: Final = LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["target"]) @@ -1339,3 +1320,927 @@ async def test_target_authorization_uses_live_policy_despite_stale_unmanaged_reg assert denied.value.status_code == 503 return assert await AgentRequestHandler.is_agent_allowed("target", auth) is allowed + + +_PERSONAL_KEY_TEAM_AGENTS: Final = { + "team-a": ("agent-a",), + "team-b": ("agent-b",), + "team-x": ("agent-x",), + "team-n": ("agent-n",), + "team-open": (), + "team-grouped": (), + "team-unreadable": (), + UI_TEAM_ID: (), +} + + +@pytest.fixture +def personal_key_owner_teams(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> UserAPIKeyAuth: + """A personal key whose owner is on ``request.param`` teams; team-a grants agent-a, team-b grants + agent-b, team-open grants no agent rule at all, team-grouped declares an access group whose + lookup fails against the mocked database and team-unreadable names a permission row that did not + load.""" + from unittest.mock import MagicMock + + import litellm + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == "team-unreadable": + return LiteLLM_TeamTable(team_id=team_id, object_permission_id=f"perm-{team_id}") + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + agent_access_groups=["grp"] if team_id == "team-grouped" else [], + ), + ) + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + prisma_client: Final = MagicMock() + prisma_client.writer_db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + user_api_key_cache: Final = MagicMock() + user_api_key_cache.async_set_cache = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=list(request.param))), + ) + return UserAPIKeyAuth(api_key="sk-personal", user_id="alice", team_id=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "personal_key_owner_teams, expected", + [ + (("team-a", "team-b"), RestrictedAgentAccess(frozenset({"agent-a", "agent-b"}))), + (("team-b",), RestrictedAgentAccess(frozenset({"agent-b"}))), + ((), RestrictedAgentAccess(frozenset())), + (("team-a", "team-open"), RestrictedAgentAccess(frozenset({"agent-a"}))), + ], + indirect=["personal_key_owner_teams"], +) +async def test_personal_key_agents_are_the_union_of_its_owner_team_grants( + personal_key_owner_teams: UserAPIKeyAuth, expected: AgentAccess +) -> None: + assert await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("", "team-open")], indirect=True) +async def test_empty_owner_team_id_does_not_restrict_agent_access( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if not team_id: + raise RuntimeError("empty team id should not be loaded") + return LiteLLM_TeamTable( + team_id=team_id, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=[], + ), + ) + + team_loader: Final = AsyncMock(side_effect=_get_team_object) + monkeypatch.setattr(auth_checks, "get_team_object", team_loader) + + owner_teams: Final = await auth_checks.personal_key_owner_teams_with_status( + valid_token=personal_key_owner_teams, + prisma_client=proxy_server.prisma_client, + user_api_key_cache=proxy_server.user_api_key_cache, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + + assert owner_teams.unreadable is False + assert tuple(team.team_id for team in owner_teams.teams) == ("team-open",) + assert await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == UnrestrictedAgentAccess() + assert all(call.kwargs["team_id"] == "team-open" for call in team_loader.await_args_list) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-open")], indirect=True) +async def test_personal_key_accessible_agents_keep_declared_grants( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.types.agents import AgentResponse + + agents: Final = ( + AgentResponse(agent_id="agent-a", agent_name="agent-a", agent_card_params={}), + AgentResponse(agent_id="agent-b", agent_name="agent-b", agent_card_params={}), + ) + accessible: Final = await accessible_agents( + personal_key_owner_teams, all_agents=agents, effective_contexts=_single_context + ) + assert tuple(agent.agent_id for agent in accessible) == ("agent-a",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-open")], indirect=True) +async def test_personal_key_strict_agent_access_keeps_declared_grants( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await AgentRequestHandler.resolve_agent_access( + personal_key_owner_teams, strict=True + ) == RestrictedAgentAccess(frozenset({"agent-a"})) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-x", "team-n")], indirect=True) +async def test_personal_key_agent_access_ignores_owner_teams_in_other_org( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + organization_id: Final = "org-1" if team_id == "team-a" else "org-2" if team_id == "team-x" else None + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=organization_id, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": "org-1"}) + assert await AgentRequestHandler.resolve_agent_access(auth) == RestrictedAgentAccess( + frozenset({"agent-a", "agent-n"}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_uses_the_team_granting_the_agent( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(personal_key_owner_teams, "agent-b") + assert billed is not personal_key_owner_teams + assert (billed.team_id, billed.org_id) == ("team-b", "org-team-b") + assert (personal_key_owner_teams.team_id, personal_key_owner_teams.org_id) == (None, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_org_id,team_org_id,expected_org_id", + [("org-1", None, "org-1"), (None, "org-2", "org-2")], +) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_preserves_key_organization( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + key_org_id: str | None, + team_org_id: str | None, + expected_org_id: str, +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=team_org_id if team_id == "team-b" else "org-team-a", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": key_org_id}) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(auth, "agent-b") + + assert billed.team_id == "team-b" + assert billed.org_id == expected_org_id + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_matches_legacy_agent_ids( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + entry: Final = { + "agent_name": "legacy-agent", + "agent_card_params": {"name": "Legacy Agent", "url": "http://localhost", "version": "1.0.0"}, + } + registry: Final = AgentRegistry() + registry.load_agents_from_config([entry]) + agent: Final = registry.get_agent_by_name("legacy-agent") + assert agent is not None + legacy_id: Final = next(iter(registry.ids_for_agent(agent.agent_id) - {agent.agent_id})) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=[legacy_id] if team_id == "team-b" else ["agent-a"], + ), + ) + + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(personal_key_owner_teams, agent.agent_id) + assert (billed.team_id, billed.org_id) == ("team-b", "org-team-b") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_inherits_adopted_organization_limits( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.proxy._types import LiteLLM_OrganizationTable, LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + organization: Final = LiteLLM_OrganizationTable( + organization_id="org-1", + organization_alias="org-one", + budget_id="budget-1", + created_by="test", + updated_by="test", + metadata={"k": "v"}, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=100, rpm_limit=5, tpm_limit=50), + ) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id="org-1" if team_id == "team-b" else "org-team-a", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + org_loader: Final = AsyncMock(return_value=organization) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) + auth: Final = personal_key_owner_teams.model_copy(update={"org_id": None}) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(auth, "agent-b") + + assert ( + billed.org_id, + billed.organization_rpm_limit, + billed.organization_tpm_limit, + billed.organization_metadata, + ) == ("org-1", 5, 50, {"k": "v"}) + assert billed.organization_max_budget == 100 + assert (auth.org_id, auth.organization_rpm_limit, auth.organization_tpm_limit, auth.organization_metadata) == ( + None, + None, + None, + None, + ) + org_loader.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_keeps_team_controls_when_org_lookup_fails( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id="org-1" if team_id == "team-b" else None, + rpm_limit=5 if team_id == "team-b" else None, + max_budget=40 if team_id == "team-b" else None, + metadata={"team_private": True} if team_id == "team-b" else None, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + org_loader: Final = AsyncMock(side_effect=RuntimeError("org lookup failed")) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object_for_request", org_loader) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(personal_key_owner_teams, "agent-b") + + assert billed is not personal_key_owner_teams + assert billed.team_id == "team-b" + assert billed.team_rpm_limit == 5 + assert billed.team_metadata == {"team_private": True} + assert billed.team_max_budget == 40 + assert billed.org_id == "org-1" + assert personal_key_owner_teams.team_id is None + assert personal_key_owner_teams.team_rpm_limit is None + org_loader.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_keeps_team_controls_when_membership_lookup_fails( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + organization_id=None, + rpm_limit=5 if team_id == "team-b" else None, + max_budget=40 if team_id == "team-b" else None, + metadata={"team_private": True} if team_id == "team-b" else None, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr( + auth_checks, + "get_team_membership", + AsyncMock(side_effect=RuntimeError("membership lookup failed")), + ) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(personal_key_owner_teams, "agent-b") + + assert billed.team_id == "team-b" + assert billed.team_rpm_limit == 5 + assert billed.team_metadata == {"team_private": True} + assert billed.team_max_budget == 40 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "strict, expected_team_rpm, expected_member_rpm, expected_metadata", + [ + pytest.param(False, 2, 3, {"policy_source": "cache"}, id="default-uses-cache"), + pytest.param(True, 7, 8, {"policy_source": "database"}, id="strict-uses-database"), + ], +) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_uses_fresh_membership_for_strict_policy( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + strict: bool, + expected_team_rpm: int, + expected_member_rpm: int, + expected_metadata: dict[str, str], +) -> None: + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **kwargs: object) -> LiteLLM_TeamTable: + database_value: Final = bool(kwargs.get("check_db_only")) + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=None, + rpm_limit=7 if database_value else 2, + metadata={"policy_source": "database" if database_value else "cache"}, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + async def _get_team_membership( + user_id: str, team_id: str, **kwargs: object + ) -> LiteLLM_TeamMembership: + database_value: Final = bool(kwargs.get("check_db_only")) + return LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=8 if database_value else 3), + ) + + team_loader: Final = AsyncMock(side_effect=_get_team_object) + monkeypatch.setattr(auth_checks, "get_team_object", team_loader) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(side_effect=_get_team_membership)) + monkeypatch.setattr(auth_checks, "_check_team_member_budget", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + auth: Final = personal_key_owner_teams.model_copy(update={"requires_fresh_policy": strict}) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(auth, "agent-b", strict=strict) + + assert (billed.team_rpm_limit, billed.team_member_rpm_limit, billed.team_metadata) == ( + expected_team_rpm, + expected_member_rpm, + expected_metadata, + ) + assert all(call.kwargs["check_db_only"] is strict for call in team_loader.await_args_list) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_agent_billing_carries_team_rate_limits( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.caching.caching import DualCache + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.utils import InternalUsageCache + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + team_alias=f"alias-{team_id}", + organization_id=f"org-{team_id}", + rpm_limit=1 if team_id == "team-b" else None, + models=["team-only-model"] if team_id == "team-b" else [], + metadata={"team_private": True} if team_id == "team-b" else None, + object_permission_id=f"perm-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + + membership: Final = LiteLLM_TeamMembership( + user_id="alice", + team_id="team-b", + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=2), + ) + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=membership)) + + billed: Final = await AgentRequestHandler.personal_key_agent_billing_auth(personal_key_owner_teams, "agent-b") + assert (billed.team_id, billed.org_id, billed.team_rpm_limit, billed.team_member_rpm_limit) == ( + "team-b", + "org-team-b", + 1, + 2, + ) + assert billed.team_models == personal_key_owner_teams.team_models + assert billed.team_metadata == {"team_private": True} + assert billed.team_object_permission == personal_key_owner_teams.team_object_permission + assert (personal_key_owner_teams.team_id, personal_key_owner_teams.team_rpm_limit) == (None, None) + + limiter: Final = PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(DualCache())) + descriptors: Final = limiter.create_rate_limit_descriptors( + user_api_key_dict=billed, + data={}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + team_limits: Final = tuple( + (descriptor["key"], descriptor["value"], descriptor["rate_limit"]["requests_per_unit"]) + for descriptor in descriptors + if descriptor["key"] in {"team", "team_member"} + ) + assert team_limits == (("team", "team-b", 1), ("team_member", "team-b:alice", 2)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_agent_billing_auth_returns_non_personal_key_unchanged( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + team_key: Final = personal_key_owner_teams.model_copy(update={"team_id": "team-a"}) + assert await AgentRequestHandler.personal_key_agent_billing_auth(team_key, "agent-a") is team_key + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-open")], indirect=True) +async def test_personal_key_non_strict_agent_check_denies_ungranted_agent( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + + monkeypatch.setattr(AgentIdentityStore, "agent", AsyncMock(return_value=None)) + assert await AgentRequestHandler.is_agent_allowed("agent-b", personal_key_owner_teams) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strict", [False, True], ids=["non-strict", "strict"]) +@pytest.mark.parametrize( + "team_a_agents, expected", + [ + (("agent-a",), RestrictedAgentAccess(frozenset({"agent-a"}))), + ((), RestrictedAgentAccess(frozenset())), + ], + ids=["eligible-grant", "eligible-unrestricted"], +) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_unreadable_withheld_team_only_adds_conservative_agent_restriction( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + team_a_agents: tuple[str, ...], + expected: AgentAccess, + strict: bool, +) -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == "team-b": + return LiteLLM_TeamTable(team_id=team_id, blocked=True, object_permission_id=f"perm-{team_id}") + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(team_a_agents), + agent_access_groups=[], + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + + assert ( + await AgentRequestHandler._personal_key_owner_team_agent_access(personal_key_owner_teams, strict=strict) + == expected + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_unreadable_eligible_owner_team_still_raises_in_strict_agent_access( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from fastapi import HTTPException + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=team_id, object_permission_id=f"perm-{team_id}") + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + + with pytest.raises(HTTPException, match="policy is unavailable") as denied: + await AgentRequestHandler._personal_key_owner_team_agent_access(personal_key_owner_teams, strict=True) + assert denied.value.status_code == 503 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-open")], indirect=True) +async def test_personal_key_unreadable_owner_team_does_not_make_access_unrestricted( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + from litellm.proxy.auth import auth_checks + + get_team_object = auth_checks.get_team_object + + async def fail_team_a_lookup(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == "team-a": + raise RuntimeError("team lookup failed") + return await get_team_object( + team_id=team_id, + prisma_client=proxy_server.prisma_client, + user_api_key_cache=proxy_server.user_api_key_cache, + parent_otel_span=personal_key_owner_teams.parent_otel_span, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + + monkeypatch.setattr(auth_checks, "get_team_object", fail_team_a_lookup) + monkeypatch.setattr(AgentIdentityStore, "agent", AsyncMock(return_value=None)) + + assert await AgentRequestHandler.resolve_agent_access(personal_key_owner_teams) == RestrictedAgentAccess( + frozenset() + ) + assert await AgentRequestHandler.is_agent_allowed("agent-b", personal_key_owner_teams) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-open",)], indirect=True) +async def test_personal_key_all_unrestricted_owner_teams_remain_unrestricted( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await AgentRequestHandler.resolve_agent_access(personal_key_owner_teams) == UnrestrictedAgentAccess() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", UI_TEAM_ID)], indirect=True) +async def test_personal_key_ui_team_does_not_expand_agent_access( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == RestrictedAgentAccess( + frozenset({"agent-a"}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_over_budget_team_grants_no_agent_access( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team: Final = LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=list(_PERSONAL_KEY_TEAM_AGENTS[team_id]), + ), + ) + if team_id == "team-b": + return team.model_copy(update={"max_budget": 1, "spend": 5}) + return team + + async def _spend_from_fallback(*, fallback_spend: float, **_: object) -> float: + return fallback_spend + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "get_current_spend", _spend_from_fallback) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "budget_alerts", AsyncMock()) + assert await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == RestrictedAgentAccess( + frozenset({"agent-a"}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blocked", [False, True], ids=["over-budget", "blocked"]) +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-open")], indirect=True) +async def test_personal_key_withheld_restricted_owner_team_denies_all_agents( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, + blocked: bool, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == "team-open": + return LiteLLM_TeamTable(team_id=team_id) + team: Final = LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=["agent-a"], + ), + ) + if blocked: + return team.model_copy(update={"blocked": True}) + return team.model_copy(update={"max_budget": 1, "spend": 5}) + + async def _spend_from_fallback(*, fallback_spend: float, **_: object) -> float: + return fallback_spend + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "get_current_spend", _spend_from_fallback) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "budget_alerts", AsyncMock()) + + access: Final = await AgentRequestHandler.resolve_agent_access(personal_key_owner_teams) + + assert access == RestrictedAgentAccess(frozenset()) + assert isinstance(access, RestrictedAgentAccess) + assert "agent-a" not in access.agent_ids + assert "agent-b" not in access.agent_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-x", "team-open")], indirect=True) +async def test_personal_key_unrestricted_over_budget_team_stays_unrestricted( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + if team_id == "team-open": + return LiteLLM_TeamTable(team_id=team_id) + return LiteLLM_TeamTable(team_id=team_id, max_budget=1, spend=5) + + async def _spend_from_fallback(*, fallback_spend: float, **_: object) -> float: + return fallback_spend + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_server, "get_current_spend", _spend_from_fallback) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "budget_alerts", AsyncMock()) + + assert await AgentRequestHandler.resolve_agent_access(personal_key_owner_teams) == UnrestrictedAgentAccess() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_personal_key_window_budget_team_is_not_an_owner_grant( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + budget_limits=( + [{"budget_duration": "24h", "max_budget": 5.0, "reset_at": None}] if team_id == "team-b" else [] + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + monkeypatch.setattr(proxy_server, "get_current_spend", AsyncMock(return_value=5.0)) + allowed: Final = await auth_checks.personal_key_owner_teams( + valid_token=personal_key_owner_teams, + prisma_client=proxy_server.prisma_client, + user_api_key_cache=proxy_server.user_api_key_cache, + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + assert tuple(team.team_id for team in allowed) == ("team-a",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_strict_personal_key_agent_access_reads_owner_teams_from_the_database( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + """Invocation (strict) must see a revoke at once, so the owner's teams are read with check_db_only + like the team-key branch, while listing keeps the cached read.""" + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.auth import auth_checks + + seen: Final[list[bool]] = [] # mutable-ok: records each read mode + + async def _get_team_object(team_id: str, **kwargs: object) -> LiteLLM_TeamTable: + seen.append(bool(kwargs.get("check_db_only"))) + return LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", agents=["agent-a"] + ), + ) + + monkeypatch.setattr(auth_checks, "get_team_object", _get_team_object) + await AgentRequestHandler._get_allowed_agents_for_team(personal_key_owner_teams, strict=True) + await AgentRequestHandler._get_allowed_agents_for_team(personal_key_owner_teams, strict=False) + assert seen == [True, False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_invocation_uses_fresh_owner_team_membership( + monkeypatch: pytest.MonkeyPatch, + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + from litellm.proxy.auth import auth_checks + + async def _get_user_object(**kwargs: object) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id="alice", + teams=[] if bool(kwargs.get("check_db_only")) else ["team-a"], + ) + + monkeypatch.setattr(auth_checks, "get_user_object", _get_user_object) + monkeypatch.setattr(AgentIdentityStore, "agent", AsyncMock(return_value=None)) + cached_auth: Final = personal_key_owner_teams.model_copy(update={"requires_fresh_policy": False}) + fresh_auth: Final = personal_key_owner_teams.model_copy(update={"requires_fresh_policy": True}) + + assert await AgentRequestHandler.is_agent_allowed("agent-a", fresh_auth) is False + assert await AgentRequestHandler.is_agent_allowed("agent-a", cached_auth) is True + + +@pytest.mark.asyncio +async def test_owner_team_agent_grant_lookup_returns_policy_unavailable_value() -> None: + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.agent_endpoints.auth.agent_permission_handler import AgentPolicyUnavailable + + team: Final = LiteLLM_TeamTable(team_id="team-unreadable", object_permission_id="perm-unreadable") + + assert await AgentRequestHandler._owner_team_agent_grants(team, strict=True) == AgentPolicyUnavailable( + team_id="team-unreadable" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_keyless_grant_source_of_the_owner_does_not_inherit_its_teams( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + """The direct-grant source of an admitted subject has the owner's user_id, no team and no api_key. + It must keep its pre-flag resolution, or a delegated token scoped to team-a reaches agent-b.""" + source: Final = UserAPIKeyAuth(user_id=personal_key_owner_teams.user_id, team_id=None) + assert await AgentRequestHandler.resolve_key_team_agent_access(source) == UnrestrictedAgentAccess() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_agent_access_is_unchanged_when_flag_off( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + import litellm + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", False) + assert ( + await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == UnrestrictedAgentAccess() + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a", "team-b")], indirect=True) +async def test_team_key_agent_access_ignores_its_owner_other_teams(personal_key_owner_teams: UserAPIKeyAuth) -> None: + from litellm.proxy.auth import auth_checks + + team_key: Final = personal_key_owner_teams.model_copy(update={"team_id": "team-b"}) + assert await AgentRequestHandler.resolve_key_team_agent_access(team_key) == RestrictedAgentAccess( + frozenset({"agent-b"}) + ) + auth_checks.get_user_object.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "personal_key_owner_teams", [("team-a", "team-grouped"), ("team-a", "team-unreadable")], indirect=True +) +async def test_owner_team_whose_grants_cannot_be_read_grants_no_agent( + personal_key_owner_teams: UserAPIKeyAuth, +) -> None: + assert await AgentRequestHandler.resolve_key_team_agent_access(personal_key_owner_teams) == RestrictedAgentAccess( + frozenset({"agent-a"}) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("personal_key_owner_teams", [("team-a",)], indirect=True) +async def test_personal_key_agent_access_is_denied_when_owner_teams_cannot_be_read( + monkeypatch: pytest.MonkeyPatch, personal_key_owner_teams: UserAPIKeyAuth +) -> None: + from litellm.proxy.auth import auth_checks + + monkeypatch.setattr( + auth_checks, "personal_key_owner_teams_with_status", AsyncMock(side_effect=RuntimeError("db down")) + ) + assert await AgentRequestHandler.resolve_agent_access(personal_key_owner_teams) == RestrictedAgentAccess( + frozenset() + ) diff --git a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py index d2f2fec073c..a2448694dd7 100644 --- a/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py @@ -418,6 +418,124 @@ def _base_patches( ] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("identity_managed", "requires_fresh_policy", "strict"), + [(True, False, True), (False, True, True), (False, False, False)], +) +async def test_invoke_agent_a2a_passes_billed_auth_to_common_processing( + monkeypatch: pytest.MonkeyPatch, + identity_managed: bool, + requires_fresh_policy: bool, + strict: bool, +) -> None: + import litellm + + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, LiteLLM_UserTable + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.proxy.auth import auth_checks + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.litellm_pre_call_utils import add_guardrails_from_auth_metadata + + agent: Final = _make_agent_mock() + agent.identity_managed = identity_managed + request: Final = _make_request_mock("message/send", _HELLO_MESSAGE_PARAMS) + auth: Final = UserAPIKeyAuth( + api_key="sk-personal", + user_id="alice", + team_id=None, + ) + auth.requires_fresh_policy = requires_fresh_policy + processed_metadata: Final[dict[str, object]] = {} + processed_request_data: Final[dict[str, object]] = {"metadata": processed_metadata} + + async def _common_processing( + *, user_api_key_dict: UserAPIKeyAuth, **_: object + ) -> tuple[dict[str, object], MagicMock]: + add_guardrails_from_auth_metadata( + user_api_key_dict=user_api_key_dict, + data=processed_request_data, + metadata_variable_name="metadata", + ) + return {}, MagicMock() + + common_processing: Final = AsyncMock(side_effect=_common_processing) + + async def _get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + team: Final = LiteLLM_TeamTable( + team_id=team_id, + object_permission_id=f"perm-{team_id}", + organization_id=f"org-{team_id}", + metadata={"guardrails": ["team-guardrail"]} if team_id == "team-b" else None, + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id=f"perm-{team_id}", + agents=["test-agent"] if team_id == "team-b" else [], + agent_access_groups=["billing-group"] if team_id == "team-b" else [], + ), + ) + return team + + database: Final = MagicMock() + database.db.litellm_agentstable.find_many = AsyncMock(return_value=[]) + database.writer_db.litellm_agentstable.find_many = AsyncMock(return_value=[]) + team_loader: Final = AsyncMock(side_effect=_get_team_object) + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr("litellm.proxy.utils.premium_user_check", MagicMock()) + monkeypatch.setattr(proxy_server, "prisma_client", database) + monkeypatch.setattr(proxy_server, "user_api_key_cache", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr(agent_registry, "global_agent_registry", AgentRegistry()) + monkeypatch.setattr( + auth_checks, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="alice", teams=["team-a", "team-b"])), + ) + monkeypatch.setattr(auth_checks, "get_team_object", team_loader) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + + with ExitStack() as stack: + for patcher in _base_patches(agent): + stack.enter_context(patcher) + stack.enter_context( + patch.object( + ProxyBaseLLMRequestProcessing, + "common_processing_pre_call_logic", + new=common_processing, + ) + ) + stack.enter_context(patch("litellm.a2a_protocol.main.A2A_SDK_AVAILABLE", False)) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response: Final = await invoke_agent_a2a( + agent_id="test-agent", + request=request, + fastapi_response=MagicMock(), + user_api_key_dict=auth, + ) + + assert json.loads(response.body.decode())["error"]["code"] == -32603 + billed_auth: Final = common_processing.await_args.kwargs["user_api_key_dict"] + assert billed_auth is not auth + assert (billed_auth.team_id, billed_auth.org_id) == ("team-b", "org-team-b") + assert billed_auth.team_metadata == {"guardrails": ["team-guardrail"]} + assert processed_metadata["guardrails"] == ["team-guardrail"] + assert (auth.team_id, auth.org_id) == (None, None) + assert auth.team_metadata is None + assert len(team_loader.await_args_list) == 2 + assert all(call.kwargs["check_db_only"] is strict for call in team_loader.await_args_list) + if strict: + database.writer_db.litellm_agentstable.find_many.assert_awaited_once() + database.db.litellm_agentstable.find_many.assert_not_awaited() + else: + database.db.litellm_agentstable.find_many.assert_awaited_once() + database.writer_db.litellm_agentstable.find_many.assert_not_awaited() + + async def _add_proxy_data(data: dict[str, object], **kwargs: object) -> dict[str, object]: return { **data, diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 72179e661a3..c583407e61e 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -27,6 +27,7 @@ from litellm.proxy._types import ( LiteLLM_EndUserTable, LiteLLM_ObjectPermissionTable, LiteLLM_TagTable, + LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_UserTable, LitellmUserRoles, @@ -48,6 +49,8 @@ from litellm.proxy.auth.auth_checks import ( _check_agent_access_group_model_access, check_end_user_budget, _check_team_member_budget, + personal_key_owner_teams, + personal_key_owner_teams_with_status, fetch_key_object_from_db_with_reconnect, _get_fuzzy_user_object, CallerTeamLoader, @@ -82,6 +85,7 @@ from litellm.constants import ( ) from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import check_api_key_for_custom_headers_or_pass_through_endpoints +from litellm.proxy.utils import ProxyLogging from litellm.proxy import proxy_server from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_bearer_token, encrypt_value_helper from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -4642,6 +4646,188 @@ async def test_check_team_member_budget_dispatches_the_configured_alert_before_t assert call_info.max_budget_alert_emails == {"50": [], "100": ["finance@co.com"]} +@pytest.mark.asyncio +async def test_personal_key_owner_teams_reports_over_budget_and_blocked_teams( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth import auth_checks + + cache: Final = UserApiKeyCache() + logging_obj: Final = ProxyLogging(user_api_key_cache=cache) + owner: Final = LiteLLM_UserTable( + user_id="owner", + teams=["team-a", "team-b", "team-blocked", "team-other-org"], + ) + loaded_teams: Final = { + "team-a": LiteLLM_TeamTable(team_id="team-a", organization_id="org-1"), + "team-b": LiteLLM_TeamTable(team_id="team-b", organization_id="org-1"), + "team-blocked": LiteLLM_TeamTable(team_id="team-blocked", organization_id="org-1", blocked=True), + "team-other-org": LiteLLM_TeamTable(team_id="team-other-org", organization_id="org-2"), + } + memberships: Final = { + "team-a": LiteLLM_TeamMembership( + user_id="owner", + team_id="team-a", + spend=1.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=5.0), + ), + "team-b": LiteLLM_TeamMembership( + user_id="owner", + team_id="team-b", + spend=1.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0), + ), + } + get_user = AsyncMock(return_value=owner) + get_membership = AsyncMock(side_effect=lambda *, team_id, **_: memberships[team_id]) + + async def get_team_object(team_id: str, **_: object) -> LiteLLM_TeamTable: + return loaded_teams[team_id] + + async def current_spend( + counter_key: str, + fallback_spend: float, + max_budget: float | None = None, + ) -> float: + assert counter_key.startswith("spend:team_member:") and max_budget is not None + return fallback_spend + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(auth_checks, "get_user_object", get_user) + monkeypatch.setattr(auth_checks, "get_team_object", get_team_object) + monkeypatch.setattr(auth_checks, "get_team_membership", get_membership) + monkeypatch.setattr(proxy_server, "get_current_spend", current_spend) + + result: Final = await personal_key_owner_teams_with_status( + valid_token=UserAPIKeyAuth(api_key="sk-personal", user_id="owner", org_id="org-1"), + prisma_client=MagicMock(), + user_api_key_cache=cache, + proxy_logging_obj=logging_obj, + ) + + assert tuple(team.team_id for team in result.teams) == ("team-a",) + assert {team.team_id for team in result.withheld} == {"team-b", "team-blocked"} + assert all(team.team_id != "team-other-org" for team in (*result.teams, *result.withheld)) + assert result.unreadable is False + assert get_user.await_count == 1 + assert {call.kwargs["team_id"] for call in get_membership.await_args_list} == {"team-a", "team-b"} + + +@pytest.mark.asyncio +async def test_personal_key_owner_teams_check_db_only_refreshes_cached_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth import auth_checks + + user_id = "owner-cache-refresh" + cached_owner: Final = LiteLLM_UserTable( + user_id=user_id, + user_email="owner@example.com", + teams=["team-a"], + ) + database_owner: Final = LiteLLM_UserTable( + user_id=user_id, + user_email="owner@example.com", + teams=[], + ) + cache: Final = UserApiKeyCache() + await cache.async_set_cache(key=user_id, value=cached_owner, model_type=LiteLLM_UserTable) + logging_obj: Final = ProxyLogging(user_api_key_cache=cache) + prisma: Final = MagicMock() + prisma.writer_db.litellm_usertable.find_unique = AsyncMock(return_value=database_owner) + monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=LiteLLM_TeamTable(team_id="team-a"))) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=None)) + valid_token: Final = UserAPIKeyAuth(api_key="sk-personal", user_id=user_id) + + cached_result: Final = await personal_key_owner_teams_with_status( + valid_token=valid_token, + prisma_client=prisma, + user_api_key_cache=cache, + proxy_logging_obj=logging_obj, + ) + fresh_result: Final = await personal_key_owner_teams_with_status( + valid_token=valid_token, + prisma_client=prisma, + user_api_key_cache=cache, + proxy_logging_obj=logging_obj, + check_db_only=True, + ) + + assert tuple(team.team_id for team in cached_result.teams) == ("team-a",) + assert fresh_result.teams == () + prisma.writer_db.litellm_usertable.find_unique.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_personal_key_owner_teams_check_db_only_uses_fresh_membership_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth import auth_checks + + user_id: Final = "owner-membership-refresh" + team_id: Final = "team-membership-refresh" + cached_membership: Final = LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + spend=1.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=5.0), + ) + database_membership: Final = LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + spend=10.0, + litellm_budget_table=LiteLLM_BudgetTable(max_budget=5.0), + ) + database_row: Final = MagicMock() + database_row.dict.return_value = database_membership.model_dump(mode="python") + prisma: Final = MagicMock() + prisma.db.litellm_teammembership.find_unique = AsyncMock() + prisma.writer_db.litellm_teammembership.find_unique = AsyncMock(return_value=database_row) + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=auth_checks.team_membership_reservation_cache_key(user_id=user_id, team_id=team_id), + value=cached_membership, + model_type=LiteLLM_TeamMembership, + ) + logging_obj: Final = ProxyLogging(user_api_key_cache=cache) + owner: Final = LiteLLM_UserTable(user_id=user_id, teams=[team_id]) + team: Final = LiteLLM_TeamTable(team_id=team_id) + + async def current_spend( + counter_key: str, + fallback_spend: float, + max_budget: float | None = None, + ) -> float: + return fallback_spend + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(auth_checks, "get_user_object", AsyncMock(return_value=owner)) + monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=team)) + monkeypatch.setattr(proxy_server, "get_current_spend", current_spend) + valid_token: Final = UserAPIKeyAuth(api_key="sk-personal", user_id=user_id) + + cached_result: Final = await personal_key_owner_teams_with_status( + valid_token=valid_token, + prisma_client=prisma, + user_api_key_cache=cache, + proxy_logging_obj=logging_obj, + ) + fresh_result: Final = await personal_key_owner_teams_with_status( + valid_token=valid_token, + prisma_client=prisma, + user_api_key_cache=cache, + proxy_logging_obj=logging_obj, + check_db_only=True, + ) + + assert tuple(team.team_id for team in cached_result.teams) == (team_id,) + assert cached_result.withheld == () + assert fresh_result.teams == () + assert tuple(team.team_id for team in fresh_result.withheld) == (team_id,) + prisma.db.litellm_teammembership.find_unique.assert_not_awaited() + prisma.writer_db.litellm_teammembership.find_unique.assert_awaited_once() + + @pytest.mark.parametrize( "spend, max_budget, expect_alert", [ @@ -8422,6 +8608,84 @@ async def test_get_team_membership_db_fetch_returns_validated_membership(): assert result.spend == 1.5 +@pytest.mark.asyncio +async def test_get_team_membership_check_db_only_bypasses_stale_cache(): + from litellm.proxy._types import LiteLLM_TeamMembership + from litellm.proxy.auth.auth_checks import get_team_membership + from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key + + cache: Final = UserApiKeyCache() + cache_key: Final = team_membership_reservation_cache_key(user_id="u-fresh", team_id="t-fresh") + stale_membership: Final = LiteLLM_TeamMembership(user_id="u-fresh", team_id="t-fresh", spend=1.0) + await cache.async_set_cache(key=cache_key, value=stale_membership, model_type=LiteLLM_TeamMembership) + + database_row: Final = MagicMock() + database_row.dict.return_value = {"user_id": "u-fresh", "team_id": "t-fresh", "spend": 9.0} + prisma: Final = MagicMock() + prisma.db.litellm_teammembership.find_unique = AsyncMock(return_value=database_row) + prisma.writer_db.litellm_teammembership.find_unique = AsyncMock(return_value=database_row) + + cached: Final = await get_team_membership( + user_id="u-fresh", + team_id="t-fresh", + prisma_client=prisma, + user_api_key_cache=cache, + ) + fresh: Final = await get_team_membership( + user_id="u-fresh", + team_id="t-fresh", + prisma_client=prisma, + user_api_key_cache=cache, + check_db_only=True, + ) + + assert cached is not None and cached.spend == 1.0 + assert fresh is not None and fresh.spend == 9.0 + prisma.db.litellm_teammembership.find_unique.assert_not_awaited() + prisma.writer_db.litellm_teammembership.find_unique.assert_awaited_once() + with pytest.raises(Exception, match=r"^No db connected$"): + await get_team_membership( + user_id="u-fresh", + team_id="t-fresh", + prisma_client=None, + user_api_key_cache=cache, + check_db_only=True, + ) + + +@pytest.mark.asyncio +async def test_get_team_membership_cache_miss_reads_replica_and_check_db_only_reads_writer(): + from litellm.proxy.auth.auth_checks import get_team_membership + + reader_row: Final = MagicMock() + reader_row.dict.return_value = {"user_id": "u-read", "team_id": "t-read", "spend": 2.0} + writer_row: Final = MagicMock() + writer_row.dict.return_value = {"user_id": "u-read", "team_id": "t-read", "spend": 9.0} + prisma_client: Final = MagicMock() + prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=reader_row) + prisma_client.writer_db.litellm_teammembership.find_unique = AsyncMock(return_value=writer_row) + cache: Final = UserApiKeyCache() + + reader_membership: Final = await get_team_membership( + user_id="u-read", + team_id="t-read", + prisma_client=prisma_client, + user_api_key_cache=cache, + ) + writer_membership: Final = await get_team_membership( + user_id="u-read", + team_id="t-read", + prisma_client=prisma_client, + user_api_key_cache=cache, + check_db_only=True, + ) + + assert reader_membership is not None and reader_membership.spend == 2.0 + assert writer_membership is not None and writer_membership.spend == 9.0 + prisma_client.db.litellm_teammembership.find_unique.assert_awaited_once() + prisma_client.writer_db.litellm_teammembership.find_unique.assert_awaited_once() + + @pytest.mark.asyncio async def test_get_team_membership_negative_caches_a_missing_row(): """ diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 9bcf0920f3e..7af747fc26d 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3938,6 +3938,126 @@ async def test_mcp_per_team_rate_limit_is_enforced_from_team_metadata_v3() -> No await handler.enforce_mcp_server_rate_limits(second_key, server) +@pytest.mark.asyncio +async def test_inherited_mcp_team_rate_limit_deduplicates_team_and_source_descriptors_v3() -> None: + handler, _ = _make_mcp_handler() + server: Final = MCPServer( + server_id="math-a", + name="math_a", + server_name="math_a", + transport=MCPTransport.http, + ) + inherited_auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-inherited"), + team_id="team-a", + team_metadata={"mcp_rpm_limit": {"math_a": 1}}, + ).model_copy(update={"mcp_source_team_rpm_limits": {"team-a": {"math_a": 1}}}) + descriptors: Final[list[RateLimitDescriptor]] = [] + handler._add_mcp_per_key_rate_limit_descriptor(inherited_auth, "math_a", descriptors) + + assert descriptors == [] + team_descriptors: Final[list[RateLimitDescriptor]] = [] + handler._add_mcp_per_team_rate_limit_descriptor(inherited_auth, "math_a", team_descriptors) + assert [(descriptor["key"], descriptor["value"]) for descriptor in team_descriptors] == [ + ("mcp_per_team", "team-a:math_a") + ] + await handler.enforce_mcp_server_rate_limits(inherited_auth, server) + + with pytest.raises(ProxyRateLimitError, match="mcp_per_team"): + await handler.enforce_mcp_server_rate_limits(inherited_auth, server) + + +def test_inherited_mcp_team_rate_limit_uses_source_limit_when_primary_metadata_is_missing() -> None: + handler, _ = _make_mcp_handler() + auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-inherited-source-limit"), + team_id="team-a", + ).model_copy(update={"mcp_source_team_rpm_limits": {"team-a": {"math_a": 1}}}) + descriptors: Final[list[RateLimitDescriptor]] = [] + + handler._add_mcp_per_team_rate_limit_descriptor(auth, "math_a", descriptors) + + assert descriptors == [ + { + "key": "mcp_per_team", + "value": "team-a:math_a", + "rate_limit": { + "requests_per_unit": 1, + "tokens_per_unit": None, + "window_size": handler.window_size, + }, + } + ] + + +@pytest.mark.asyncio +async def test_billed_personal_key_auth_produces_team_rpm_descriptors(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import AsyncMock, MagicMock + + from litellm.models.budget import LiteLLM_BudgetTable + from litellm.models.team_membership import LiteLLM_TeamMembership + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.auth import auth_checks + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable + + server: Final = MCPServer( + server_id="math_a", + name="math_a", + server_name="math_a", + transport=MCPTransport.http, + ) + team: Final = LiteLLM_TeamTable( + team_id="team-a", + organization_id="org-team-a", + rpm_limit=7, + object_permission_id="perm-team-a", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="perm-team-a", + mcp_servers=["math_a"], + mcp_tool_permissions={"math_a": ["add"]}, + ), + ) + membership: Final = LiteLLM_TeamMembership( + user_id="owner", + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable(rpm_limit=3), + ) + user_api_key_auth: Final = UserAPIKeyAuth( + api_key=hash_token("sk-billed-personal"), + user_id="owner", + ) + + monkeypatch.setattr(litellm, "personal_key_model_access_from_teams", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "should_load_db_object", lambda _: False) + monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock()) + monkeypatch.setattr(auth_checks, "personal_key_owner_teams", AsyncMock(return_value=(team,))) + monkeypatch.setattr(auth_checks, "get_team_object", AsyncMock(return_value=team)) + monkeypatch.setattr(auth_checks, "get_team_membership", AsyncMock(return_value=membership)) + monkeypatch.setattr(auth_checks, "get_org_object", AsyncMock(return_value=None)) + monkeypatch.setitem(global_mcp_server_manager.registry, server.server_id, server) + + billed_auth: Final = await MCPRequestHandler.billing_auth_for_tool_call(user_api_key_auth, "math_a-add") + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + + descriptors: Final = handler.create_rate_limit_descriptors( + user_api_key_dict=billed_auth, + data={}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + + assert [ + (descriptor["key"], descriptor["value"], descriptor["rate_limit"]["requests_per_unit"]) + for descriptor in descriptors + if descriptor["key"] in ("team", "team_member") + ] == [("team", "team-a", 7), ("team_member", "team-a:owner", 3)] + + @pytest.mark.asyncio async def test_mcp_server_rpm_is_shared_across_keys_v3() -> None: handler, _ = _make_mcp_handler() From f4347d4f5015e9cc54681b4dca2e2d60532ae414 Mon Sep 17 00:00:00 2001 From: Vansh Singh Date: Tue, 20 Oct 2026 04:32:11 +0530 Subject: [PATCH 029/124] fix(speech)!: correctly propagate spend logging and usage for Gemini TTS bridge (#44551) * fix(speech): correctly propagate spend logging and usage for Gemini TTS bridge * fix(speech): call Gemini once per TTS request and log the bridge's token usage aspeech re-ran speech whenever the first result was not a coroutine, so the speech-to-completion bridge sent every Gemini TTS request upstream twice. It now returns the sync result. The bridge's inner completion runs as an internal sub call, so only the speech call fires success callbacks, and the completion's usage rides on the audio response so the spend log and usage_object carry real token counts. Replaces the separate inner Logging object and the cost_calculator fallback, which never reached the spend row. * fix(speech): flag a cached Gemini TTS reply as a cache hit so it logs at no charge * fix(speech): price a cached Gemini TTS reply's savings from its token usage and keep its cache key * test(integration): cover Gemini TTS speech billing, logging, cache hits and deployment TPM * test(integration): start the speech TPM cell in a minute's first 10 s so the two requests never cross a minute --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 2 + .../speech_to_completion_bridge/handler.py | 9 +- .../transformation.py | 15 +- litellm/main.py | 8 +- litellm/types/llms/openai.py | 11 +- .../sdk/test_gemini_tts_speech_logging_sdk.py | 69 ++++ .../spend/test_gemini_tts_speech_billing.py | 334 ++++++++++++++++++ tests/unit/test_main.py | 65 +++- 8 files changed, 497 insertions(+), 16 deletions(-) create mode 100644 tests/integration/sdk/test_gemini_tts_speech_logging_sdk.py create mode 100644 tests/integration/spend/test_gemini_tts_speech_billing.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4fcd3bde843..8c6e2fead2d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1103,6 +1103,8 @@ def get_usage_object( if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)): return None if completion_response.usage is None else _decisions_usage(completion_response.usage) + if isinstance(completion_response, HttpxBinaryResponseContent): + return completion_response.usage if usage_obj is None: return None if isinstance(usage_obj, Usage): diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 6c33621ec89..00d112773f1 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Final from typing_extensions import TypedDict +from litellm._internal_context import internal_sub_call + if TYPE_CHECKING: from litellm import LiteLLMLoggingObj from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -111,9 +113,10 @@ class SpeechToCompletionBridgeHandler: voice=voice, ) - result: Final = completion( - **request_data, - ) + with internal_sub_call(): + result: Final = completion( + **request_data, + ) requested_response_format: Final = optional_params.get("response_format") if isinstance(result, ModelResponse): diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 2ed140c0208..9c1e5ae7a29 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -5,11 +5,12 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param if TYPE_CHECKING: from litellm import Logging as LiteLLMLoggingObj from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent - from litellm.types.utils import ModelResponse + from litellm.types.utils import ModelResponse, Usage def _completion_response_cost(model_response: "ModelResponse") -> float | None: @@ -20,6 +21,13 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None: return response_cost if isinstance(response_cost, float) else None +def _completion_usage(model_response: "ModelResponse") -> "Usage | None": + from litellm.types.utils import Usage + + usage: Final = getattr(model_response, "usage", None) + return usage if isinstance(usage, Usage) else None + + GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16" GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm" GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT}) @@ -179,4 +187,9 @@ class SpeechToCompletionBridgeTransformationHandler: ) binary_response: Final = HttpxBinaryResponseContent(response) binary_response.set_response_cost(_completion_response_cost(model_response)) + binary_response.usage = _completion_usage(model_response) + cached_hidden_params: Final = get_hidden_params(model_response) if served_from_cache(model_response) else None + if cached_hidden_params is not None: + set_hidden_param(binary_response, "cache_hit", True) + set_hidden_param(binary_response, "cache_key", cached_hidden_params.get("cache_key")) return binary_response diff --git a/litellm/main.py b/litellm/main.py index 38591441e01..9430af34401 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8500,14 +8500,8 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent: api_base=api_base_hint, ) - # Await normally init_response: Final = await loop.run_in_executor(None, func_with_context) - if asyncio.iscoroutine(init_response): - response = await init_response - else: - # Call the synchronous function using run_in_executor - response = await loop.run_in_executor(None, func_with_context) - return response + return await init_response if asyncio.iscoroutine(init_response) else init_response except Exception as e: custom_llm_provider = custom_llm_provider or "openai" raise exception_type( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8cc9e912e80..8dbc0d97d54 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2,7 +2,7 @@ import builtins from collections.abc import Iterable, Mapping from enum import Enum from os import PathLike -from typing import IO, Any, Final, Generic, Literal, Optional, TypeAlias, Union +from typing import IO, TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, Union import httpx from openai import Omit @@ -96,6 +96,9 @@ from litellm.types.responses.main import ( from .base import CachedTokensDetails +if TYPE_CHECKING: + from litellm.types.utils import Usage + FileContent = IO[bytes] | bytes | PathLike FileTypes = ( @@ -119,6 +122,7 @@ class BinaryResponseSummary(TypedDict): object: ReadOnly[Literal["binary"]] content_type: ReadOnly[str | None] num_bytes: ReadOnly[int] + usage: NotRequired[ReadOnly[Mapping[str, builtins.object]]] _ResponseT = TypeVar("_ResponseT", bound=httpx.Response | SDKResponse, default=httpx.Response) @@ -126,6 +130,7 @@ _ResponseT = TypeVar("_ResponseT", bound=httpx.Response | SDKResponse, default=h class HttpxBinaryResponseContent(_HttpxBinaryResponseContent, Generic[_ResponseT]): _hidden_params: dict + usage: "Usage | None" response: _ResponseT # pyright: ignore[reportIncompatibleVariableOverride] # SDK accepts both backends at runtime @property @@ -139,6 +144,7 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent, Generic[_ResponseT def __init__(self, response: _ResponseT) -> None: super().__init__(response) # pyright: ignore[reportArgumentType] # SDK accepts both backends at runtime self._hidden_params = {} + self.usage = None @property def content_type(self) -> str | None: @@ -146,11 +152,12 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent, Generic[_ResponseT return headers.get("content-type") def logging_summary(self) -> BinaryResponseSummary: - return { + summary: Final[BinaryResponseSummary] = { "object": "binary", "content_type": self.content_type, "num_bytes": self._num_bytes(), } + return summary if self.usage is None else {**summary, "usage": self.usage.model_dump()} def _num_bytes(self) -> int: try: diff --git a/tests/integration/sdk/test_gemini_tts_speech_logging_sdk.py b/tests/integration/sdk/test_gemini_tts_speech_logging_sdk.py new file mode 100644 index 00000000000..3df10da8fda --- /dev/null +++ b/tests/integration/sdk/test_gemini_tts_speech_logging_sdk.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import threading +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import eventually +from integration._support.wire import wire_server +from integration.providers._gemini_tts_voice_support import ( + AUDIO_TOKENS, + BACKEND, + GEMINI_API_KEY, + PCM, + PROMPT_TOKENS, + gemini_peer, + marker, + received_text, + wav_payload, +) + +import litellm +from litellm.integrations.custom_logger import CustomLogger + +BINARY_RESPONSE: Final = "HttpxBinaryResponseContent" + + +@dataclass(frozen=True, slots=True) +class _Logged: + call_type: object + response_type: str + tokens: tuple[object, object] + + +class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self._lock: Final = threading.Lock() + self.events: tuple[_Logged, ...] = () + + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + payload: Final = kwargs.get("standard_logging_object") + logged: Final = payload if isinstance(payload, dict) else {} + event: Final = _Logged( + call_type=kwargs.get("call_type"), + response_type=type(response_obj).__name__, + tokens=(logged.get("prompt_tokens"), logged.get("completion_tokens")), + ) + with self._lock: + self.events = (*self.events, event) + + +def test_sync_speech_logs_one_success_event_carrying_the_reply_token_usage(monkeypatch: pytest.MonkeyPatch) -> None: + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with wire_server(gemini_peer) as wire: + monkeypatch.setenv("GEMINI_API_BASE", wire.url) + text: Final = marker() + audio: Final = litellm.speech(model=f"gemini/{BACKEND}", input=text, voice="Kore", api_key=GEMINI_API_KEY) + assert wav_payload(audio.content) == PCM + assert sum(1 for request in wire.drain() if received_text(request) == text) == 1 + events: Final = eventually( + lambda: recorder.events, + lambda found: any(event.response_type == BINARY_RESPONSE for event in found), + seconds=30, + ) + assert events == (_Logged("speech", BINARY_RESPONSE, (PROMPT_TOKENS, AUDIO_TOKENS)),) diff --git a/tests/integration/spend/test_gemini_tts_speech_billing.py b/tests/integration/spend/test_gemini_tts_speech_billing.py new file mode 100644 index 00000000000..f1bf3de91b3 --- /dev/null +++ b/tests/integration/spend/test_gemini_tts_speech_billing.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path +from typing import Final, Literal + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Wire, wire_server +from integration.providers._gemini_tts_voice_support import ( + AUDIO_TOKENS, + BACKEND, + GEMINI_API_KEY, + PCM, + PROMPT_TOKENS, + REJECTED_PREFIX, + VERTEX_LOCATION, + VERTEX_PROJECT, + gemini_peer, + marker, + received_text, + wav_payload, +) +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +Provider = Literal["gemini", "vertex_ai"] +MODELS: Final[Mapping[Provider, str]] = {"gemini": "tts-billing-gemini", "vertex_ai": "tts-billing-vertex"} +TPM_MODEL: Final = "tts-billing-tpm" +TPM_LIMIT: Final = PROMPT_TOKENS + AUDIO_TOKENS +BINARY_RESPONSE: Final = "HttpxBinaryResponseContent" +CALLBACK_LOG: Final = "TTS_BILLING_CALLBACK_LOG" +CALLBACK_MODULE: Final = """ +import json +import os + +from litellm.integrations.custom_logger import CustomLogger + + +def _event(kind, kwargs, response_obj): + payload = kwargs.get("standard_logging_object") or {} + metadata = payload.get("metadata") or {} + response = payload.get("response") + usage = response.get("usage") if isinstance(response, dict) else None + return { + "kind": kind, + "key": metadata.get("user_api_key_hash"), + "call_type": kwargs.get("call_type"), + "response_type": type(response_obj).__name__, + "response_cost": payload.get("response_cost"), + "prompt_tokens": payload.get("prompt_tokens"), + "completion_tokens": payload.get("completion_tokens"), + "cache_hit": payload.get("cache_hit"), + "saved_cache_cost": payload.get("saved_cache_cost"), + "cache_key": payload.get("cache_key"), + "logged_usage_total": usage.get("total_tokens") if isinstance(usage, dict) else None, + } + + +class SpeechEvents(CustomLogger): + def _write(self, line): + with open(os.environ["TTS_BILLING_CALLBACK_LOG"], "a") as sink: + sink.write(json.dumps(line) + "\\n") + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self._write(_event("sync_success", kwargs, response_obj)) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self._write(_event("async_success", kwargs, response_obj)) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._write(_event("sync_failure", kwargs, response_obj)) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self._write(_event("async_failure", kwargs, response_obj)) + + +speech_events = SpeechEvents() +""" +SPEND_ROWS_SQL: Final = ( + 'SELECT call_type, spend, prompt_tokens, completion_tokens, cache_hit FROM "LiteLLM_SpendLogs"' + ' WHERE api_key = %s ORDER BY "startTime"' +) +KEY_SPEND_SQL: Final = 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s' + +Event = dict[str, JsonValue] + + +@dataclass(frozen=True, slots=True) +class BillingProxy: + proxy: OwnedProxy + callback_log: Path + + +@dataclass(frozen=True, slots=True) +class Sent: + status: int + body: bytes + cache_key: str + + +def _hashed(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _model_list(token_url: str) -> list[JsonValue]: + return [ + {"model_name": MODELS["gemini"], "litellm_params": {"model": f"gemini/{BACKEND}", "api_key": GEMINI_API_KEY}}, + { + "model_name": TPM_MODEL, + "litellm_params": {"model": f"gemini/{BACKEND}", "api_key": GEMINI_API_KEY, "tpm": TPM_LIMIT}, + }, + { + "model_name": MODELS["vertex_ai"], + "litellm_params": { + "model": f"vertex_ai/{BACKEND}", + "vertex_project": VERTEX_PROJECT, + "vertex_location": VERTEX_LOCATION, + "vertex_credentials": service_account_json(VERTEX_PROJECT, token_url.rstrip("/")), + }, + }, + ] + + +def _config(directory: Path, token_url: str, *, cached: bool) -> Path: + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + settings: Final = object_value(stock["litellm_settings"]) + kept: Final = {name: value for name, value in settings.items() if name not in {"cache", "cache_params"}} + cache: Final[dict[str, JsonValue]] = ( + {"cache": True, "cache_params": {**object_value(settings["cache_params"]), "namespace": marker()}} + if cached + else {"cache": False} + ) + routing: Final[dict[str, JsonValue]] = ( + {} + if cached + else { + "routing_strategy": "usage-based-routing-v2", + "redis_host": os.environ["REDIS_HOST"], + "redis_port": int(os.environ["REDIS_PORT"]), + } + ) + (directory / "tts_billing_callbacks.py").write_text(CALLBACK_MODULE) + path: Final = directory / "tts-billing.yaml" + path.write_text( + yaml.safe_dump( + { + **stock, + "model_list": _model_list(token_url), + "litellm_settings": {**kept, **cache, "callbacks": "tts_billing_callbacks.speech_events"}, + "router_settings": {**object_value(stock["router_settings"]), "num_retries": 0, **routing}, + } + ) + ) + return path + + +def _billing_proxy(wire: Wire, directory: Path, *, cached: bool) -> Iterator[BillingProxy]: + callback_log: Final = directory / "callback_events.jsonl" + callback_log.touch() + with gateway_from_environment() as gateway: + config: Final = _config(directory, gateway.upstream_url, cached=cached) + overrides: Final = {"GEMINI_API_BASE": wire.url, "VERTEXAI_API_BASE": wire.url, CALLBACK_LOG: str(callback_log)} + with owned_proxy_process(gateway, directory, overrides, config=config, workers=2) as proxy: + yield BillingProxy(proxy, callback_log) + + +@pytest.fixture(scope="module") +def tts_wire() -> Iterator[Wire]: + with wire_server(gemini_peer) as wire: + yield wire + + +@pytest.fixture(scope="module") +def uncached(tts_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[BillingProxy]: + yield from _billing_proxy(tts_wire, tmp_path_factory.mktemp("tts-billing-uncached"), cached=False) + + +@pytest.fixture(scope="module") +def cached(tts_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[BillingProxy]: + yield from _billing_proxy(tts_wire, tmp_path_factory.mktemp("tts-billing-cached"), cached=True) + + +def _speech(billing: BillingProxy, model: str, text: str, key: str, voice: str = "Kore") -> Sent: + response: Final = billing.proxy.gateway.request( + "POST", "/v1/audio/speech", {"model": model, "input": text, "voice": voice}, key=key + ) + return Sent(response.status_code, response.content, response.headers.get("x-litellm-cache-key", "")) + + +def _upstream_calls(wire: Wire, text: str) -> int: + return sum(1 for request in wire.drain() if received_text(request) == text) + + +def _spend_rows(key: str, count: int) -> list[dict[str, JsonValue]]: + return eventually(lambda: read_rows(SPEND_ROWS_SQL, (_hashed(key),)), lambda rows: len(rows) == count, seconds=70) + + +def _key_spend(key: str, expected: float) -> JsonValue: + rows: Final = eventually( + lambda: read_rows(KEY_SPEND_SQL, (_hashed(key),)), + lambda found: len(found) == 1 and found[0]["spend"] == pytest.approx(expected), + seconds=70, + return_last_on_timeout=True, + ) + return rows[0]["spend"] + + +def _chat_spend(billing: BillingProxy, model: str, key: str) -> float: + body: Final[dict[str, JsonValue]] = { + "model": model, + "messages": [{"role": "user", "content": marker()}], + "modalities": ["audio"], + "audio": {"voice": "Kore", "format": "pcm16"}, + } + response: Final = billing.proxy.gateway.request("POST", "/v1/chat/completions", body, key=key) + assert response.status_code == 200, response.text + row: Final = _spend_rows(key, 1)[0] + assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row + spend: Final = row["spend"] + assert isinstance(spend, float) and spend > 0, row + return spend + + +def _events(billing: BillingProxy, key: str) -> tuple[Event, ...]: + complete_lines: Final = billing.callback_log.read_text().split("\n")[:-1] + events: Final = tuple(JSON_OBJECT.validate_json(line) for line in complete_lines) + return tuple(event for event in events if event["key"] == _hashed(key)) + + +def _speech_events(billing: BillingProxy, key: str, count: int) -> tuple[Event, ...]: + return eventually( + lambda: _events(billing, key), + lambda found: sum(1 for event in found if event["response_type"] == BINARY_RESPONSE) >= count, + seconds=70, + ) + + +def _summary(event: Event) -> tuple[JsonValue, ...]: + return (event["call_type"], event["response_type"], event["prompt_tokens"], event["completion_tokens"]) + + +@pytest.mark.parametrize("provider", ["gemini", "vertex_ai"]) +def test_speech_calls_google_once_and_bills_and_logs_its_token_usage_once( + uncached: BillingProxy, tts_wire: Wire, provider: Provider +) -> None: + with uncached.proxy.gateway.scenario() as scenario: + speech_key: Final = scenario.key() + text: Final = marker() + sent: Final = _speech(uncached, MODELS[provider], text, speech_key) + assert sent.status == 200, sent.body[:300] + assert wav_payload(sent.body) == PCM + assert _upstream_calls(tts_wire, text) == 1 + chat_spend: Final = _chat_spend(uncached, MODELS[provider], scenario.key()) + row: Final = _spend_rows(speech_key, 1)[0] + assert row["call_type"] == "aspeech", row + assert (row["prompt_tokens"], row["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), row + assert row["spend"] == pytest.approx(chat_spend), row + assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend) + events: Final = _speech_events(uncached, speech_key, 1) + assert [_summary(event) for event in events] == [("aspeech", BINARY_RESPONSE, PROMPT_TOKENS, AUDIO_TOKENS)] + assert events[0]["response_cost"] == pytest.approx(chat_spend), events + assert events[0]["logged_usage_total"] == PROMPT_TOKENS + AUDIO_TOKENS, events + + +def test_cached_speech_repeat_is_a_free_cache_hit_after_a_billed_first_call( + cached: BillingProxy, tts_wire: Wire +) -> None: + model: Final = MODELS["gemini"] + with cached.proxy.gateway.scenario() as scenario: + speech_key: Final = scenario.key() + text: Final = marker() + first: Final = _speech(cached, model, text, speech_key) + repeat: Final = _speech(cached, model, text, speech_key) + assert (first.status, repeat.status) == (200, 200), (first.body[:300], repeat.body[:300]) + assert wav_payload(first.body) == PCM and repeat.body == first.body + assert _upstream_calls(tts_wire, text) == 1 + assert repeat.cache_key, "the cached repeat carried no x-litellm-cache-key header" + chat_spend: Final = _chat_spend(cached, model, scenario.key()) + billed, hit = _spend_rows(speech_key, 2) + assert billed["cache_hit"] != "True" and billed["spend"] == pytest.approx(chat_spend), billed + assert (billed["prompt_tokens"], billed["completion_tokens"]) == (PROMPT_TOKENS, AUDIO_TOKENS), billed + assert (hit["cache_hit"], hit["spend"]) == ("True", 0.0), hit + assert _key_spend(speech_key, chat_spend) == pytest.approx(chat_spend) + events: Final = _speech_events(cached, speech_key, 2) + assert len(events) == 2, events + hits: Final = tuple(event for event in events if event["cache_hit"] is True) + assert len(hits) == 1, events + assert (hits[0]["response_cost"], hits[0]["cache_key"]) == (0.0, repeat.cache_key), hits + assert hits[0]["saved_cache_cost"] == pytest.approx(chat_spend), hits + paid: Final = tuple(event for event in events if event["cache_hit"] is not True) + assert len(paid) == 1 and paid[0]["response_cost"] == pytest.approx(chat_spend), events + + +def test_speech_rejected_by_google_reaches_the_caller_and_logs_one_failure_after_one_call( + uncached: BillingProxy, tts_wire: Wire +) -> None: + with uncached.proxy.gateway.scenario() as scenario: + key: Final = scenario.key() + text: Final = marker() + sent: Final = _speech(uncached, MODELS["gemini"], text, key, voice="Custom-Voice") + assert sent.status == 400, sent.body[:300] + assert f"{REJECTED_PREFIX}Custom-Voice" in sent.body.decode(), sent.body[:300] + assert _upstream_calls(tts_wire, text) == 1 + events: Final = eventually(lambda: _events(uncached, key), lambda found: len(found) >= 2, seconds=70) + logged: Final = sorted((str(event["kind"]), event["call_type"]) for event in events) + assert logged == [("async_failure", "aspeech"), ("sync_failure", "aspeech")], events + + +def test_a_speech_reply_fills_the_deployment_tpm_limit_so_the_next_request_gets_429( + uncached: BillingProxy, tts_wire: Wire +) -> None: + eventually(lambda: datetime.now(UTC), lambda now: now.second < 10, seconds=65) + with uncached.proxy.gateway.scenario() as scenario: + key: Final = scenario.key() + first_text: Final = marker() + first: Final = _speech(uncached, TPM_MODEL, first_text, key) + assert first.status == 200, first.body[:300] + _spend_rows(key, 1) + second_text: Final = marker() + second: Final = _speech(uncached, TPM_MODEL, second_text, key) + assert second.status == 429, second.body[:300] + assert b"No deployments available" in second.body, second.body[:300] + received: Final = tuple(received_text(request) for request in tts_wire.drain()) + assert (received.count(first_text), received.count(second_text)) == (1, 0) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 63f440d4aba..aa15893e395 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -4425,18 +4425,27 @@ class _RecordedSpeechSuccess: spend_metadata: Mapping[str, object] response_cost: float | None logged_response_cost: float | None + logged_tokens: tuple[object, object] + logged_cache_hit: object + logged_saved_cache_cost: object + logged_cache_key: object def _record_speech_success(payload: dict[str, object]) -> _RecordedSpeechSuccess: call_type: Final = payload.get("call_type") response_cost: Final = payload.get("response_cost") logging_payload: Final = payload.get("standard_logging_object") - logged_cost: Final = logging_payload.get("response_cost") if isinstance(logging_payload, dict) else None + logged: Final[Mapping[str, object]] = logging_payload if isinstance(logging_payload, dict) else {} + logged_cost: Final = logged.get("response_cost") return _RecordedSpeechSuccess( call_type=call_type if isinstance(call_type, str) else None, spend_metadata=get_litellm_metadata_from_kwargs(payload), response_cost=response_cost if isinstance(response_cost, float) else None, logged_response_cost=logged_cost if isinstance(logged_cost, float) else None, + logged_tokens=(logged.get("prompt_tokens"), logged.get("completion_tokens")), + logged_cache_hit=logged.get("cache_hit"), + logged_saved_cache_cost=logged.get("saved_cache_cost"), + logged_cache_key=logged.get("cache_key"), ) @@ -4445,6 +4454,11 @@ class _SuccessEventRecorder(CustomLogger): super().__init__() self.events: list[_RecordedSpeechSuccess] = [] # mutable-ok: test recorder of success-callback events + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.events.append(_record_speech_success(kwargs)) + async def async_log_success_event( self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object ) -> None: @@ -4459,6 +4473,16 @@ async def _wait_for_success_event(recorder: _SuccessEventRecorder, call_type: st pytest.fail(f"no {call_type} success event; got {[e.call_type for e in recorder.events]}") +async def _wait_for_success_events( + recorder: _SuccessEventRecorder, call_type: str, count: int +) -> tuple[_RecordedSpeechSuccess, ...]: + for _ in range(100): + if len(events := tuple(e for e in recorder.events if e.call_type == call_type)) >= count: + return events + await asyncio.sleep(0.05) + pytest.fail(f"expected {count} {call_type} success events; got {[e.call_type for e in recorder.events]}") + + def _gemini_tts_generate_content_response() -> dict[str, object]: return { "candidates": [ @@ -4490,7 +4514,7 @@ def _gemini_tts_generate_content_response() -> dict[str, object]: @pytest.mark.asyncio -async def test_aspeech_gemini_bridge_keeps_proxy_metadata_for_spend_tracking( +async def test_aspeech_gemini_bridge_bills_one_call_with_proxy_metadata_and_completion_tokens( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -4510,9 +4534,11 @@ async def test_aspeech_gemini_bridge_keeps_proxy_metadata_for_spend_tracking( metadata={"user_api_key": "hashed-virtual-key", "user_api_key_user_id": "user-1"}, ) - assert mock_route.called + assert mock_route.call_count == 1 assert mock_route.calls.last.request.headers["x-goog-api-key"] == "fake-gemini-key" speech_event: Final = await _wait_for_success_event(recorder, call_type="aspeech") + assert recorder.events == [speech_event] + assert speech_event.logged_tokens == (5, 60) assert speech_event.spend_metadata["user_api_key"] == "hashed-virtual-key" assert speech_event.spend_metadata["user_api_key_user_id"] == "user-1" expected_prompt_cost, expected_completion_cost = litellm.cost_per_token( @@ -4525,6 +4551,39 @@ async def test_aspeech_gemini_bridge_keeps_proxy_metadata_for_spend_tracking( assert speech_event.logged_response_cost == pytest.approx(expected_cost) +@pytest.mark.asyncio +async def test_aspeech_gemini_bridge_logs_a_cached_reply_as_a_free_cache_hit( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + recorder: Final = _SuccessEventRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + mock_route: Final = respx_mock.post( + url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-preview-tts:generateContent.*" + ).mock(return_value=httpx.Response(200, json=_gemini_tts_generate_content_response())) + + for _ in range(2): + await litellm.aspeech( + model="gemini/gemini-2.5-flash-preview-tts", + input="cache hit check", + voice="Kore", + api_key="fake-gemini-key", + ) + + first, cached = await _wait_for_success_events(recorder, call_type="aspeech", count=2) + assert mock_route.call_count == 1 + assert len(recorder.events) == 2 + assert first.logged_cache_hit is not True + assert first.logged_response_cost is not None and first.logged_response_cost > 0 + assert cached.logged_cache_hit is True + assert cached.logged_response_cost == 0.0 + assert cached.logged_saved_cache_cost == pytest.approx(first.logged_response_cost) + assert isinstance(cached.logged_cache_key, str) and cached.logged_cache_key + + def _stream_builder_text_chunk(model: str, content: str, finish_reason: str | None = None) -> ModelResponseStream: return ModelResponseStream( id="chatcmpl-cost", From 93b79d2587d186fb7266556a34a22d03f187a3da Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 16:26:41 -0700 Subject: [PATCH 030/124] fix(anthropic): apply tool_config cache-control points on chat and /v1/messages (#47394) * fix(anthropic): apply tool_config cache-control points on chat and /v1/messages A cache_control_injection_points entry with location tool_config was spread into the Anthropic request body on /v1/chat/completions, which Anthropic rejected with 400 Extra inputs are not permitted, and the native /v1/messages path dropped the point silently. Both paths now put the point's control on the last tool Anthropic caches and never forward the key. Co-authored-by: Tan Nguyen * refactor(anthropic): validate tool_config inputs instead of casting them Keeps the lint gates flat: the helper takes untyped tools and points and narrows them with the existing object validators, the chat transform builds the body with the marked tools instead of writing into optional_params, and the handler reads a typed view of kwargs. * fix(openai): keep tool_config cache points out of extra_body, add integration test The cache hook hands a tool_config point on for the provider step, and for providers that call through the OpenAI SDK every non-OpenAI param rides in extra_body, so OpenAI answered 400 Unrecognized request argument supplied. The new integration file drives a live proxy against scripted Anthropic, Bedrock Converse and OpenAI upstreams for /v1/chat/completions and /v1/messages. * test(integration): cover tool_config cache points on bridged /v1/messages Three new cases on /v1/messages for models without a native Anthropic messages path: OpenAI through the Responses bridge and Gemini through the chat bridge never see cache_control_injection_points in the body, and Bedrock Nova through the chat bridge gets the cachePoint on toolConfig --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Tan Nguyen --- litellm/llms/anthropic/chat/transformation.py | 15 + litellm/llms/anthropic/common_utils.py | 47 ++- .../pass_through/messages/handler.py | 10 +- .../anthropic_cache_control_hook.py | 2 +- litellm/utils.py | 5 +- .../test_tool_config_cache_point_wire.py | 312 ++++++++++++++++++ .../test_anthropic_chat_transformation.py | 96 +++++- ...erimental_pass_through_messages_handler.py | 99 ++++-- tests/unit/test_utils.py | 205 ++++++------ 9 files changed, 633 insertions(+), 158 deletions(-) create mode 100644 tests/integration/providers/test_tool_config_cache_point_wire.py diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 38af6e7998b..9624dbb582b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -125,6 +125,7 @@ from ..common_utils import ( refusal_provider_specific_fields, requires_native_compaction_beta, strip_advisor_blocks_from_messages, + tools_marked_for_a_tool_config_point, ) if TYPE_CHECKING: @@ -2408,6 +2409,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) optional_params.pop("client_metadata", None) + request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature + dict[str, object], optional_params + ) + marked_tools: Final = tools_marked_for_a_tool_config_point( + request_params.get("tools"), request_params.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + + request_litellm_params: Final = cast( # cast-ok: litellm_params is the untyped dict of the shared signature + Mapping[str, object], litellm_params + ) + AnthropicCacheControlHook.record_gateway_injection(request_litellm_params, 1) # ``top_k`` is a provider-specific kwarg that bypasses # ``map_openai_params``; gate it here, the single boundary shared by @@ -2427,6 +2441,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "model": model, "messages": anthropic_messages, **optional_params, + **({} if marked_tools is None else {"tools": list(marked_tools)}), } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index dd9b22f3d5d..578329c4c11 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -50,6 +50,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_OAUTH_TOKEN_PREFIX, ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -111,7 +112,7 @@ _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) -_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DECODED_JSON: Final = TypeAdapter(object) @@ -160,6 +161,40 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None ) +def _tool_config_point_control(points: object) -> Mapping[object, object] | None: + point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ()) + tool_config_point: Final = next( + (fields for fields in point_fields if fields is not None and fields.get("location") == "tool_config"), None + ) + if tool_config_point is None: + return None + control: Final = tool_config_point.get("control") + return {"type": "ephemeral"} if control is None else validated_object_mapping(control) + + +def tools_marked_for_a_tool_config_point(tools: object, points: object) -> tuple[object, ...] | None: + """Put the ``tool_config`` point's breakpoint on the last tool Anthropic caches. + + Anthropic caches tools as one prefix ending at the marked tool, so the last one carries + the mark, skipping tool-search tools whose transform drops cache markers. A client that + already marked a tool keeps its own breakpoint. Returns ``None`` when nothing is marked. + """ + control: Final = _tool_config_point_control(points) + tool_list: Final = validated_object_list(tools) or () + tool_fields: Final = tuple(validated_object_mapping(tool) for tool in tool_list) + if control is None or any(fields is not None and "cache_control" in fields for fields in tool_fields): + return None + cacheable: Final = tuple( + (position, fields) + for position, fields in enumerate(tool_fields) + if fields is not None and fields.get("type") not in ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + ) + if not cacheable: + return None + position, fields = cacheable[-1] + return (*tool_list[:position], {**fields, "cache_control": control}, *tool_list[position + 1 :]) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code sends its API calls through the Anthropic SDK as `claude-cli/` and its own fetches, such as gateway model discovery, as `claude-code/`""" @@ -173,9 +208,9 @@ def validated_object_mapping(value: object) -> dict[object, object] | None: return None -def _validated_claude_code_list(value: object) -> list[object] | None: +def validated_object_list(value: object) -> Sequence[object] | None: try: - return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value) + return OBJECT_LIST_ADAPTER.validate_python(value) except ValidationError: return None @@ -202,7 +237,7 @@ def _claude_code_billing_fields(text: str) -> tuple[tuple[str, str], ...] | None def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None: if isinstance(system, str): return (system,) - blocks: Final = _validated_claude_code_list(system) + blocks: Final = validated_object_list(system) if blocks is None: return None block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks) @@ -1963,8 +1998,8 @@ def tool_call_is_rebuilt_as_server_tool_use( return ( find_anthropic_server_tool_result( tool_call_id, - _validated_claude_code_list(fields.get("web_search_results")), - _validated_claude_code_list(fields.get("tool_results")), + validated_object_list(fields.get("web_search_results")), + validated_object_list(fields.get("tool_results")), ) is not None ) diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 002f39c121c..d0f0fa4d901 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -15,10 +15,12 @@ from pydantic import TypeAdapter import litellm from litellm.anthropic_beta_headers_manager import aload_beta_headers_config +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( strip_provider_specific_fields_from_anthropic_messages, + tools_marked_for_a_tool_config_point, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -701,7 +703,13 @@ def anthropic_messages_handler( f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}" ) - local_vars.update(kwargs) + request_kwargs: Final = cast(dict[str, object], kwargs) # cast-ok: **kwargs of the handler is untyped + marked_tools: Final = tools_marked_for_a_tool_config_point( + local_vars.get("tools"), request_kwargs.pop("cache_control_injection_points", None) + ) + if marked_tools is not None: + AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) + local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( params=local_vars, diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 564dc1db527..636bb0b84ee 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -23,7 +23,7 @@ class CacheControlMessageInjectionPoint(TypedDict): class CacheControlToolConfigInjectionPoint(TypedDict): - """Type for tool_config-level injection points (Bedrock).""" + """Type for tool_config-level injection points (Anthropic and Bedrock).""" location: Literal["tool_config"] control: ChatCompletionCachedContent | None diff --git a/litellm/utils.py b/litellm/utils.py index 143c2470d33..2b7abae358b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5329,6 +5329,9 @@ def get_optional_params( EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"}) +LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -5352,7 +5355,7 @@ def add_provider_specific_params_to_optional_params( if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) for k in passed_params: - if k not in openai_params and passed_params[k] is not None: + if k not in openai_params and k not in LITELLM_ONLY_PARAMS and passed_params[k] is not None: extra_body[k] = passed_params[k] if not isinstance(optional_params.get("extra_body"), dict): optional_params["extra_body"] = {} diff --git a/tests/integration/providers/test_tool_config_cache_point_wire.py b/tests/integration/providers/test_tool_config_cache_point_wire.py new file mode 100644 index 00000000000..6373e122975 --- /dev/null +++ b/tests/integration/providers/test_tool_config_cache_point_wire.py @@ -0,0 +1,312 @@ +import json +import uuid +from typing import Final +from urllib.parse import unquote, urlsplit + +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_ANTHROPIC_MODEL: Final = "claude-sonnet-4-6" +_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-6" +_OPENAI_MODEL: Final = "gpt-4.1" +_GEMINI_MODEL: Final = "gemini-3.8-flash" +_NOVA_MODEL: Final = "us.amazon.nova-pro-v1:0" +_API_KEY: Final = "synthetic-provider-key" +_CONTROL: Final = {"type": "ephemeral", "ttl": "1h"} +_TOOL_CONFIG_POINT: Final = [{"location": "tool_config", "control": _CONTROL}] +_AWS: Final = { + "aws_access_key_id": "AKIAINTEGRATIONTEST", + "aws_secret_access_key": "integration-secret", + "aws_region_name": "us-east-1", +} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_TOOL_LIST: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _openai_tools() -> list[JsonValue]: + return [ + { + "type": "function", + "function": { + "name": name, + "description": f"{name} tool", + "parameters": {"type": "object", "properties": {}}, + }, + } + for name in ("get_weather", "get_forecast") + ] + + +def _anthropic_tools() -> list[JsonValue]: + return [ + {"name": name, "description": f"{name} tool", "input_schema": {"type": "object", "properties": {}}} + for name in ("get_weather", "get_forecast") + ] + + +def _anthropic_reply(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": _ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3, "cache_creation_input_tokens": 10}, + } + ).encode() + + +def _converse_reply() -> bytes: + return json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "done"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 12, "outputTokens": 3, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + } + ).encode() + + +def _assert_last_anthropic_tool_marked(body: dict[str, JsonValue]) -> None: + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(body["tools"]) + assert [tool["name"] for tool in tools] == ["get_weather", "get_forecast"], tools + assert "cache_control" not in tools[0], tools + assert tools[1]["cache_control"] == _CONTROL, tools + + +def test_chat_completions_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None: + identity: Final = f"anthropic-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + _assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body)) + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_ANTHROPIC_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_marks_the_last_anthropic_tool(gateway: Gateway) -> None: + identity: Final = f"anthropic-native-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + _assert_last_anthropic_tool_marked(_JSON_OBJECT.validate_json(request.body)) + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_ANTHROPIC_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["usage"]["cache_creation_input_tokens"] == 10, response.text + assert len(wire.drain()) == 1 + + +def test_chat_completions_tool_config_point_appends_a_bedrock_converse_cache_point(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + path: Final = unquote(urlsplit(request.target).path) + assert request.method == "POST" and path == f"/model/{_BEDROCK_MODEL}/converse", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"]) + assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools + assert tools[2] == {"cachePoint": {"type": "default", "ttl": "1h"}}, tools + return Reply(body=_converse_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{_BEDROCK_MODEL}", + api_base=wire.url, + api_key=None, + cache_control_injection_points=_TOOL_CONFIG_POINT, + **_AWS, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_chat_completions_tool_config_point_never_reaches_openai(gateway: Gateway) -> None: + identity: Final = f"openai-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/chat/completions", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(body["tools"]) + assert [tuple(sorted(tool)) for tool in tools] == [("function", "type"), ("function", "type")], tools + return chat_reply(identity, _OPENAI_MODEL, "done", stream=False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{_OPENAI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _openai_tools(), + }, + ) + assert response.status_code == 200, response.text + assert response.json()["id"] == identity, response.text + assert len(wire.drain()) == 1 + + +def _gemini_reply() -> bytes: + return json.dumps( + { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "done"}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 12, "candidatesTokenCount": 3, "totalTokenCount": 15}, + "modelVersion": _GEMINI_MODEL, + } + ).encode() + + +def test_messages_tool_config_point_never_reaches_openai_through_the_responses_bridge(gateway: Gateway) -> None: + identity: Final = f"openai-bridge-tool-config-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + return responses_reply(identity, _OPENAI_MODEL, "done", stream=False) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{_OPENAI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_never_reaches_gemini_through_the_chat_bridge(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + path: Final = urlsplit(request.target).path + assert request.method == "POST" and path == f"/models/{_GEMINI_MODEL}:generateContent", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + return Reply(body=_gemini_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"gemini/{_GEMINI_MODEL}", + api_base=wire.url, + api_key=_API_KEY, + cache_control_injection_points=_TOOL_CONFIG_POINT, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_messages_tool_config_point_appends_a_bedrock_converse_cache_point_through_the_chat_bridge( + gateway: Gateway, +) -> None: + def respond(request: Request) -> Reply: + path: Final = unquote(urlsplit(request.target).path) + assert request.method == "POST" and path == f"/model/{_NOVA_MODEL}/converse", request.target + body: Final = _JSON_OBJECT.validate_json(request.body) + assert "cache_control_injection_points" not in body, body + tools: Final = _TOOL_LIST.validate_python(_JSON_OBJECT.validate_python(body["toolConfig"])["tools"]) + assert [tuple(tool) for tool in tools] == [("toolSpec",), ("toolSpec",), ("cachePoint",)], tools + assert tools[2] == {"cachePoint": {"type": "default"}}, tools + return Reply(body=_converse_reply()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"bedrock/{_NOVA_MODEL}", + api_base=wire.url, + api_key=None, + cache_control_injection_points=_TOOL_CONFIG_POINT, + **_AWS, + ) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": _anthropic_tools(), + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 1308c5aa01f..ef8447712d6 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1375,7 +1375,6 @@ def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(fiel assert ([tool["name"] for tool in optional_params.get("tools", [])] == ["json_tool_call"]) is not native - @pytest.mark.parametrize( ("model", "thinking_params"), [ @@ -1388,7 +1387,13 @@ def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(fiel {"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]}, ), ], - ids=["adaptive-thinking", "reasoning-effort", "reasoning-effort-none", "model-without-forced-tools", "caller-tools"], + ids=[ + "adaptive-thinking", + "reasoning-effort", + "reasoning-effort-none", + "model-without-forced-tools", + "caller-tools", + ], ) def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(model, thinking_params): optional_params: Final = AnthropicConfig().map_openai_params( @@ -1414,6 +1419,7 @@ def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(m assert "output_format" in optional_params assert "json_tool_call" not in [tool["name"] for tool in optional_params.get("tools", [])] + def test_native_structured_output_uses_bundled_capability_when_remote_map_lags( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -3704,7 +3710,9 @@ def _block_ids(message: dict[str, object], block_type: str, id_key: str) -> list return sorted(block[id_key] for block in message["content"] if block["type"] == block_type) -_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn([], {"type": "text", "text": "I have no tool that books flights."}) +_SEARCH_THAT_FOUND_NOTHING: Final = _tool_search_turn( + [], {"type": "text", "text": "I have no tool that books flights."} +) @pytest.mark.parametrize( @@ -9528,7 +9536,9 @@ def test_map_openai_params_runs_tool_search_on_a_custom_named_deployment_whose_m AzureAnthropicConfig().map_openai_params( non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False ) - litellm.register_model({"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}}) + litellm.register_model( + {"azure_ai/my-claude-deployment": {"supports_tool_search": True, "litellm_provider": "azure_ai"}} + ) mapped: Final = AzureAnthropicConfig().map_openai_params( non_default_params={"tools": tools}, optional_params={}, model="my-claude-deployment", drop_params=False ) @@ -9653,3 +9663,81 @@ def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> Non ) provider_specific_fields = result.choices[0].message.provider_specific_fields or {} assert "stop_details" not in provider_specific_fields + + +def _two_function_tools() -> list[dict]: + return [ + { + "type": "function", + "function": {"name": f"tool_{i}", "description": f"Tool {i}", "parameters": {"type": "object"}}, + } + for i in range(2) + ] + + +@pytest.mark.parametrize( + "control, expected_control", + [ + (None, {"type": "ephemeral"}), + ({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}), + ], +) +def test_transform_request_tool_config_point_marks_last_tool_and_drops_the_key(control, expected_control): + """LIT-9880: a tool_config injection point used to be spread into the Anthropic body, which + Anthropic rejected with 400 Extra inputs are not permitted.""" + tools = _two_function_tools() + tools_before = copy.deepcopy(tools) + point = {"location": "tool_config", **({"control": control} if control is not None else {})} + litellm_params = {"metadata": {}, "model_info": {"id": "deployment-1"}} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [point]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][-1]["cache_control"] == expected_control + assert "cache_control" not in data["tools"][0] + assert tools == tools_before + assert litellm_params["metadata"]["litellm_gateway_injected_cache"] == "deployment-1" + + +def test_transform_request_tool_config_point_keeps_the_clients_tool_marker(): + tools = _two_function_tools() + tools[0]["cache_control"] = {"type": "ephemeral"} + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert "cache_control_injection_points" not in data + assert data["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][1] + + +def test_transform_request_tool_config_point_skips_tool_search_tools(): + tool_search_type = next(iter(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)) + tools = [*_two_function_tools(), {"type": tool_search_type, "name": "tool_search"}] + + data = _make_transform_request( + optional_params={"tools": tools, "cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params={}, + ) + + assert data["tools"][1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in data["tools"][2] + assert "cache_control" not in data["tools"][0] + + +def test_transform_request_tool_config_point_without_tools_drops_the_key(): + litellm_params = {"metadata": {}} + + data = _make_transform_request( + optional_params={"cache_control_injection_points": [{"location": "tool_config"}]}, + litellm_params=litellm_params, + ) + + assert "cache_control_injection_points" not in data + assert "tools" not in data + assert litellm_params["metadata"] == {} diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index ffe9e46bf56..1f02b3aaa5a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1103,7 +1103,9 @@ async def test_messages_honors_a_chat_pin_on_a_prefixed_responses_deployment( litellm.in_memory_llm_clients_cache.flush_cache() with respx.mock(assert_all_called=True) as respx_mock: - upstream: Final = respx_mock.route(host="api.perplexity.ai").mock(side_effect=httpx.ConnectError("cut at the wire")) + upstream: Final = respx_mock.route(host="api.perplexity.ai").mock( + side_effect=httpx.ConnectError("cut at the wire") + ) with pytest.raises(litellm.exceptions.InternalServerError): await litellm.anthropic.messages.acreate( max_tokens=100, @@ -1524,6 +1526,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"] + @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" @@ -1638,7 +1641,9 @@ def _claude_code_auto_mode_request() -> tuple[list[dict[str, object]], list[dict return safeguards, safeguard_results -def _upstream_answering_with(safeguard_results: list[dict[str, object]], captured: dict[str, object]) -> AsyncHTTPHandler: +def _upstream_answering_with( + safeguard_results: list[dict[str, object]], captured: dict[str, object] +) -> AsyncHTTPHandler: def upstream_records_the_request(request: httpx.Request) -> httpx.Response: captured["body"] = json.loads(request.content) captured["anthropic-beta"] = request.headers.get("anthropic-beta") @@ -1664,7 +1669,9 @@ def _upstream_answering_with(safeguard_results: list[dict[str, object]], capture _CLIENT_BETA_HEADERS: Final = ( - pytest.param({"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta"), + pytest.param( + {"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"}, id="client_sends_beta" + ), pytest.param({"anthropic-beta": "interleaved-thinking-2025-05-14"}, id="client_omits_beta"), pytest.param({}, id="client_sends_no_beta_header"), ) @@ -1789,7 +1796,11 @@ def _tool_use_sse(model: str) -> bytes: "delta": {"type": "input_json_delta", "partial_json": json.dumps(_CURL_TOOL_USE["input"])}, }, {"type": "content_block_stop", "index": 0}, - {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 4}}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 4}, + }, {"type": "message_stop"}, ) return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() @@ -1957,26 +1968,20 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking(): print("Call kwargs:", json.dumps(call_kwargs, indent=2, default=str)) # Verify that litellm_metadata was passed and contains _latency_per_deployment - assert ( - "litellm_metadata" in call_kwargs - ), "litellm_metadata should be passed to anthropic_messages" + assert "litellm_metadata" in call_kwargs, "litellm_metadata should be passed to anthropic_messages" litellm_metadata = call_kwargs["litellm_metadata"] assert litellm_metadata is not None, "litellm_metadata should not be None" - assert isinstance( - litellm_metadata, dict - ), "litellm_metadata should be a dictionary" + assert isinstance(litellm_metadata, dict), "litellm_metadata should be a dictionary" # Verify _latency_per_deployment is present - assert ( - "_latency_per_deployment" in litellm_metadata - ), "litellm_metadata should contain _latency_per_deployment field" + assert "_latency_per_deployment" in litellm_metadata, ( + "litellm_metadata should contain _latency_per_deployment field" + ) # Verify the structure of _latency_per_deployment latency_per_deployment = litellm_metadata["_latency_per_deployment"] - assert isinstance( - latency_per_deployment, dict - ), "_latency_per_deployment should be a dictionary" + assert isinstance(latency_per_deployment, dict), "_latency_per_deployment should be a dictionary" print(f"✅ Latency per deployment data: {latency_per_deployment}") @@ -2134,9 +2139,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): when using anthropic.messages.acreate with a bedrock model """ # Mock the get_credentials method - with unittest.mock.patch.object( - BaseAWSLLM, "get_credentials" - ) as mock_get_credentials: + with unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials: # Create a proper mock for credentials with the necessary attributes mock_credentials = unittest.mock.MagicMock() mock_credentials.access_key = "mock_access_key" @@ -2147,9 +2150,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): # We also need to mock the actual AWS request signing to avoid real API calls with unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"): # Set up mock for AsyncHTTPHandler.post to avoid actual API calls - with unittest.mock.patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post" - ) as mock_post: + with unittest.mock.patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post: # Configure mock response mock_response = unittest.mock.MagicMock() mock_response.raise_for_status = unittest.mock.MagicMock() @@ -2191,9 +2192,7 @@ async def test_anthropic_messages_bedrock_credentials_passthrough(): # Assert that our test credentials were passed correctly for param_name, param_value in aws_params.items(): - assert ( - call_args[param_name] == param_value - ), f"Parameter {param_name} was not passed correctly" + assert call_args[param_name] == param_value, f"Parameter {param_name} was not passed correctly" @pytest.mark.asyncio @@ -2221,11 +2220,8 @@ async def test_anthropic_messages_bedrock_dynamic_region(): # Patch necessary AWS components with ( unittest.mock.patch("botocore.auth.SigV4Auth.add_auth"), - unittest.mock.patch.object( - BaseAWSLLM, "get_credentials" - ) as mock_get_credentials, + unittest.mock.patch.object(BaseAWSLLM, "get_credentials") as mock_get_credentials, ): - # Setup mock credentials mock_credentials = unittest.mock.MagicMock() mock_credentials.access_key = "test_access_key" @@ -2254,9 +2250,9 @@ async def test_anthropic_messages_bedrock_dynamic_region(): # Check that the URL contains the correct region url = call_args.kwargs.get("url", "") - assert ( - f"bedrock-runtime.{test_region}.amazonaws.com" in url - ), f"URL does not contain the correct region. URL: {url}" + assert f"bedrock-runtime.{test_region}.amazonaws.com" in url, ( + f"URL does not contain the correct region. URL: {url}" + ) # Verify get_credentials was called with the correct region mock_get_credentials.assert_called_once() @@ -2294,3 +2290,42 @@ async def test_anthropic_messages_with_safeguards_and_no_classifier_keep_the_ups assert rejected.value.status_code == 400 assert "invalid_request_error" in str(rejected.value) + + +def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): + """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the + native path dropped it silently, so the tools went out without any cache breakpoint.""" + from litellm.llms.anthropic.pass_through.messages import handler + + captured = {} + + def fake_native(**kwargs): + captured["params"] = kwargs["anthropic_messages_optional_request_params"] + captured["kwargs"] = kwargs["kwargs"] + captured["messages"] = kwargs["messages"] + return "native-passthrough" + + monkeypatch.setattr(handler.base_llm_http_handler, "anthropic_messages_handler", fake_native) + tools = [{"name": f"tool_{i}", "description": f"Tool {i}", "input_schema": {"type": "object"}} for i in range(2)] + tools_before = copy.deepcopy(tools) + + result = handler.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="anthropic/claude-opus-4-6", + api_key="sk-test", + tools=tools, + litellm_metadata={}, + cache_control_injection_points=[ + {"location": "message", "role": "user"}, + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + ) + + assert result == "native-passthrough" + assert "cache_control_injection_points" not in captured["params"] + assert "cache_control_injection_points" not in captured["kwargs"] + assert captured["params"]["tools"][-1]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert "cache_control" not in captured["params"]["tools"][0] + assert captured["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert tools == tools_before diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 69ef2caf8f4..6ffb589fd41 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1465,8 +1465,6 @@ def test_get_model_info_bedrock_inference_profile_arn_keeps_regional_row(local_m assert bare["input_cost_per_token"] == base["input_cost_per_token"] - - @pytest.mark.parametrize( "bare_key", [ @@ -2302,9 +2300,7 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None: tree: Final = ast.parse(Path(__file__).read_text()) defined: Final = frozenset( - node.name - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) ) invoked: Final = tuple( node.value.func.id @@ -3619,9 +3615,7 @@ class TestExtraBodyCannotOverrideModel: } ] - untouched = litellm.get_optional_params( - model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools() - ) + untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()) assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched result = litellm.get_optional_params( @@ -4661,7 +4655,9 @@ def test_is_prompt_caching_valid_prompt_stops_counting_once_the_minimum_is_reach assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=1024) is True assert sum(counted_messages) < len(long_prompt), sum(counted_messages) - full_count = litellm.token_counter(model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True) + full_count = litellm.token_counter( + model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True + ) assert ( is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=full_count) is True @@ -4977,7 +4973,9 @@ class _EntryReadDuringARegistration(dict[str, object]): @pytest.mark.parametrize("walk", ["fal_ai", "fireworks_ai", "ollama", "gpt-5", "case-insensitive scan"]) -def test_request_path_cost_map_walks_survive_a_registration_mid_walk(monkeypatch: pytest.MonkeyPatch, walk: str) -> None: +def test_request_path_cost_map_walks_survive_a_registration_mid_walk( + monkeypatch: pytest.MonkeyPatch, walk: str +) -> None: from litellm.llms.fal_ai.cost_calculator import _keyed_rows from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.ollama.common_utils import OllamaModelInfo @@ -5226,7 +5224,13 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = ( pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"), pytest.param( ResponsesAPIResponse( - id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6" + id="resp_abc", + created_at=1, + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + model="gpt-5.6", ), CallTypes.aresponses, id="responses", @@ -5254,7 +5258,9 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo assert result is response assert second_hook.seen_responses == (response,) - failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message) + failure_logs: Final = tuple( + r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message + ) assert len(failure_logs) == 1 assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message assert str(call_type) in failure_logs[0].message @@ -5391,17 +5397,13 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi ({"metadata": None}, True, False), ], ) -def test_is_litellm_router_call_is_async_aware( - kwargs: Mapping[str, object], is_async: bool, expected: bool -) -> None: +def test_is_litellm_router_call_is_async_aware(kwargs: Mapping[str, object], is_async: bool, expected: bool) -> None: assert _is_litellm_router_call(kwargs, is_async=is_async) is expected @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"]) -async def test_router_aresponses_does_not_run_sdk_retries( - monkeypatch: pytest.MonkeyPatch, stream: bool -) -> None: +async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() model_list: Final = [ @@ -5415,9 +5417,7 @@ async def test_router_aresponses_does_not_run_sdk_retries( }, } ] - router: Final = litellm.Router( - model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True - ) + router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True) try: with respx.mock(assert_all_called=True) as respx_mock: @@ -5453,9 +5453,7 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call( }, } ] - router: Final = litellm.Router( - model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True - ) + router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True) try: with respx.mock(assert_all_called=True) as respx_mock: @@ -7208,7 +7206,9 @@ def test_get_model_info_gemini(monkeypatch): assert info.get("rpm") is not None, f"{model} does not have rpm" -def test_register_model_without_a_provider_never_inherits_another_providers_entry(monkeypatch: pytest.MonkeyPatch) -> None: +def test_register_model_without_a_provider_never_inherits_another_providers_entry( + monkeypatch: pytest.MonkeyPatch, +) -> None: from litellm.utils import _invalidate_model_cost_lowercase_map key: Final = "azure/claude-sonnet-4-6" @@ -7359,7 +7359,9 @@ def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summa @pytest.mark.parametrize("original_function", ("atext_completion", "text_completion")) -def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(original_function: str) -> None: +def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation( + original_function: str, +) -> None: assert _logged_request_messages(original_function, model="gpt-4o") is None @@ -7401,6 +7403,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -7414,6 +7417,7 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + MODEL: Final = "anthropic/claude-haiku-4-5" @@ -7423,6 +7427,7 @@ def test_validate_tool_choice_none(): result = validate_chat_completion_tool_choice(None, model=MODEL) assert result is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_string(): """Test that string values are returned as-is.""" @@ -7430,6 +7435,7 @@ def test_validate_tool_choice_string(): assert validate_chat_completion_tool_choice("none", model=MODEL) == "none" assert validate_chat_completion_tool_choice("required", model=MODEL) == "required" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_standard_dict(): """Test standard OpenAI format with function.""" @@ -7437,6 +7443,7 @@ def test_validate_tool_choice_standard_dict(): result = validate_chat_completion_tool_choice(tool_choice, model=MODEL) assert result == tool_choice + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_cursor_format(): """Cursor IDE format {"type": "auto"} is unwrapped to the bare string.""" @@ -7444,6 +7451,7 @@ def test_validate_tool_choice_cursor_format(): assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none" assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "tool_choice", @@ -7465,6 +7473,7 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.status_code == 400 assert exc_info.value.model == MODEL + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("tool_choice", [123, []]) def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): @@ -7475,6 +7484,7 @@ def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): validate_chat_completion_tool_choice(tool_choice, model=MODEL) assert exc_info.value.status_code == 400 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_without_model_is_still_a_400(): """Callers that predate the model argument keep getting a 400, with an empty model on the error.""" @@ -7483,12 +7493,14 @@ def test_validate_tool_choice_without_model_is_still_a_400(): assert exc_info.value.status_code == 400 assert exc_info.value.model == "" + @pytest.fixture() def _vcr_outcome_gate_local_testing(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -7541,6 +7553,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -7584,6 +7597,7 @@ def setup_and_teardown_local_testing(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7609,6 +7623,7 @@ def test_vertex_projects(): assert "vertex_ai_project" in optional_params assert "vertex_ai_location" in optional_params + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7624,6 +7639,7 @@ def test_bedrock_embed_v2_regular(): print(f"received optional_params: {optional_params}") assert optional_params == {"dimensions": 512} + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7642,6 +7658,7 @@ def test_bedrock_embed_v2_with_drop_params(): print(f"received optional_params: {optional_params}") assert optional_params == {"dimensions": 512, "embeddingTypes": ["binary"]} + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7663,6 +7680,7 @@ def test_openai_non_text_embedding_3_with_allowed_openai_params(): print(f"received optional_params: {optional_params}") assert optional_params.get("dimensions") == 1024 + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7689,6 +7707,7 @@ def test_openai_non_text_embedding_3_without_allowed_openai_params_raises(): finally: litellm.drop_params = prev_drop_params + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7717,6 +7736,7 @@ def test_openai_non_text_embedding_3_drop_params_per_call(): finally: litellm.drop_params = prev_drop_params + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -7744,12 +7764,14 @@ def test_openai_non_text_embedding_3_drop_params_global(): finally: litellm.drop_params = prev_drop_params + @pytest.fixture() def _vcr_outcome_gate_search_tests(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.mark.usefixtures("_vcr_outcome_gate_search_tests") def test_search_tool_name_in_all_litellm_params(): """ @@ -7759,6 +7781,7 @@ def test_search_tool_name_in_all_litellm_params(): """ assert "search_tool_name" in all_litellm_params + @pytest.mark.usefixtures("_vcr_outcome_gate_search_tests") def test_filter_out_search_tool_name(): """ @@ -7785,6 +7808,7 @@ def test_filter_out_search_tool_name(): assert filtered["query"] == "latest ai developments" assert filtered["max_results"] == 5 + @pytest.mark.asyncio async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None: """Chat over the Responses bridge exits two @client wrappers with one logging object. Issue @@ -7937,9 +7961,7 @@ def test_trimming_with_system_message_within_max_tokens(): "content": "This is a medium normal message, let's say litellm is awesome.", }, ] - trimmed_messages = trim_messages( - messages, max_tokens=30, model="gpt-4-0613" - ) + trimmed_messages = trim_messages(messages, max_tokens=30, model="gpt-4-0613") assert len(trimmed_messages) == 2 assert trimmed_messages[0]["content"] == "This is a short system message" @@ -8122,9 +8144,7 @@ def test_aget_valid_models(): print(valid_models) # list of openai supported llms on litellm - expected_models = ( - litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models - ) + expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models assert set(valid_models) == set(expected_models) @@ -8213,9 +8233,7 @@ def test_get_chat_completion_prompt(): prompt_variables=None, ) - assert litellm_logging_obj.messages == [ - {"role": "user", "content": updated_message} - ] + assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}] def test_redact_msgs_from_logs(): @@ -8287,9 +8305,7 @@ def test_redact_embedding_response(): litellm.turn_off_message_logging = True # Create a test EmbeddingResponse with usage data - original_usage = litellm.Usage( - prompt_tokens=10, completion_tokens=0, total_tokens=10 - ) + original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) original_data = [ {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]}, {"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]}, @@ -8325,9 +8341,7 @@ def test_redact_embedding_response(): # Assert the redacted response preserves critical metadata assert _redacted_response_obj.usage == original_usage # usage should be preserved - assert ( - _redacted_response_obj.model == "text-embedding-3-small" - ) # model should be preserved + assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved assert _redacted_response_obj.object == "list" # object should be preserved # Assert sensitive data is cleared @@ -8381,12 +8395,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): ) # Test Case 1: standard_callback_dynamic_params = False (or not set) - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=False - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8395,12 +8405,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): assert _redacted_response_obj.choices[0].message.content == test_content # Test Case 2: standard_callback_dynamic_params = True - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=True - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8411,9 +8417,7 @@ def test_redact_msgs_from_logs_with_dynamic_params(): # Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging # since litellm.turn_off_message_logging is True redaction should occur standard_callback_dynamic_params = StandardCallbackDynamicParams() - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -8755,10 +8759,7 @@ def test_models_by_provider(): continue elif k == "sample_spec": continue - elif ( - v["litellm_provider"] == "sagemaker" - or v["litellm_provider"] == "bedrock_converse" - ): + elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse": continue elif v.get("mode") in ("search", "decisions", "evaluation"): continue @@ -8766,9 +8767,7 @@ def test_models_by_provider(): providers.add(v["litellm_provider"]) for provider in providers: - assert provider in models_by_provider.keys() or JSONProviderRegistry.exists( - provider - ) + assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider) @pytest.fixture @@ -8790,16 +8789,11 @@ def restore_end_user_cost_tracking_flags(monkeypatch: pytest.MonkeyPatch) -> Non ({"user_api_key_end_user_id": "123"}, True, None), ], ) -def test_get_end_user_id_for_cost_tracking( - litellm_params, disable_end_user_cost_tracking, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id): from litellm.utils import get_end_user_id_for_cost_tracking litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking - assert ( - get_end_user_id_for_cost_tracking(litellm_params=litellm_params) - == expected_end_user_id - ) + assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id @pytest.mark.usefixtures("restore_end_user_cost_tracking_flags") @@ -8816,13 +8810,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ): from litellm.utils import get_end_user_id_for_cost_tracking - litellm.enable_end_user_cost_tracking_prometheus_only = ( - enable_end_user_cost_tracking_prometheus_only - ) + litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only assert ( - get_end_user_id_for_cost_tracking( - litellm_params=litellm_params, service_type="prometheus" - ) + get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus") == expected_end_user_id ) @@ -8837,20 +8827,14 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ), # Test with only litellm_metadata field (new behavior) ( - { - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - } - }, + {"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}}, "user_from_litellm_metadata", ), # Test with both fields - metadata should take precedence for user_api_key fields ( { "metadata": {"user_api_key_end_user_id": "user_from_metadata"}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_metadata", ), @@ -8866,9 +8850,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ( { "metadata": {}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_litellm_metadata", ), @@ -8876,9 +8858,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ({}, None), ], ) -def test_get_end_user_id_for_cost_tracking_metadata_handling( - litellm_params, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id): """ Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata fields using the get_litellm_metadata_from_kwargs helper function. @@ -9136,9 +9116,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch): assert len(litellm.success_callback) == curr_len_success_callback assert len(litellm.failure_callback) == curr_len_failure_callback - assert any( - isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback - ) + assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback) @pytest.mark.asyncio @@ -9235,20 +9213,14 @@ def test_validate_user_messages_invalid_content_type(): [ { "name": "default_on_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=True) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)], "kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}}, "expected": ["test_guardrail"], }, { "name": "request_specific_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], - "kwargs": { - "metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}} - }, + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}}, "expected": ["test_guardrail"], }, { @@ -9257,18 +9229,12 @@ def test_validate_user_messages_invalid_content_type(): CustomGuardrail(guardrail_name="default_guardrail", default_on=True), CustomGuardrail(guardrail_name="request_guardrail", default_on=False), ], - "kwargs": { - "metadata": { - "requester_metadata": {"guardrails": ["request_guardrail"]} - } - }, + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}}, "expected": ["default_guardrail", "request_guardrail"], }, { "name": "empty_metadata", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], "kwargs": {}, "expected": [], }, @@ -10074,9 +10040,7 @@ def test_pre_call_rules_see_messages_passed_positionally(monkeypatch: pytest.Mon ) with pytest.raises(litellm.APIResponseValidationError, match="failed post-call-rule check"): - litellm.completion( - "gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test" - ) + litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test") allowed: Final = litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say hi"}], api_key="sk-test") assert route.call_count == 1 @@ -10094,3 +10058,18 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): prefixed_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") assert litellm.get_model_info(model=short_name, custom_llm_provider="vertex_ai") == prefixed_info, model assert prefixed_info["key"] == f"vertex_ai/{model}", model + + +def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body(): + """LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no + step that consumes it, so it must not ride along in extra_body and 400 at OpenAI.""" + optional_params = litellm.utils.get_optional_params( + model="gpt-4.1", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[{"location": "tool_config", "control": {"type": "ephemeral"}}], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True} + assert "cache_control_injection_points" not in optional_params From 7b76b2b15fe6826e091e289a574b23c1e96fd627 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 16:55:13 -0700 Subject: [PATCH 031/124] fix(proxy): price the estimated input tokens on a read-timed-out request's spend row (#47435) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 60 +++++++++- ..._utils_model_creation_and_error_logging.py | 107 ++++++++++++++++-- 2 files changed, 154 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 29749afb429..1bd856803eb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -50,6 +50,7 @@ from typing import ( ) import anyio +import httpx from typing_extensions import Never, ReadOnly, TypedDict from litellm import _custom_logger_compatible_callbacks_literal @@ -1013,12 +1014,17 @@ def _failure_usage_to_lift( model_call_details: Mapping[str, object], request_body: Mapping[str, object], dispatched: bool, + provider_billed: bool = False, ) -> tuple[object, object] | None: """A stream that broke mid-flight still billed the provider for the chunks already delivered; the streaming handler stashes that recovered usage and cost in model_call_details, so prefer it. Otherwise a request that was dispatched to a provider and failed without upstream usage gets an - estimated input-side Usage with zero cost. The raw request body backfills + estimated input-side Usage. When the failure is one the provider bills + anyway (``provider_billed``: a read timeout, where the provider had the + prompt and kept working after LiteLLM hung up) that + usage is priced at the model's input rate; any other post-dispatch failure + stays at zero cost. The raw request body backfills the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a chat-completions provider) without filling optional_params. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" @@ -1046,7 +1052,27 @@ def _failure_usage_to_lift( ) if estimated_usage is None: return None - return estimated_usage, 0.0 + if not provider_billed: + return estimated_usage, 0.0 + return estimated_usage, _estimated_input_cost(model_call_details, estimated_usage.prompt_tokens) + + +def _estimated_input_cost(model_call_details: Mapping[str, object], prompt_tokens: int) -> float: + """Input price for the estimated prompt at the cost map's rate; an unmapped model keeps the + row at $0 rather than dropping the failure row.""" + try: + model: Final = str(model_call_details.get("model") or "") + estimated_input_cost, _ = litellm.cost_per_token( + model=model, + custom_llm_provider=str(model_call_details.get("custom_llm_provider") or "") or None, + prompt_tokens=prompt_tokens, + ) + except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written + verbose_proxy_logger.debug( + "failure spend estimate: could not price %s: %s", model_call_details.get("model"), exc + ) + return 0.0 + return estimated_input_cost _EMPTY_LIFT: Final = MappingProxyType({}) @@ -1161,7 +1187,28 @@ _PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRai _LOG_DB_METRICS_CALL_TYPES: Final = frozenset(("get_data", "insert_data", "update_data", "delete_data")) -def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]: +_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16 + + +def _exception_chain(exception: BaseException) -> tuple[BaseException, ...]: + chain: Final[list[BaseException]] = [exception] # mutable-ok: built once, returned frozen + for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH): + following = chain[-1].__cause__ or chain[-1].__context__ + if following is None or following in chain: + break + chain.append(following) + return tuple(chain) + + +def _timed_out_reading_from_provider(exception: Exception) -> bool: + """A read timeout means the request was sent and the provider was working on it; a connect + timeout (``httpx.ConnectTimeout``) never reached the provider, so it is not billed.""" + return isinstance(exception, litellm.Timeout) and any( + isinstance(cause, httpx.ReadTimeout) for cause in _exception_chain(exception) + ) + + +def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]: """Failure-path callbacks run after ``litellm_logging_obj`` is popped from request_data (it is not serialisable), so the caller merges these fields onto request_data first: the request start and first-handoff instants for @@ -1177,6 +1224,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, model_call_details=_model_call_details, request_body=request_data, dispatched=_first_handoff is not None, + provider_billed=provider_billed, ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), @@ -3400,7 +3448,11 @@ class ProxyLogging: original_exception=original_exception, ) - request_data.update(await offload_token_count(_failure_fields_to_lift)(request_data)) + request_data.update( + await offload_token_count(_failure_fields_to_lift)( + request_data, provider_billed=_timed_out_reading_from_provider(original_exception) + ) + ) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index ec7104fa459..336d2f3e7c2 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2,6 +2,7 @@ import datetime as real_datetime import smtplib from typing import Final +import httpx import pytest from fastapi import HTTPException @@ -687,11 +688,13 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: """A non-stream request that failed after dispatch (timeout, provider error) consumed provider-billed input tokens but recovered no usage. post_call_failure_hook must estimate the input side onto request_data so - the spend log's failure row records what was sent instead of zero, while - never charging spend for the failure (LIT-5690). + the spend log's failure row records what was sent instead of zero + (LIT-5690). On a timeout that estimate is priced at the model's input + rate since the provider bills the prompt it accepted (LIT-145); other + post-dispatch failures keep zero spend. """ - async def _run(self, request_data): + async def _run(self, request_data, original_exception: Exception | None = None): from unittest.mock import AsyncMock, patch from litellm.proxy._types import UserAPIKeyAuth @@ -701,17 +704,81 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data=request_data, - original_exception=Exception("boom"), + original_exception=original_exception or Exception("boom"), user_api_key_dict=UserAPIKeyAuth(), ) + def _timeout(self, model: str, cause: Exception) -> Exception: + try: + try: + raise cause + except Exception as inner: + raise litellm.Timeout("Request timed out.", model=model, llm_provider="openai") from inner + except litellm.Timeout as timed_out: + return timed_out + def _logging_obj(self, model_call_details): logging_obj = MagicMock() logging_obj.model_call_details = model_call_details return logging_obj @pytest.mark.asyncio - async def test_dispatched_failure_estimates_input_tokens_with_zero_cost(self): + async def test_timed_out_request_estimates_input_tokens_priced_at_input_rate(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + "response_cost": 123.0, + } + await self._run( + request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ReadTimeout("no bytes in time")) + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert estimated.completion_tokens == 0 + assert estimated.total_tokens == estimated.prompt_tokens + expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens) + assert expected_input_cost > 0 + assert request_data["response_cost"] == pytest.approx(expected_input_cost) + + @pytest.mark.asyncio + async def test_connect_timeout_never_reached_the_provider_stays_zero_cost(self): + from datetime import datetime + + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + } + await self._run( + request_data, original_exception=self._timeout("gpt-3.5-turbo", httpx.ConnectTimeout("no route")) + ) + + assert isinstance(request_data["combined_usage_object"], Usage) + assert request_data["response_cost"] == 0.0 + + @pytest.mark.asyncio + async def test_dispatched_non_timeout_failure_estimates_input_tokens_with_zero_cost(self): from datetime import datetime from litellm.types.utils import Usage @@ -733,8 +800,6 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: estimated = request_data["combined_usage_object"] assert isinstance(estimated, Usage) assert estimated.prompt_tokens > 0 - assert estimated.completion_tokens == 0 - assert estimated.total_tokens == estimated.prompt_tokens assert request_data["response_cost"] == 0.0 @pytest.mark.asyncio @@ -1901,7 +1966,28 @@ def test_usage_the_stream_already_recovered_beats_an_estimate(): assert lifted == (recovered, 0.25) -def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost(): +def test_a_timed_out_request_on_an_unmapped_model_keeps_its_row_at_zero_cost(): + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": "my-unmapped-model", + "custom_llm_provider": "openai", + "messages": ONE_USER_MESSAGE, + }, + request_body={}, + dispatched=True, + provider_billed=True, + ) + + assert lifted is not None + usage, response_cost = lifted + assert usage.prompt_tokens > 0 + assert response_cost == 0.0 + + +def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rate(): from litellm.proxy.utils import _failure_usage_to_lift lifted = _failure_usage_to_lift( @@ -1912,13 +1998,16 @@ def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost(): }, request_body={}, dispatched=True, + provider_billed=True, ) assert lifted is not None usage, response_cost = lifted assert usage.prompt_tokens > 0 assert usage.completion_tokens == 0 - assert response_cost == 0.0 + expected_input_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens) + assert expected_input_cost > 0 + assert response_cost == pytest.approx(expected_input_cost) @pytest.mark.parametrize( From 497f31d43abb4f975f998915cbd0b5fc051f3f8b Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 19 Oct 2026 17:07:00 -0700 Subject: [PATCH 032/124] perf(savings): read deployment pricing from the router's cached model info (#47400) Savings and the savings baseline priced deployments through Router.get_deployment_model_info, which deep-copies the cost map entries on every call. The router already keeps an lru_cache of the same lookup that is cleared whenever the model list or cost map changes, and budget reservation reads from it, so the savings path now reads from it too --- litellm/proxy/spend_tracking/savings.py | 2 +- litellm/router_strategy/savings_baseline.py | 2 +- .../unit/proxy/spend_tracking/test_savings.py | 26 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index bca91d710e1..2aab8deb034 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -90,7 +90,7 @@ def _effective_model_info(router: "Router | None", deployment_id: str | None, mo if router is None or deployment_id is None: return None try: - return router.get_deployment_model_info(deployment_id, model) + return router.cached_deployment_model_info(deployment_id, model) except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) return None diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index a2e983a8369..b5b80ddae6d 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -115,7 +115,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N provider, _, model_name = candidate.model.partition("/") try: - info: Final = router.get_deployment_model_info(candidate.deployment_id or "", candidate.model) + info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model) if info is None: return None prompt_cost, completion_cost = generic_cost_per_token( diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index de413a86521..f9566fd1ba0 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -14,6 +14,7 @@ from litellm.proxy.spend_tracking.savings import ( prompt_caching_savings_for_request, ) from litellm.router import Router +from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import Usage pytestmark = pytest.mark.usefixtures("local_model_cost_map") @@ -1070,6 +1071,31 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): assert with_deployment_rate.autorouter > at_public_rate.autorouter +def test_savings_follow_a_deployments_updated_price() -> None: + def deployment(input_cost_per_token: float) -> Deployment: + return Deployment( + model_name="negotiated", + litellm_params=LiteLLM_Params( + model="anthropic/claude-sonnet-5", input_cost_per_token=input_cost_per_token + ), + model_info={"id": "repriced"}, + ) + + router: Final = Router(model_list=[deployment(1e-6).model_dump(exclude_none=True)]) + + def compression_savings() -> float: + return compute_savings_spend( + "claude-sonnet-5", "anthropic", 1000, False, model_id="repriced", llm_router=lambda: router + ).compression + + assert compression_savings() == pytest.approx(1000 * 1e-6) + assert compression_savings() == pytest.approx(1000 * 1e-6) + + router.upsert_deployment(deployment(3e-6)) + + assert compression_savings() == pytest.approx(1000 * 3e-6) + + def _routed_decision() -> dict: return {"savings_baseline_model": "anthropic/claude-opus-5", "conversation_continuing": True} From c0cdab071bc815e0ee25d4a4fd7e2143f5663dfd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:35:39 +0000 Subject: [PATCH 033/124] chore(hooks): drop Conventional Branches pre-push hook in favor of litellm_ prefix (#44951) * chore(hooks): drop Conventional Branches pre-push hook in favor of litellm_ prefix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(makefile): describe install-hooks as Conventional Commits only * docs(contributing): scope the litellm_ branch prefix to internal contributors --------- Co-authored-by: nate Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .githooks/pre-push | 90 -------------------------- CONTRIBUTING.md | 10 +-- Makefile | 4 +- scripts/install_git_hooks.sh | 2 +- tests/unit/test_git_hooks.py | 119 ++--------------------------------- 5 files changed, 14 insertions(+), 211 deletions(-) delete mode 100755 .githooks/pre-push diff --git a/.githooks/pre-push b/.githooks/pre-push deleted file mode 100755 index dc1a73a7ba2..00000000000 --- a/.githooks/pre-push +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# -# pre-push — enforce Conventional Branches -# https://conventional-branch.github.io/ -# -# Branch format: / -# must be one of: feature, bugfix, hotfix, release, chore -# -# Protected branches (always allowed): -# - main -# - dependabot/* -# - gh-readonly-queue/* -# -# Tag pushes and branch deletions are skipped. -# Bypass: git push --no-verify. - -set -eu - -ZERO_OID="0000000000000000000000000000000000000000" -ZERO_OID_SHA256="0000000000000000000000000000000000000000000000000000000000000000" -ALLOWED_TYPES="feature|bugfix|hotfix|release|chore" -BRANCH_PATTERN="^(${ALLOWED_TYPES})/.+" - -PROTECTED_NAMES="main" -PROTECTED_PREFIXES="dependabot/ gh-readonly-queue/" - -is_protected() { - branch="$1" - for name in $PROTECTED_NAMES; do - if [ "$branch" = "$name" ]; then - return 0 - fi - done - for prefix in $PROTECTED_PREFIXES; do - case "$branch" in "$prefix"*) return 0 ;; esac - done - return 1 -} - -invalid="" - -while read -r local_ref local_oid remote_ref remote_oid; do - # Branch deletion (no local commit being pushed). - if [ "$local_oid" = "$ZERO_OID" ] || [ "$local_oid" = "$ZERO_OID_SHA256" ]; then - continue - fi - - # Only validate branch pushes; ignore tags and other ref namespaces. - case "$remote_ref" in - refs/heads/*) ;; - *) continue ;; - esac - - branch="${remote_ref#refs/heads/}" - - if is_protected "$branch"; then - continue - fi - - if ! printf '%s' "$branch" | grep -Eq "$BRANCH_PATTERN"; then - invalid="$invalid $branch" - fi -done - -if [ -n "$invalid" ]; then - cat >&2 </ - - Allowed types: feature, bugfix, hotfix, release, chore - Examples: - feature/weighted-round-robin - bugfix/streaming-empty-chunks - chore/bump-deps - hotfix/auth-bypass - - Protected (always allowed): main, dependabot/*, gh-readonly-queue/*. - -See https://conventional-branch.github.io/ - -Rename with: git branch -m -To bypass (use sparingly): git push --no-verify -EOF - exit 1 -fi - -exit 0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0af12bd5318..89a1b544b82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,12 +39,12 @@ git clone https://github.com/YOUR_USERNAME/litellm.git cd litellm # Create a new branch for your feature (see "Commit and Branch Conventions" below) -git checkout -b feature/your-feature +git checkout -b litellm_your_feature # Install development dependencies make install-dev -# Install git hooks that enforce commit + branch conventions (one-time, opt-in) +# Install git hooks that enforce commit conventions (one-time, opt-in) make install-hooks # Verify your setup works @@ -55,7 +55,7 @@ That's it! Your local development environment is ready. ## Commit and Branch Conventions -Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and branches follow [Conventional Branches](https://conventional-branch.github.io/). Run `make install-hooks` once per clone to enable the local git hooks that enforce these — see the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for the full type list, examples, the protected-branch bypass list, and how to opt out. +Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Internal contributors name branches with the `litellm_` prefix and no `/`, and fork branches can use any name. Run `make install-hooks` once per clone to enable the local hook that enforces the commit convention. See the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for details ### 2. Development Workflow @@ -78,8 +78,8 @@ uv run pytest tests/unit/.py -v git add . git commit -m "feat(scope): your descriptive commit message" -# Push and create a PR (branch must follow Conventional Branches — see above) -git push origin feature/your-feature +# Push and create a PR +git push origin litellm_your_feature ``` ## Adding Testing diff --git a/Makefile b/Makefile index a7a12a8635a..dec853515b9 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ help: @echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)" @echo " make install-test-deps - Install the full local test environment" @echo " make install-helm-unittest - Install helm unittest plugin" - @echo " make install-hooks - Install git hooks (Conventional Commits + Branches)" + @echo " make install-hooks - Install git hooks (Conventional Commits)" @echo " make check - Run CI-equivalent lint on staged files, or on the diff vs the base branch when nothing is staged" @echo " make pre-commit - Legacy alias for make check" @echo " make format - Apply ruff format code formatting" @@ -120,7 +120,7 @@ install-helm-unittest: helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \ } -# Install git hooks that enforce Conventional Commits and Conventional Branches. +# Install git hooks that enforce Conventional Commits. # Opt-in: not chained into install-dev. install-hooks: ./scripts/install_git_hooks.sh diff --git a/scripts/install_git_hooks.sh b/scripts/install_git_hooks.sh index 8f4e79e4ddd..cccc8d6afc4 100755 --- a/scripts/install_git_hooks.sh +++ b/scripts/install_git_hooks.sh @@ -31,7 +31,7 @@ cat < subprocess.CompletedProcess: @@ -54,21 +47,6 @@ def _run_commit_msg(subject: str, tmp_path: Path) -> subprocess.CompletedProcess ) -def _run_pre_push(stdin: str) -> subprocess.CompletedProcess: - return subprocess.run( - ["bash", str(_PRE_PUSH_HOOK)], - input=stdin, - capture_output=True, - text=True, - check=False, - ) - - -def _ref_line(branch: str, local_oid: str = _NONZERO_OID, remote_oid: str = _ZERO_OID) -> str: - ref = f"refs/heads/{branch}" - return f"{ref} {local_oid} {ref} {remote_oid}\n" - - # ----- commit-msg ----------------------------------------------------------- @@ -198,88 +176,3 @@ def test_commit_msg_uses_first_non_comment_line(tmp_path): check=False, ) assert result.returncode == 0, result.stderr - - -# ----- pre-push ------------------------------------------------------------- - - -@pytest.mark.parametrize( - "branch", - [ - "feature/weighted-round-robin", - "bugfix/streaming-empty-chunks", - "hotfix/auth-bypass", - "release/v1.45.0", - "chore/bump-deps", - "feature/nested/path/ok", # nested slashes after type are fine - ], -) -def test_pre_push_accepts_conventional_branches(branch): - result = _run_pre_push(_ref_line(branch)) - assert result.returncode == 0, ( - f"hook rejected a valid branch:\n branch: {branch!r}\n" - f" stderr: {result.stderr}" - ) - - -@pytest.mark.parametrize( - "branch", - [ - "random-branch-name", - "litellm_fix/optimize-streaming", # legacy pattern is now rejected - "ui/navbar-notifications", # not in the allow list - "feature/", # empty description - "Feature/foo", # type is case-sensitive - "feat/foo", # angular commit type, not branch type - ], -) -def test_pre_push_rejects_non_conventional_branches(branch): - result = _run_pre_push(_ref_line(branch)) - assert result.returncode == 1, ( - f"hook accepted an invalid branch:\n branch: {branch!r}\n" - f" stderr: {result.stderr}" - ) - assert "Conventional Branches" in result.stderr - - -@pytest.mark.parametrize( - "branch", - [ - "main", - "dependabot/github_actions/foo", - "gh-readonly-queue/main/abc123", - ], -) -def test_pre_push_bypasses_protected_branches(branch): - result = _run_pre_push(_ref_line(branch)) - assert result.returncode == 0, ( - f"protected branch was rejected:\n branch: {branch!r}\n" - f" stderr: {result.stderr}" - ) - - -def test_pre_push_skips_tag_pushes(): - line = f"refs/tags/v1 {_NONZERO_OID} refs/tags/v1 {_ZERO_OID}\n" - result = _run_pre_push(line) - assert result.returncode == 0, result.stderr - - -def test_pre_push_skips_branch_deletions(): - # local oid all zeros = deletion - line = f"refs/heads/whatever {_ZERO_OID} refs/heads/whatever {_NONZERO_OID}\n" - result = _run_pre_push(line) - assert result.returncode == 0, result.stderr - - -def test_pre_push_fails_if_any_ref_is_invalid(): - # Mixed batch: one valid, one invalid — entire push should fail. - stdin = _ref_line("feature/ok") + _ref_line("random-bad") - result = _run_pre_push(stdin) - assert result.returncode == 1 - assert "random-bad" in result.stderr - - -def test_pre_push_no_refs_passes(): - # Empty stdin (no refs being pushed) should pass. - result = _run_pre_push("") - assert result.returncode == 0, result.stderr From a81379bd4dbbf48b175e32a79c633cbff8b15978 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 17:38:01 -0700 Subject: [PATCH 034/124] chore: add yassin-berriai as code owner for helm/ and terraform/ (#47452) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 582cf0f5217..a707eb8044c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,4 @@ /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri +/helm/ @yassin-berriai +/terraform/ @yassin-berriai From ece162740855edb0a66437eeb5b8ff3be672049a Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:09 -0700 Subject: [PATCH 035/124] fix(vertex-ai): drop floor deprecation_date from gemini-3.8-live (#47447) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 7 +++---- model_prices_and_context_window.json | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1cadc93969b..e53f8fa55dd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29836,7 +29836,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { - "deprecation_date": "2027-09-25", + "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -29849,6 +29849,7 @@ "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, + "output_cost_per_video_token": 1e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/vertex_ai/live", @@ -29869,9 +29870,7 @@ "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, - "supports_web_search": true, - "gemini_audio_only_live": true, - "output_cost_per_video_token": 1e-06 + "supports_web_search": true }, "vertex_ai/gemini-3.1-pro-preview": { "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1cadc93969b..e53f8fa55dd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29836,7 +29836,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { - "deprecation_date": "2027-09-25", + "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -29849,6 +29849,7 @@ "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, + "output_cost_per_video_token": 1e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_endpoints": [ "/vertex_ai/live", @@ -29869,9 +29870,7 @@ "supports_function_calling": true, "supports_response_schema": false, "supports_vision": true, - "supports_web_search": true, - "gemini_audio_only_live": true, - "output_cost_per_video_token": 1e-06 + "supports_web_search": true }, "vertex_ai/gemini-3.1-pro-preview": { "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, From b2eda10befe8402daaa914d30ff3f49d5fd7a506 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 19 Oct 2026 17:52:05 -0700 Subject: [PATCH 036/124] perf(router): resolve deployment usage limits once per deployment instead of every request (#47393) _increment_deployment_usage looked up the deployment, resolved its model info and model_dump'ed it on every call just to decide whether the deployment has any tpm/rpm/itpm/otpm limit. That answer only changes when the model list or discovered limits change, so it is now memoized per deployment and cleared in _invalidate_model_group_info_cache --- litellm/router.py | 53 ++++++++------- .../test_router/test_router_helper_utils.py | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5bf46c68b20..d8da69f7789 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1389,6 +1389,9 @@ class Router: ) self._ptu_share_teams = lru_cache(maxsize=1)(self._collect_ptu_share_teams) self._cached_team_ptu_ceiling = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._resolve_team_ptu_ceiling) + self._cached_deployment_tracks_usage = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._deployment_tracks_usage + ) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -9053,31 +9056,10 @@ class Router: ) -> str | None: from litellm.types.caching import RedisPipelineIncrementOperation - deployment_info: Final = self.get_deployment(model_id=deployment_id) - if deployment_info is None: - return None - deployment_model_info: Final = self.get_router_model_info( - deployment=deployment_info, - received_model_name=model_group, - ) - configured_limits: Final = ( - deployment_info.get("tpm", None), - deployment_info.get("rpm", None), - deployment_info.litellm_params.tpm, - deployment_info.litellm_params.rpm, - deployment_model_info.get("tpm", None), - deployment_model_info.get("rpm", None), - ) - ## Nothing to track only when neither tpm/rpm nor itpm/otpm limits are - ## set. IO deployments still record TPM/RPM usage here so TPM-aware - ## routing strategies see their real load in mixed model groups; their - ## itpm/otpm enforcement runs separately in ModelRateLimitingCheck. - if all(limit is None for limit in configured_limits) and not deployment_has_io_token_limits( - deployment_info.model_dump() - ): - return None if total_tokens <= 0 and rpm_increment <= 0: return None + if not self._cached_deployment_tracks_usage(deployment_id, model_group): + return None current_minute: Final = get_utc_datetime().strftime("%H-%M") # use the same timezone regardless of system clock tpm_key: Final = RouterCacheEnum.TPM.value.format( @@ -9101,6 +9083,30 @@ class Router: ) return tpm_key + def _deployment_tracks_usage(self, deployment_id: str, model_group: str) -> bool: + deployment_info: Final = self.get_deployment(model_id=deployment_id) + if deployment_info is None: + return False + deployment_model_info: Final = self.get_router_model_info( + deployment=deployment_info, + received_model_name=model_group, + ) + configured_limits: Final = ( + deployment_info.get("tpm", None), + deployment_info.get("rpm", None), + deployment_info.litellm_params.tpm, + deployment_info.litellm_params.rpm, + deployment_model_info.get("tpm", None), + deployment_model_info.get("rpm", None), + ) + ## Nothing to track only when neither tpm/rpm nor itpm/otpm limits are + ## set. IO deployments still record TPM/RPM usage here so TPM-aware + ## routing strategies see their real load in mixed model groups; their + ## itpm/otpm enforcement runs separately in ModelRateLimitingCheck. + return any(limit is not None for limit in configured_limits) or deployment_has_io_token_limits( + deployment_info.model_dump() + ) + def sync_deployment_callback_on_success( self, kwargs, # kwargs to completion @@ -13386,6 +13392,7 @@ class Router: self.cached_deployment_model_info.cache_clear() self._ptu_share_teams.cache_clear() self._cached_team_ptu_ceiling.cache_clear() + self._cached_deployment_tracks_usage.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index ce60cce8408..e3061294bef 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1,3 +1,4 @@ +import functools import json import os from datetime import datetime, timezone @@ -1276,6 +1277,69 @@ async def test_increment_deployment_usage_writes_only_positive_deltas_for_limite assert await unlimited.get_model_group_usage("gpt-5-mini") == (None, None) +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_increment_deployment_usage_starts_tracking_after_limit_is_added(): + router = Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake"}, + "model_info": {"id": "lit-3058-upsert"}, + } + ] + ) + increment = functools.partial( + router._increment_deployment_usage, + deployment_id="lit-3058-upsert", + deployment_name="gpt-5-mini", + model_group="gpt-5-mini", + total_tokens=25, + rpm_increment=1, + parent_otel_span=None, + ) + + assert await increment() is None + + router.upsert_deployment( + Deployment( + model_name="gpt-5-mini", + litellm_params=LiteLLM_Params(model="gpt-5-mini", api_key="sk-fake", rpm=10), + model_info={"id": "lit-3058-upsert"}, + ) + ) + + assert await increment() is not None + assert await router.get_model_group_usage("gpt-5-mini") == (25, 1) + + +@pytest.mark.parametrize( + "litellm_params, model_info, deployment_id, tracks", + [ + ({}, {}, "lit-3058-tracks", False), + ({"rpm": 10}, {}, "lit-3058-tracks", True), + ({"tpm": 1000}, {}, "lit-3058-tracks", True), + ({"otpm": 500}, {}, "lit-3058-tracks", True), + ({}, {"itpm": 500}, "lit-3058-tracks", True), + ({"rpm": 10}, {}, "lit-3058-unknown", False), + ], +) +def test_deployment_tracks_usage_for_every_limit_source( + litellm_params: dict, model_info: dict, deployment_id: str, tracks: bool +): + router = Router( + model_list=[ + { + "model_name": "my-unmapped-model", + "litellm_params": {"model": "openai/my-unmapped-model", "api_key": "sk-fake", **litellm_params}, + "model_info": {"id": "lit-3058-tracks", **model_info}, + } + ] + ) + + assert router._deployment_tracks_usage(deployment_id, "my-unmapped-model") is tracks + + def _shared_redis_stub(store: dict) -> MagicMock: from litellm.caching.redis_cache import RedisCache From f98168e73d8b075f26341a73516ff70456bdd560 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 17:56:50 -0700 Subject: [PATCH 037/124] test(e2e/perf): read the memory baseline after a one minute warm-up (#47455) memory_growth_megabytes compared end-of-run RSS with RSS read right after boot, before a worker had freed its startup allocations, so a light case (small_bodies_5rps) read negative growth. The case now replays its first minute of traffic before the baseline read, so the metric is the memory the measured traffic added Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/test_perf_cases.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/e2e/perf/test_perf_cases.py b/tests/e2e/perf/test_perf_cases.py index 902483f8a2f..5d0e25b7616 100644 --- a/tests/e2e/perf/test_perf_cases.py +++ b/tests/e2e/perf/test_perf_cases.py @@ -7,6 +7,7 @@ from __future__ import annotations import os import time +from collections.abc import Mapping from datetime import UTC, datetime from typing import Final @@ -38,14 +39,15 @@ from setup.observe import ( reset_mock, set_mock_latency, ) -from setup.performance_test_case import PerformanceTestCase -from setup.plan import build_plan +from setup.performance_test_case import MockLatency, PerformanceTestCase +from setup.plan import PlannedRequest, build_plan from setup.podcpu import gateway_cpu_sampler from setup.provision import MOCK_PROVIDERS, build_setup, guardrail_url, mock_url from setup.run import run_plan pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] _SPEND_SAMPLE: Final = 40 +_WARMUP_SECONDS: Final = 60 def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float | None: @@ -60,6 +62,20 @@ def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float return found / len(call_ids) +def _warmup_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...]) -> tuple[PlannedRequest, ...]: + """The plan's first minute of traffic. It is sent before the memory baseline is read so a worker + has shed its startup allocations and `memory_growth_megabytes` is what the measured traffic added, + not startup churn (a lightly loaded case read negative growth).""" + return plan[: max(1, round(case.traffic.rps * _WARMUP_SECONDS))] + + +def _arm_mocks(mock_urls: Mapping[str, str], latency: MockLatency) -> None: + for url in (*mock_urls.values(), guardrail_url()): + _ = reset_mock(url) + for url in mock_urls.values(): + _ = set_mock_latency(url, latency) + + def _event_loop_rows(case: PerformanceTestCase, since: datetime) -> tuple[StallRow, ...]: """The gateway's asyncio debug holds for the traffic window; a case that asks for them cannot pass without a log.""" assert case.event_loop is not None @@ -86,12 +102,11 @@ def test_perf_case( ) plan: Final = build_plan(case) mock_urls: Final = {provider: mock_url(provider) for provider in MOCK_PROVIDERS} - for url in (*mock_urls.values(), guardrail_url()): - _ = reset_mock(url) - for url in mock_urls.values(): - _ = set_mock_latency(url, case.mock_latency) + _arm_mocks(mock_urls, case.mock_latency) setup_started: Final = time.monotonic() setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources) + _ = run_plan(case, _warmup_plan(case, plan), setup) + _arm_mocks(mock_urls, case.mock_latency) rss_before: Final = read_rss_everywhere(proxy) db_before: Final = read_db_counters() redis_before: Final = read_redis_counters() From 3fa1587f4367fad5706d58545c47f413a9dea8b8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 17:59:53 -0700 Subject: [PATCH 038/124] fix(logging): do not stamp a provider handoff when pre_call runs for a proxy-only failure (#47456) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 24 +++++++---- .../test_litellm_logging.py | 15 +++++++ ..._utils_model_creation_and_error_logging.py | 41 +++++++++++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b065d4dd91a..7203a176884 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -44,6 +44,7 @@ from litellm.constants import ( EMPTY_MAPPING, INTERNAL_CALLBACK_NAMES, KUBERNETES_POD_ROUTING_KEY, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, ) @@ -1531,15 +1532,20 @@ class Logging(LiteLLMLoggingBaseClass): "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) - self.model_call_details["api_call_start_time"] = datetime.datetime.now() - # Set-once first provider-handoff instant. api_call_start_time - # is overwritten on every retry, so it can't measure one-time - # preprocessing; pinning the first attempt excludes retry loops - # + backoff. Logging object only — must NOT go into - # litellm_params["metadata"] (caller request metadata, typed - # Dict[str, str], echoed downstream; a datetime breaks it). - if self.model_call_details.get("first_api_call_start_time") is None: - self.model_call_details["first_api_call_start_time"] = self.model_call_details["api_call_start_time"] + # A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks + # without any provider handoff, so it must not stamp the handoff instants. + if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details: + self.model_call_details["api_call_start_time"] = datetime.datetime.now() + # Set-once first provider-handoff instant. api_call_start_time + # is overwritten on every retry, so it can't measure one-time + # preprocessing; pinning the first attempt excludes retry loops + # + backoff. Logging object only — must NOT go into + # litellm_params["metadata"] (caller request metadata, typed + # Dict[str, str], echoed downstream; a datetime breaks it). + if self.model_call_details.get("first_api_call_start_time") is None: + self.model_call_details["first_api_call_start_time"] = self.model_call_details[ + "api_call_start_time" + ] # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or []) for callback in callbacks: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 1d3bb2d0f34..041ab2a61b2 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -32,6 +32,7 @@ from litellm._service_logger import ServiceLogging from litellm.caching.caching import DualCache from litellm.constants import ( KUBERNETES_POD_ROUTING_KEY, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM, SENTRY_PII_DENYLIST, @@ -408,6 +409,20 @@ def test_get_masked_api_base(logging_obj): assert type(masked_api_base) == str +@pytest.mark.parametrize("no_upstream_call", [False, True]) +def test_pre_call_stamps_the_provider_handoff_only_when_a_provider_call_is_made(logging_obj, no_upstream_call): + """The proxy fires pre_call for a failure it raised itself (auth, rate limit, client disconnect) + so callbacks see it; that call never reached a provider, so neither handoff instant may be set.""" + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + if no_upstream_call: + logging_obj.model_call_details[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + + logging_obj.pre_call(input="", api_key="") + + assert ("api_call_start_time" in logging_obj.model_call_details) is not no_upstream_call + assert (logging_obj.model_call_details.get("first_api_call_start_time") is not None) is not no_upstream_call + + def test_pre_call_tolerates_missing_api_base(logging_obj): """Presigned batch retrieves (Mistral, Bedrock) build their own URL and pass api_base=None to pre_call; masking must not raise or the request's pre-call logging is silently lost.""" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 336d2f3e7c2..34d6c165182 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -444,6 +444,47 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime: assert "litellm_logging_obj" not in request_data +class TestPostCallFailureHookDoesNotInventAProviderHandoff: + """A failure the proxy raised before any provider call (here an auth error) is logged through + _handle_logging_proxy_only_error, which fires pre_call for the callbacks. That must not stamp a + handoff instant onto the request, or every downstream reader (preprocessing latency, the + dispatched check behind the failure-row estimate) sees a provider call that never happened.""" + + @pytest.mark.asyncio + async def test_proxy_only_failure_lifts_no_first_api_call_start_time(self): + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth + + logging_obj = Logging( + model="gpt-4.1", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="pre-handoff-auth-failure", + function_id="pre-handoff", + ) + request_data: Final[dict[str, object]] = {"litellm_logging_obj": logging_obj, "model": "gpt-4.1", "metadata": {}} + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("Authentication Error, invalid key"), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + error_type=ProxyErrorTypes.auth_error, + ) + + assert request_data.get("first_api_call_start_time") is None + assert "api_call_start_time" not in logging_obj.model_call_details + + class TestPostCallFailureHookLiftsRecoveredPartialSpend: """A stream that broke mid-flight still billed the provider for the chunks already delivered. The streaming handler stashes that recovered usage and From 8b9cd12a8930062709f7ba0dc0e5559ef6613885 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:06:37 +0000 Subject: [PATCH 039/124] test: catch regressions in the validation lines #44798 added (#47420) * test: pin the validation lines #44798 kept with failing cases * fix(containers): accept endpoint table entries that omit their param lists * test: keep the endpoint table and xAI adapter as on main The endpoint table ships inside the package, so optional param lists fixed no reachable state and turned a loud failure into silently dropped params. Drops the Gemini case that main's tests already cover. * test: cover the cost map backup and Gemini image config validation lines The cost map backup adapter's hide flag, and the validation itself, could be removed with every test still green. The Gemini imageConfig adapter could be narrowed with only the integration suite noticing, which does not run on PRs. The endpoint table cases now read a real file through the module's location instead of replacing open, so reading the file another way keeps them green. * test: pin the container handler's loud table failure and loosen two pinned drops A container endpoint table entry without path_params or query_params fails the call on main, and no test held that. The new case trims each key from the packaged table and expects the validation error. The Gemini imageConfig case now accepts a drop or UnsupportedParamsError for a non-object value, and the Bedrock beta case checks only that numbers come back as strings, so neither pins behavior the drop_params convention may change. * test: check a numeric beta array is never split as text and use the latest Gemini image model The Bedrock case now also fails when a JSON array of numbers falls through to the comma split, which returns bracketed fragments, while a drop or a string coercion both stay allowed. --- .../unit/containers/test_endpoint_factory.py | 24 +++++++++++++++++ .../test_litellm_responses_bridge.py | 7 +++++ .../test_get_model_cost_map.py | 16 ++++++++++++ .../llms/bedrock/test_bedrock_common_utils.py | 24 +++++++++++++++++ .../custom_httpx/test_container_handler.py | 24 +++++++++++++++++ .../llms/gemini/test_gemini_common_utils.py | 26 ++++++++++++++++++- .../gemini/test_image_usage_transformation.py | 9 +++++++ 7 files changed, 129 insertions(+), 1 deletion(-) diff --git a/tests/unit/containers/test_endpoint_factory.py b/tests/unit/containers/test_endpoint_factory.py index 8de0d039afc..cf640f0e281 100644 --- a/tests/unit/containers/test_endpoint_factory.py +++ b/tests/unit/containers/test_endpoint_factory.py @@ -1,4 +1,9 @@ +import json +from pathlib import Path +from typing import Final + import pytest +from pydantic import ValidationError from litellm.containers import endpoint_factory from litellm.containers.endpoint_factory import ( @@ -52,6 +57,25 @@ class TestEndpointsConfig: assert first["endpoints"] is not second["endpoints"] assert first == second + @pytest.mark.parametrize("document", ['["only-in-the-file"]', '"only-in-the-file"', "null"]) + def test_a_packaged_file_that_is_not_a_json_object_is_rejected_without_quoting_it( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, document: str + ): + (tmp_path / "endpoints.json").write_text(document) + monkeypatch.setattr(endpoint_factory, "__file__", str(tmp_path / "endpoint_factory.py")) + + with pytest.raises(ValidationError) as rejected: + _load_endpoints_config() + + assert "only-in-the-file" not in str(rejected.value) + + def test_a_packaged_json_object_is_returned_as_written(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + document: Final = {"endpoints": [{"name": "list_container_files"}], "schema_version": 2} + (tmp_path / "endpoints.json").write_text(json.dumps(document)) + monkeypatch.setattr(endpoint_factory, "__file__", str(tmp_path / "endpoint_factory.py")) + + assert _load_endpoints_config() == document + class TestResponseTypeMapping: def test_mapping_resolves_every_named_response_type(self): diff --git a/tests/unit/interactions/test_litellm_responses_bridge.py b/tests/unit/interactions/test_litellm_responses_bridge.py index 58f2756e7fb..9330ae44ad8 100644 --- a/tests/unit/interactions/test_litellm_responses_bridge.py +++ b/tests/unit/interactions/test_litellm_responses_bridge.py @@ -10,6 +10,7 @@ from datetime import datetime from typing import Final import pytest +from pydantic import ValidationError from litellm.interactions.litellm_responses_transformation.transformation import ( LiteLLMResponsesInteractionsConfig, @@ -142,3 +143,9 @@ class TestBridgeResponseTransformation: assert interaction.created is not None assert datetime.fromisoformat(interaction.created).timestamp() == 1700000000 assert interaction.updated == interaction.created + + def test_responses_response_with_a_status_that_is_not_text_is_refused(self): + reply: Final = _responses_api_response("completed", None).model_copy(update={"status": 7}) + + with pytest.raises(ValidationError): + LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(reply, None) diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index eca9bbdc170..84e785ac99a 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -8,12 +8,15 @@ import asyncio, importlib, importlib.resources as importlib_get_model, json, lit import os import sys import threading +from pathlib import Path from typing import Final import boto3 import pytest +from pydantic import ValidationError from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.litellm_core_utils import get_model_cost_map as get_model_cost_map_module from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, match_capability_generalizations, @@ -1246,3 +1249,16 @@ def test_get_backup_model_cost_map(): print("inside backup") content = json.load(f) print("content", content) + + +@pytest.mark.parametrize("backup", [b'["only-in-the-backup"]', b'"only-in-the-backup"', b"null"]) +def test_a_bundled_backup_that_is_not_a_json_object_is_rejected_without_quoting_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, backup: bytes +): + (tmp_path / "model_prices_and_context_window_backup.json").write_bytes(backup) + monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path) + + with pytest.raises(ValidationError) as rejected: + GetModelCostMap.load_local_model_cost_map_with_revision() + + assert "only-in-the-backup" not in str(rejected.value) diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index d5bd7d44223..7bde1805e18 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1197,6 +1197,15 @@ def test_get_anthropic_beta_from_headers_reads_a_json_array_header(header_value: assert get_anthropic_beta_from_headers({"anthropic-beta": header_value}) == expected +def test_get_anthropic_beta_from_headers_never_splits_a_json_array_of_numbers_as_text(): + from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers + + betas: Final = get_anthropic_beta_from_headers({"anthropic-beta": "[1, 2]"}) + + assert all(isinstance(beta, str) for beta in betas) + assert not any("[" in beta or "]" in beta for beta in betas) + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) @@ -2126,3 +2135,18 @@ def test_without_bedrock_native_output_opt_in_turns_the_opt_in_off_and_keeps_the assert turned_off["api_base"] == "https://bedrock.example" assert deployment_model_info(turned_off)["id"] == "deployment-1" assert litellm_params["model_info"] == {"id": "deployment-1"} + + +def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_the_service_as_a_list(): + import litellm.llms.bedrock.common_utils as mod + + pytest.importorskip("botocore") + with ( + patch("botocore.loaders.Loader.load_service_model", return_value=["only-in-the-description"]), + patch.object(mod.verbose_logger, "warning") as warning, + ): + shape: Final = mod._load_bedrock_response_stream_shape() + + assert shape is None + (logged,) = warning.call_args_list + assert "only-in-the-description" not in logged.args[0] % logged.args[1:] diff --git a/tests/unit/llms/custom_httpx/test_container_handler.py b/tests/unit/llms/custom_httpx/test_container_handler.py index 1b09165087a..0393f12564e 100644 --- a/tests/unit/llms/custom_httpx/test_container_handler.py +++ b/tests/unit/llms/custom_httpx/test_container_handler.py @@ -1,11 +1,16 @@ +import json +from importlib.resources import files +from pathlib import Path from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx import container_handler from litellm.llms.custom_httpx.container_handler import generic_container_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.containers.main import DeleteContainerFileResponse @@ -124,3 +129,22 @@ def test_json_endpoint_sends_the_configured_route_and_parses_its_response_model( assert type(response) is DeleteContainerFileResponse assert response.id.endswith("/containers/cntr_real/files/cfile_nonexistent") assert response.deleted is True + + +@pytest.mark.parametrize("missing_key", ["path_params", "query_params"]) +def test_an_endpoint_table_without_a_param_list_fails_the_call( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, missing_key: str +): + packaged: Final = json.loads(files("litellm.containers").joinpath("endpoints.json").read_text()) + trimmed: Final = { + "endpoints": [ + {key: value for key, value in entry.items() if key != missing_key} for entry in packaged["endpoints"] + ] + } + (tmp_path / "containers").mkdir() + (tmp_path / "containers" / "endpoints.json").write_text(json.dumps(trimmed)) + monkeypatch.setattr(container_handler, "__file__", str(tmp_path / "llms" / "custom_httpx" / "container_handler.py")) + empty_list: Final = {"object": "list", "data": [], "first_id": None, "last_id": None, "has_more": False} + + with pytest.raises(ValidationError): + _handle("list_container_files", _sync_client(httpx.Response(200, json=empty_list))) diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index 918a7475eec..196b912752b 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -1,3 +1,4 @@ +import contextlib from typing import Final from unittest.mock import AsyncMock, patch @@ -8,7 +9,11 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.llms.gemini.common_utils import GeminiModelInfo, GoogleAIStudioTokenCounter +from litellm.llms.gemini.common_utils import ( + GeminiModelInfo, + GoogleAIStudioTokenCounter, + map_openai_image_params_to_gemini, +) from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter as GeminiCountTokensHandler @@ -473,3 +478,22 @@ class TestGoogleAIStudioTokenCounter: # Verify the contents are unchanged assert cleaned_contents == contents_without_function_response + + +def _map_image_config(raw_image_config: str) -> dict[str, object]: + return map_openai_image_params_to_gemini( + {"imageConfig": raw_image_config}, + "gemini-3.1-flash-image", + ["imageConfig"], + parse_image_config_string=True, + ) + + +def test_map_openai_image_params_to_gemini_forwards_a_json_object_image_config(): + assert _map_image_config('{"aspectRatio": "1:1"}') == {"imageConfig": {"aspectRatio": "1:1"}} + + +@pytest.mark.parametrize("raw_image_config", ["[1, 2]", '"16:9"', "null"]) +def test_map_openai_image_params_to_gemini_drops_or_refuses_a_non_object_image_config(raw_image_config: str): + with contextlib.suppress(litellm.UnsupportedParamsError): + assert "imageConfig" not in _map_image_config(raw_image_config) diff --git a/tests/unit/llms/gemini/test_image_usage_transformation.py b/tests/unit/llms/gemini/test_image_usage_transformation.py index 54e0668d2d9..f2f1c3d999f 100644 --- a/tests/unit/llms/gemini/test_image_usage_transformation.py +++ b/tests/unit/llms/gemini/test_image_usage_transformation.py @@ -1,6 +1,7 @@ from typing import Final import pytest +from pydantic import ValidationError from litellm.llms.gemini.image_usage_transformation import transform_gemini_image_usage @@ -45,3 +46,11 @@ def test_transform_gemini_image_usage_reports_image_and_chat_style_counts( "completion_tokens_details": expected_output_details, "output_tokens_details": expected_output_details, } + + +@pytest.mark.parametrize("count_field", ["promptTokenCount", "totalTokenCount"]) +def test_transform_gemini_image_usage_rejects_a_count_that_is_not_a_number(count_field: str): + with pytest.raises(ValidationError): + transform_gemini_image_usage( + {"promptTokenCount": 35, "candidatesTokenCount": 4, "totalTokenCount": 39, count_field: "many"} + ) From d29200fa0511ae50f41b692de5922659a1f176c0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 18:17:29 -0700 Subject: [PATCH 040/124] fix(vertex_ai): bill per-character text embeddings on Google's billableCharacterCount (#47460) Vertex text embedding predict responses report metadata.billableCharacterCount, the unit Google bills text-embedding-005 and text-multilingual-embedding-002 on. Carry it into usage.prompt_tokens_details.character_count and, when the model carries input_cost_per_character, bill that count instead of the token count so the prompt is charged under one unit. Per-token embeddings (gemini-embedding-001) are unchanged. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 8 ++++- .../vertex_embeddings/transformation.py | 34 +++++++++++++++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 25 ++++++++++++++ .../test_embedding_handler.py | 18 ++++++++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index c6141be008e..1b836e935d7 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1118,8 +1118,14 @@ def _calculate_input_cost( ) -> float: """ Calculates the input cost for a given model, prompt tokens, and completion tokens. + + A model priced per character (`input_cost_per_character`) bills the reported character count + in place of its text tokens, so the same prompt is never charged under both units. """ - prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost + bills_characters: Final = ( + prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None + ) + prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index cab2bed8a60..ddc83bc53a5 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,13 +1,35 @@ import types from typing import Final, Literal +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict + from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import EmbeddingResponse, Usage +from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, Usage from .types import * +class _PredictMetadata(TypedDict, total=False): + billableCharacterCount: ReadOnly[int] + + +class _PredictResponseMetadata(TypedDict, total=False): + metadata: ReadOnly[_PredictMetadata] + + +_PREDICT_RESPONSE_METADATA: Final = TypeAdapter(_PredictResponseMetadata) + + +def billable_character_count(response: _PredictResponseMetadata) -> int | None: + """ + `metadata.billableCharacterCount` of a Vertex text embedding predict response, the count Google + bills per-character embedding models (text-embedding-005, text-multilingual-embedding-002) on. + """ + return response.get("metadata", {}).get("billableCharacterCount") + + class VertexAITextEmbeddingConfig(LiteLLMBaseModel): """ Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#TextEmbeddingInput @@ -247,7 +269,15 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): model_response.object = "list" model_response.data = embedding_response model_response.model = model - usage: Final = Usage(prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens) + billable_characters: Final = billable_character_count(_PREDICT_RESPONSE_METADATA.validate_python(response)) + usage: Final = Usage( + prompt_tokens=input_tokens, + completion_tokens=0, + total_tokens=input_tokens, + prompt_tokens_details=( + None if billable_characters is None else PromptTokensDetailsWrapper(character_count=billable_characters) + ), + ) setattr(model_response, "usage", usage) return model_response diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 15712cc7d9c..f14f3c5f7d1 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -99,6 +99,31 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) +@pytest.mark.parametrize( + ("model_info", "expected_prompt_cost"), + [ + ({"input_cost_per_token": 1e-7, "input_cost_per_character": 2.5e-8, "output_cost_per_token": 0.0}, 35 * 2.5e-8), + ({"input_cost_per_token": 1.5e-7, "output_cost_per_token": 0.0}, 11 * 1.5e-7), + ], + ids=["character_priced_model_bills_characters_instead_of_tokens", "token_priced_model_ignores_the_character_count"], +) +def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( + model_info: ModelInfo, expected_prompt_cost: float +) -> None: + usage = Usage( + prompt_tokens=11, + completion_tokens=0, + total_tokens=11, + prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info + ) + + assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) + + def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: openai_info: ModelInfo = {"input_cost_per_token": 2e-6} fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} diff --git a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py index 4ceae8d407a..1058d4fb82d 100644 --- a/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py +++ b/tests/unit/llms/vertex_ai/vertex_embeddings/test_embedding_handler.py @@ -287,3 +287,21 @@ async def test_vertex_async_embedding_call_maps_a_malformed_predict_body_to_a_co ) assert (failure.value.status_code, "vertex-secret" in str(failure.value)) == (500, False) + + +@BOTH_CALL_STYLES +@pytest.mark.parametrize( + ("metadata", "character_count"), + [({"metadata": {"billableCharacterCount": 10}}, 10), ({}, None)], + ids=["billable_character_count_reported", "no_metadata"], +) +async def test_vertex_embedding_usage_carries_the_billable_character_count_google_prices_on( + run_async: bool, predict_endpoint: respx.Route, metadata: dict[str, object], character_count: int | None +) -> None: + payload: Final = {"predictions": [_gecko_prediction([0.1, 0.2, 0.3], 3), _gecko_prediction([1, 2.0, 3], 5)], **metadata} + response: Final = await _embedding_response(run_async, predict_endpoint, GECKO, payload, _embedding_logging()) + + assert (response.usage.prompt_tokens, getattr(response.usage.prompt_tokens_details, "character_count", None)) == ( + 8, + character_count, + ) From 2754da5391d398d0023539fbb6edc6f2d9e0361f Mon Sep 17 00:00:00 2001 From: Joshua Wang Date: Mon, 19 Oct 2026 21:22:07 -0400 Subject: [PATCH 041/124] fix(friendliai): drop litellm-internal message fields before sending (#45092) * fix(friendliai): drop litellm-internal assistant fields before sending Claude Code replays thinking blocks on every turn after the first. The Anthropic adapter turns them into thinking_blocks plus reasoning_content on the chat assistant message, and FriendliAI rejects any message key it does not know, so multi-turn Claude Code sessions on friendliai/ models failed with a 500 after the first tool call FriendliAI accepts reasoning_content and GLM reads it back, so only thinking_blocks and provider_specific_fields are dropped. together_ai already had this exact helper as a private function, so it moves to common_utils next to strip_litellm_internal_message_fields and both providers share it Related to #32188 * test(friendliai): cover a replayed tool call turn The regression fixture is now the shape Claude Code replays after a tool call: an assistant turn with thinking_blocks, tool_calls and no content, followed by the matching tool result. The test asserts both survive the sync and async request paths unchanged * ci: retrigger checks * ci: retrigger benchmarks after runner timeout * chore(friendliai): mark inherited list signatures as mutable-ok The type-discipline gate on main now caps LIT001 at the merge-base count, so the _transform_messages overloads need the same inherited-contract marker together_ai uses for its identical override * ci(test-unit): add tests/unit/llms/friendliai to the llms-providers shard * test(friendliai): check the outbound FriendliAI body through the proxy for chat and messages replays Both replay shapes go through a real proxy to a scripted FriendliAI upstream, and the test reads the request body FriendliAI received. It fails at the merge base, where thinking_blocks and provider_specific_fields reach the upstream, and passes with the field stripping in place. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + .../llms/friendliai/chat/transformation.py | 34 ++++- .../providers/test_friendliai_replay_wire.py | 122 ++++++++++++++++++ tests/unit/llms/friendliai/__init__.py | 0 tests/unit/llms/friendliai/chat/__init__.py | 0 .../friendliai/chat/test_transformation.py | 42 ++++++ 6 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 tests/integration/providers/test_friendliai_replay_wire.py create mode 100644 tests/unit/llms/friendliai/__init__.py create mode 100644 tests/unit/llms/friendliai/chat/__init__.py create mode 100644 tests/unit/llms/friendliai/chat/test_transformation.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 5e42f182345..1514c58d022 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -266,6 +266,7 @@ jobs: tests/unit/llms/featherless_ai tests/unit/llms/firecrawl tests/unit/llms/fireworks_ai + tests/unit/llms/friendliai tests/unit/llms/gdc tests/unit/llms/gemini tests/unit/llms/gigachat diff --git a/litellm/llms/friendliai/chat/transformation.py b/litellm/llms/friendliai/chat/transformation.py index 168549142f3..d4df4636218 100644 --- a/litellm/llms/friendliai/chat/transformation.py +++ b/litellm/llms/friendliai/chat/transformation.py @@ -2,8 +2,40 @@ Translate from OpenAI's `/v1/chat/completions` to Friendliai's `/v1/chat/completions` """ +from collections.abc import Coroutine +from typing import Literal, overload + +from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields +from litellm.types.llms.openai import AllMessageValues + from ...openai_like.chat.handler import OpenAILikeChatConfig class FriendliaiChatConfig(OpenAILikeChatConfig): - pass + @overload + def _transform_messages( + self, + messages: list[AllMessageValues], # mutable-ok: inherited contract + model: str, + is_async: Literal[True], + ) -> Coroutine[object, object, list[AllMessageValues]]: ... # mutable-ok: inherited contract + + @overload + def _transform_messages( + self, + messages: list[AllMessageValues], # mutable-ok: inherited contract + model: str, + is_async: Literal[False] = False, + ) -> list[AllMessageValues]: ... # mutable-ok: inherited contract + + def _transform_messages( + self, + messages: list[AllMessageValues], # mutable-ok: inherited contract + model: str, + is_async: bool = False, + ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: # mutable-ok: inherited contract + return super()._transform_messages( + messages=[strip_litellm_response_only_fields(message) for message in messages], + model=model, + is_async=is_async, + ) diff --git a/tests/integration/providers/test_friendliai_replay_wire.py b/tests/integration/providers/test_friendliai_replay_wire.py new file mode 100644 index 00000000000..1e41b5489f6 --- /dev/null +++ b/tests/integration/providers/test_friendliai_replay_wire.py @@ -0,0 +1,122 @@ +import json +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "zai-org/GLM-5.3" +_API_KEY: Final = "synthetic-friendliai-key" +_PROMPT: Final = "What is the weather in Paris? Use the tool." +_REASONING: Final = "The user wants the weather in Paris, so call get_weather." +_TOOL_RESULT: Final = "18C and sunny" +_ANSWER: Final = "It is 18C and sunny in Paris." +_LITELLM_ONLY_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_items", "provider_specific_fields"}) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]) +_CALL: Final[JsonValue] = { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, +} +_PARAMETERS: Final[JsonValue] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} + + +def _respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/chat/completions") + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + return Reply( + body=json.dumps( + { + "id": "friendliai-replay", + "object": "chat.completion", + "created": 1, + "model": _MODEL, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": _ANSWER}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 40, "completion_tokens": 9, "total_tokens": 49}, + } + ).encode() + ) + + +def _sent_messages(requests: tuple[Request, ...]) -> tuple[dict[str, JsonValue], ...]: + assert len(requests) == 1 + body: Final = _JSON_OBJECT.validate_json(requests[0].body) + assert body["model"] == _MODEL + return _MESSAGES.validate_python(body["messages"]) + + +def test_chat_replay_of_returned_assistant_message_reaches_friendliai_without_litellm_fields(gateway: Gateway) -> None: + with wire_server(_respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"friendliai/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "user", "content": _PROMPT}, + { + "role": "assistant", + "content": None, + "tool_calls": [{**_CALL, "provider_specific_fields": {"thought_signature": "sig"}}], + "reasoning_content": _REASONING, + "thinking_blocks": [{"type": "thinking", "thinking": _REASONING, "signature": "sig"}], + "provider_specific_fields": {"reasoning": _REASONING}, + }, + { + "role": "tool", + "tool_call_id": "call_paris", + "content": _TOOL_RESULT, + "provider_specific_fields": {}, + }, + ], + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": _PARAMETERS}}], + }, + ) + assert response.status_code == 200, response.text + assert _sent_messages(wire.drain()) == ( + {"role": "user", "content": _PROMPT}, + {"role": "assistant", "tool_calls": [_CALL], "reasoning_content": _REASONING}, + {"role": "tool", "tool_call_id": "call_paris", "content": _TOOL_RESULT}, + ) + + +def test_messages_replay_of_thinking_and_tool_use_reaches_friendliai_without_litellm_fields(gateway: Gateway) -> None: + with wire_server(_respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"friendliai/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 256, + "messages": [ + {"role": "user", "content": _PROMPT}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": _REASONING, "signature": "sig"}, + {"type": "tool_use", "id": "call_paris", "name": "get_weather", "input": {"city": "Paris"}}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "call_paris", "content": _TOOL_RESULT}], + }, + ], + "tools": [{"name": "get_weather", "input_schema": _PARAMETERS}], + }, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["content"] == [{"type": "text", "text": _ANSWER}] + messages: Final = _sent_messages(wire.drain()) + assert [sorted(_LITELLM_ONLY_FIELDS & message.keys()) for message in messages] == [[]] * len(messages) + assistant: Final = next(message for message in messages if message["role"] == "assistant") + assert assistant["reasoning_content"] == _REASONING diff --git a/tests/unit/llms/friendliai/__init__.py b/tests/unit/llms/friendliai/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/friendliai/chat/__init__.py b/tests/unit/llms/friendliai/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/friendliai/chat/test_transformation.py b/tests/unit/llms/friendliai/chat/test_transformation.py new file mode 100644 index 00000000000..48fc08a3ccd --- /dev/null +++ b/tests/unit/llms/friendliai/chat/test_transformation.py @@ -0,0 +1,42 @@ +from typing import Final + +from litellm.llms.friendliai.chat.transformation import FriendliaiChatConfig +from litellm.types.llms.openai import AllMessageValues + +_TOOL_CALLS: Final = [ + {"id": "toolu_01", "type": "function", "function": {"name": "Bash", "arguments": '{"command": "echo hi"}'}} +] + + +def _replayed_thinking_tool_turn() -> list[AllMessageValues]: + return [ + {"role": "user", "content": "run echo hi"}, + { + "role": "assistant", + "thinking_blocks": [{"type": "thinking", "thinking": "run the Bash tool", "signature": ""}], + "tool_calls": _TOOL_CALLS, + "reasoning_content": "run the Bash tool", + "provider_specific_fields": {}, + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "hi"}, + ] + + +async def test_transform_request_drops_internal_fields_friendliai_rejects(): + messages: Final = _replayed_thinking_tool_turn() + config: Final = FriendliaiChatConfig() + + sync_request: Final = config.transform_request( + model="zai-org/GLM-5.3", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + async_request: Final = await config.async_transform_request( + model="zai-org/GLM-5.3", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + expected_assistant: Final = { + "role": "assistant", + "tool_calls": _TOOL_CALLS, + "reasoning_content": "run the Bash tool", + } + assert sync_request["messages"] == async_request["messages"] == [messages[0], expected_assistant, messages[2]] + assert messages == _replayed_thinking_tool_turn() From ed26fc5ce59fe190ccb4b07d46d0f6fafd2fefa9 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Mon, 19 Oct 2026 18:26:09 -0700 Subject: [PATCH 042/124] test(images): stop writing image edit responses to disk (#47463) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/images/test_image_edit_utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unit/images/test_image_edit_utils.py b/tests/unit/images/test_image_edit_utils.py index 109c4b89101..2f240b0fed4 100644 --- a/tests/unit/images/test_image_edit_utils.py +++ b/tests/unit/images/test_image_edit_utils.py @@ -536,15 +536,6 @@ async def test_azure_image_edit_litellm_sdk(): # Validate the response meets expected schema ImageResponse.model_validate(result) - if isinstance(result, ImageResponse) and result.data: - image_base64 = result.data[0].b64_json - if image_base64: - image_bytes = base64.b64decode(image_base64) - - # Save the image to a file - with open("test_image_edit.png", "wb") as f: - f.write(image_bytes) - @pytest.mark.asyncio async def test_openai_image_edit_cost_tracking(): """Test OpenAI image edit cost tracking with custom logger""" @@ -607,15 +598,6 @@ async def test_openai_image_edit_cost_tracking(): # Validate the response meets expected schema ImageResponse.model_validate(result) - if isinstance(result, ImageResponse) and result.data: - image_base64 = result.data[0].b64_json - if image_base64: - image_bytes = base64.b64decode(image_base64) - - # Save the image to a file - with open("test_image_edit.png", "wb") as f: - f.write(image_bytes) - await asyncio.sleep(5) print( "standard logging payload", From ca329a7f13d2e86cbf28a88010bae923e873ef92 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:27:38 -0700 Subject: [PATCH 043/124] chore(bedrock): add Model EOL deprecation dates for Palmyra X4, X5 and Pegasus v1.2 (#47462) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 ++++++++++++++--- model_prices_and_context_window.json | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e53f8fa55dd..e923369d8fe 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -293,6 +293,7 @@ "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { + "deprecation_date": "2027-03-09", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -300,11 +301,12 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, "supports_pdf_input": true }, "us.writer.palmyra-x5-v1:0": { + "deprecation_date": "2027-03-21", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -317,6 +319,7 @@ "supports_pdf_input": true }, "writer.palmyra-x4-v1:0": { + "deprecation_date": "2027-03-09", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -324,11 +327,12 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, "supports_pdf_input": true }, "writer.palmyra-x5-v1:0": { + "deprecation_date": "2027-03-21", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -812,33 +816,40 @@ "supports_image_input": true }, "twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "us.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "eu.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "global.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", "supports_video_input": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md" }, "amazon.titan-text-express-v1": { "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e53f8fa55dd..e923369d8fe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -293,6 +293,7 @@ "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { + "deprecation_date": "2027-03-09", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -300,11 +301,12 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, "supports_pdf_input": true }, "us.writer.palmyra-x5-v1:0": { + "deprecation_date": "2027-03-21", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -317,6 +319,7 @@ "supports_pdf_input": true }, "writer.palmyra-x4-v1:0": { + "deprecation_date": "2027-03-09", "input_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -324,11 +327,12 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, "supports_pdf_input": true }, "writer.palmyra-x5-v1:0": { + "deprecation_date": "2027-03-21", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -812,33 +816,40 @@ "supports_image_input": true }, "twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "us.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "eu.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", "supports_video_input": true }, "global.twelvelabs.pegasus-1-2-v1:0": { + "deprecation_date": "2027-03-22", "input_cost_per_video_per_second": 0.00049, "output_cost_per_token": 7.5e-06, "litellm_provider": "bedrock", "mode": "chat", "supports_video_input": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md" }, "amazon.titan-text-express-v1": { "input_cost_per_token": 2e-07, From 923b7d1784d9dda5047c87c1323ec2f68f7674c1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 19 Oct 2026 18:39:16 -0700 Subject: [PATCH 044/124] test(integration): read router cooldown failure rows from the proxy that served them (#47457) test_router_cooldown_failure_row.py boots an owned proxy, which writes to a private database since #47213. The file was written on a base without #47213 and read its rows from the worker database through read_rows, so all 13 cases timed out with State did not converge: [] Every row read now goes through Gateway.rows on the owned proxy's gateway, the same fix #47410 made for the stream stall cooldown cases --- .../spend/test_router_cooldown_failure_row.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/integration/spend/test_router_cooldown_failure_row.py b/tests/integration/spend/test_router_cooldown_failure_row.py index 04e36785823..5466a4dff92 100644 --- a/tests/integration/spend/test_router_cooldown_failure_row.py +++ b/tests/integration/spend/test_router_cooldown_failure_row.py @@ -15,7 +15,6 @@ import openai import pytest import yaml from integration._support.client import GATEWAY_LIMITS, Gateway, eventually, gateway_from_environment, object_value -from integration._support.database import read_rows from integration._support.process import owned_proxy_process from integration._support.redis_process import OwnedRedis, owned_redis from integration._support.wire import Reply, Request, Wire, wire_server @@ -166,9 +165,9 @@ def send_chat(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) -def failure_row(call_id: str) -> dict[str, JsonValue]: +def failure_row(gateway: Gateway, call_id: str) -> dict[str, JsonValue]: rows: Final = eventually( - lambda: read_rows(FAILURE_ROW, (call_id,)), lambda found: len(found) == 1, seconds=ROW_SECONDS + lambda: gateway.rows(FAILURE_ROW, (call_id,)), lambda found: len(found) == 1, seconds=ROW_SECONDS ) assert rows[0]["status"] == "failure", rows return rows[0] @@ -204,8 +203,8 @@ def cooldown_entries(error: Mapping[str, JsonValue]) -> list[JsonValue] | None: return sorted(entries, key=lambda entry: str(object_value(entry)["model_id"])) -def assert_refusal_row(call_id: str, deployments: Mapping[str, str]) -> Mapping[str, JsonValue]: - row: Final = failure_row(call_id) +def assert_refusal_row(gateway: Gateway, call_id: str, deployments: Mapping[str, str]) -> Mapping[str, JsonValue]: + row: Final = failure_row(gateway, call_id) error: Final = object_value(row["error"]) assert (error["error_class"], error["error_code"]) == ("RouterRateLimitError", "429"), row assert cooldown_entries(error) == expected_cooldown(deployments), error.get("error_message") @@ -213,18 +212,18 @@ def assert_refusal_row(call_id: str, deployments: Mapping[str, str]) -> Mapping[ return error -def provider_row(call_id: str) -> tuple[str, str, str, JsonValue]: - row: Final = failure_row(call_id) +def provider_row(gateway: Gateway, call_id: str) -> tuple[str, str, str, JsonValue]: + row: Final = failure_row(gateway, call_id) error: Final = object_value(row["error"]) return str(row["model_id"]), str(row["api_base"]), str(error["error_code"]), error.get("error_cooldown_deployments") -def assert_provider_rows(history: tuple[Outcome, ...], deployments: Mapping[str, str]) -> None: +def assert_provider_rows(gateway: Gateway, history: tuple[Outcome, ...], deployments: Mapping[str, str]) -> None: provider: Final = history[:-1] assert all(outcome.status == 429 and NO_DEPLOYMENTS not in outcome.detail for outcome in provider), [ (o.status, o.detail[:200]) for o in provider ] - rows: Final = tuple(provider_row(outcome.call_id) for outcome in provider) + rows: Final = tuple(provider_row(gateway, outcome.call_id) for outcome in provider) assert {identity for identity, _, _, _ in rows} == set(deployments), rows assert all(base.startswith(deployments[identity]) for identity, base, _, _ in rows), rows assert all((code, cooldown) == ("429", None) for _, _, code, cooldown in rows), rows @@ -236,8 +235,8 @@ def test_all_cooled_refusal_row_records_429_and_every_cooled_deployment(rig: Rig history: Final = until_router_refusal(lambda: send(rig.gateway, path, rig.group(name), stream)) refusal: Final = history[-1] assert refusal.status == 429, (refusal.status, refusal.detail[:300]) - assert_refusal_row(refusal.call_id, rig.deployments(name)) - assert_provider_rows(history, rig.deployments(name)) + assert_refusal_row(rig.gateway, refusal.call_id, rig.deployments(name)) + assert_provider_rows(rig.gateway, history, rig.deployments(name)) def openai_chat(rig: Rig, group: str) -> Outcome: @@ -283,8 +282,8 @@ SDK_CLIENTS: Final = { @pytest.mark.parametrize("client", SDK_CELLS) def test_sdk_clients_get_429_and_the_row_names_the_cooled_deployments(rig: Rig, client: str) -> None: history: Final = until_router_refusal(lambda: SDK_CLIENTS[client](rig, rig.group(client))) - assert_refusal_row(history[-1].call_id, rig.deployments(client)) - assert_provider_rows(history, rig.deployments(client)) + assert_refusal_row(rig.gateway, history[-1].call_id, rig.deployments(client)) + assert_provider_rows(rig.gateway, history, rig.deployments(client)) def test_the_row_lists_only_the_requested_groups_deployments_while_another_group_cools(rig: Rig) -> None: @@ -297,7 +296,7 @@ def test_the_row_lists_only_the_requested_groups_deployments_while_another_group and all(identity in outcome.detail for identity in rig.deployments("neighbour")) ), ) - assert_refusal_row(refusal.call_id, rig.deployments("scoped")) + assert_refusal_row(rig.gateway, refusal.call_id, rig.deployments("scoped")) def test_a_concurrent_burst_on_a_cooled_group_writes_one_complete_row_per_refusal(rig: Rig) -> None: @@ -319,9 +318,9 @@ def test_a_concurrent_burst_on_a_cooled_group_writes_one_complete_row_per_refusa assert refusals, [(o.status, o.detail[:200]) for o in outcomes] assert len(tuple(request for request in rig.wire.drain() if not is_model_probe(request))) == len(reached) for outcome in refusals: - assert_refusal_row(outcome.call_id, rig.deployments("burst")) + assert_refusal_row(rig.gateway, outcome.call_id, rig.deployments("burst")) for outcome in reached: - identity, api_base, code, cooldown = provider_row(outcome.call_id) + identity, api_base, code, cooldown = provider_row(rig.gateway, outcome.call_id) assert api_base.startswith(rig.deployments("burst")[identity]), (identity, api_base) assert (code, cooldown) == ("429", None), outcome @@ -343,8 +342,8 @@ def test_a_tag_routed_refusal_row_names_only_the_cooled_deployments_its_tags_rea tenant: Final = f"{rig.group(TAGGED)}-a" cool_lane(rig, TAGGED, "a", TENANT_A) cool_lane(rig, TAGGED, "b", UNTAGGED) - assert_refusal_row(router_refusal(rig, TAGGED, TENANT_A).call_id, {tenant: deployments[tenant]}) - assert_refusal_row(router_refusal(rig, TAGGED, UNTAGGED).call_id, deployments) + assert_refusal_row(rig.gateway, router_refusal(rig, TAGGED, TENANT_A).call_id, {tenant: deployments[tenant]}) + assert_refusal_row(rig.gateway, router_refusal(rig, TAGGED, UNTAGGED).call_id, deployments) def test_a_region_restricted_refusal_row_names_only_the_cooled_deployments_in_its_region(rig: Rig) -> None: @@ -355,4 +354,6 @@ def test_a_region_restricted_refusal_row_names_only_the_cooled_deployments_in_it rig.gateway.post("/customer/new", {"user_id": customer, "allowed_model_region": "eu"}) cleanups.callback(rig.gateway.post, "/customer/delete", {"user_ids": [customer]}) cool_group(rig, REGIONAL) - assert_refusal_row(router_refusal(rig, REGIONAL, {"user": customer}).call_id, {european: deployments[european]}) + assert_refusal_row( + rig.gateway, router_refusal(rig, REGIONAL, {"user": customer}).call_id, {european: deployments[european]} + ) From 6c4c134f2333db4dd2cfac9c66237f7bde17d101 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:43:59 +0000 Subject: [PATCH 045/124] fix(secret_managers)!: resolve oidc/file and oidc/env refs in admin-configured api_key per request (#47341) * fix(secret_managers): resolve oidc/file and oidc/env refs in admin-configured api_key per request An api_key set to oidc/file/ or oidc/env/ in a credential_list entry or a deployment's litellm_params was sent upstream as literal text, so every call answered 401. The SDK client wrapper now reads the referenced file or env var on each call and never writes the token back, so a rotated token is used on the next call. The existing server-owned federation gates treat an oidc/ api_key as server-owned by value, so a request body, a non-admin credential or model write, user_config, fallback targets and routing tier params can never carry or name one. Other oidc/ kinds answer 400 * fix(secret_managers): strip only the leading oidc/ from a secret reference A token file under a directory named oidc resolved to the wrong path, because every oidc/ in the reference was removed * fix(auth): refuse oidc/ api_key refs in advisor tools and scope the route gates to them The advisor tool's own api_key now answers a 400 when it holds an oidc/ reference, since its sub-call runs through the client wrapper that resolves one. Stored fallback targets, auto-router tiers and user_config refuse only an oidc/ api_key, inline or through the credential they name, so their workload identity federation handling stays as it was. Provider model discovery for a wildcard deployment resolves the reference before calling the provider. * fix(auth): allow admin oidc api_key on deployment writes, resolve refs on discovery, batch probe and realtime health, refuse client api_base on oidc deployments * fix(utils): resolve oidc api_key refs only from admin-configured sources Mark an oidc/ api_key read from a proxy-level deployment or a stored credential, and resolve only that marked value. Team deployments, prompt templates, nested configs and a direct SDK api_key send the text as written. An unsupported reference kind raises a 401 so the router fails over to the group's next deployment. * fix(utils): send a credential's oidc token only to its own api_base host A stored credential's oidc/ api_key now resolves only for the scheme, host and port of the credential's own api_base or base_url, so a team model or vector store that names the credential with another api_base gets a 401 and that host gets nothing. Saving a team model with an inline oidc/ api_key answers 400, since team rows never resolve one. * fix(proxy): send an admin oidc api_key only to the hosts configured beside it Every admin oidc/ reference now carries the origins of the host params set beside it, checked for api_base, base_url and the Bedrock, S3, SageMaker and deployment_url params. Token counting reads the reference only when it calls the provider, answering 401 when it cannot, and pass-through 401s redact internal paths. * fix(proxy): redact the token path from token count auth errors * test(security): cover token count, Bedrock endpoint, api_base update and pass-through path redaction for oidc api_key refs * fix(proxy): redact the token path from secret reference errors where they are raised * fix(proxy): keep the secret reference's path out of the error a caller sees The 401 for an unreadable oidc api_key or pass-through credential now names only the reference kind and a fixed reason, so a token path under /run/secrets or a custom credential dir never reaches the caller, while the server log keeps the full reference for the operator * fix(secret_managers): strip line breaks from the logged secret reference A reference holding a newline can no longer forge a server log line * fix(secret_managers): label an unknown oidc reference kind instead of echoing it An api_key such as oidc/env_MY_VAR (a typo for oidc/env/MY_VAR) came back in full in the caller's 401 and in the spend log, since the kind was whatever text followed oidc/. A kind get_secret does not know now reads oidc/. The relative-path hint uses oidc/file//, which the client-message redaction on chat routes leaves whole. * test(health): run the real model auth check in the oidc test connection test * test(secret_managers): read an oidc/file token stored under a directory named oidc * fix(secret_managers): count a URL in the model name as an upstream host for oidc api_key references A huggingface/* wildcard call could name its own endpoint URL as the model and receive the resolved token. The origin check now treats a URL in the model as a host the reference must allow. The integration contract also reads the proxy's own database instead of the pytest worker's. * fix(secret_managers): treat a default port as explicit when matching an oidc api_key origin A credential on https://api.mistral.ai refused a call to https://api.mistral.ai:443, though both reach the same server. The origin check now compares scheme, host and effective port, as assert_same_origin does, and an unparseable api_base is refused instead of raising. The advisor test sets the real proxy flag instead of replacing our own helpers. * test: type the oidc request-body helpers and drive the tier test through mock_response * fix(secret_managers): refuse an oidc api_key reference whose call drops the configured host --- .../litellm_core_utils/credential_accessor.py | 27 +- .../messages/interceptors/advisor.py | 3 + litellm/proxy/auth/auth_utils.py | 67 +- .../litellm_executed_batches.py | 7 +- .../common_utils/credential_hydration.py | 20 +- .../proxy/credential_endpoints/endpoints.py | 11 +- .../health_endpoints/_health_endpoints.py | 20 +- litellm/proxy/litellm_pre_call_utils.py | 5 + .../model_management_endpoints.py | 34 +- .../passthrough_endpoint_router.py | 31 +- litellm/proxy/proxy_server.py | 37 +- litellm/realtime_api/main.py | 13 +- litellm/router.py | 16 +- .../clientside_credential_handler.py | 12 + litellm/router_utils/common_utils.py | 20 +- .../router_utils/fallback_event_handlers.py | 2 + litellm/secret_managers/main.py | 58 +- litellm/types/router.py | 46 +- litellm/types/utils.py | 106 +++ litellm/utils.py | 131 ++- litellm/vector_store_files/main.py | 3 +- litellm/vector_stores/main.py | 3 +- .../security/test_api_key_secret_reference.py | 783 ++++++++++++++++++ .../messages/test_advisor_orchestration.py | 18 +- tests/unit/proxy/auth/test_auth_utils.py | 221 +++++ .../test_litellm_executed_batches.py | 33 + .../common_utils/test_credential_hydration.py | 51 +- .../credential_endpoints/test_endpoints.py | 50 ++ .../health_endpoints/test_health_endpoints.py | 68 ++ .../test_model_management_endpoints.py | 94 +++ .../test_passthrough_endpoint_router.py | 12 +- .../unit/proxy/test_litellm_pre_call_utils.py | 70 ++ tests/unit/proxy/test_proxy_token_counter.py | 70 ++ tests/unit/realtime_api/test_main.py | 23 + .../router_strategy/test_complexity_router.py | 42 + .../test_fallback_event_handlers.py | 37 + .../test_secret_managers_main.py | 29 +- .../unit/test_router_model_cost_isolation.py | 30 + tests/unit/test_utils.py | 550 ++++++++++++ tests/unit/types/test_router.py | 18 + .../test_vector_store_registry.py | 120 +++ 41 files changed, 2901 insertions(+), 90 deletions(-) create mode 100644 tests/integration/security/test_api_key_secret_reference.py diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py index 95e36a64ef7..b2310894c3a 100644 --- a/litellm/litellm_core_utils/credential_accessor.py +++ b/litellm/litellm_core_utils/credential_accessor.py @@ -1,9 +1,15 @@ """Utils for accessing credentials.""" +from collections.abc import Mapping from typing import Final import litellm -from litellm.types.utils import CredentialItem +from litellm.types.router import holds_api_key_secret_reference +from litellm.types.utils import ( + API_KEY_SECRET_REFERENCE_REJECTION, + CredentialItem, + credential_api_key_secret_reference, +) class CredentialAccessor: @@ -21,7 +27,7 @@ class CredentialAccessor: """Safe accessor for credentials.""" credential: Final = CredentialAccessor.find_credential(credential_name) - return {} if credential is None else credential.credential_values.copy() + return {} if credential is None else dict(credential_api_key_secret_reference(credential.credential_values)) @staticmethod def upsert_credentials(credentials: list[CredentialItem]): @@ -38,3 +44,20 @@ class CredentialAccessor: break else: litellm.credential_list.append(credential) + + +def sends_api_key_secret_reference(params: Mapping[str, object]) -> bool: + named: Final = params.get("litellm_credential_name") + credential_values: Final = CredentialAccessor.get_credential_values(named) if isinstance(named, str) else {} + return holds_api_key_secret_reference({**credential_values, **params}) + + +def reject_api_key_secret_reference(params: Mapping[str, object]) -> None: + if holds_api_key_secret_reference(params): + raise ValueError(API_KEY_SECRET_REFERENCE_REJECTION) + named: Final = params.get("litellm_credential_name") + if isinstance(named, str) and holds_api_key_secret_reference(CredentialAccessor.get_credential_values(named)): + raise ValueError( + f"Rejected Request: litellm_credential_name={named!r} names a credential holding an oidc/ api_key " + "secret reference, which only a deployment a proxy admin configures can attach." + ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py index 0f833ccb4b8..03b27201398 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py @@ -27,6 +27,7 @@ from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +from litellm.types.utils import API_KEY_SECRET_REFERENCE_REJECTION, OIDC_SECRET_REFERENCE_PREFIX if TYPE_CHECKING: from litellm.router import Router @@ -226,6 +227,8 @@ def _resolve_advisor_credentials(advisor_tool: dict) -> tuple[str | None, str | return None, None api_key: Final[str | None] = advisor_tool.get("api_key") api_base: Final[str | None] = advisor_tool.get("api_base") + if isinstance(api_key, str) and api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX): + raise litellm.BadRequestError(message=API_KEY_SECRET_REFERENCE_REJECTION, model=None, llm_provider=None) if api_base is None: return api_key, None if not api_key: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 94cf7ff4dbe..7a05121ff2b 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -21,7 +21,7 @@ from litellm.constants import ( MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import ( SSRFError, @@ -36,7 +36,12 @@ from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_me from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, ) -from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, Deployment, server_owned_wif_fields_present +from litellm.types.router import ( + CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, + Deployment, + reject_server_owned_wif_fields, + server_owned_wif_fields_present, +) from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params @@ -246,16 +251,19 @@ reject_server_owned_wif_params = _reject_server_owned_wif_params _CREDENTIAL_VALUES: Final = TypeAdapter(dict[str, object]) _DECODED_JSON: Final = TypeAdapter(object) +_LIST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) def reject_federated_credential_reference(body: Mapping[str, object]) -> None: - """Raise ``ValueError`` if a request body picks the federated identity by credential name. + """Raise ``ValueError`` if a request body picks a server-owned identity by credential name. ``load_credentials_from_list`` merges a named credential's values into the call, so a body naming a federated credential moves the token exchange onto that credential's federation rule and organization exactly as sending the federation fields inline would, which - ``reject_server_owned_wif_params`` already refuses. Attaching one is a deployment decision, so - it is refused with no client-side opt-in to relax it, the same as the inline form. + ``reject_server_owned_wif_params`` already refuses, and a body naming one whose ``api_key`` is + an ``oidc/`` secret reference gets the proxy to read that secret for a call it shapes. Attaching + either is a deployment decision, so it is refused with no client-side opt-in to relax it, the + same as the inline form. Only credentials already loaded into memory can be resolved here, which is every credential the proxy would resolve for the call itself: ``load_credentials_from_list`` reads the same list. @@ -271,8 +279,8 @@ def reject_federated_credential_reference(body: Mapping[str, object]) -> None: if wif_fields: raise ValueError( f"Rejected Request: litellm_credential_name={named!r} names a credential configured for " - f"workload identity federation or OAuth token exchange ({wif_fields[0]}), which a request body " - "cannot choose. " + f"workload identity federation or OAuth token exchange, or holding an oidc/ api_key secret " + f"reference ({wif_fields[0]}), which a request body cannot choose. " "A proxy admin attaches it to a deployment." ) @@ -685,19 +693,52 @@ def is_request_body_safe( _reject_client_chosen_federation(user_config_params, manages_deployments=manages_deployments) litellm_params: Final = _coerce_metadata_to_dict(request_body.get("litellm_params")) if litellm_params is not None: - _reject_client_chosen_federation(litellm_params, manages_deployments=manages_deployments) + _reject_request_credential_params(litellm_params, manages_deployments=manages_deployments) litellm_params_metadata: Final = _coerce_metadata_to_dict(litellm_params.get("metadata")) if litellm_params_metadata is not None: _check_banned_params( - litellm_params_metadata, - general_settings, - llm_router, - model, - manages_deployments=manages_deployments, + litellm_params_metadata, general_settings, llm_router, model, manages_deployments=manages_deployments ) + for user_config_params in _user_config_litellm_params(request_body): # pyright: ignore[reportUnknownArgumentType] # request_body is an untyped dict + reject_api_key_secret_reference(user_config_params) return True +def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: + if manages_deployments: + reject_server_owned_wif_fields(litellm_params) + else: + reject_server_owned_wif_params(litellm_params) + reject_federated_credential_reference(litellm_params) + for nested_params in _nested_configs(litellm_params): + reject_api_key_secret_reference(nested_params) + + +def _user_config_litellm_params(request_body: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + user_config: Final = _coerce_metadata_to_dict(request_body.get("user_config")) + if user_config is None: + return () + entries: Final = _mappings_in(user_config.get("model_list")) + _mappings_in(user_config.get("search_tools")) + nested: Final = tuple( + litellm_params + for entry in entries + if (litellm_params := _coerce_metadata_to_dict(entry.get("litellm_params"))) is not None + ) + defaults: Final = _coerce_metadata_to_dict(user_config.get("default_litellm_params")) + return nested if defaults is None else (*nested, defaults) + + +def _nested_configs(params: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + nested for key in _NESTED_CONFIG_KEYS if (nested := _coerce_metadata_to_dict(params.get(key))) is not None + ) + + +def _mappings_in(value: object) -> tuple[Mapping[str, object], ...]: + entries: Final = _LIST_ENTRIES.validate_python(value) if isinstance(value, list) else () + return tuple(mapping for entry in entries if (mapping := _coerce_metadata_to_dict(entry)) is not None) + + def carries_client_side_overrides(request_body: object) -> bool: if not isinstance(request_body, dict): return False diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 984aed71b1c..e1a5fdd5a71 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -35,6 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders +from litellm.utils import resolve_api_key_secret_reference if TYPE_CHECKING: from prisma import types as prisma_types @@ -241,7 +242,11 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, ) except Exception: # noqa: BLE001 # get_llm_provider raises on a model it cannot map, which means nothing to probe return None - return None if resolved_api_base is None else (resolved_api_base, resolved_api_key) + if resolved_api_base is None: + return None + return resolved_api_base, resolve_api_key_secret_reference( + resolved_api_key, model=model, llm_provider=provider, upstream=credentials + ) async def litellm_executed_provider_for( diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index a4e858692e6..f6a6c682cf0 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -18,8 +18,8 @@ from litellm.repositories.credentials_repository import CredentialsRepository from litellm.router_utils.clientside_credential_handler import clientside_credential_keys from litellm.types.router import ( GenericLiteLLMParams, - server_owned_wif_fields_named, server_owned_wif_fields_present, + server_owned_wif_fields_written, ) from litellm.types.utils import CredentialItem, LlmProviders, server_owned_wif_litellm_params @@ -68,7 +68,7 @@ def decrypted_or_stored(key: str, value: str) -> str: return value if decrypted is None else decrypted -def _decrypted(db_credential: CredentialItem) -> CredentialItem: +def decrypted_credential(db_credential: CredentialItem) -> CredentialItem: """The stored credential with every value decrypted, leaving already-plaintext values alone.""" decrypted_values: Final = MappingProxyType( { @@ -99,7 +99,7 @@ async def hydrate_named_credential_authoritative( db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) if db_credential is None: return await hydrate_named_credential(credential_name, prisma_client) - return _decrypted(db_credential) + return decrypted_credential(db_credential) async def hydrate_named_credential( @@ -114,7 +114,7 @@ async def hydrate_named_credential( db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) if db_credential is None: return None - return _decrypted(db_credential) + return decrypted_credential(db_credential) async def named_credential_wif_fields( @@ -129,11 +129,17 @@ async def named_credential_wif_fields( refuses whenever either side says the credential is server-owned. """ matching: Final = tuple(c for c in litellm.credential_list if c.credential_name == credential_name) - in_memory: Final = tuple(chain.from_iterable(server_owned_wif_fields_named(c.credential_values) for c in matching)) + in_memory: Final = tuple( + chain.from_iterable(server_owned_wif_fields_written(c.credential_values) for c in matching) + ) if prisma_client is None: return in_memory db_credential: Final = await CredentialsRepository(prisma_client).find_by_name(credential_name) - stored: Final = () if db_credential is None else server_owned_wif_fields_named(db_credential.credential_values) + stored: Final = ( + () + if db_credential is None + else server_owned_wif_fields_written(decrypted_credential(db_credential).credential_values) + ) return tuple(dict.fromkeys(in_memory + stored)) @@ -165,7 +171,7 @@ async def effective_server_owned_wif_fields( touching the field. """ from_stored: Final = () if stored is None else server_owned_wif_fields_present(stored) - from_incoming: Final = () if incoming is None else server_owned_wif_fields_named(incoming.keys()) + from_incoming: Final = () if incoming is None else server_owned_wif_fields_written(incoming) from_credential: Final = tuple( chain.from_iterable( await asyncio.gather( diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index f51d71a4c20..953c944bfb2 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.credential_hydration import ( + decrypted_credential, hydrate_named_credential_authoritative, named_credential_wif_fields, stored_credential_provider, @@ -61,7 +62,7 @@ from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object from litellm.repositories.base_repository import is_unique_violation from litellm.repositories.credentials_repository import CredentialsRepository -from litellm.types.router import server_owned_wif_fields_named +from litellm.types.router import server_owned_wif_fields_named, server_owned_wif_fields_written from litellm.types.utils import CreateCredentialItem, CredentialItem from .user_provider_credentials import ( @@ -97,7 +98,7 @@ def _reject_non_admin_wif_fields( raise ProxyException( message=( f"Only proxy admins can change {wif_fields[0]!r}, a server-owned workload identity federation " - "or OAuth token exchange parameter." + "or OAuth token exchange parameter or oidc/ api_key secret reference." ), type=ProxyErrorTypes.auth_error.value, code=status.HTTP_403_FORBIDDEN, @@ -112,13 +113,13 @@ def _incoming_wif_fields(incoming_values: Mapping[str, object], credential: Upda names in ``credential_values_to_delete``, since dropping a federation field off the stored credential breaks every deployment referencing it just as installing one would redirect them. """ - return server_owned_wif_fields_named(incoming_values) + server_owned_wif_fields_named( + return server_owned_wif_fields_written(incoming_values) + server_owned_wif_fields_named( credential.credential_values_to_delete or () ) def _stored_wif_fields(stored_credential: CredentialItem) -> tuple[str, ...]: - return server_owned_wif_fields_named(stored_credential.credential_values) + return server_owned_wif_fields_written(decrypted_credential(stored_credential).credential_values) def _reject_overlapping_credential_values(credential: UpdateCredentialItem) -> None: @@ -317,7 +318,7 @@ async def create_credential( status_code=400, detail="Credential values are required. Unable to infer credential values from model ID.", ) - _reject_non_admin_wif_fields(server_owned_wif_fields_named(credential_values), user_api_key_dict) + _reject_non_admin_wif_fields(server_owned_wif_fields_written(credential_values), user_api_key_dict) _reject_non_admin_wif_fields( await named_credential_wif_fields(credential.credential_name, prisma_client), user_api_key_dict ) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 3c8dd7890fc..1e7f67392e1 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -8,6 +8,7 @@ import time import traceback from collections.abc import Iterable, Mapping, Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final, Literal, TypedDict import fastapi @@ -42,7 +43,6 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check - reject_server_owned_wif_params, ) from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -80,6 +80,8 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields +from litellm.types.router import reject_server_owned_wif_fields +from litellm.types.utils import AdminApiKeySecretReference, admin_api_key_secret_reference #### Health ENDPOINTS #### @@ -161,6 +163,16 @@ def _request_inherits_config_credentials( return not any(param in request_params for param in _BANNED_REQUEST_BODY_PARAMS) +def _admin_api_key_from_request( + request_litellm_params: Mapping[str, object], call_params: Mapping[str, object] +) -> Mapping[str, object]: + """Only reached once the model auth check passed, which refuses an ``oidc/`` api_key from anyone but a proxy + admin.""" + marked: Final = admin_api_key_secret_reference({**call_params, "api_key": request_litellm_params.get("api_key")}) + api_key: Final = marked.get("api_key") + return MappingProxyType({"api_key": api_key} if isinstance(api_key, AdminApiKeySecretReference) else {}) + + def _config_base_for_health_check( config_params: Mapping[str, object], request_params: Mapping[str, object], @@ -2291,7 +2303,7 @@ async def test_model_connection( ) selected_by_id: Final = deployment_by_id is not None - reject_server_owned_wif_params(request_litellm_params) + reject_server_owned_wif_fields(request_litellm_params) # Merge: config params (from proxy config) as base, request params override litellm_params = { **_config_base_for_health_check( @@ -2341,6 +2353,10 @@ async def test_model_connection( ) health_check_params: Final = { **_OBJECT_MAPPING.validate_python(litellm_params), + **_admin_api_key_from_request( + _OBJECT_MAPPING.validate_python(request_litellm_params), + _OBJECT_MAPPING.validate_python(litellm_params), + ), "secret_fields": SecretFields(raw_headers=RedactedDict(raw_headers)), } diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index dbeda4e27e3..54882288b49 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -254,8 +254,10 @@ from litellm.router import Router from litellm.secret_managers.main import get_secret_bool from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS +from litellm.types.router import holds_api_key_secret_reference from litellm.types.services import ServiceTypes from litellm.types.utils import ( + API_KEY_SECRET_REFERENCE_REJECTION, CAPTURE_MESSAGE_CONTENT_VAR, EXCLUDED_SERVICES_VAR, CustomPricingLiteLLMParams, @@ -2735,6 +2737,9 @@ async def add_litellm_data_to_request( ), ) + if holds_api_key_secret_reference(data): # pyright: ignore[reportUnknownArgumentType] # data is an untyped dict + raise HTTPException(status_code=400, detail=API_KEY_SECRET_REFERENCE_REJECTION) + ## ENFORCED PARAMS CHECK # loop through each enforced param # example enforced_params ['user', 'metadata', 'metadata.generation_name'] diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7eae193dd8a..fc0868a5fc9 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -157,6 +157,7 @@ from litellm.types.router import ( ) from litellm.types.utils import ( COST_MAP_LOOKUP_KEY, + OIDC_SECRET_REFERENCE_PREFIX, echoed_cost_map_fields, echoed_cost_map_pricing_fields, without_server_derived_pricing, @@ -2298,7 +2299,8 @@ class ModelManagementAuthChecks: raise ProxyException( message=( f"Only proxy admins can change the credentials of a deployment configured for " - f"workload identity federation or OAuth token exchange ({wif_fields[0]!r})." + f"workload identity federation or OAuth token exchange, or holding an oidc/ api_key secret " + f"reference ({wif_fields[0]!r})." ), type=ProxyErrorTypes.auth_error.value, code=status.HTTP_403_FORBIDDEN, @@ -2315,6 +2317,33 @@ class ModelManagementAuthChecks: param="litellm_credential_name", ) + @staticmethod + def _reject_team_inline_api_key_secret_reference( + *, + model_params: Deployment, + incoming_params: Mapping[str, object] | None, + incoming_model_params: updateDeployment | None, + ) -> None: + api_key: Final = None if incoming_params is None else incoming_params.get("api_key") + incoming_model_info: Final = None if incoming_model_params is None else incoming_model_params.model_info + team_id: Final = ( + incoming_model_info.team_id + if incoming_model_info is not None and incoming_model_info.team_id is not None + else model_params.model_info.team_id + ) + if team_id is None or not isinstance(api_key, str) or not api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX): + return + raise ProxyException( + message=( + "A team model cannot hold an oidc/ api_key secret reference, because LiteLLM resolves one only on a " + "proxy-level model or in a stored credential. Store the reference in a credential and set " + "litellm_credential_name on this model." + ), + type=ProxyErrorTypes.bad_request_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="api_key", + ) + @staticmethod async def can_user_make_model_call( model_params: Deployment, @@ -2332,6 +2361,9 @@ class ModelManagementAuthChecks: LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, ): raise HTTPException(status_code=403, detail="View-only users cannot manage models.") + ModelManagementAuthChecks._reject_team_inline_api_key_secret_reference( + model_params=model_params, incoming_params=incoming_params, incoming_model_params=incoming_model_params + ) # Federation fields choose which server-side secret is read and where the org-scoped token # it buys is sent, so only a proxy admin may point a federated deployment somewhere else. # Evaluated on the RESULTING deployment: a patch attaching a credential by name inherits diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 3d1be882779..270a1cc2f3b 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -10,7 +10,11 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor -from litellm.secret_managers.main import get_secret_str +from litellm.secret_managers.main import ( + UnresolvableSecretReference, + get_secret_str, + resolve_local_oidc_reference, +) from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict @@ -39,22 +43,17 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | return credentials -# Network I/O prohibited; get_credentials() is synchronous inside async routes. -# oidc/env_path/ is excluded because it bypasses the oidc/file/ credential-directory allowlist. -_LOCAL_OIDC_PREFIXES: Final = ("oidc/file/", "oidc/env/") - - def _resolve_oidc_reference(credential: str | None) -> str | None: - """Resolve local ``oidc/...`` credentials on every call to pick up rotated tokens.""" - if credential is None or not credential.startswith(_LOCAL_OIDC_PREFIXES): - return credential - try: - return get_secret_str(credential) - except (OSError, ValueError) as exc: - raise HTTPException( - status_code=401, - detail=f"pass-through credential {credential} could not be resolved: {exc}", - ) from exc + if credential is None: + return None + match resolve_local_oidc_reference(credential): + case UnresolvableSecretReference(kind=kind, reason=reason): + raise HTTPException( + status_code=401, + detail=f"pass-through credential {kind} secret reference could not be resolved: {reason}", + ) + case str() as resolved: + return resolved class PassthroughEndpointRouter: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5030b1512a6..689a5f38416 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -202,7 +202,7 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import cost_map_omits_token_price, load_credentials_from_list +from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential if TYPE_CHECKING: from aiohttp import ClientSession @@ -302,7 +302,11 @@ import litellm import litellm._redis from litellm import Router from litellm._internal_context import service_target, with_service_target -from litellm._logging import redact_string, verbose_proxy_logger, verbose_router_logger +from litellm._logging import ( + redact_string, + verbose_proxy_logger, + verbose_router_logger, +) from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import DeclaredBatchRead from litellm.caching.redis_batch import ( @@ -1350,6 +1354,7 @@ class _WebsearchInterceptionLitellmSettings(TypedDict, total=False): _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody) _LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody) _ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig) +_LITELLM_PARAMS_COPY: Final = TypeAdapter(dict[str, object]) _WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings) _UNTYPED_VALUE: Final = TypeAdapter(object) @@ -14712,6 +14717,20 @@ def _get_provider_token_counter( return None, None, None +def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: + call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params) + try: + load_credentials_from_list(call_params) + except litellm.AuthenticationError as e: + raise ProxyException( + message=e.message, + type="auth_error", + param="model", + code=401, + ) from e + return call_params + + _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -14771,7 +14790,12 @@ async def _try_provider_token_count( model_to_use=model_to_use or "", messages=messages, contents=contents, - deployment=deployment_in_region, + deployment=None + if deployment_in_region is None + else { + **deployment_in_region, + "litellm_params": _with_resolved_api_key(deployment_in_region.get("litellm_params", {})), + }, request_model=request_model, tools=tools, system=system, @@ -15207,7 +15231,12 @@ async def count_request_tokens( if deployment is not None: litellm_model_name = deployment.get("litellm_params", {}).get("model") model_info = deployment.get("model_info", {}) - load_credentials_from_list(deployment.get("litellm_params", {})) + deployment = { # rebind-ok: merge the named credential into a copy + **deployment, + "litellm_params": with_named_credential( + _LITELLM_PARAMS_COPY.validate_python(deployment.get("litellm_params", {})) + ), + } # remove the custom_llm_provider_prefix in the litellm_model_name if "/" in litellm_model_name: litellm_model_name = litellm_model_name.split("/", 1)[1] diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index ea47d744cae..088dfb54155 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -799,12 +799,17 @@ async def realtime_health_check( import websockets resolved_params: Final = _model_params_with_stored_credentials(model_params or _EMPTY_MODEL_PARAMS) - resolved_api_key: Final = cast( # cast-ok: provider parameters expose optional string credentials - str | None, api_key or resolved_params.get("api_key") - ) resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints str | None, api_base or resolved_params.get("api_base") ) + resolved_api_key: Final = resolve_api_key_secret_reference( + cast( # cast-ok: provider parameters expose optional string credentials + str | None, api_key or resolved_params.get("api_key") + ), + model=model, + llm_provider=custom_llm_provider, + upstream=MappingProxyType({**resolved_params, "api_base": resolved_api_base}), + ) resolved_api_version: Final = cast( # cast-ok: provider parameters expose optional string versions str | None, api_version or resolved_params.get("api_version") ) diff --git a/litellm/router.py b/litellm/router.py index d8da69f7789..5a34a00e204 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_parent_otel_span_from_kwargs, ) from litellm.litellm_core_utils.coroutine_checker import coroutine_checker -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, @@ -214,6 +214,7 @@ from litellm.router_utils.common_utils import ( truncate_fallback_error_detail, warn_on_ignored_model_capability_flags, warn_on_provider_credential_mismatch, + with_admin_api_key_secret_reference, without_model_capability_flags, without_router_only_kwargs, ) @@ -413,6 +414,7 @@ from litellm.utils import ( is_generalized_model_info, is_region_allowed, provider_rejectable_params, + resolve_api_key_secret_reference, set_live_deployment_replay, ) @@ -10073,7 +10075,7 @@ class Router: f"Invalid regex in tag_regex for model '{deployment.model_name}': {pattern!r} — {exc}" ) from exc - deployment = self._add_deployment(deployment=deployment) + deployment = self._add_deployment(deployment=with_admin_api_key_secret_reference(deployment)) model: Final = deployment.to_json(exclude_none=True) @@ -10841,7 +10843,7 @@ class Router: declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) - served: Final = without_model_capability_flags(deployment) + served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) # add to model list _deployment: Final = served.to_json(exclude_none=True) @@ -11414,7 +11416,12 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return - api_key: Final = params.api_key or dynamic_api_key + api_key: Final = resolve_api_key_secret_reference( + params.api_key or dynamic_api_key, + model=model, + llm_provider=provider, + upstream=MappingProxyType({"api_base": params.api_base}), + ) headers: Final = TypeAdapter(Mapping[str, str]).validate_python( params.get("extra_headers") or params.get("headers") or MappingProxyType({}) ) @@ -14604,6 +14611,7 @@ class Router: back on a routing pass that pins none.""" if not tier_litellm_params: return False + reject_api_key_secret_reference(tier_litellm_params) accepted_tier_params: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs) surface_tier_params: Final = self._tier_ceiling_under_the_surface_name( accepted_tier_params, responses_call=responses_call diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index 65980449937..2bd58405693 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -14,6 +14,8 @@ Ensures cooldowns are applied correctly. from collections.abc import MutableMapping from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.credential_accessor import sends_api_key_secret_reference from litellm.types.utils import server_owned_wif_litellm_params clientside_credential_keys: Final = ["api_key", "api_base", "base_url"] @@ -115,6 +117,16 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di if "api_base" in request_kwargs or "base_url" in request_kwargs: from litellm.llms.github_copilot.per_user_auth import github_copilot_auth_mode + if sends_api_key_secret_reference(litellm_params): # pyright: ignore[reportUnknownArgumentType] # litellm_params is the untyped request dict + raise BadRequestError( + message=( + "Rejected Request: this deployment's api_key is an oidc/ secret reference, so a request cannot " + "send it to an api_base of its own. Send your own api_key with the api_base." + ), + model=None, + llm_provider=None, + ) + # A per-user OAuth credential must survive the clear: dropping # litellm_credential_name / github_copilot_auth_type would flip the call to # shared mode and send the admin's Copilot token to the redirected host. diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 9b9a3d185cc..caab6e572de 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -23,7 +23,12 @@ from litellm.types.router import ( LiteLLM_Params, SkippedFallbackTarget, ) -from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo +from litellm.types.utils import ( + AdminApiKeySecretReference, + LlmProviders, + ProviderSpecificModelInfo, + admin_api_key_secret_reference, +) _V = TypeVar("_V") @@ -32,6 +37,19 @@ ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_er _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment: + if deployment.model_info.get("team_id") is not None or isinstance( + deployment.litellm_params.api_key, AdminApiKeySecretReference + ): + return deployment + params: Final = admin_api_key_secret_reference(deployment.litellm_params.model_dump(exclude_none=True)) + if not isinstance(params.get("api_key"), AdminApiKeySecretReference): + return deployment + return deployment.model_copy( + update={"litellm_params": deployment.litellm_params.model_copy(update={"api_key": params["api_key"]})} + ) + + def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index bc870572f38..e1201df1a13 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy +from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( @@ -816,6 +817,7 @@ async def run_async_fallback( for target in fallback_model_group: if isinstance(target, dict): reject_server_owned_wif_params(target) + reject_api_key_secret_reference(target) for mg in fallback_model_group: if mg == failed_model_group: diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index d12e200772f..f4234e26c60 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -4,6 +4,7 @@ import os import time import traceback from collections.abc import Callable +from dataclasses import dataclass from functools import lru_cache from typing import Final @@ -89,7 +90,7 @@ def _resolve_oidc_file_path(requested_path: str) -> str: if not os.path.isabs(requested_path): raise OidcPathNotAllowedError( "oidc/file path must be absolute. Use the format " - "'oidc/file//var/run/secrets/' (note the leading slash " + "'oidc/file//' (note the leading slash " "after 'oidc/file/')." ) resolved: Final = os.path.realpath(requested_path) @@ -164,6 +165,59 @@ def get_secret_str( return value +LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/") + + +@dataclass(frozen=True, slots=True) +class UnresolvableSecretReference: + kind: str + reason: str + + +_OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"}) + + +def oidc_reference_kind(reference: str) -> str: + provider: Final = reference.removeprefix("oidc/").split("/", 1)[0] + return f"oidc/{provider}" if provider in _OIDC_PROVIDERS else "oidc/" + + +def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReference: + """Only local kinds resolve, since request paths must not do network I/O, and ``oidc/env_path/`` is + excluded because it bypasses the ``oidc/file/`` credential-directory allowlist.""" + if not value.startswith(LOCAL_OIDC_SECRET_PREFIXES): + return value + kind: Final = oidc_reference_kind(value) + try: + secret: Final = get_secret_str(value) + except (OSError, ValueError) as exc: + return _logged_unresolvable(value, kind, _unresolvable_reason(kind, exc)) + if not secret: + return _logged_unresolvable(value, kind, "it resolved to an empty value") + return secret + + +def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference: + verbose_logger.warning( + "Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason + ) + return UnresolvableSecretReference(kind=kind, reason=reason) + + +def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str: + match exc: + case UnicodeDecodeError(): + return "the file is not UTF-8 text" + case OidcPathNotAllowedError(): + return str(exc) + case OSError(strerror=str() as strerror): + return strerror + case _ if kind == "oidc/env": + return "the environment variable is not set" + case _: + return "the file could not be read" + + def normalize_nonempty_secret_str(val: str | None) -> str | None: """ Strip whitespace and treat None, '', and whitespace-only strings as unset. @@ -213,7 +267,7 @@ def get_secret( # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke if secret_name.startswith("oidc/"): - secret_name_split: Final = secret_name.replace("oidc/", "") + secret_name_split: Final = secret_name.removeprefix("oidc/") oidc_provider, oidc_aud = secret_name_split.split("/", 1) oidc_aud = "/".join(secret_name_split.split("/")[1:]) # TODO: Add caching for HTTP requests diff --git a/litellm/types/router.py b/litellm/types/router.py index f83127806db..36d298b2bc2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -42,6 +42,16 @@ from .litellm_params import RoutingStrategyName from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider +from .utils import ( + # private alias: `from .types.router import *` would rebind a public Final in litellm/__init__.py + API_KEY_SECRET_REFERENCE_REJECTION as _API_KEY_SECRET_REFERENCE_REJECTION, +) +from .utils import ( + OIDC_SECRET_REFERENCE_PREFIX as _OIDC_SECRET_REFERENCE_PREFIX, +) +from .utils import ( + AdminApiKeySecretReference as _AdminApiKeySecretReference, +) from .utils import ( CooldownDeploymentInfo, CustomPricingLiteLLMParams, @@ -50,7 +60,6 @@ from .utils import ( StandardLoggingRoutingDecision, ) from .utils import ( - # private alias: `from .types.router import *` would rebind a public Final in litellm/__init__.py server_owned_wif_litellm_params as _server_owned_wif_litellm_params, ) @@ -362,7 +371,7 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): def __init__(self, /, **data: object) -> None: ... # kwargs-ok: credential fields vary by provider - api_key: str | None = None + api_key: _AdminApiKeySecretReference | str | None = None api_base: str | None = None api_version: str | None = None ## AZURE OAUTH ## @@ -451,15 +460,29 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): token_exchange_audience: str | None = None +def _api_key_secret_reference(fields: Mapping[str, object]) -> tuple[str, ...]: + """The call path reads an ``oidc/`` api_key reference from the proxy's own filesystem or environment + and sends the result upstream, so it is server-owned the same way a federation field is.""" + api_key: Final = fields.get("api_key") + return ("api_key",) if isinstance(api_key, str) and api_key.startswith(_OIDC_SECRET_REFERENCE_PREFIX) else () + + +def holds_api_key_secret_reference(fields: Mapping[str, object]) -> bool: + return bool(_api_key_secret_reference(fields)) + + def server_owned_wif_fields_present(fields: Mapping[str, object]) -> tuple[str, ...]: - """Server-owned federation or OAuth token-exchange field names set in ``fields``. + """Server-owned federation or OAuth token-exchange field names set in ``fields``, plus ``api_key`` + when it holds an ``oidc/`` secret reference. ``fields`` is a ``litellm_params`` dict (or a credential's ``credential_values`` mapping, which feeds the same resolution when referenced by name). Derived from ``server_owned_wif_litellm_params`` rather than hand-copied, so a persistence gate built on this stays correct when a server-owned field is added there. """ - return tuple(name for name in _server_owned_wif_litellm_params if fields.get(name) is not None) + return tuple( + name for name in _server_owned_wif_litellm_params if fields.get(name) is not None + ) + _api_key_secret_reference(fields) def server_owned_wif_fields_named(keys: Container[str]) -> tuple[str, ...]: @@ -475,6 +498,10 @@ def server_owned_wif_fields_named(keys: Container[str]) -> tuple[str, ...]: return tuple(name for name in _server_owned_wif_litellm_params if name in keys) +def server_owned_wif_fields_written(values: Mapping[str, object]) -> tuple[str, ...]: + return server_owned_wif_fields_named(values) + _api_key_secret_reference(values) + + _WIF_POINTER_FIELDS: Final = frozenset(name for name in _server_owned_wif_litellm_params if name.endswith("_ref")) @@ -1404,7 +1431,8 @@ class AdaptiveRouterPreferences(LiteLLMBaseModel): def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: """Raise ``ValueError`` if a mapping that did not come from deployment config carries a - server-owned workload identity federation or OAuth token-exchange field. + server-owned workload identity federation or OAuth token-exchange field, or an ``api_key`` + holding an ``oidc/`` secret reference. These are never settable inline on a client surface, with or without a client-side credential opt-in. Naming a stored credential that already holds them is the other way in and has its own @@ -1413,6 +1441,14 @@ def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: This lives here rather than under ``litellm.proxy`` so the router can call it on a post-authentication merge without core importing from the proxy package. """ + reject_server_owned_wif_fields(body) + if holds_api_key_secret_reference(body): + raise ValueError(_API_KEY_SECRET_REFERENCE_REJECTION) + + +def reject_server_owned_wif_fields(body: Mapping[str, object]) -> None: + """``reject_server_owned_wif_params`` without the ``oidc/`` api_key check, for the deployment writes whose + role gate already refuses a non-admin setting one.""" submitted_names: Final = frozenset(key.removesuffix("[]") for key in body) for param in _server_owned_wif_litellm_params: if param in submitted_names: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 41581d702f3..99788dad4e1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -13,6 +13,7 @@ from typing import ( TypeAlias, get_args, ) +from urllib.parse import urlsplit import httpx from openai._models import BaseModel as OpenAIObject @@ -39,6 +40,7 @@ from pydantic import ( ConfigDict, Field, FieldSerializationInfo, + GetCoreSchemaHandler, JsonValue, PrivateAttr, SkipValidation, @@ -46,6 +48,7 @@ from pydantic import ( field_validator, ) from pydantic.main import IncEx +from pydantic_core import core_schema from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -4186,6 +4189,109 @@ oauth_token_exchange_litellm_params: Final = ( "token_exchange_scope", ) github_copilot_oauth_litellm_params: Final = ("github_copilot_auth_type",) +OIDC_SECRET_REFERENCE_PREFIX: Final = "oidc/" +API_KEY_SECRET_REFERENCE_REJECTION: Final = ( + "Rejected Request: api_key holds an oidc/ secret reference, which only a deployment or credential a proxy " + "admin configures can carry." +) + + +API_KEY_UPSTREAM_PARAMS: Final = ( + "api_base", + "base_url", + "aws_bedrock_runtime_endpoint", + "s3_endpoint_url", + "sagemaker_base_url", + "deployment_url", +) + + +_DEFAULT_PORTS: Final = MappingProxyType({"http": 80, "ws": 80, "https": 443, "wss": 443}) + + +def _origin(url: str) -> str: + text: Final = url.strip() + try: + parts: Final = urlsplit(text) + explicit_port: Final = parts.port + except ValueError: + return text.rstrip("/").lower() + host: Final = (parts.hostname or "").rstrip(".") + if not host: + return text.rstrip("/").lower() + scheme: Final = parts.scheme.lower() + port: Final = explicit_port if explicit_port is not None else _DEFAULT_PORTS.get(scheme) + return f"{scheme}://{host}" if port is None else f"{scheme}://{host}:{port}" + + +_URL_IN_MODEL: Final = re.compile(r"https?://\S+", re.IGNORECASE) + + +def _model_url(model: object) -> str | None: + found: Final = _URL_IN_MODEL.search(model) if isinstance(model, str) else None + return None if found is None else found.group(0) + + +def upstream_origins(params: Mapping[str, object]) -> frozenset[str]: + values: Final = (*(params.get(key) for key in API_KEY_UPSTREAM_PARAMS), _model_url(params.get("model"))) + return frozenset(_origin(value) for value in values if isinstance(value, str) and value) + + +class AdminApiKeySecretReference(str): + """An ``oidc/`` api_key reference read from a source only a proxy admin writes: a proxy-level deployment or a + stored credential. The call path swaps only this type for the secret it points to, so the same text arriving + in a request, a prompt template, or a row a non-admin wrote is sent as written, and the secret goes only to the + origins the admin configured beside the reference.""" + + def __new__(cls, reference: str, origins: frozenset[str]) -> "AdminApiKeySecretReference": + return super().__new__(cls, reference) + + def __init__(self, reference: str, origins: frozenset[str]) -> None: + super().__init__() + self.origins: Final = origins + + def __reduce__(self) -> tuple[type["AdminApiKeySecretReference"], tuple[str, frozenset[str]]]: + return type(self), (str(self), self.origins) + + def may_reach(self, upstream: Mapping[str, object]) -> bool: + destinations: Final = upstream_origins(upstream) + return destinations <= self.origins if destinations else not self.origins + + @classmethod + def __get_pydantic_core_schema__(cls, source: object, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + return core_schema.is_instance_schema(cls, serialization=core_schema.to_string_ser_schema()) + + +class CredentialApiKeySecretReference(AdminApiKeySecretReference): + """An admin ``oidc/`` api_key reference stored in a named credential, so its origins are the credential's own.""" + + +def _unmarked_api_key_secret_reference(params: Mapping[str, object]) -> str | None: + api_key: Final = params.get("api_key") + if ( + not isinstance(api_key, str) + or isinstance(api_key, AdminApiKeySecretReference) + or not api_key.startswith(OIDC_SECRET_REFERENCE_PREFIX) + ): + return None + return api_key + + +def admin_api_key_secret_reference(params: Mapping[str, object]) -> Mapping[str, object]: + api_key: Final = _unmarked_api_key_secret_reference(params) + if api_key is None: + return params + return {**params, "api_key": AdminApiKeySecretReference(api_key, upstream_origins(params))} + + +def credential_api_key_secret_reference(credential_values: Mapping[str, object]) -> Mapping[str, object]: + api_key: Final = _unmarked_api_key_secret_reference(credential_values) + if api_key is None: + return credential_values + reference: Final = CredentialApiKeySecretReference(api_key, upstream_origins(credential_values)) + return {**credential_values, "api_key": reference} + + server_owned_wif_litellm_params: Final = ( anthropic_wif_litellm_params + openai_wif_litellm_params diff --git a/litellm/utils.py b/litellm/utils.py index 2b7abae358b..59e0d2d8180 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -228,12 +228,15 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( ABOVE_THRESHOLD_COST_KEY_PATTERN, + API_KEY_UPSTREAM_PARAMS, OPENAI_RESPONSE_HEADERS, + AdminApiKeySecretReference, CallTypes, ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Choices, CostPerToken, + CredentialApiKeySecretReference, CredentialItem, CustomHuggingfaceTokenizer, Delta, @@ -261,6 +264,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + credential_api_key_secret_reference, is_litellm_owned_kwarg, ) @@ -457,7 +461,13 @@ from litellm.llms.base_llm.completion.transformation import BaseTextCompletionCo from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig -from litellm.secret_managers.main import get_secret +from litellm.secret_managers.main import ( + LOCAL_OIDC_SECRET_PREFIXES, + UnresolvableSecretReference, + get_secret, + oidc_reference_kind, + resolve_local_oidc_reference, +) from ._logging import is_debugging_on, verbose_logger from .caching.caching import ( @@ -721,16 +731,13 @@ def get_applied_guardrails(kwargs: dict[str, object]) -> list[str]: return applied_guardrails -def load_credentials_from_list(kwargs: dict): - """ - Updates kwargs with the credentials if credential_name in kwarg - """ +def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, object]: # Access CredentialAccessor via module to trigger lazy loading if needed credential_accessor: Final[type[CredentialAccessor]] = litellm_utils.CredentialAccessor credential_name: Final = kwargs.get("litellm_credential_name") - if not credential_name: - return + if not isinstance(credential_name, str) or not credential_name: + return MappingProxyType({}) credential: Final = credential_accessor.find_credential(credential_name) if credential is None: verbose_logger.warning( @@ -738,10 +745,93 @@ def load_credentials_from_list(kwargs: dict): credential_name, len(litellm.credential_list), ) - return - for key, value in credential.credential_values.items(): - if key not in kwargs: - kwargs[key] = value + return MappingProxyType({}) + return MappingProxyType( + { + key: value + for key, value in credential_api_key_secret_reference(credential.credential_values).items() + if key not in kwargs + } + ) + + +def resolve_api_key_secret_reference( + api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] +) -> str | None: + if not isinstance(api_key, AdminApiKeySecretReference): + return api_key + return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream) + + +def _secret_behind_api_key_reference( + reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object] +) -> str: + if not reference.may_reach(upstream): + raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model) + if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES): + raise _secret_reference_error( + f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. " + "Use oidc/file/ or oidc/env/.", + llm_provider=llm_provider, + model=model, + ) + match resolve_local_oidc_reference(reference): + case UnresolvableSecretReference(kind=kind, reason=reason): + raise _secret_reference_error( + f"api_key {kind} secret reference could not be resolved: {reason}", + llm_provider=llm_provider, + model=model, + ) + case str() as secret: + return secret + + +def _secret_reference_error(message: str, llm_provider: object, model: object) -> AuthenticationError: + return AuthenticationError(message=message, llm_provider=llm_provider, model=model) + + +def _unreachable_upstream_message(reference: AdminApiKeySecretReference) -> str: + if isinstance(reference, CredentialApiKeySecretReference): + return ( + "The api_key secret reference comes from a stored credential, so LiteLLM sends it only to " + "that credential's own api_base. Set this api_base on the credential, or give this model its own " + "credential." + ) + return ( + "The api_key secret reference goes only to the host its deployment configures, and this call " + "points at another one. Send your own api_key with that host." + ) + + +def with_named_credential(kwargs: Mapping[str, object]) -> Mapping[str, object]: + return {**kwargs, **_named_credential_values(kwargs)} + + +def load_credentials_from_list(kwargs: dict[str, object]) -> None: + """ + Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key + reference for the secret it points to, read now so a rotated token is used on the next call + """ + kwargs.update(_credential_updates(kwargs)) + + +def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: + credential_values: Final = _named_credential_values(kwargs) + api_key: Final = kwargs.get("api_key", credential_values.get("api_key")) + if not isinstance(api_key, AdminApiKeySecretReference): + return credential_values + secret: Final = _secret_behind_api_key_reference( + api_key, + model=kwargs.get("model"), + llm_provider=kwargs.get("custom_llm_provider"), + upstream=MappingProxyType( + { + "model": kwargs.get("model"), + **{key: kwargs.get(key, credential_values.get(key)) for key in API_KEY_UPSTREAM_PARAMS}, + } + ), + ) + return MappingProxyType({**credential_values, "api_key": secret}) def get_dynamic_callbacks( @@ -8150,6 +8240,23 @@ def _infer_valid_provider_from_env_vars( return valid_providers +def _provider_listing_params( + custom_llm_provider: str, litellm_params: LiteLLM_Params | None +) -> Mapping[str, object] | None: + if litellm_params is None: + return None + params: Final = litellm_params.model_dump(exclude_none=True) + api_key: Final = litellm_params.api_key + if not isinstance(api_key, AdminApiKeySecretReference): + return params + return { + **params, + "api_key": _secret_behind_api_key_reference( + api_key, model=litellm_params.model, llm_provider=custom_llm_provider, upstream=params + ), + } + + def _get_valid_models_from_provider_api( provider_config: BaseLLMModelInfo, custom_llm_provider: str, @@ -8161,7 +8268,7 @@ def _get_valid_models_from_provider_api( if cached_result is not None: return cached_result models: Final = provider_config.discover_models( - litellm_params=litellm_params.model_dump(exclude_none=True) if litellm_params is not None else None + litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) diff --git a/litellm/vector_store_files/main.py b/litellm/vector_store_files/main.py index c03963980de..792157b09ce 100644 --- a/litellm/vector_store_files/main.py +++ b/litellm/vector_store_files/main.py @@ -23,7 +23,7 @@ from litellm.types.vector_store_files import ( VectorStoreFileObject, VectorStoreFileUpdateRequest, ) -from litellm.utils import ProviderConfigManager, client +from litellm.utils import ProviderConfigManager, client, load_credentials_from_list from litellm.vector_store_files.utils import VectorStoreFileRequestUtils base_llm_http_handler = BaseLLMHTTPHandler() @@ -49,6 +49,7 @@ def _prepare_registry_credentials( kwargs.update(registry_credentials) except Exception: pass + load_credentials_from_list(kwargs) @client diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 4d945310293..8b33b1ce12c 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -31,7 +31,7 @@ from litellm.types.vector_stores import ( VectorStoreSearchResponse, VectorStoreSearchResult, ) -from litellm.utils import ProviderConfigManager, client +from litellm.utils import ProviderConfigManager, client, load_credentials_from_list from litellm.vector_stores.utils import VectorStoreRequestUtils if TYPE_CHECKING: @@ -403,6 +403,7 @@ def search( kwargs.update(registry_credentials) except Exception: pass + load_credentials_from_list(kwargs) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(vector_store_id=vector_store_id, **kwargs) diff --git a/tests/integration/security/test_api_key_secret_reference.py b/tests/integration/security/test_api_key_secret_reference.py new file mode 100644 index 00000000000..0b1187d2e0e --- /dev/null +++ b/tests/integration/security/test_api_key_secret_reference.py @@ -0,0 +1,783 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.process import ( + OwnedProxy, + WorkerHealthcheck, + graceful_stop_seconds, + owned_proxy_process, +) +from integration._support.wire import Reply, Request, wire_server +from integration.security._sinks import Recorder +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120)) + +_CLOSE: Final = MappingProxyType({"Connection": "close"}) +_FRESH: Final = httpx.Limits(max_keepalive_connections=0) +_CHAT_MODEL: Final = "openai/gpt-4o-mini" +_EMBEDDING_MODEL: Final = "openai/text-embedding-3-small" +_REALTIME_MODEL: Final = "openai/gpt-4o-realtime-preview" +_LISTED_MODEL: Final = "wif-listed-model" +_VECTOR_STORE: Final = "vs_wif_reference" +_SEARCH_PATH: Final = f"/v1/vector_stores/{_VECTOR_STORE}/search" +_BATCH_DEPLOYMENT: Final = "wif-vllm-batch" +_BEDROCK_MODEL: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" +_PROVIDER_INPUT_TOKENS: Final = 7919 +_UNSUPPORTED_REFERENCE: Final = "oidc/google/https://example.invalid" +_RUN_SECRETS_TOKEN: Final = "/run/secrets/wif-absent-token" +_SOURCES: Final = ("config_credential", "config_inline", "db_credential", "db_inline") +_UNREADABLE: Final = MappingProxyType( + {"missing_file": "wif-missing", "outside_allowlist": "wif-outside", "unsupported_kind": "wif-unsupported"} +) +_ATTACKS: Final = ( + "client_api_base_inline", + "client_api_base_credential", + "body_reference", + "body_credential_name", + "user_config", +) +_CREDENTIAL_ROW: Final = ( + 'SELECT credential_values::text AS stored, updated_at::text AS updated_at FROM "LiteLLM_CredentialsTable" ' + "WHERE credential_name = %s" +) +_MODEL_ROW: Final = ( + 'SELECT litellm_params::text AS stored, updated_at::text AS updated_at FROM "LiteLLM_ProxyModelTable" ' + "WHERE model_id = %s" +) +_ENDPOINTS: Final = MappingProxyType( + {"embeddings": ("/v1/embeddings", "wif-embedding"), "responses": ("/v1/responses", "wif-config-inline")} +) +_TEAM_WRITES: Final = MappingProxyType( + { + ("team_admin", "plain_key"): 200, + ("team_admin", "names_credential"): 403, + ("team_admin", "inline_reference"): 400, + ("proxy_admin", "inline_reference"): 400, + } +) +_MODEL_ROWS: Final = ( + "SELECT model_id FROM \"LiteLLM_ProxyModelTable\" WHERE model_info::jsonb ->> 'team_public_model_name' = %s" +) + + +@dataclass(frozen=True, slots=True) +class _Secrets: + allowed: Path + token: Path + missing: Path + outside: Path + + def current(self) -> str: + return self.token.read_text() + + def rotate(self) -> str: + value: Final = f"wif-token-{uuid.uuid4().hex}" + self.token.write_text(value) + return value + + def overrides(self) -> Mapping[str, str]: + return MappingProxyType( + {"LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS": str(self.allowed), "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"} + ) + + +def _reference(path: Path) -> str: + return f"oidc/file/{path}" + + +def _secrets(directory: Path) -> _Secrets: + allowed: Final = directory / "secrets" + outside_dir: Final = directory / "outside" + allowed.mkdir() + outside_dir.mkdir() + secrets: Final = _Secrets( + allowed=allowed, token=allowed / "token", missing=allowed / "absent-token", outside=outside_dir / "token" + ) + secrets.rotate() + secrets.outside.write_text(f"outside-secret-{uuid.uuid4().hex}") + return secrets + + +def _json_reply(body: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _served(request: Request) -> Reply: + path: Final = request.target.split("?", 1)[0] + match request.method, path: + case "POST", "/v1/chat/completions": + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "served", stream=False) + case "POST", "/v1/responses": + return responses_reply(f"resp_{uuid.uuid4().hex}", "gpt-4o-mini", "served", stream=False) + case "POST", "/v1/responses/input_tokens": + return _json_reply({"object": "response.input_tokens", "input_tokens": _PROVIDER_INPUT_TOKENS}) + case "POST", converse if converse.startswith("/model/") and converse.endswith("/converse"): + return _json_reply( + { + "output": {"message": {"role": "assistant", "content": [{"text": "served"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + } + ) + case "POST", "/v1/embeddings": + return _json_reply( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + ) + case "GET", "/v1/models": + return _json_reply( + {"object": "list", "data": [{"id": _LISTED_MODEL, "object": "model", "created": 0, "owned_by": "it"}]} + ) + case "POST", searched if searched == _SEARCH_PATH: + return _json_reply( + { + "object": "vector_store.search_results.page", + "search_query": "served", + "data": [], + "has_more": False, + "next_page": None, + } + ) + case _: + return openai_error(404) + + +def _provider(secrets: _Secrets) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.headers.get("authorization") != f"Bearer {secrets.current()}": + return openai_error(401) + return _served(request) + + return respond + + +def _intruder(request: Request) -> Reply: + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "intercepted", stream=False) + + +def _deployment( + name: str, params: Mapping[str, JsonValue], info: Mapping[str, JsonValue] = MappingProxyType({}) +) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": dict(params), "model_info": dict(info)} + + +def _credential(name: str, api_key: str, api_base: str) -> dict[str, JsonValue]: + return { + "credential_name": name, + "credential_values": {"api_key": api_key, "api_base": api_base}, + "credential_info": {"custom_llm_provider": "openai"}, + } + + +def _named(model: str, credential: str) -> dict[str, JsonValue]: + return {"model": model, "litellm_credential_name": credential} + + +def _config(directory: Path, secrets: _Secrets, provider: str) -> Path: + inline: Final = {"api_key": _reference(secrets.token), "api_base": provider} + configuration: Final[dict[str, JsonValue]] = { + "model_list": [ + _deployment("wif-config-credential", _named(_CHAT_MODEL, "wif-config")), + _deployment("wif-config-inline", {"model": _CHAT_MODEL, **inline}), + _deployment("wif-embedding", _named(_EMBEDDING_MODEL, "wif-config"), {"mode": "embedding"}), + _deployment("wif-missing", _named(_CHAT_MODEL, "wif-missing")), + _deployment("wif-outside", _named(_CHAT_MODEL, "wif-outside")), + _deployment("wif-unsupported", _named(_CHAT_MODEL, "wif-unsupported")), + _deployment("wif-run-secrets", _named(_CHAT_MODEL, "wif-run-secrets")), + _deployment("wif-missing-fallback", _named(_CHAT_MODEL, "wif-missing")), + _deployment("wif-realtime", _named(_REALTIME_MODEL, "wif-config"), {"mode": "realtime"}), + _deployment("wif-vllm", {"model": "hosted_vllm/wif-local", **inline}, {"id": _BATCH_DEPLOYMENT}), + _deployment("openai/*", _named("openai/*", "wif-config")), + _deployment( + "wif-bedrock-endpoint", + { + "model": _BEDROCK_MODEL, + "aws_region_name": "us-east-1", + "api_key": _reference(secrets.token), + "aws_bedrock_runtime_endpoint": provider.removesuffix("/v1"), + }, + ), + _deployment( + "wif-mistral-passthrough", + { + "model": "mistral/mistral-small-latest", + "api_key": _reference(secrets.missing), + "use_in_pass_through": True, + }, + ), + ], + "credential_list": [ + _credential("wif-config", _reference(secrets.token), provider), + _credential("wif-missing", _reference(secrets.missing), provider), + _credential("wif-outside", _reference(secrets.outside), provider), + _credential("wif-unsupported", _UNSUPPORTED_REFERENCE, provider), + _credential("wif-run-secrets", f"oidc/file/{_RUN_SECRETS_TOKEN}", provider), + ], + "vector_store_registry": [ + { + "vector_store_name": "wif-vector-store", + "litellm_params": { + "vector_store_id": _VECTOR_STORE, + "custom_llm_provider": "openai", + "litellm_credential_name": "wif-config", + }, + } + ], + "router_settings": { + "fallbacks": [{"wif-missing-fallback": ["wif-config-credential"]}], + "disable_cooldowns": True, + }, + "litellm_settings": {"check_provider_endpoint": True}, + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "allow_client_side_credentials": True}, + } + path: Final = directory / "api_key_secret_reference_config.yaml" + path.write_text(yaml.safe_dump(configuration, sort_keys=False)) + return path + + +@dataclass(frozen=True, slots=True) +class _Stored: + credential: str + models: Mapping[str, str] + model_ids: Mapping[str, str] + + +def _delete_credential(gateway: Gateway, name: str) -> None: + response: Final = gateway.request("DELETE", f"/credentials/{name}") + assert response.status_code in (200, 404), response.text + + +def _created_model(gateway: Gateway, scenario: Scenario, body: Mapping[str, JsonValue]) -> tuple[str, str]: + created: Final = gateway.post("/model/new", body) + identity: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return string_value(created["model_name"]), identity + + +def _stored(owned: OwnedProxy, scenario: Scenario, secrets: _Secrets, provider: str) -> _Stored: + credential: Final = f"db-oidc-{uuid.uuid4().hex}" + owned.gateway.post("/credentials", _credential(credential, _reference(secrets.token), provider)) + scenario.cleanups.callback(_delete_credential, owned.gateway, credential) + by_credential: Final = _created_model( + owned.gateway, + scenario, + _deployment(f"db-credential-{uuid.uuid4().hex}", _named(_CHAT_MODEL, credential)), + ) + inline: Final = _created_model( + owned.gateway, + scenario, + _deployment( + f"db-inline-{uuid.uuid4().hex}", + {"model": _CHAT_MODEL, "api_key": _reference(secrets.token), "api_base": provider}, + ), + ) + for name, identity in (by_credential, inline): + eventually(partial(owned.serves_only_on_every_worker, name, identity), bool, seconds=120) + return _Stored( + credential=credential, + models=MappingProxyType({"db_credential": by_credential[0], "db_inline": inline[0]}), + model_ids=MappingProxyType({"db_credential": by_credential[1], "db_inline": inline[1]}), + ) + + +@dataclass(frozen=True, slots=True) +class _Rig: + owned: OwnedProxy + provider: Recorder + intruder: Recorder + provider_base: str + intruder_base: str + secrets: _Secrets + stored: _Stored + boot: tuple[Request, ...] + boot_token: str + + def model(self, source: str) -> str: + match source: + case "config_credential": + return "wif-config-credential" + case "config_inline": + return "wif-config-inline" + case "db_credential" | "db_inline": + return self.stored.models[source] + case _: + pytest.fail(f"unknown source {source}") + + +def _is_discovery(request: Request) -> bool: + return (request.method, request.target) == ("GET", "/v1/models") + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("api_key_secret_reference").resolve() + secrets: Final = _secrets(directory) + with ( + gateway_from_environment() as gateway, + wire_server(_provider(secrets)) as provider_wire, + wire_server(_intruder) as intruder_wire, + ): + provider: Final = Recorder(provider_wire) + provider_base: Final = f"{provider_wire.url}/v1" + with owned_proxy_process( + gateway, + directory, + {**secrets.overrides(), "MISTRAL_API_BASE": provider_wire.url}, + config=_config(directory, secrets, provider_base), + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + ) as owned: + with owned.gateway.scenario() as scenario: + stored: Final = _stored(owned, scenario, secrets, provider_base) + boot: Final = eventually(provider.requests, lambda seen: any(_is_discovery(r) for r in seen)) + yield _Rig( + owned=owned, + provider=provider, + intruder=Recorder(intruder_wire), + provider_base=provider_base, + intruder_base=f"{intruder_wire.url}/v1", + secrets=secrets, + stored=stored, + boot=boot, + boot_token=secrets.current(), + ) + + +def _marker() -> str: + return f"wif-marker-{uuid.uuid4().hex}" + + +def _chat( + gateway: Gateway, + model: str, + marker: str, + extra: Mapping[str, JsonValue] = MappingProxyType({}), + *, + key: str | None = None, +) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"say {marker}"}], **extra}, + key=key, + headers=_CLOSE, + ) + + +def _carrying(requests: Sequence[Request], marker: str) -> tuple[Request, ...]: + return tuple(request for request in requests if marker in request.body.decode(errors="replace")) + + +def _bearers(requests: Sequence[Request]) -> tuple[str | None, ...]: + return tuple(request.headers.get("authorization") for request in requests) + + +def _anywhere(requests: Sequence[Request], value: str) -> bool: + return any( + value in json.dumps(dict(request.headers)) + request.body.decode(errors="replace") for request in requests + ) + + +def _reached_on_worker(rig: _Rig, model: str, idle: int) -> tuple[int, tuple[Request, ...]]: + marker: Final = _marker() + gateway: Final = rig.owned.gateway + with ( + rig.owned.suspended(idle), + httpx.Client(base_url=gateway.client.base_url, timeout=60, trust_env=False, limits=_FRESH) as client, + ): + response: Final = client.post( + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + return response.status_code, _carrying(rig.provider.requests(), marker) + + +def _chat_on_worker(rig: _Rig, model: str, idle: int) -> tuple[int, tuple[str | None, ...]]: + status, reached = _reached_on_worker(rig, model, idle) + return status, _bearers(reached) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_admin_reference_sends_the_current_token_from_every_worker(rig: _Rig, source: str) -> None: + first, second = sorted(eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2)) + model: Final = rig.model(source) + before: Final = rig.secrets.current() + sent_before: Final = (_chat_on_worker(rig, model, second), _chat_on_worker(rig, model, first)) + rotated: Final = rig.secrets.rotate() + sent_after: Final = (_chat_on_worker(rig, model, second), _chat_on_worker(rig, model, first)) + assert sent_before == ((200, (f"Bearer {before}",)),) * 2, sent_before + assert sent_after == ((200, (f"Bearer {rotated}",)),) * 2, sent_after + + +@pytest.mark.parametrize("endpoint", tuple(_ENDPOINTS)) +def test_other_call_types_send_the_current_token(rig: _Rig, endpoint: str) -> None: + marker: Final = _marker() + path, model = _ENDPOINTS[endpoint] + response: Final = rig.owned.gateway.request( + "POST", path, {"model": model, "input": f"say {marker}"}, headers=_CLOSE + ) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + + +@pytest.mark.parametrize("shape", ("credential_name", "inline")) +def test_connection_with_an_admin_reference_reaches_the_provider(rig: _Rig, shape: str) -> None: + params: Final[dict[str, JsonValue]] = ( + _named(_CHAT_MODEL, "wif-config") + if shape == "credential_name" + else {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), "api_base": rig.provider_base} + ) + seen: Final = len(rig.provider.requests()) + response: Final = rig.owned.gateway.request( + "POST", "/health/test_connection", {"litellm_params": params, "mode": "chat"}, headers=_CLOSE + ) + sent: Final = frozenset(_bearers(rig.provider.requests()[seen:])) + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content)["status"] == "success", response.text + assert sent == {f"Bearer {rig.secrets.current()}"}, sent + + +@pytest.mark.parametrize("case", tuple(_UNREADABLE)) +def test_unreadable_reference_fails_closed_before_any_upstream_call(rig: _Rig, case: str) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, _UNREADABLE[case], marker) + outside_secret: Final = rig.secrets.outside.read_text() + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert outside_secret not in response.text + assert not _anywhere(rig.provider.requests(), outside_secret) + + +def test_unreadable_reference_falls_over_to_the_next_deployment(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-missing-fallback", marker) + sent: Final = frozenset(_bearers(_carrying(rig.provider.requests(), marker))) + assert (response.status_code, sent) == (200, {f"Bearer {rig.secrets.current()}"}), response.text + + +def test_local_token_count_does_not_need_an_unreadable_reference(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/utils/token_counter", + {"model": _UNREADABLE["missing_file"], "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers=_CLOSE, + ) + assert response.status_code == 200, response.text + counted: Final = JSON_OBJECT.validate_json(response.content)["total_tokens"] + assert isinstance(counted, int) and counted > 0, response.text + assert _carrying(rig.provider.requests(), marker) == () + + +def test_input_token_count_with_an_unreadable_reference_fails_closed(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/v1/responses/input_tokens", + {"model": _UNREADABLE["missing_file"], "input": f"say {marker}"}, + headers=_CLOSE, + ) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_messages_with_an_unreadable_reference_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/v1/messages", + { + "model": _UNREADABLE["missing_file"], + "max_tokens": 16, + "messages": [{"role": "user", "content": f"say {marker}"}], + }, + headers=_CLOSE, + ) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_unreadable_reference_under_run_secrets_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-run-secrets", marker) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert _RUN_SECRETS_TOKEN not in response.text, response.text + + +@pytest.mark.parametrize("source", ("config_credential", "config_inline")) +def test_provider_token_count_sends_the_current_token(rig: _Rig, source: str) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", "/v1/responses/input_tokens", {"model": rig.model(source), "input": f"say {marker}"}, headers=_CLOSE + ) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + assert JSON_OBJECT.validate_json(response.content)["input_tokens"] == _PROVIDER_INPUT_TOKENS, response.text + + +def test_pass_through_with_an_unreadable_reference_hides_the_token_path(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.owned.gateway.request( + "POST", + "/mistral/v1/chat/completions", + {"model": "mistral-small-latest", "messages": [{"role": "user", "content": f"say {marker}"}]}, + headers=_CLOSE, + ) + assert (response.status_code, _carrying(rig.provider.requests(), marker)) == (401, ()), response.text + assert str(rig.secrets.allowed) not in response.text, response.text + + +def test_admin_configured_endpoint_receives_the_current_token(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-bedrock-endpoint", marker) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.secrets.current()}",)), response.text + + +def _hosts_and_bearers(requests: Sequence[Request]) -> tuple[tuple[str | None, str | None], ...]: + return tuple((request.headers.get("host"), request.headers.get("authorization")) for request in requests) + + +def test_updated_api_base_carries_the_token_to_the_new_host(rig: _Rig) -> None: + first, second = sorted(eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2)) + moved_base: Final = rig.provider_base.replace("127.0.0.1", "localhost") + with rig.owned.gateway.scenario() as scenario: + name, identity = _created_model( + rig.owned.gateway, + scenario, + _deployment( + f"db-moved-{uuid.uuid4().hex}", + {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), "api_base": rig.provider_base}, + ), + ) + eventually(partial(rig.owned.serves_only_on_every_worker, name, identity), bool, seconds=120) + patched: Final = rig.owned.gateway.request( + "PATCH", f"/model/{identity}/update", {"litellm_params": {"api_base": moved_base}} + ) + assert patched.status_code == 200, patched.text + expected: Final = ((200, ((httpx.URL(moved_base).netloc.decode(), f"Bearer {rig.secrets.current()}"),)),) * 2 + reached: Final = eventually( + lambda: tuple( + (status, _hosts_and_bearers(requests)) + for status, requests in (_reached_on_worker(rig, name, idle) for idle in (second, first)) + ), + lambda seen: seen == expected, + seconds=120, + return_last_on_timeout=True, + ) + assert reached == expected, reached + + +def _attack(rig: _Rig, attack: str) -> tuple[str, dict[str, JsonValue]]: + redirected: Final[dict[str, JsonValue]] = {"api_base": rig.intruder_base} + match attack: + case "client_api_base_inline": + return "wif-config-inline", redirected + case "client_api_base_credential": + return "wif-config-credential", redirected + case "body_reference": + return "wif-config-inline", {**redirected, "api_key": _reference(rig.secrets.token)} + case "body_credential_name": + return "wif-config-inline", {**redirected, "litellm_credential_name": "wif-config"} + case "user_config": + injected: Final = {"model": _CHAT_MODEL, "api_key": _reference(rig.secrets.token), **redirected} + return "wif-injected", { + "user_config": {"model_list": [{"model_name": "wif-injected", "litellm_params": injected}]} + } + case _: + pytest.fail(f"unknown attack {attack}") + + +@pytest.mark.parametrize("attack", _ATTACKS) +def test_caller_cannot_send_the_admin_token_to_its_own_host(rig: _Rig, attack: str) -> None: + marker: Final = _marker() + model, extra = _attack(rig, attack) + response: Final = _chat(rig.owned.gateway, model, marker, extra) + intruded: Final = rig.intruder.requests() + assert not _anywhere(intruded, rig.secrets.current()) + assert _carrying(intruded, marker) == (), _bearers(_carrying(intruded, marker)) + assert _carrying(rig.provider.requests(), marker) == () + assert response.status_code in (400, 401), response.text + + +def test_caller_with_its_own_key_still_reaches_its_own_host(rig: _Rig) -> None: + marker: Final = _marker() + own_key: Final = f"sk-caller-{uuid.uuid4().hex}" + response: Final = _chat( + rig.owned.gateway, "wif-config-inline", marker, {"api_base": rig.intruder_base, "api_key": own_key} + ) + intruded: Final = rig.intruder.requests() + assert (response.status_code, _bearers(_carrying(intruded, marker))) == (200, (f"Bearer {own_key}",)), response.text + assert not _anywhere(intruded, rig.secrets.current()) + + +def _team_key(scenario: Scenario, team_id: str, role: str) -> str: + return scenario.key(user_id=scenario.member(team_id, role=role), team_id=team_id) + + +def _team_body(name: str, team_id: str, params: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return _deployment(name, {"model": _CHAT_MODEL, **params}, {"team_id": team_id}) + + +def _team_write_params(rig: _Rig, shape: str) -> dict[str, JsonValue]: + match shape: + case "plain_key": + return {"api_key": f"sk-team-{uuid.uuid4().hex}", "api_base": rig.provider_base} + case "names_credential": + return {"litellm_credential_name": "wif-config"} + case "inline_reference": + return {"api_key": _reference(rig.secrets.token), "api_base": rig.provider_base} + case _: + pytest.fail(f"unknown shape {shape}") + + +def _delete_written(scenario: Scenario, response: httpx.Response) -> None: + if response.status_code == 200: + scenario.delete_model( + string_value(object_value(JSON_OBJECT.validate_json(response.content)["model_info"])["id"]) + ) + + +@pytest.mark.parametrize(("writer", "shape"), tuple(_TEAM_WRITES)) +def test_team_model_write_carrying_a_reference_is_refused(rig: _Rig, writer: str, shape: str) -> None: + with rig.owned.gateway.scenario() as scenario: + team_id: Final = scenario.team() + name: Final = f"wif-team-{uuid.uuid4().hex}" + key: Final = None if writer == "proxy_admin" else _team_key(scenario, team_id, "admin") + response: Final = rig.owned.gateway.request( + "POST", "/model/new", _team_body(name, team_id, _team_write_params(rig, shape)), key=key + ) + stored: Final = len(rig.owned.gateway.rows(_MODEL_ROWS, (name,))) + _delete_written(scenario, response) + expected: Final = _TEAM_WRITES[(writer, shape)] + assert (response.status_code, stored) == (expected, int(expected == 200)), response.text + + +@pytest.mark.parametrize("host", ("credential_host", "other_host")) +def test_team_row_naming_the_credential_sends_the_token_only_to_its_host(rig: _Rig, host: str) -> None: + redirected: Final[dict[str, JsonValue]] = {} if host == "credential_host" else {"api_base": rig.intruder_base} + with rig.owned.gateway.scenario() as scenario: + team_id: Final = scenario.team() + name: Final = f"wif-team-{uuid.uuid4().hex}" + stored_name, identity = _created_model( + rig.owned.gateway, + scenario, + _team_body(name, team_id, {"litellm_credential_name": "wif-config", **redirected}), + ) + eventually(lambda: rig.owned.serves_only_on_every_worker(stored_name, identity), bool, seconds=120) + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, name, marker, key=_team_key(scenario, team_id, "user")) + intruded: Final = rig.intruder.requests() + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert not _anywhere(intruded, rig.secrets.current()) + assert _carrying(intruded, marker) == () + expected: Final = (200, (f"Bearer {rig.secrets.current()}",)) if host == "credential_host" else (401, ()) + assert (response.status_code, sent) == expected, response.text + + +def test_startup_discovery_sends_the_boot_token(rig: _Rig) -> None: + sent: Final = frozenset(_bearers(tuple(request for request in rig.boot if _is_discovery(request)))) + assert sent == {f"Bearer {rig.boot_token}"}, sent + + +def _side_call(rig: _Rig, seam: str, marker: str) -> tuple[httpx.Response, Callable[[Request], bool]]: + gateway: Final = rig.owned.gateway + match seam: + case "realtime_health": + return ( + gateway.request("GET", "/health", params={"model": "wif-realtime"}, headers=_CLOSE), + lambda request: request.target.startswith("/v1/realtime"), + ) + case "vector_store_search": + return ( + gateway.request( + "POST", f"/v1/vector_stores/{_VECTOR_STORE}/search", {"query": f"say {marker}"}, headers=_CLOSE + ), + lambda request: marker in request.body.decode(errors="replace"), + ) + case "batch_probe": + return ( + gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": f"file-{marker}", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": _BATCH_DEPLOYMENT, + }, + headers=_CLOSE, + ), + lambda request: request.target == "/v1/files", + ) + case _: + pytest.fail(f"unknown seam {seam}") + + +_SIDE_STATUS: Final = MappingProxyType({"realtime_health": 503, "vector_store_search": 200, "batch_probe": 400}) + + +@pytest.mark.parametrize("seam", tuple(_SIDE_STATUS)) +def test_calls_outside_the_client_wrapper_send_the_current_token(rig: _Rig, seam: str) -> None: + seen: Final = len(rig.provider.requests()) + response, matches = _side_call(rig, seam, _marker()) + sent: Final = frozenset(_bearers(tuple(request for request in rig.provider.requests()[seen:] if matches(request)))) + assert (response.status_code, sent) == (_SIDE_STATUS[seam], {f"Bearer {rig.secrets.current()}"}), response.text + + +def test_wildcard_listing_shows_the_models_the_provider_lists(rig: _Rig) -> None: + response: Final = rig.owned.gateway.request("GET", "/v1/models", headers=_CLOSE) + assert response.status_code == 200, response.text + entries: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + listed: Final = frozenset(string_value(object_value(entry)["id"]) for entry in entries) + asked: Final = frozenset(_bearers(tuple(request for request in rig.provider.requests() if _is_discovery(request)))) + assert any(_LISTED_MODEL in identity for identity in listed), sorted(listed) + assert not any(bearer.startswith("Bearer oidc/") for bearer in asked if bearer is not None), asked + + +def _stored_rows(rig: _Rig) -> tuple[list[dict[str, JsonValue]], ...]: + return ( + rig.owned.gateway.rows(_CREDENTIAL_ROW, (rig.stored.credential,)), + *(rig.owned.gateway.rows(_MODEL_ROW, (identity,)) for identity in rig.stored.model_ids.values()), + ) + + +def test_resolved_token_is_never_written_back(rig: _Rig) -> None: + before: Final = _stored_rows(rig) + calls: Final = tuple( + _chat(rig.owned.gateway, rig.model(source), _marker()).status_code for source in ("db_credential", "db_inline") + ) + after: Final = _stored_rows(rig) + assert tuple(len(rows) for rows in before) == (1, 1, 1), before + assert after == before, (calls, before, after) diff --git a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py index 91647472f81..5b0f743b836 100644 --- a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py @@ -8,11 +8,14 @@ Tests: 4. strip_advisor_blocks_from_messages with replace_with_text=True """ -from typing import Dict +import sys +from typing import Dict, Final from unittest.mock import AsyncMock, patch import pytest +import litellm + ADVISOR_TOOL = { "type": "advisor_20260301", "name": "advisor", @@ -776,6 +779,19 @@ def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): _resolve_advisor_credentials(tool) +@pytest.mark.parametrize("api_base", [None, "https://other.example"]) +def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(api_base): + from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + _resolve_advisor_credentials, + ) + + tool = {**ADVISOR_TOOL, "api_key": "oidc/env/LITELLM_MASTER_KEY", "api_base": api_base} + proxy_settings: Final = {"litellm.proxy.proxy_server": _fake_proxy_server({"allow_client_side_credentials": True})} + with patch.dict(sys.modules, proxy_settings): + with pytest.raises(litellm.BadRequestError, match="oidc/ secret reference"): + _resolve_advisor_credentials(tool) + + def test_resolve_advisor_credentials_validates_api_base_before_use(): from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( _resolve_advisor_credentials, diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 64d057dd006..02be27d921e 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -363,6 +363,227 @@ def test_configuring_a_deployment_still_cannot_carry_federation_fields_inline(ro ) +_SERVICE_ACCOUNT_TOKEN_REFERENCE: Final = "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token" + + +def _user_config_with(litellm_params: dict[str, str]) -> dict[str, object]: + return {"model_list": [{"model_name": "devstral", "litellm_params": {"model": "mistral/devstral-latest", **litellm_params}}]} + + +def _user_config_defaulting_to(litellm_params: dict[str, str]) -> dict[str, object]: + return { + "model_list": [ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_base": "https://attacker.example"}, + } + ], + "default_litellm_params": litellm_params, + } + + +def _user_config_search_tool_with(litellm_params: dict[str, str]) -> dict[str, object]: + return { + "search_tools": [ + { + "search_tool_name": "web", + "litellm_params": {"search_provider": "tavily", "api_base": "https://attacker.example", **litellm_params}, + } + ] + } + + +@pytest.fixture +def oidc_api_key_credential(monkeypatch): + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + credential_info={"custom_llm_provider": "mistral"}, + ) + ], + ) + + +def _vector_store_embedding_config_with(embedding_config: dict[str, str]) -> dict[str, object]: + return { + "vector_store_id": "docs", + "custom_llm_provider": "milvus", + "litellm_params": { + "api_base": "https://milvus.example", + "litellm_embedding_config": {"api_base": "https://attacker.example", **embedding_config}, + }, + } + + +def test_a_vector_store_embedding_config_may_carry_a_plain_api_key(): + assert is_request_body_safe( + request_body=_vector_store_embedding_config_with({"api_key": "sk-embedding"}), + general_settings={}, + llm_router=None, + model="", + route="/vector_store/new", + ) + + +@pytest.mark.parametrize("route", [None, "/model/new", "/health/test_connection"]) +@pytest.mark.parametrize( + "body", + [ + {"model": "devstral", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + {"model": "devstral", "extra_body": {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}}, + {"model": "devstral", "fallbacks": [{"model": "other", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}]}, + { + "model": "devstral", + "user_config": _user_config_with( + {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE, "api_base": "https://attacker.example"} + ), + }, + {"model": "devstral", "user_config": _user_config_defaulting_to({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE})}, + {"model": "web", "user_config": _user_config_search_tool_with({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE})}, + _vector_store_embedding_config_with({"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}), + ], + ids=[ + "top_level", + "extra_body", + "fallback_target", + "user_config", + "user_config_defaults", + "user_config_search_tool", + "vector_store_embedding_config", + ], +) +def test_a_request_body_cannot_carry_an_oidc_api_key_reference(route: str | None, body: dict): + """The call path reads an oidc/ api_key from the proxy's own filesystem or environment, so a body that set one + could send the proxy's service account token to a host the caller picks. No opt-in or route relaxes it.""" + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + route=route, + ) + + +@pytest.mark.parametrize( + "body", + [ + {"model": "devstral", "litellm_credential_name": "admin-oidc"}, + { + "model": "devstral", + "user_config": _user_config_with( + {"litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"} + ), + }, + {"model": "devstral", "user_config": _user_config_defaulting_to({"litellm_credential_name": "admin-oidc"})}, + _vector_store_embedding_config_with({"litellm_credential_name": "admin-oidc"}), + ], + ids=["top_level", "user_config", "user_config_defaults", "vector_store_embedding_config"], +) +def test_a_request_body_cannot_name_a_credential_holding_an_oidc_api_key(oidc_api_key_credential, body: dict): + with pytest.raises(ValueError, match=r"names a credential .* oidc/ api_key secret reference"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + ) + + +@pytest.mark.parametrize("route", [None, "/chat/completions"]) +def test_a_call_cannot_carry_an_oidc_api_key_in_litellm_params(route: str | None): + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + is_request_body_safe( + request_body={"model": "devstral", "litellm_params": {"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + route=route, + ) + + +@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/deployment-1/update", "/health/test_connection"]) +def test_a_deployment_write_may_carry_an_oidc_api_key_in_litellm_params(route: str): + """A proxy admin configures an oidc/ api_key on a deployment through these routes, and the model management role + gate refuses anyone else with a 403.""" + assert ( + is_request_body_safe( + request_body={ + "model": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + }, + general_settings={}, + llm_router=None, + model="devstral", + route=route, + ) + is True + ) + + +_OIDC_DEPLOYMENT_PARAMS: Final = ( + pytest.param({"model": "mistral/devstral-latest", "api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, id="inline"), + pytest.param({"model": "mistral/devstral-latest", "litellm_credential_name": "admin-oidc"}, id="named_credential"), +) + + +@pytest.mark.parametrize("override", ["api_base", "base_url"]) +@pytest.mark.parametrize("deployment_params", _OIDC_DEPLOYMENT_PARAMS) +def test_a_base_override_cannot_send_the_deployments_oidc_api_key( + oidc_api_key_credential, deployment_params: dict, override: str +): + from litellm.exceptions import BadRequestError + from litellm.router_utils.clientside_credential_handler import get_dynamic_litellm_params + + with pytest.raises(BadRequestError, match="api_key is an oidc/ secret reference"): + get_dynamic_litellm_params( + litellm_params=dict(deployment_params), request_kwargs={override: "https://caller.example/v1"} + ) + + +@pytest.mark.parametrize("deployment_params", _OIDC_DEPLOYMENT_PARAMS) +def test_a_base_override_with_the_callers_own_api_key_is_allowed(oidc_api_key_credential, deployment_params: dict): + from litellm.router_utils.clientside_credential_handler import get_dynamic_litellm_params + + out: Final = get_dynamic_litellm_params( + litellm_params=dict(deployment_params), + request_kwargs={"api_base": "https://caller.example/v1", "api_key": "sk-caller"}, + ) + assert (out["api_base"], out["api_key"]) == ("https://caller.example/v1", "sk-caller") + + +def test_test_connection_may_name_a_credential_holding_an_oidc_api_key(oidc_api_key_credential): + """The admin UI tests a stored credential by name, which is how a proxy admin checks an oidc/ api_key works.""" + assert ( + is_request_body_safe( + request_body={"model": "devstral", "litellm_params": {"litellm_credential_name": "admin-oidc"}}, + general_settings={}, + llm_router=None, + model="devstral", + route="/health/test_connection", + ) + is True + ) + + +def test_a_user_config_with_its_own_api_key_is_still_allowed(): + assert ( + is_request_body_safe( + request_body={"model": "devstral", "user_config": _user_config_with({"api_key": "sk-client-owned"})}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="devstral", + ) + is True + ) + + class TestCustomAuthCommonChecksWarning: """custom_auth_common_checks_warning only warns when custom auth is configured and the common-checks opt-in is off, since that is the only state where diff --git a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py index 00f84e75664..4189a8dba91 100644 --- a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -3,6 +3,7 @@ import json from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from pathlib import Path from types import MappingProxyType from typing import Final, Literal, cast from unittest.mock import AsyncMock, MagicMock @@ -594,6 +595,38 @@ async def test_litellm_executed_provider_for_leaves_a_server_with_its_own_files_ assert probe.upstreams == [("http://vllm.test/v1/", "vllm-key")] +@pytest.mark.parametrize( + ("model_info", "expected_key"), + [({"id": "vllm-batch"}, "vllm-secret"), ({"id": "vllm-batch", "team_id": "team-1"}, None)], + ids=["proxy-level deployment", "team-scoped deployment"], +) +async def test_resolve_litellm_executed_provider_probes_with_the_secret_only_behind_an_admin_oidc_api_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, model_info: Mapping[str, str], expected_key: str | None +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("vllm-secret") + reference: Final = f"oidc/file/{token}" + router: Final = Router( + model_list=[ + { + "model_name": BATCH_MODEL, + "litellm_params": {**VLLM_CREDENTIALS, "api_key": reference}, + "model_info": dict(model_info), + } + ] + ) + probe: Final = FakeFilesApiProbe(True, []) + + try: + provider: Final = await resolve_litellm_executed_provider(router, "vllm-batch", "team-1", probe) + finally: + router.discard() + + assert provider == "hosted_vllm" + assert probe.upstreams == [("http://vllm.test/v1/", expected_key or reference)] + + @pytest.mark.parametrize( "credentials", [{"custom_llm_provider": "openai", "model": "gpt-4o", "api_base": "http://openai.test/v1"}, {"model": 7}], diff --git a/tests/unit/proxy/common_utils/test_credential_hydration.py b/tests/unit/proxy/common_utils/test_credential_hydration.py index f40b0114d71..ad75b5b8760 100644 --- a/tests/unit/proxy/common_utils/test_credential_hydration.py +++ b/tests/unit/proxy/common_utils/test_credential_hydration.py @@ -1,10 +1,16 @@ +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm -from litellm.proxy.common_utils.credential_hydration import hydrate_named_credential_authoritative +from litellm.proxy.common_utils.credential_hydration import ( + effective_server_owned_wif_fields, + hydrate_named_credential_authoritative, + named_credential_wif_fields, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.types.utils import CredentialItem @pytest.mark.asyncio @@ -26,3 +32,46 @@ async def test_authoritative_hydrate_returns_an_encrypted_empty_value_as_empty(m assert resolved is not None assert resolved.credential_values == {"api_base": "", "openai_service_account_id": "user-1"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "credential_api_key, expected", + [ + ("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", ("api_key",)), + ("sk-static", ()), + ], + ids=["oidc_reference", "static_key"], +) +async def test_attaching_a_credential_is_server_owned_only_when_its_api_key_is_an_oidc_reference( + credential_api_key: str, expected: tuple[str, ...] +): + credential: Final = CredentialItem( + credential_name="admin-oidc", credential_values={"api_key": credential_api_key}, credential_info={} + ) + + with patch.object(litellm, "credential_list", [credential]): # test-quality-ok: the in-memory list is the input + fields = await effective_server_owned_wif_fields( + {"model": "mistral/devstral-latest"}, {"litellm_credential_name": "admin-oidc"}, None + ) + + assert fields == expected + + +@pytest.mark.asyncio +async def test_a_stored_credential_whose_encrypted_api_key_is_an_oidc_reference_is_server_owned(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-hydration-test-salt") + row = { + "credential_name": "admin-oidc", + "credential_values": { + "api_key": encrypt_value_helper("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token") + }, + "credential_info": {"custom_llm_provider": "mistral"}, + } + prisma = MagicMock() + prisma.db.litellm_credentialstable.find_unique = AsyncMock(return_value=row) + + with patch.object(litellm, "credential_list", []): # test-quality-ok: a pod that has not loaded the row yet + fields = await named_credential_wif_fields("admin-oidc", prisma) + + assert fields == ("api_key",) diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 4359ea6592b..6aeceaf4356 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -17,6 +17,7 @@ import litellm from litellm.models.credentials import CredentialSource from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.credential_endpoints.endpoints import get_llm_router from litellm.proxy.proxy_server import app from litellm.types.utils import CredentialItem @@ -659,6 +660,29 @@ class TestNonAdminCannotPersistWifFieldsOnCredential: assert response.status_code == 403, response.text + @pytest.mark.parametrize( + "auth, expected_status", + [(_as_non_admin, 403), (_as_admin, 200)], + ids=["non_admin", "proxy_admin"], + ) + def test_only_a_proxy_admin_can_create_a_credential_with_an_oidc_api_key( + self, restore_credential_list, auth, expected_status: int + ): + """The call path reads an oidc/ api_key from the proxy's own filesystem, so it is a server-owned value the + same way a federation field is.""" + with _repository_holding(None) as repository: + response = _post_credential( + { + "credential_name": "mistral-sa", + "credential_values": {"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + "credential_info": {"custom_llm_provider": "mistral"}, + }, + auth=auth, + ) + + assert response.status_code == expected_status, response.text + assert repository.create.await_count == (1 if expected_status == 200 else 0) + def test_non_admin_can_create_a_credential_without_wif_fields(self, restore_credential_list): with _repository_holding(None) as repository: response = _post_credential( @@ -921,6 +945,32 @@ class TestNonAdminCannotTouchAStoredWifCredential: assert "anthropic_keycloak_token_url" in response.text repository.update_by_name.assert_not_awaited() + def test_non_admin_cannot_repoint_a_stored_credential_whose_encrypted_api_key_is_an_oidc_reference( + self, restore_credential_list, monkeypatch + ): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-credential-endpoints-test-salt") + stored = CredentialItem( + credential_name="mistral-sa", + credential_values={ + "api_key": encrypt_value_helper("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token") + }, + credential_info={"custom_llm_provider": "mistral"}, + ) + with _repository_holding(stored) as repository: + response = _patch_credential( + "mistral-sa", + { + "credential_name": "mistral-sa", + "credential_values": {"api_base": "https://attacker.example"}, + "credential_info": {}, + }, + auth=_as_non_admin, + ) + + assert response.status_code == 403, response.text + assert "'api_key'" in response.text + repository.update_by_name.assert_not_awaited() + def test_proxy_admin_can_delete_a_wif_field_off_a_credential(self, restore_credential_list): with _repository_holding(_wif_credential("federated-cred")) as repository: response = _patch_credential( diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 441c52b6a5c..d5288a7fc12 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -6,6 +6,7 @@ from collections.abc import Awaitable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime, timedelta from importlib import import_module +from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -740,6 +741,73 @@ def _test_connection_probe( yield ahealth_check +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_api_key", "request_params"), + [ + ("oidc/file/{token}", {"model": "mistral/devstral-latest"}), + ("sk-stale", {"model": "mistral/devstral-latest", "api_key": "oidc/file/{token}"}), + ], + ids=["stored on the deployment", "typed into the probe by an admin"], +) +async def test_test_model_connection_sends_the_token_behind_an_admin_oidc_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + deployment_api_key: str, + request_params: Mapping[str, str], +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = tmp_path / "token" + token.write_text("mistral-sa-token") + router: Final = Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": { + "model": "mistral/devstral-latest", + "api_key": deployment_api_key.format(token=token), + }, + "model_info": {"id": "devstral-id"}, + } + ] + ) + + try: + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.premium_user", False), + respx.mock(assert_all_called=True) as upstream, + ): + upstream.post("https://api.mistral.ai/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-probe", + "object": "chat.completion", + "created": 1, + "model": "devstral-latest", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + result: Final = await health_test_model_connection( + request=MagicMock(), + mode="chat", + litellm_params={key: value.format(token=token) for key, value in request_params.items()}, + model_info={"id": "devstral-id"}, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, token="admin-token"), + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + finally: + router.discard() + + assert result["status"] == "success" + assert sent == "Bearer mistral-sa-token" + + MANTLE_CLAUDE_DEPLOYMENT: Final = MappingProxyType( { "model_name": "claude-haiku-4-5", diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index bc714ad1cbc..303b8e060ed 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -518,6 +518,43 @@ class TestModelManagementAuthChecks: ) assert result is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model_info", "api_key", "refused"), + [ + ({"team_id": "test_team"}, "oidc/env/MISTRAL_SA_TOKEN", True), + ({}, "oidc/env/MISTRAL_SA_TOKEN", False), + ({"team_id": "test_team"}, "sk-static", False), + ], + ids=["team model with a reference", "proxy-level model with a reference", "team model with a plain key"], + ) + async def test_an_admin_cannot_save_an_oidc_api_key_reference_on_a_team_model( + self, model_info: dict, api_key: str, refused: bool + ): + """A team model's inline api_key is never resolved, so saving an oidc/ reference there would only ever send + the reference text upstream.""" + model_params = Deployment( + model_name="test_model", + litellm_params=LiteLLM_Params(model="mistral/devstral-latest", api_key=api_key), + model_info=model_info, + ) + + async def save() -> object: + return await ModelManagementAuthChecks.can_user_make_model_call( + model_params=model_params, + user_api_key_dict=self.admin_user, + prisma_client=MockPrismaClient(team_exists=True), + premium_user=True, + incoming_params={"model": "mistral/devstral-latest", "api_key": api_key}, + ) + + if not refused: + assert await save() is True + return + with pytest.raises(Exception, match="team model cannot hold an oidc/ api_key secret reference") as exc_info: + await save() + assert (getattr(exc_info.value, "param", None), getattr(exc_info.value, "code", None)) == ("api_key", "400") + @pytest.mark.asyncio async def test_can_user_make_model_call_normal_user_fails(self): """Test that normal users cannot make model calls""" @@ -8838,6 +8875,63 @@ class TestNonAdminCannotPersistWifFieldsOnModel: authorized for a team-scoped model must not be able to set one via /model/new, /model/update, or PATCH /model/{id}/update; a proxy admin still can.""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "stored_api_key, patched_params", + [ + ("sk-static", {"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}), + ("oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", {"api_base": "https://attacker.example"}), + ], + ids=["sets_the_reference", "redirects_a_stored_reference"], + ) + async def test_patch_model_non_admin_cannot_touch_an_oidc_api_key_reference( + self, stored_api_key: str, patched_params: dict + ): + """The call path reads an oidc/ api_key from the proxy's own filesystem, so a team admin who could set one, or + move a deployment that has one to another api_base, could send the proxy's token to a host they pick.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + patch_model, + ) + + non_admin = UserAPIKeyAuth(user_id="team_admin", user_role=LitellmUserRoles.INTERNAL_USER) + existing_row = MagicMock() + existing_row.litellm_params = {"model": "mistral/devstral-latest", "api_key": stored_api_key} + existing_row.model_dump.return_value = { + "model_name": "devstral", + "litellm_params": existing_row.litellm_params, + "model_info": {"id": "m1"}, + } + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + + with ( + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.prisma_client", + mock_prisma, + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.llm_router", + MagicMock(**{"get_model_ids.return_value": ["m1"]}), + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.store_model_in_db", + True, + ), + patch( # test-quality-ok: the proxy wiring under test is what this patches + "litellm.proxy.proxy_server.premium_user", + True, + ), + ): + with pytest.raises(Exception, match="oidc/ api_key secret reference") as exc_info: + await patch_model( + model_id="m1", + patch_data=updateDeployment(litellm_params=updateLiteLLMParams(**patched_params)), + user_api_key_dict=non_admin, + ) + assert getattr(exc_info.value, "param", "") == "api_key" + mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() + @pytest.mark.asyncio async def test_patch_model_non_admin_cannot_set_wif_field(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 941accff314..9c50a6d4df6 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -449,19 +449,21 @@ def test_oidc_file_reference_outside_allowed_dirs_is_refused(tmp_path, monkeypat _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) assert excinfo.value.status_code == 401 - assert f"oidc/file/{outside}" in str(excinfo.value.detail) + assert str(outside) not in str(excinfo.value.detail) assert "outside the allowed credential directories" in str(excinfo.value.detail) -def test_missing_oidc_token_file_is_a_clear_auth_error(tmp_path, monkeypatch): - monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) - monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{tmp_path / 'missing-token'}") +def test_missing_oidc_token_file_is_a_clear_auth_error(monkeypatch): + monkeypatch.delenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "oidc/file//run/secrets/lit-missing-token") with pytest.raises(HTTPException) as excinfo: _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) assert excinfo.value.status_code == 401 - assert f"oidc/file/{tmp_path / 'missing-token'}" in str(excinfo.value.detail) + assert "pass-through credential oidc/file" in str(excinfo.value.detail) + assert "No such file or directory" in str(excinfo.value.detail) + assert "/run/secrets" not in str(excinfo.value.detail) def test_oidc_env_reference_is_resolved(monkeypatch): diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 2943fe79979..4647952312a 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -7181,6 +7181,76 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_ assert value not in logged +_SERVICE_ACCOUNT_TOKEN_REFERENCE: Final = "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token" + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_refuses_a_forwarded_x_api_key_holding_an_oidc_reference(): + """A forwarded provider key lands in api_key after auth, so the body check cannot see it and a client could get + the proxy to read its own service account token for the call.""" + from fastapi import HTTPException + + request_mock = _make_request_mock( + "/v1/messages", + { + "Content-Type": "application/json", + "x-litellm-api-key": "Bearer sk-virtual-key", + "x-api-key": _SERVICE_ACCOUNT_TOKEN_REFERENCE, + }, + ) + + with pytest.raises(HTTPException) as exc_info: + await add_litellm_data_to_request( + data={"model": "claude-sonnet-5", "messages": [{"role": "user", "content": "hello"}]}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"forward_llm_provider_auth_headers": True}, + version="test-version", + ) + + assert exc_info.value.status_code == 400 + assert "api_key holds an oidc/ secret reference" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_refuses_a_model_config_credential_holding_an_oidc_reference(monkeypatch): + """Team and project metadata are written by non-admins, so a model_config naming a credential whose api_key is an + oidc/ reference must not get that reference resolved for a deployment the team controls.""" + from fastapi import HTTPException + + monkeypatch.setattr(litellm, "enable_model_config_credential_overrides", True) + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_info={}, + credential_values={"api_key": _SERVICE_ACCOUNT_TOKEN_REFERENCE}, + ) + ], + ) + team_key: Final = UserAPIKeyAuth( + api_key="hashed-key", + team_id="team-1", + team_metadata={"model_config": {"azure/gpt-4o": {"azure": {"litellm_credentials": "admin-oidc"}}}}, + ) + + with pytest.raises(HTTPException) as exc_info: + await add_litellm_data_to_request( + data={"model": "azure/gpt-4o", "messages": [{"role": "user", "content": "hello"}]}, + request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}), + user_api_key_dict=team_key, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.status_code == 400 + assert "api_key holds an oidc/ secret reference" in str(exc_info.value.detail) + + @pytest.mark.parametrize( "header, expected_redacted", [ diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 18fe5a2ad32..5cbda9d320c 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -199,6 +199,76 @@ async def test_gpt_token_counting(): assert response.request_model == "gpt-4" + +@pytest.mark.asyncio +async def test_token_counter_leaves_the_routers_deployment_without_the_credentials_it_resolves(tmp_path, monkeypatch): + """token_counter reads the router's own deployment, so loading the named credential into it would leave the + token read from the file on the shared deployment for every later request.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token_file = tmp_path / "token" + token_file.write_text("sa-token-from-file") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="mistral-sa", + credential_values={"api_key": f"oidc/file/{token_file}"}, + credential_info={}, + ) + ], + ) + llm_router = Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "litellm_credential_name": "mistral-sa"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + + response = await token_counter( + request=TokenCountRequest(model="devstral", messages=[{"role": "user", "content": "hello"}]) + ) + + assert response.total_tokens > 0 + stored = (await llm_router.async_get_available_deployment(model="devstral", request_kwargs={}))["litellm_params"] + assert "api_key" not in stored + assert stored["litellm_credential_name"] == "mistral-sa" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_endpoint", [False, True], ids=["local tokenizer", "provider count api"]) +async def test_token_counter_reads_an_oidc_api_key_only_when_it_calls_the_provider( + tmp_path, monkeypatch, call_endpoint +): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + llm_router = Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": f"oidc/file/{tmp_path / 'missing-token'}", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + request = TokenCountRequest(model="claude", messages=[{"role": "user", "content": "hello"}]) + + if not call_endpoint: + assert (await token_counter(request=request, call_endpoint=call_endpoint)).total_tokens > 0 + return + with pytest.raises(ProxyException) as excinfo: + await token_counter(request=request, call_endpoint=call_endpoint) + assert excinfo.value.code == "401" + assert str(tmp_path) not in excinfo.value.message + + @pytest.mark.asyncio async def test_anthropic_messages_count_tokens_endpoint(): """ diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 527d3d599b1..4d89ee497a5 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -527,6 +527,29 @@ async def test_bearer_health_check_sends_stored_credential_as_bearer_token( assert connect.url == expected_url +@pytest.mark.asyncio +async def test_bearer_health_check_sends_the_secret_behind_an_oidc_credential( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("sk-from-token-file") + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={})], + ) + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key=None, + model_params={"model": "openai/gpt-realtime", "litellm_credential_name": "voice-sa"}, + ) + assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer sk-from-token-file"} + + @pytest.mark.asyncio async def test_azure_health_check_probes_ga_transcription_url_for_transcription_model(local_model_cost_map): """Regression for LIT-6240: transcription-only models (mode audio_transcription diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index e11c5d6c2f3..f956bb66e6c 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -5511,6 +5511,48 @@ class TestRouterPreRoutingSharedAliasName: assert sent["aws_region_name"] == "us-west-2" assert sent["drop_params"] is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + "tier_params", + [ + { + "api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", + "api_base": "https://attacker.example", + }, + {"litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"}, + ], + ids=["inline_reference", "credential_holding_a_reference"], + ) + async def test_a_tier_cannot_make_the_proxy_read_an_oidc_api_key(self, monkeypatch, tier_params: dict): + """A team admin can write an auto-router's tiers, and tier params beat the target deployment's own, so a + tier could otherwise send the proxy's own token file to a host the tier picks.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + credential_info={}, + ) + ], + ) + marker = self._region_marker_entry() + marker["litellm_params"]["complexity_router_config"] = { + "tiers": { + tier: {"model_name": "bedrock-tier", "litellm_params": tier_params} + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + } + } + tier_target = self._bedrock_tier_entry() + tier_target["litellm_params"]["mock_response"] = "sent upstream" + router = Router(model_list=[marker, tier_target]) + + with pytest.raises(ValueError, match="oidc/"): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + @pytest.mark.asyncio async def test_a_markers_explicit_flag_beats_the_tiers_pydantic_default(self): """Every deployment materializes `LiteLLM_Params` defaults such as diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 383a1e25e98..09ce002353f 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -1265,6 +1265,43 @@ async def test_a_stored_fallback_target_cannot_carry_an_openai_federation_field( ) +@pytest.mark.parametrize( + "target", + [ + {"model": "mistral-backup", "api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + {"model": "mistral-backup", "litellm_credential_name": "admin-oidc", "api_base": "https://attacker.example"}, + ], + ids=["inline_reference", "credential_holding_a_reference"], +) +@pytest.mark.asyncio +async def test_a_stored_fallback_target_cannot_make_the_proxy_read_an_oidc_api_key(monkeypatch, target: dict): + """A stored key or team fallback is merged into kwargs ahead of the call, so it could otherwise point the + proxy's own token file at a host the fallback picks.""" + from litellm.types.utils import CredentialItem + + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="admin-oidc", + credential_values={"api_key": "oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token"}, + credential_info={"custom_llm_provider": "mistral"}, + ) + ], + ) + + with pytest.raises(ValueError, match="oidc/"): + await run_async_fallback( + litellm_router=FakeRouter(), + fallback_model_group=[target], + original_model_group="primary-model", + original_exception=RuntimeError("upstream limited request"), + max_fallbacks=3, + fallback_depth=0, + ) + + @pytest.mark.asyncio async def test_the_refusal_is_not_swallowed_as_a_fallback_error(): """Checked before the per-target loop on purpose: inside it, the refusal would be caught as diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index d32555c3669..7c6e1de6949 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -9,10 +9,12 @@ from unittest.mock import Mock, patch import pytest import litellm +from litellm._logging import redact_internal_details_from_client_message from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.secret_managers.main import ( get_secret, normalize_nonempty_secret_str, + resolve_local_oidc_reference, secret_manager_would_be_consulted, ) from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem @@ -322,6 +324,15 @@ def test_oidc_file_success(tmp_path, monkeypatch): assert result == "file_token" +def test_oidc_file_reads_a_token_under_a_directory_named_oidc(tmp_path, monkeypatch): + token_dir: Final = tmp_path / "oidc" + token_dir.mkdir() + (token_dir / "token").write_text("nested_token") + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + + assert get_secret(f"oidc/file/{token_dir}/token") == "nested_token" + + def test_oidc_file_rejects_path_outside_allowlist(tmp_path, monkeypatch): outside_file = tmp_path / "outside.txt" outside_file.write_text("should_not_read") @@ -334,10 +345,12 @@ def test_oidc_file_rejects_path_outside_allowlist(tmp_path, monkeypatch): get_secret(f"oidc/file/{outside_file}") -def test_oidc_file_rejects_relative_path(tmp_path, monkeypatch): +def test_oidc_file_rejects_relative_path_with_a_hint_client_redaction_leaves_whole(tmp_path, monkeypatch): monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) - with pytest.raises(ValueError, match="must be absolute"): + with pytest.raises(ValueError, match="must be absolute") as excinfo: get_secret("oidc/file/relative/path/token") + hint: Final = str(excinfo.value) + assert redact_internal_details_from_client_message(hint) == hint def test_oidc_env_success(mock_env): @@ -474,3 +487,15 @@ def test_get_secret_turns_only_boolean_literals_from_the_secret_manager_into_boo assert secret == expected assert type(secret) is type(expected) + + +def test_an_unresolvable_reference_cannot_forge_a_server_log_line( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.delenv("LIT_UNSET_SA_TOKEN", raising=False) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + resolve_local_oidc_reference("oidc/env/LIT_UNSET_SA_TOKEN\nFORGED ENTRY") + + assert "LIT_UNSET_SA_TOKEN" in caplog.text + assert "\nFORGED" not in caplog.text diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 094a13c37b2..4cce5b50f0b 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -18,6 +18,7 @@ import queue import re import sys import threading +from pathlib import Path from types import MappingProxyType from typing import Final, cast from unittest.mock import Mock, patch @@ -172,6 +173,35 @@ async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch _invalidate_model_cost_lowercase_map() +async def test_discovery_sends_the_secret_behind_an_oidc_api_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text("local-secret") + router: Final = Router(model_list=[{ + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://discovery.test/v1", + "api_key": f"oidc/file/{token}", + }, + "model_info": {"id": "oidc-deployment"}, + }]) + authorizations: Final[list[str | None]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + authorizations.append(request.headers.get("Authorization")) + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await router._arefresh_deployment_model_info(router.model_list[0], client=handler) + assert authorizations == ["Bearer local-secret"] + _invalidate_model_cost_lowercase_map() + + async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) first, second = tuple( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6ffb589fd41..7df4910c24f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -59,6 +59,7 @@ from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_K from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams, LiteLLM_Params from litellm.types.utils import ( + CredentialItem, ADDRESSED_RESPONSE_ID_FIELD, CallTypes, Choices, @@ -7004,6 +7005,555 @@ def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_with assert _credential_warnings(caplog) == [] +_MISTRAL_CHAT_URL: Final = "https://api.mistral.ai/v1/chat/completions" +_MISTRAL_REPLY: Final = { + "id": "chatcmpl-oidc", + "object": "chat.completion", + "created": 0, + "model": "devstral-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "pong"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _allowed_token_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, content: str) -> Path: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token: Final = tmp_path / "token" + token.write_text(content) + return token + + +def _mistral_credential(api_key: str) -> CredentialItem: + return CredentialItem(credential_name="mistral-sa", credential_values={"api_key": api_key}, credential_info={}) + + +_PING: Final = [{"role": "user", "content": "ping"}] + + +def _ping_mistral_through(credential_name: str) -> None: + litellm.completion(model="mistral/devstral-latest", messages=_PING, litellm_credential_name=credential_name) + + +def test_named_credential_oidc_file_api_key_sends_the_token_read_on_each_call( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "token-before-rotation") + reference: Final = f"oidc/file/{token}" + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + _ping_mistral_through("mistral-sa") + token.write_text("token-after-rotation") + _ping_mistral_through("mistral-sa") + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + + assert sent == ["Bearer token-before-rotation", "Bearer token-after-rotation"] + assert litellm.credential_list[0].credential_values["api_key"] == reference + + +@pytest.mark.asyncio +async def test_deployment_oidc_env_api_key_resolves_per_call_and_the_router_keeps_the_reference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token-one") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "oidc/env/MISTRAL_SA_TOKEN"}, + } + ] + ) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + await router.acompletion(model="devstral", messages=[{"role": "user", "content": "ping"}]) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token-two") + await router.acompletion(model="devstral", messages=[{"role": "user", "content": "ping"}]) + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + + assert sent == ["Bearer env-token-one", "Bearer env-token-two"] + assert router.get_model_list(model_name="devstral")[0]["litellm_params"]["api_key"] == "oidc/env/MISTRAL_SA_TOKEN" + + +def test_explicit_api_key_wins_over_a_named_credential_reference( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(f"oidc/file/{tmp_path / 'absent'}")]) + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_key="sk-explicit", + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == "Bearer sk-explicit" + + +@pytest.mark.parametrize( + ("make_reference", "reason"), + [ + (lambda tmp_path: f"oidc/file/{tmp_path / 'absent'}", "No such file or directory"), + (lambda tmp_path: f"oidc/file/{tmp_path.parent}/outside", "outside the allowed credential directories"), + (lambda tmp_path: "oidc/file//run/secrets/lit-absent-token", "outside the allowed credential directories"), + (lambda tmp_path: "oidc/file/relative/token", "must be absolute"), + (lambda tmp_path: "oidc/env/LIT_UNSET_SA_TOKEN", "environment variable is not set"), + (lambda tmp_path: f"oidc/file/{tmp_path / 'empty'}", "empty value"), + (lambda tmp_path: f"oidc/file/{tmp_path / 'binary'}", "not UTF-8 text"), + ], +) +def test_unresolvable_oidc_api_key_fails_closed_naming_the_reason_without_its_target_or_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + make_reference: Callable[[Path], str], + reason: str, +) -> None: + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + monkeypatch.delenv("LIT_UNSET_SA_TOKEN", raising=False) + (tmp_path / "empty").write_text("") + (tmp_path / "binary").write_bytes(b"\xffFILE-CONTENT") + (tmp_path.parent / "outside").write_text("FILE-CONTENT") + reference: Final = make_reference(tmp_path) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(litellm.AuthenticationError) as excinfo: + _ping_mistral_through("mistral-sa") + + assert not route.called + assert excinfo.value.status_code == 401 + assert reason in excinfo.value.message + assert reference.split("/", 2)[2] not in excinfo.value.message + assert reference in caplog.text + assert "FILE-CONTENT" not in excinfo.value.message + assert "0xff" not in excinfo.value.message + + +@pytest.mark.parametrize( + ("reference", "kind"), + [ + ("oidc/env_path/MISTRAL_SA_TOKEN_PATH", "oidc/env_path"), + ("oidc/google/https://api.mistral.ai", "oidc/google"), + ("oidc/github/mistral", "oidc/github"), + ("oidc/env_MISTRAL_SA_TOKEN_PATH", "oidc/"), + ("oidc/LIT-UNKNOWN-KIND/MISTRAL_SA_TOKEN_PATH", "oidc/"), + ], +) +def test_oidc_api_key_of_a_kind_api_key_cannot_resolve_fails_closed_as_an_authentication_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, reference: str, kind: str +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "must-not-be-sent") + monkeypatch.setenv("MISTRAL_SA_TOKEN_PATH", str(token)) + monkeypatch.setattr(litellm, "credential_list", [_mistral_credential(reference)]) + + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + with pytest.raises(litellm.AuthenticationError) as excinfo: + _ping_mistral_through("mistral-sa") + + assert not route.called + assert excinfo.value.status_code == 401 + assert f"{kind} secret reference" in excinfo.value.message + assert reference.removeprefix(kind).lstrip("/") not in excinfo.value.message + assert "MISTRAL_SA_TOKEN_PATH" not in excinfo.value.message + + +@pytest.mark.asyncio +async def test_router_fails_over_from_a_deployment_whose_oidc_api_key_kind_cannot_resolve( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": { + "model": "mistral/devstral-latest", + "api_key": "oidc/google/https://api.mistral.ai", + "order": 1, + }, + }, + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "sk-healthy", "order": 2}, + }, + ], + num_retries=1, + retry_after=0, + ) + + try: + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + response: Final = await router.acompletion(model="devstral", messages=_PING) + sent: Final = [call.request.headers["Authorization"] for call in upstream.calls] + finally: + router.discard() + + assert response.choices[0].message.content == "pong" + assert sent == ["Bearer sk-healthy"] + + +def test_an_oidc_api_key_sent_with_the_call_goes_upstream_as_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "server-secret") + reference: Final = f"oidc/file/{token}" + + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + litellm.completion(model="mistral/devstral-latest", messages=_PING, api_key=reference) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == f"Bearer {reference}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "expected"), + [({}, "Bearer env-token"), ({"team_id": "team-1"}, "Bearer oidc/env/MISTRAL_SA_TOKEN")], + ids=["proxy-level deployment", "team-scoped deployment"], +) +async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( + monkeypatch: pytest.MonkeyPatch, model_info: Mapping[str, str], expected: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {"model": "mistral/devstral-latest", "api_key": "oidc/env/MISTRAL_SA_TOKEN"}, + "model_info": dict(model_info), + } + ] + ) + + try: + with respx.mock(assert_all_called=True) as upstream: + upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) + await router.acompletion( + model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + finally: + router.discard() + + assert sent == expected + + +_DEPLOYMENT_HOST_REFERENCE_REFUSAL: Final = "goes only to the host its deployment configures" +_BEDROCK_RUNTIME: Final = "https://bedrock-runtime.us-east-1.amazonaws.com" +_BEDROCK_CLAUDE: Final = "us.anthropic.claude-sonnet-5-5" +_BEDROCK_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "pong"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, +} + + +def _bedrock_router(api_key: str, endpoint: str | None) -> litellm.Router: + params: Final = {"model": f"bedrock/converse/{_BEDROCK_CLAUDE}", "aws_region_name": "us-east-1", "api_key": api_key} + return litellm.Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {**params, "aws_bedrock_runtime_endpoint": endpoint} if endpoint else params, + } + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_endpoint", "call_endpoint"), + [(None, "https://caller.example"), (_BEDROCK_RUNTIME, f"{_BEDROCK_RUNTIME}:8443")], + ids=["deployment without an endpoint", "endpoint on another port"], +) +async def test_an_inline_oidc_api_key_is_refused_before_its_file_is_read_when_the_call_picks_another_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, deployment_endpoint: str | None, call_endpoint: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + router: Final = _bedrock_router("oidc/file//run/secrets/lit-missing-token", deployment_endpoint) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_BEDROCK_REPLY) + with pytest.raises(litellm.AuthenticationError, match=_DEPLOYMENT_HOST_REFERENCE_REFUSAL) as excinfo: + await router.acompletion(model="claude", messages=_PING, aws_bedrock_runtime_endpoint=call_endpoint) + finally: + router.discard() + + assert upstream.calls.call_count == 0 + assert "/run/secrets" not in excinfo.value.message + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_configures_when_the_call_names_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + router: Final = _bedrock_router(f"oidc/file/{token}", _BEDROCK_RUNTIME) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_BEDROCK_REPLY) + await router.acompletion( + model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + ) + sent: Final = tuple( + (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls + ) + finally: + router.discard() + + assert sent == (("bedrock-runtime.us-east-1.amazonaws.com", "Bearer file-token"),) + + +_HUGGINGFACE_EMBEDDING: Final = [[0.1, 0.2]] + + +def _authorizations(calls: respx.models.CallList) -> tuple[tuple[str, str], ...]: + return tuple( + (call.request.url.host, call.request.headers["Authorization"]) + for call in calls + if "Authorization" in call.request.headers + ) + + +def _huggingface_router(model: str, api_key: str) -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": model, "litellm_params": {"model": model, "api_key": api_key}}], + num_retries=0, + ) + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_is_refused_when_a_wildcard_call_names_its_own_url_as_the_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + router: Final = _huggingface_router("huggingface/*", f"oidc/file/{token}") + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_HUGGINGFACE_EMBEDDING) + with pytest.raises(litellm.AuthenticationError, match=_DEPLOYMENT_HOST_REFERENCE_REFUSAL): + await router.aembedding( + model="huggingface/https://caller.example/embed", input=["hello"], input_type="feature-extraction" + ) + finally: + router.discard() + + assert _authorizations(upstream.calls) == () + + +@pytest.mark.asyncio +async def test_an_inline_oidc_api_key_reaches_the_url_its_deployment_names_as_the_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + token: Final = _allowed_token_file(tmp_path, monkeypatch, "file-token") + model: Final = "huggingface/https://endpoint.example/embed" + router: Final = _huggingface_router(model, f"oidc/file/{token}") + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_HUGGINGFACE_EMBEDDING) + await router.aembedding(model=model, input=["hello"], input_type="feature-extraction") + sent: Final = _authorizations(upstream.calls) + finally: + router.discard() + + assert sent == (("endpoint.example", "Bearer file-token"),) + + +_CALLER_HOST_REFERENCE_REFUSAL: Final = "sends it only to that credential's own api_base" + + +@pytest.mark.parametrize( + ("credential_api_base", "call_api_base"), + [ + ("https://api.mistral.ai/v1", "https://caller.example/v1"), + (None, "https://caller.example/v1"), + ("https://api.mistral.ai/v1", "http://api.mistral.ai/v1"), + ("https://api.mistral.ai/v1", "https://api.mistral.ai@caller.example/v1"), + ("https://api.mistral.ai/v1", "https://api.mistral.ai:8443/v1"), + ("https://api.mistral.ai/v1", "http://[api.mistral.ai/v1"), + ("https://mistral-gateway.example/v1", None), + ], + ids=[ + "another host", + "credential without api_base", + "another scheme", + "userinfo trick", + "another port", + "unparseable", + "call clears the api_base", + ], +) +def test_a_named_credentials_oidc_api_key_never_goes_to_an_api_base_the_credential_does_not_set( + monkeypatch: pytest.MonkeyPatch, credential_api_base: str | None, call_api_base: str | None +) -> None: + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + credential_values: Final = {"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": credential_api_base} + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="mistral-sa", credential_values=credential_values, credential_info={})], + ) + + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + with pytest.raises(litellm.AuthenticationError, match=_CALLER_HOST_REFERENCE_REFUSAL): + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_base=call_api_base, + ) + + assert upstream.calls.call_count == 0 + + +@pytest.mark.parametrize( + ("credential_api_base", "call_api_base"), + [ + ("https://api.mistral.ai/v1", "https://api.mistral.ai:443/v1"), + ("https://api.mistral.ai:443/v1", "https://API.mistral.ai./v1"), + ], + ids=["explicit default port on the call", "explicit default port on the credential"], +) +def test_a_named_credentials_oidc_api_key_reaches_an_api_base_on_the_same_origin( + monkeypatch: pytest.MonkeyPatch, credential_api_base: str, call_api_base: str +) -> None: + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + credential_values: Final = {"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": credential_api_base} + monkeypatch.setattr( + litellm, + "credential_list", + [CredentialItem(credential_name="mistral-sa", credential_values=credential_values, credential_info={})], + ) + + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + litellm.completion( + model="mistral/devstral-latest", + messages=_PING, + litellm_credential_name="mistral-sa", + api_base=call_api_base, + ) + sent: Final = tuple(authorization for _, authorization in _authorizations(upstream.calls)) + + assert sent == ("Bearer env-token",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("row_api_base", "refused"), + [(None, False), ("https://API.mistral.ai/v1/", False), ("https://team-admin-host.example/v1", True)], + ids=["row without api_base", "row on the credential's host", "row on another host"], +) +async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials_host( + monkeypatch: pytest.MonkeyPatch, row_api_base: str | None, refused: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("MISTRAL_SA_TOKEN", "env-token") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="mistral-sa", + credential_values={"api_key": "oidc/env/MISTRAL_SA_TOKEN", "api_base": "https://api.mistral.ai"}, + credential_info={}, + ) + ], + ) + row_params: Final = {"model": "mistral/devstral-latest", "litellm_credential_name": "mistral-sa"} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "devstral", + "litellm_params": {**row_params, "api_base": row_api_base} if row_api_base else row_params, + "model_info": {"team_id": "team-1"}, + } + ], + num_retries=0, + ) + + try: + with respx.mock(assert_all_called=False) as upstream: + upstream.route().respond(200, json=_MISTRAL_REPLY) + if refused: + with pytest.raises(litellm.AuthenticationError, match=_CALLER_HOST_REFERENCE_REFUSAL): + await router.acompletion( + model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} + ) + else: + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) + sent: Final = tuple( + (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls + ) + finally: + router.discard() + + assert sent == (() if refused else (("api.mistral.ai", "Bearer env-token"),)) + + +@pytest.mark.parametrize( + ("listing_params", "expected"), + [ + ( + lambda reference: LiteLLM_Params( + **litellm.Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + ).get_model_list(model_name="openai/*")[0]["litellm_params"] + ), + lambda reference: "Bearer listing-token", + ), + ( + lambda reference: LiteLLM_Params(model="openai/*", api_key=reference), + lambda reference: f"Bearer {reference}", + ), + ], + ids=["proxy-level deployment", "caller-built params"], +) +def test_provider_model_listing_sends_the_token_only_behind_a_deployment_oidc_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + listing_params: Callable[[str], LiteLLM_Params], + expected: Callable[[str], str], +) -> None: + token: Final = _allowed_token_file(tmp_path, monkeypatch, "listing-token") + reference: Final = f"oidc/file/{token}" + + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://api.openai.com/v1/models").respond( + 200, json={"object": "list", "data": [{"id": "gpt-listed", "object": "model"}]} + ) + listed: Final = get_valid_models( + check_provider_endpoint=True, custom_llm_provider="openai", litellm_params=listing_params(reference) + ) + sent: Final = upstream.calls.last.request.headers["Authorization"] + + assert sent == expected(reference) + assert any(model.endswith("gpt-listed") for model in listed) + + _MOCK_STREAM_ID: Final = "chatcmpl-mock-stream" _ChunkSnapshot = tuple[str, tuple[str | None, ...], Usage | None] diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 0db122b927f..e15e95f9d55 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -16,6 +16,7 @@ from litellm.types.router import ( holds_secret_pointer, reject_server_owned_wif_params, server_owned_wif_fields_named, + server_owned_wif_fields_written, server_owned_wif_fields_present, ) from litellm.types.utils import ( @@ -312,6 +313,23 @@ def test_server_owned_wif_fields_named_is_derived_from_the_shared_list(): ) +@pytest.mark.parametrize( + "api_key", ["oidc/file//var/run/secrets/kubernetes.io/serviceaccount/token", "oidc/env/SA_TOKEN", "oidc/google/aud"] +) +def test_an_oidc_reference_api_key_is_server_owned(api_key: str): + assert server_owned_wif_fields_present({"api_key": api_key}) == ("api_key",) + assert server_owned_wif_fields_written({"api_key": api_key}) == ("api_key",) + with pytest.raises(ValueError, match="api_key holds an oidc/ secret reference"): + reject_server_owned_wif_params({"api_key": api_key}) + + +@pytest.mark.parametrize("api_key", ["sk-static", "os.environ/MISTRAL_API_KEY", None]) +def test_a_plain_api_key_stays_client_settable(api_key: str | None): + assert server_owned_wif_fields_present({"api_key": api_key}) == () + assert server_owned_wif_fields_written({"api_key": api_key}) == () + reject_server_owned_wif_params({"api_key": api_key}) + + @pytest.mark.parametrize("param_name", ["anthropic_issuer_signing_key_ref", "anthropic_keycloak_client_secret_ref"]) def test_wif_ref_fields_hold_secret_pointers(param_name: str): assert holds_secret_pointer(param_name) diff --git a/tests/unit/vector_stores/test_vector_store_registry.py b/tests/unit/vector_stores/test_vector_store_registry.py index 83b59d77aaf..1a158771f29 100644 --- a/tests/unit/vector_stores/test_vector_store_registry.py +++ b/tests/unit/vector_stores/test_vector_store_registry.py @@ -20,6 +20,7 @@ from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreIndex, ) +from litellm.types.utils import CredentialItem from litellm.vector_stores.main import search from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry, VectorStoreRegistry @@ -192,6 +193,125 @@ def test_search_uses_registry_credentials(): litellm.vector_store_registry = original_registry +@pytest.fixture +def registry_with_an_oidc_env_credential(monkeypatch): + monkeypatch.setenv("VS_PROJECTED_TOKEN", "projected-token-1") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="openai-sa", + credential_values={"api_key": "oidc/env/VS_PROJECTED_TOKEN", "api_base": "https://api.openai.com/v1"}, + credential_info={"custom_llm_provider": "openai"}, + ) + ], + ) + monkeypatch.setattr( + litellm, + "vector_store_registry", + VectorStoreRegistry( + [ + LiteLLM_ManagedVectorStore( + vector_store_id="vs1", + custom_llm_provider="openai", + litellm_credential_name="openai-sa", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + ] + ), + ) + + +_SEARCH_PAGE: Final = { + "object": "vector_store.search_results.page", + "search_query": ["test"], + "data": [], + "has_more": False, + "next_page": None, +} +_VECTOR_STORE_FILE: Final = { + "id": "file-1", + "object": "vector_store.file", + "created_at": 0, + "vector_store_id": "vs1", + "status": "completed", + "usage_bytes": 0, + "last_error": None, +} + + +def _search_vs1(**params: str) -> None: + search(vector_store_id="vs1", query="test", custom_llm_provider="openai", **params) + + +def _create_vs1_file(**params: str) -> None: + import litellm.vector_store_files.main as vector_store_files_main + + vector_store_files_main.create(vector_store_id="vs1", file_id="file-1", **params) + + +def _sent_by(call, **params: str) -> Sequence[httpx.Request]: + with respx.mock(assert_all_called=False) as provider: + provider.post(url__regex=r"/vector_stores/vs1/search$").respond(200, json=_SEARCH_PAGE) + provider.post(url__regex=r"/vector_stores/vs1/files$").respond(200, json=_VECTOR_STORE_FILE) + call(**params) + return tuple(sent.request for sent in provider.calls) + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_registry_credentials_oidc_api_key_reaches_the_provider_as_the_token_it_names( + registry_with_an_oidc_env_credential, monkeypatch, call +): + (first,) = _sent_by(call) + monkeypatch.setenv("VS_PROJECTED_TOKEN", "projected-token-2") + (second,) = _sent_by(call) + + assert (first.headers["authorization"], second.headers["authorization"]) == ( + "Bearer projected-token-1", + "Bearer projected-token-2", + ) + assert litellm.credential_list[0].credential_values["api_key"] == "oidc/env/VS_PROJECTED_TOKEN" + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_caller_api_base_never_redirects_a_registry_credential_that_sets_its_own( + registry_with_an_oidc_env_credential, call +): + (sent,) = _sent_by(call, api_base="https://caller.example/v1") + + assert (sent.url.host, sent.headers["authorization"]) == ("api.openai.com", "Bearer projected-token-1") + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_a_registry_credential_without_an_api_base_never_sends_its_oidc_token_to_a_caller_api_base( + registry_with_an_oidc_env_credential, monkeypatch, call +): + credential: Final = litellm.credential_list[0] + monkeypatch.setattr( + litellm, + "credential_list", + [credential.model_copy(update={"credential_values": {"api_key": "oidc/env/VS_PROJECTED_TOKEN"}})], + ) + + with respx.mock(assert_all_called=False) as provider: + provider.route().respond(200, json=_SEARCH_PAGE) + with pytest.raises(litellm.AuthenticationError, match="sends it only to that credential's own api_base"): + call(api_base="https://caller.example/v1") + assert provider.calls.call_count == 0 + + +@pytest.mark.parametrize("call", [_search_vs1, _create_vs1_file], ids=["search", "file_create"]) +def test_an_unresolvable_registry_credential_reference_never_reaches_the_provider( + registry_with_an_oidc_env_credential, monkeypatch, call +): + monkeypatch.delenv("VS_PROJECTED_TOKEN") + + with pytest.raises(litellm.AuthenticationError, match="oidc/env secret reference could not be resolved"): + _sent_by(call) + + def _config_registry(vector_store_id: str = "vs_from_config") -> VectorStoreRegistry: registry = VectorStoreRegistry(vector_stores=[]) registry.load_vector_stores_from_config( From 783cf06f31e55e8f7ab84c2d7c708b0d90e68001 Mon Sep 17 00:00:00 2001 From: MFJG <59931828+MFJG-dev@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:44:54 -0400 Subject: [PATCH 046/124] fix(sdk): decide MCP gateway routing without importing the proxy handler (#45961) * fix(mcp): keep the gateway handler off the SDK path when no tool is an MCP tool completion(), responses() and anthropic messages imported the MCP gateway handler whenever tools were passed, and responses() did so on every call. Since the handler began importing the MCP catalog at module level in #42568, that import needs the optional mcp package, so a base SDK install failed with ModuleNotFoundError on any function-tool call and on every responses() call A new import-safe pre-check skips the handler unless some tool is a dict of type mcp, which is the only shape the handler's gateway decision can accept. MCP gateway tools still reach the handler as before Adds in-process regression tests for the three entry points and a function-tool completion to the base SDK install check * fix(mcp): leave one-shot tool iterables unread in the gateway pre-check responses() declares tools as an Iterable and the handler scans the same object a second time, so a pre-check that read a generator would hand the handler one tool fewer. The pre-check now scans only collections and reports anything else as possibly naming the gateway, which keeps the previous path The Anthropic messages regression test now mocks the httpx layer with respx and turns off the aiohttp transport, instead of replacing AsyncHTTPHandler * fix(mcp): scan only lists and tuples in the gateway pre-check The pre-check read any Collection, but a Collection that yields its items only once would still be exhausted by the handler's second scan. Only the list and tuple builtins are guaranteed to iterate the same way twice, so they are the only inputs read now. Everything else keeps the previous path and is reported as possibly naming the gateway The unread-input test now covers a generator, an iterator, a one-shot collection and a deque * fix(mcp): scan only exact list and tuple types in the gateway pre-check isinstance accepted list subclasses, and a subclass that iterates only once would still be consumed by the handler's second scan. The fast path now reads only objects whose type is exactly list or tuple. Every other input, subclasses included, keeps the previous path and is reported as possibly naming the gateway The unread-input test gains a one-shot list subclass and a tuple subclass * fix(sdk): decide MCP gateway routing without importing the proxy handler The pre-check now holds the handler's exact gateway test, so remote MCP tools, mcp 1.x installs and any tools iterable stay off the handler, and the base SDK check calls responses() with a remote MCP tool. * fix(sdk): send /mcp/ remote MCP URLs to the provider when the gateway registry cannot load A path-only /mcp/ match now counts as a gateway tool only when the MCP server registry imports, so SDK installs without the proxy extras pass remote MCP servers like Zapier to the provider. Also drops a dead test patch. * test(sdk): send a /mcp/ remote MCP tool in the base SDK install check * style(sdk): drop docstrings from the MCP gateway pre-check and its tests * test(mcp): cover /mcp/ gateway routing on the proxy and Messages tools on a base SDK install --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../pass_through/messages/handler.py | 53 ++- litellm/main.py | 103 +++--- litellm/responses/main.py | 7 +- .../mcp/litellm_proxy_mcp_handler.py | 44 +-- litellm/responses/mcp/tool_precheck.py | 43 +++ .../base_sdk_tests/check_base_sdk_install.py | 24 ++ tests/base_sdk_tests/check_sdk_http.py | 55 ++- .../integration/mcp/test_mcp_llm_endpoints.py | 55 +++ .../unit/responses/mcp/test_tool_precheck.py | 318 ++++++++++++++++++ .../test_responses_prompt_management.py | 35 +- 10 files changed, 593 insertions(+), 144 deletions(-) create mode 100644 litellm/responses/mcp/tool_precheck.py create mode 100644 tests/unit/responses/mcp/test_tool_precheck.py diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index d0f0fa4d901..e0d10cd37f4 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -28,6 +28,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -603,38 +604,34 @@ def anthropic_messages_handler( # downstream path (native passthrough and both bridges) gets real tools rather than a # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider. skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler and tools: + if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( anthropic_messages_with_mcp, ) - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools): - return anthropic_messages_with_mcp( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower - "str | None", system - ), - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - container=container, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, - ) + return anthropic_messages_with_mcp( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + container=container, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) anthropic_messages_provider_config: Final = _native_messages_config( model, custom_llm_provider, kwargs.get("model_info"), api_base diff --git a/litellm/main.py b/litellm/main.py index 9430af34401..3ecb325c5e6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -135,6 +135,7 @@ from litellm.llms.vertex_ai.common_utils import ( get_vertex_ai_model_route, ) from litellm.realtime_api.main import realtime_health_check +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.completion import ( CompletionDispatchContext, @@ -5434,63 +5435,55 @@ def completion( control_options: Final = _resolve_control_options(kwargs, model) skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler and tools: + if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - from litellm.types.llms.openai import ToolParam - # Check if MCP tools are present (following responses pattern) - # Cast tools to Optional[Iterable[ToolParam]] for type checking - tools_for_mcp: Final = cast(Iterable[ToolParam] | None, tools) - if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools_for_mcp): - return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it - model=model, - messages=messages, - functions=functions, - function_call=function_call, - timeout=timeout, - temperature=temperature, - top_p=top_p, - n=n, - stream=stream, - stream_options=stream_options, - stop=stop, - max_tokens=max_tokens, - max_completion_tokens=max_completion_tokens, - modalities=modalities, - prediction=prediction, - audio=audio, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - logit_bias=logit_bias, - user=user, - response_format=response_format, - seed=seed, - tools=tools, - tool_choice=tool_choice, - parallel_tool_calls=parallel_tool_calls, - logprobs=logprobs, - top_logprobs=top_logprobs, - deployment_id=deployment_id, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - safety_identifier=safety_identifier, - service_tier=service_tier, - store=store, - prompt_cache_key=prompt_cache_key, - base_url=base_url, - api_version=api_version, - api_key=api_key, - model_list=model_list, - extra_headers=extra_headers, - thinking=thinking, - web_search_options=web_search_options, - shared_session=shared_session, - enable_json_schema_validation=enable_json_schema_validation, - **kwargs, - ) + return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it + model=model, + messages=messages, + functions=functions, + function_call=function_call, + timeout=timeout, + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_tokens=max_tokens, + max_completion_tokens=max_completion_tokens, + modalities=modalities, + prediction=prediction, + audio=audio, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + parallel_tool_calls=parallel_tool_calls, + logprobs=logprobs, + top_logprobs=top_logprobs, + deployment_id=deployment_id, + reasoning_effort=reasoning_effort, + verbosity=verbosity, + safety_identifier=safety_identifier, + service_tier=service_tier, + store=store, + prompt_cache_key=prompt_cache_key, + base_url=base_url, + api_version=api_version, + api_key=api_key, + model_list=model_list, + extra_headers=extra_headers, + thinking=thinking, + web_search_options=web_search_options, + shared_session=shared_session, + enable_json_schema_validation=enable_json_schema_validation, + **kwargs, + ) api_base = kwargs.get("api_base", None) mock_response: Final[MOCK_RESPONSE_TYPE | None] = kwargs.get("mock_response", None) mock_tool_calls: Final = kwargs.get("mock_tool_calls", None) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 727a5ccfedf..98475e42b99 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -49,6 +49,7 @@ from litellm.responses.mcp.round_billing import ( first_round_billed_on_failure, summed_cost_breakdown, ) +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PromptObject, @@ -1220,11 +1221,7 @@ def _responses_try_dispatch_mcp_gateway( | None ): """Return a response when MCP gateway handles the call; otherwise None.""" - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - - if skip_mcp_handler or not LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools): + if skip_mcp_handler or not may_name_litellm_mcp_gateway(tools): return None mcp_call_kwargs: Final = { "model": model, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index b4dbf4c3d4e..e66a7d1102b 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,4 +1,3 @@ -import re import traceback from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping, Sequence from datetime import datetime @@ -20,6 +19,12 @@ from litellm.proxy._experimental.mcp_server.utils import ( strip_known_server_prefix, ) from litellm.responses.main import aresponses # noqa: TID251 # inner call must skip the MCP gateway that invoked it +from litellm.responses.mcp.tool_precheck import ( + LITELLM_PROXY_MCP_SERVER_URL_PREFIX, + may_name_litellm_mcp_gateway, + names_gateway_explicitly, + proxy_path_mcp_name, +) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( ResponseInputParam, @@ -65,29 +70,6 @@ class _AresponsesOptions(ExtraItemsTypedDict, total=False, extra_items=object): pass -LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" -LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" - -_PROXY_MCP_PATH_RE: Final = re.compile(r"^https?://.+/mcp/([^/]+)$") - - -def _mcp_server_url(tool: ToolParam) -> str | None: - if not isinstance(tool, dict) or tool.get("type") != "mcp": - return None - server_url: Final = tool.get("server_url") - return server_url if isinstance(server_url, str) else None - - -def _names_gateway_explicitly(tool: ToolParam) -> bool: - return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) - - -def _proxy_path_mcp_name(tool: ToolParam) -> str | None: - server_url: Final = _mcp_server_url(tool) - match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) - return None if match is None else match.group(1) - - def _registered_mcp_servers() -> Collection[MCPServer]: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, @@ -133,7 +115,7 @@ async def _gateway_served_names( async def _served_mcp_path_names( tools: Collection[ToolParam], served_names: Callable[[Collection[str]], Awaitable[frozenset[str]]] ) -> frozenset[str]: - names: Final = frozenset(name for name in map(_proxy_path_mcp_name, tools) if name is not None) + names: Final = frozenset(name for name in map(proxy_path_mcp_name, tools) if name is not None) return await served_names(names) if names else frozenset[str]() @@ -162,17 +144,15 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def should_use_litellm_mcp_gateway(tools: Iterable[ToolParam] | None) -> bool: - """True when a tool may name this gateway: server_url "litellm_proxy..." or an http(s) URL ending in - /mcp/. `_split_mcp_tools` then settles which of the latter the gateway actually serves.""" - return any(_names_gateway_explicitly(tool) or _proxy_path_mcp_name(tool) is not None for tool in tools or ()) + return may_name_litellm_mcp_gateway(tools) _should_use_litellm_mcp_gateway = should_use_litellm_mcp_gateway @staticmethod def _parse_mcp_tools(tools: Iterable[Mapping[str, object]] | None) -> SplitTools: items: Final = tuple(tools or ()) - gateway_tools: Final[list[ToolParam]] = [tool for tool in items if _names_gateway_explicitly(tool)] - other_tools: Final[list[ToolParam]] = [tool for tool in items if not _names_gateway_explicitly(tool)] + gateway_tools: Final[list[ToolParam]] = [tool for tool in items if names_gateway_explicitly(tool)] + other_tools: Final[list[ToolParam]] = [tool for tool in items if not names_gateway_explicitly(tool)] return gateway_tools, other_tools @staticmethod @@ -185,7 +165,7 @@ class LiteLLM_Proxy_MCP_Handler: return LiteLLM_Proxy_MCP_Handler._parse_mcp_tools( [ {**tool, "server_url": f"{LITELLM_PROXY_MCP_SERVER_URL_PREFIX}{name}"} - if (name := _proxy_path_mcp_name(tool)) in served + if (name := proxy_path_mcp_name(tool)) in served else tool for tool in items ] @@ -201,7 +181,7 @@ class LiteLLM_Proxy_MCP_Handler: ) -> tuple[bool, ...]: items: Final = tuple(tools or ()) served: Final = await _served_mcp_path_names(items, served_names) - return tuple(_names_gateway_explicitly(tool) or _proxy_path_mcp_name(tool) in served for tool in items) + return tuple(names_gateway_explicitly(tool) or proxy_path_mcp_name(tool) in served for tool in items) @staticmethod async def _apply_toolset_permissions( diff --git a/litellm/responses/mcp/tool_precheck.py b/litellm/responses/mcp/tool_precheck.py new file mode 100644 index 00000000000..b76f4066e6d --- /dev/null +++ b/litellm/responses/mcp/tool_precheck.py @@ -0,0 +1,43 @@ +import importlib +import re +from collections.abc import Iterable +from typing import Final + +LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" +LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" + +_PROXY_MCP_PATH_RE: Final = re.compile(r"^https?://.+/mcp/([^/]+)$") +_GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" + + +def _mcp_server_url(tool: object) -> str | None: + match tool: + case {"type": "mcp", "server_url": str() as server_url} if isinstance(tool, dict): + return server_url + case _: + return None + + +def names_gateway_explicitly(tool: object) -> bool: + return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) + + +def proxy_path_mcp_name(tool: object) -> str | None: + server_url: Final = _mcp_server_url(tool) + match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) + return None if match is None else match.group(1) + + +def _gateway_registry_loads() -> bool: + try: + importlib.import_module(_GATEWAY_REGISTRY_MODULE) + except ImportError: + return False + return True + + +def may_name_litellm_mcp_gateway(tools: Iterable[object] | None) -> bool: + items: Final = tuple(tools or ()) + if any(map(names_gateway_explicitly, items)): + return True + return any(proxy_path_mcp_name(tool) is not None for tool in items) and _gateway_registry_loads() diff --git a/tests/base_sdk_tests/check_base_sdk_install.py b/tests/base_sdk_tests/check_base_sdk_install.py index 3abea2a9707..4abd71e39a6 100644 --- a/tests/base_sdk_tests/check_base_sdk_install.py +++ b/tests/base_sdk_tests/check_base_sdk_install.py @@ -102,6 +102,29 @@ def check_completion() -> str: return "mock completion round-trips" +def check_completion_with_function_tool() -> str: + import litellm + + response: Final = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "ping"}], + tools=[ + { + "type": "function", + "function": { + "name": "noop", + "description": "No operation", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + mock_response="pong", + ) + content: Final = response.choices[0].message.content + _require(content == "pong", f"mock completion with a function tool returned {content!r}") + return "mock completion with a function tool round-trips" + + def check_mcp_install_guidance() -> str: try: import litellm.experimental_mcp_client @@ -209,6 +232,7 @@ CHECKS: tuple[tuple[str, Callable[[], str]], ...] = ( ("import litellm", check_import), ("optional MCP installation guidance", check_mcp_install_guidance), ("chat completion", check_completion), + ("chat completion with a function tool", check_completion_with_function_tool), ("embedding", check_embedding), ("bundled model metadata", check_bundled_model_metadata), ("token counter", check_token_counter), diff --git a/tests/base_sdk_tests/check_sdk_http.py b/tests/base_sdk_tests/check_sdk_http.py index c93306cdced..f7a1e30ac99 100644 --- a/tests/base_sdk_tests/check_sdk_http.py +++ b/tests/base_sdk_tests/check_sdk_http.py @@ -89,7 +89,6 @@ def payload(response: object) -> Mapping[str, JsonValue]: return JSON_OBJECT.validate_json(response.model_dump_json(exclude_none=True)) -@skipIf(importlib.util.find_spec("mcp") is None, "BUG: LIT-9333 completion(tools=...) imports optional mcp") def check_tools(base: str, asynchronous: bool = False) -> None: tools: Final = [ { @@ -122,7 +121,6 @@ def check_tools(base: str, asynchronous: bool = False) -> None: assert json.loads(body) == {"model": "gpt-6", "messages": MESSAGES, "tools": tools} -@skipIf(importlib.util.find_spec("mcp") is None, "BUG: LIT-9333 responses() imports optional mcp") def check_responses(base: str, asynchronous: bool = False) -> None: output: Final[Sequence[JsonValue]] = [ { @@ -144,14 +142,62 @@ def check_responses(base: str, asynchronous: bool = False) -> None: "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, } ) - arguments: Final = dict(model="openai/gpt-6", input="ping", api_key="test-key", api_base=base + "/v1") + tools: Final = [ + {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}, + {"type": "mcp", "server_label": "repos", "server_url": "https://mcp.example.com/mcp/repos"}, + ] + arguments: Final = dict(model="openai/gpt-6", input="ping", tools=tools, api_key="test-key", api_base=base + "/v1") response: Final = asyncio.run(litellm.aresponses(**arguments)) if asynchronous else litellm.responses(**arguments) assert payload(response)["output"] == output assert payload(response)["usage"] == {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5} path, headers, body = REQUESTS.get(timeout=10) assert path == "/v1/responses" assert headers["authorization"] == "Bearer test-key" - assert json.loads(body) == {"model": "gpt-6", "input": "ping"} + assert json.loads(body) == {"model": "gpt-6", "input": "ping", "tools": tools} + + +def check_messages_tools(base: str) -> None: + tools: Final = [ + { + "name": "lookup", + "description": "Look up the forecast for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ] + content: Final = [{"type": "tool_use", "id": "toolu-http-check", "name": "lookup", "input": {"city": "Paris"}}] + enqueue( + { + "id": "msg-tools-http-check", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + } + ) + + async def send() -> object: + create: Final = cast( # cast-ok: SDK facade exposes untyped kwargs + Callable[..., Awaitable[object]], litellm.anthropic.messages.acreate + ) + return await create( + model="anthropic/claude-sonnet-5-5", + messages=MESSAGES, + tools=tools, + max_tokens=16, + api_key="test-key", + api_base=base, + ) + + response: Final = JSON_OBJECT.validate_python(asyncio.run(send())) + assert response["content"] == content, response + path, headers, body = REQUESTS.get(timeout=10) + assert path == "/v1/messages" + assert headers["x-api-key"] == "test-key" + sent: Final = JSON_OBJECT.validate_json(body) + assert (sent["model"], sent["messages"], sent["tools"]) == ("claude-sonnet-5-5", MESSAGES, tools), sent @skipIf(importlib.util.find_spec("fastapi") is None, "BUG: LIT-9333 cache hits import proxy-only fastapi") @@ -392,6 +438,7 @@ def main() -> None: partial(check_tools, asynchronous=True), check_responses, partial(check_responses, asynchronous=True), + check_messages_tools, check_cache_hit, check_messages_stream_and_callback, ): diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index 9f6101423bf..4edf7b620f2 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -566,6 +566,61 @@ def test_server_scoped_gateway_url_exposes_only_that_servers_tools(gateway: Gate assert [call["body"]["params"]["name"] for call in _peer_add_calls(rig.peer)] == ["add"] +@pytest.mark.parametrize("surface", ("chat", "responses", "messages")) +def test_proxy_path_url_of_a_served_server_is_listed_and_executed_by_the_gateway( + gateway: Gateway, surface: Surface +) -> None: + with _rig(gateway, surface) as rig: + server_url: Final = f"{str(gateway.client.base_url).rstrip('/')}/mcp/{rig.alias}" + response: Final = rig.send(_granted_key(rig), [{**AUTO, "server_url": server_url}]) + assert response.status_code == 200, response.text + requests: Final = rig.upstream_tools() + assert len(requests) == 2 and all(rig.tool in names for names in requests), requests + peer_calls: Final = tuple(JSON_OBJECT.validate_python(call["body"]) for call in _peer_add_calls(rig.peer)) + assert [object_value(call["params"])["name"] for call in peer_calls] == ["add"], peer_calls + assert rig.final_text(JSON_OBJECT.validate_json(response.content)) == ANSWER, response.text + + +LOOKUP_PARAMETERS: Final = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} +CALLER_FUNCTION_TOOLS: Final[Mapping[Surface, Mapping[str, object]]] = { + "chat": {"type": "function", "function": {"name": "lookup", "parameters": LOOKUP_PARAMETERS}}, + "responses": {"type": "function", "name": "lookup", "parameters": LOOKUP_PARAMETERS}, + "messages": {"name": "lookup", "input_schema": LOOKUP_PARAMETERS}, +} + + +@pytest.mark.parametrize("surface", ("chat", "responses", "messages")) +def test_tools_that_do_not_name_the_gateway_reach_the_provider_unchanged_and_never_the_peer( + gateway: Gateway, surface: Surface +) -> None: + with _rig(gateway, surface) as rig: + remote_mcp_tools: Final = [ + { + "type": "mcp", + "server_label": "docs", + "server_url": "https://mcp.example.com/sse", + "require_approval": "never", + }, + { + "type": "mcp", + "server_label": "repos", + "server_url": f"https://mcp.example.com/mcp/unserved{uuid.uuid4().hex[:8]}", + "require_approval": "never", + }, + ] + response: Final = rig.send(_granted_key(rig), [CALLER_FUNCTION_TOOLS[surface], *remote_mcp_tools]) + assert response.status_code == 200, response.text + bodies: Final = tuple( + JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST" + ) + assert len(bodies) == 1, bodies + forwarded: Final = bodies[0].get("tools") + assert isinstance(forwarded, list) and len(forwarded) == 3, forwarded + assert [tool for tool in forwarded if isinstance(tool, dict) and tool.get("type") == "mcp"] == remote_mcp_tools + assert "lookup" in json.dumps([tool for tool in forwarded if tool not in remote_mcp_tools]), forwarded + assert tool_calls(rig.peer.drain()) == (), "a tool that does not name the gateway reached the peer" + + def test_streaming_chat_executes_the_tool_once_and_streams_the_follow_up(gateway: Gateway) -> None: with _rig(gateway, "chat") as rig: key: Final = _granted_key(rig) diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py new file mode 100644 index 00000000000..832151ce2a6 --- /dev/null +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -0,0 +1,318 @@ +import json +import sys +from collections import deque +from collections.abc import Callable, Iterable, Mapping, Sequence +from importlib.abc import MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway + +GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" +GATEWAY_HANDLER_MODULES: Final = ( + "litellm.responses.mcp.litellm_proxy_mcp_handler", + "litellm.responses.mcp.chat_completions_handler", + "litellm.llms.anthropic.pass_through.messages.mcp_handler", +) + +FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "noop", "description": "No operation", "parameters": {"type": "object", "properties": {}}}, +} +RESPONSES_FUNCTION_TOOL: Final = { + "type": "function", + "name": "noop", + "description": "No operation", + "parameters": {"type": "object", "properties": {}}, +} +ANTHROPIC_FUNCTION_TOOL: Final = { + "name": "noop", + "description": "No operation", + "input_schema": {"type": "object", "properties": {}}, +} +ANTHROPIC_MESSAGE_PAYLOAD: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} +GATEWAY_MCP_TOOL: Final = {"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"} +GATEWAY_SERVER_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "litellm_proxy/mcp/github"} +GATEWAY_PATH_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "https://gw.example.com/mcp/github"} +REMOTE_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/sse", + "require_approval": "never", +} + +RESPONSES_PAYLOAD: Final = { + "id": "resp_1", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "tools": [], +} + + +@pytest.fixture +def gateway_handler_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + for module_name in GATEWAY_HANDLER_MODULES: + monkeypatch.setitem(sys.modules, module_name, None) + + +class _FailingImport(MetaPathFinder): + def __init__(self, module_name: str, error: ImportError) -> None: + self.module_name: Final = module_name + self.error: Final = error + + def find_spec( + self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = None + ) -> ModuleSpec | None: + if fullname == self.module_name: + raise self.error + return None + + +@pytest.fixture( + params=[ + ModuleNotFoundError("No module named 'mcp'"), + ImportError("cannot import name 'CacheableResult' from 'mcp.types'"), + ModuleNotFoundError("No module named 'fastapi'"), + ], + ids=["mcp-missing", "mcp-1x", "proxy-extras-missing"], +) +def gateway_registry_unavailable( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, gateway_handler_unavailable: None +) -> None: + monkeypatch.delitem(sys.modules, GATEWAY_REGISTRY_MODULE, raising=False) + monkeypatch.setattr(sys, "meta_path", [_FailingImport(GATEWAY_REGISTRY_MODULE, request.param), *sys.meta_path]) + + +class _ToolList(list[Mapping[str, object]]): + pass + + +@pytest.mark.parametrize( + "tools", + [[FUNCTION_TOOL], [REMOTE_MCP_TOOL], _ToolList([FUNCTION_TOOL]), (FUNCTION_TOOL, REMOTE_MCP_TOOL)], + ids=["function-tool", "remote-mcp-tool", "list-subclass", "tuple"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_completion_without_a_gateway_tool_does_not_import_the_gateway_handler(tools: Iterable[Mapping[str, object]]): + response: Final = litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=tools, + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gateway_handler_unavailable") +async def test_acompletion_with_function_tool_does_not_import_the_gateway_handler(): + response: Final = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[FUNCTION_TOOL], + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +@pytest.mark.parametrize( + "tool", + [GATEWAY_MCP_TOOL, GATEWAY_SERVER_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], + ids=["litellm-proxy", "litellm-proxy-server", "gateway-path-url"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_completion_with_gateway_mcp_tool_still_reaches_the_handler(tool: Mapping[str, object]): + with pytest.raises(ModuleNotFoundError, match=r"litellm_proxy_mcp_handler|chat_completions_handler"): + litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[tool], + mock_response="ok", + ) + + +@pytest.mark.parametrize( + "tools", + [None, [RESPONSES_FUNCTION_TOOL], [REMOTE_MCP_TOOL]], + ids=["no-tools", "function-tool", "remote-mcp-tool"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_responses_without_a_gateway_tool_does_not_import_the_gateway_handler( + tools: list[Mapping[str, object]] | None, respx_mock: respx.MockRouter +): + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=RESPONSES_PAYLOAD) + ) + + response: Final = litellm.responses(model="openai/gpt-4o-mini", input="hi", tools=tools, api_key="sk-test") + + assert upstream.called + assert response.output[0].content[0].text == "Done." + + +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_responses_with_gateway_mcp_tool_still_reaches_the_handler(): + with pytest.raises(litellm.APIConnectionError, match=r"litellm_proxy_mcp_handler"): + litellm.responses(model="openai/gpt-4o-mini", input="hi", tools=[GATEWAY_MCP_TOOL], api_key="sk-test") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gateway_handler_unavailable") +async def test_anthropic_messages_with_function_tool_does_not_import_the_gateway_handler( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_MESSAGE_PAYLOAD) + ) + + response: Final = await litellm.anthropic_interface.messages.acreate( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + tools=[ANTHROPIC_FUNCTION_TOOL], + api_key="sk-test", + ) + + assert upstream.called + assert response["content"][0]["text"] == "ok" + + +@pytest.mark.parametrize( + "tools, expected", + [ + (None, False), + ([], False), + ([FUNCTION_TOOL], False), + ([RESPONSES_FUNCTION_TOOL], False), + ([{"type": "web_search"}], False), + (["not-a-mapping"], False), + ([REMOTE_MCP_TOOL], False), + ([{"type": "mcp", "server_label": "x"}], False), + ([{"type": "mcp", "server_label": "x", "server_url": None}], False), + ([{"server_url": "litellm_proxy"}], False), + ([FUNCTION_TOOL, GATEWAY_MCP_TOOL], True), + ([GATEWAY_SERVER_MCP_TOOL], True), + ([GATEWAY_PATH_MCP_TOOL], True), + ([REMOTE_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], True), + ], + ids=[ + "none", + "empty", + "chat-function", + "responses-function", + "web-search", + "non-mapping", + "remote-mcp", + "mcp-without-url", + "mcp-null-url", + "gateway-url-without-mcp-type", + "gateway-among-functions", + "gateway-server", + "gateway-path-url", + "gateway-path-after-remote-mcp", + ], +) +def test_may_name_litellm_mcp_gateway_matches_only_gateway_mcp_tools(tools: list[object] | None, expected: bool): + assert may_name_litellm_mcp_gateway(tools) is expected + + +@pytest.mark.parametrize( + "tools, expected", + [ + ([GATEWAY_PATH_MCP_TOOL], False), + ([FUNCTION_TOOL, GATEWAY_PATH_MCP_TOOL], False), + ([GATEWAY_MCP_TOOL], True), + ([GATEWAY_SERVER_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], True), + ], + ids=["gateway-path-url", "gateway-path-among-functions", "litellm-proxy", "litellm-proxy-server-and-path-url"], +) +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_proxy_path_url_names_the_gateway_only_when_its_registry_loads(tools: list[object], expected: bool): + assert may_name_litellm_mcp_gateway(tools) is expected + + +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_responses_sends_a_proxy_path_mcp_url_to_the_provider_without_the_gateway_registry( + respx_mock: respx.MockRouter, +): + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=RESPONSES_PAYLOAD) + ) + + response: Final = litellm.responses( + model="openai/gpt-4o-mini", input="hi", tools=[GATEWAY_PATH_MCP_TOOL], api_key="sk-test" + ) + + assert response.output[0].content[0].text == "Done." + assert json.loads(upstream.calls.last.request.content)["tools"] == [GATEWAY_PATH_MCP_TOOL] + + +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_completion_sends_a_proxy_path_mcp_url_to_the_provider_without_the_gateway_registry(): + response: Final = litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[GATEWAY_PATH_MCP_TOOL], + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +class _ToolTuple(tuple[Mapping[str, object], ...]): + pass + + +@pytest.mark.parametrize( + "make_tools", + [ + lambda items: (item for item in items), + lambda items: iter(items), + lambda items: deque(items), + lambda items: _ToolList(items), + lambda items: _ToolTuple(items), + ], + ids=["generator", "iterator", "deque", "list-subclass", "tuple-subclass"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_precheck_reads_every_iterable_shape_without_the_gateway_handler( + make_tools: Callable[[list[Mapping[str, object]]], Iterable[Mapping[str, object]]], +): + assert may_name_litellm_mcp_gateway(make_tools([FUNCTION_TOOL, REMOTE_MCP_TOOL])) is False + assert may_name_litellm_mcp_gateway(make_tools([FUNCTION_TOOL, GATEWAY_MCP_TOOL])) is True diff --git a/tests/unit/responses/test_responses_prompt_management.py b/tests/unit/responses/test_responses_prompt_management.py index 1e8880424ea..9af92e7f9ca 100644 --- a/tests/unit/responses/test_responses_prompt_management.py +++ b/tests/unit/responses/test_responses_prompt_management.py @@ -70,10 +70,6 @@ def _patch_responses_dispatch(): import_module("litellm.responses.main").litellm, "get_llm_provider", side_effect=_provider_by_model, ), - patch.object( - import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, "should_use_litellm_mcp_gateway", - return_value=False, - ), patch.object( import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config", return_value=None, @@ -158,7 +154,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -193,7 +189,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -216,7 +212,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -247,7 +243,7 @@ class TestResponsesAPIPromptManagement: logging_obj.model_call_details = {} patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -273,7 +269,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -305,7 +301,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -328,7 +324,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -368,7 +364,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -399,8 +395,7 @@ class TestResponsesAPIPromptManagement: side_effect=_provider_by_model, ), patches[1], - patches[2], - patches[3] as mock_handler, + patches[2] as mock_handler, ): import litellm @@ -437,7 +432,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm await litellm.aresponses( @@ -467,7 +462,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm await litellm.aresponses( @@ -498,7 +493,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm await litellm.aresponses( @@ -524,7 +519,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm await litellm.aresponses( @@ -657,7 +652,7 @@ async def test_async_guardrail_block_from_prompt_hook_reaches_caller_unwrapped() logging_obj.async_get_chat_completion_prompt = AsyncMock(side_effect=block) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3], pytest.raises(HTTPException) as exc_info: + with patches[0], patches[1], patches[2], pytest.raises(HTTPException) as exc_info: await litellm.aresponses(input="Hi", model="gpt-4o", prompt_id="blocked", litellm_logging_obj=logging_obj) assert exc_info.value is block @@ -669,7 +664,7 @@ def test_sync_guardrail_block_from_prompt_hook_reaches_caller_unwrapped(): logging_obj.get_chat_completion_prompt.side_effect = block patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3], pytest.raises(HTTPException) as exc_info: + with patches[0], patches[1], patches[2], pytest.raises(HTTPException) as exc_info: litellm.responses(input="Hi", model="gpt-4o", prompt_id="blocked", litellm_logging_obj=logging_obj) assert exc_info.value is block From c6c303a6181bf0b59f6b30beb708deade0d955d0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 18:51:51 -0700 Subject: [PATCH 047/124] fix(batches): apply the Vertex regional endpoint uplift to batch cost (#47466) * fix(batches): apply the Vertex regional endpoint uplift to batch cost batch_cost_calculator takes vertex_location and multiplies by the model's regional_endpoint_uplift_multiplier through get_vertex_regional_endpoint_uplift, and every batch cost path (aretrieve_batch pricing, completed-batch logging, native Vertex batch rows, the proxy batch cost poll and its per-line items) resolves the deployment's location and passes it down Co-authored-by: Mrun25 <165297479+Mrun25@users.noreply.github.com> * test(logging): point the resolver tests at resolve_vertex_location_for_cost --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Mrun25 <165297479+Mrun25@users.noreply.github.com> --- .../proxy/common_utils/check_batch_cost.py | 23 +++++++- litellm/batches/batch_line_items.py | 7 ++- litellm/batches/batch_utils.py | 37 ++++++++++--- litellm/cost_calculator.py | 7 ++- litellm/litellm_core_utils/litellm_logging.py | 12 ++++- .../llms/vertex_ai/batches/transformation.py | 9 +++- tests/unit/batches/test_batch_line_items.py | 25 +++++++++ tests/unit/batches/test_batch_utils.py | 37 +++++++++++-- .../test_litellm_logging.py | 54 ++++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 4 +- .../common_utils/test_check_batch_cost.py | 3 +- tests/unit/test_cost_calculator.py | 23 ++++++++ 12 files changed, 220 insertions(+), 21 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 2d2bc92dbd2..7449ec9882f 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -619,7 +619,11 @@ class CheckBatchCost: from litellm.files.types import FileContentCallOptions from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging - from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials + from litellm.litellm_core_utils.litellm_logging import ( + deployment_pricing_model_info, + mask_api_base_credentials, + resolve_vertex_location_for_cost, + ) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, ) @@ -748,6 +752,12 @@ class CheckBatchCost: custom_llm_provider=batch_file_provider, model_name=model_name, model_info=deployment_model_info, + vertex_location=resolve_vertex_location_for_cost( + custom_llm_provider=llm_provider, + litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True), + optional_params=None, + model=model_name, + ), ) error_file_failed_requests: Final = await count_error_file_failed_requests( response, @@ -889,7 +899,10 @@ class CheckBatchCost: ) from litellm.batches.batch_utils import fetch_batch_managed_file_content from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + from litellm.litellm_core_utils.litellm_logging import ( + deployment_pricing_model_info, + resolve_vertex_location_for_cost, + ) try: deployment_info: Final = self.llm_router.get_deployment(model_id=model_id) @@ -933,6 +946,12 @@ class CheckBatchCost: cost_provider=cost_provider, model_info=deployment_pricing_model_info(model_id=model_id, deployment_model=litellm_model_name), metadata=line_item_metadata(await self._creator_attribution.batch_metadata(job, batch_id)), + vertex_location=resolve_vertex_location_for_cost( + custom_llm_provider=llm_provider, + litellm_params=deployment_info.litellm_params.model_dump(exclude_none=True), + optional_params=None, + model=model_name, + ), ) input_content: Final = ( b"" diff --git a/litellm/batches/batch_line_items.py b/litellm/batches/batch_line_items.py index 3d175b22b90..120c1d958e1 100644 --- a/litellm/batches/batch_line_items.py +++ b/litellm/batches/batch_line_items.py @@ -38,6 +38,7 @@ class BatchLineItemSource: cost_provider: BatchFileProvider model_info: ModelInfo | None metadata: StandardBatchLineItemMetadata + vertex_location: str | None = None def line_item_metadata(attribution: Mapping[str, object]) -> StandardBatchLineItemMetadata: @@ -140,7 +141,11 @@ def _line_item( source: BatchLineItemSource, custom_id: str | None, request: Mapping[str, object] | None, entry: dict[str, object] ) -> StandardBatchLineItem: priced: Final = batch_cost_and_usage( - (entry,), source.cost_provider, model_name=source.model, model_info=source.model_info + (entry,), + source.cost_provider, + model_name=source.model, + model_info=source.model_info, + vertex_location=source.vertex_location, ) succeeded: Final = priced.successful_requests > 0 return StandardBatchLineItem( diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 45b0eaa0b57..6a2fa41830f 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -77,9 +77,10 @@ async def calculate_batch_cost_and_usage( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """Calculate the cost and usage of a batch, see batch_cost_and_usage.""" - return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info) + return batch_cost_and_usage(file_content_dictionary, custom_llm_provider, model_name, model_info, vertex_location) def batch_cost_and_usage( @@ -87,6 +88,7 @@ def batch_cost_and_usage( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """ Calculate the cost and usage of a batch. @@ -96,16 +98,21 @@ def batch_cost_and_usage( pricing. Threaded through to batch_cost_calculator so that deployment-specific pricing (e.g. input_cost_per_token_batches) is used instead of the global cost map. + vertex_location: The Vertex AI location the batch ran on; non-global + locations apply the model's regional-endpoint uplift multiplier. """ first_row: Final = file_content_dictionary[0] if file_content_dictionary else None if _uses_native_vertex_output(custom_llm_provider, model_name, first_row): - return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name, model_info=model_info) + return calculate_vertex_ai_batch_cost_and_usage( + file_content_dictionary, model_name, model_info=model_info, vertex_location=vertex_location + ) return _aggregate_batch_cost_usage_models( entries=file_content_dictionary, custom_llm_provider=custom_llm_provider, model_name=model_name, model_info=model_info, + vertex_location=vertex_location, ) @@ -115,6 +122,7 @@ async def handle_completed_batch( model_name: str | None = None, litellm_params: dict[str, object] | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """Fetch a completed batch's output file and aggregate its cost, usage, and models in a single pass over the JSONL lines, so the parsed file content is @@ -128,6 +136,8 @@ async def handle_completed_batch( model_info: Optional deployment-level model info with custom pricing, threaded through so a deployment's configured rates win over the global cost map. + vertex_location: The Vertex AI location the batch ran on; non-global + locations apply the model's regional-endpoint uplift multiplier. """ # A completed batch whose request lines all failed has no output file - the # results are written to a separate error_file_id and output_file_id is None. @@ -155,7 +165,10 @@ async def handle_completed_batch( output_file_result: Final = ( calculate_vertex_ai_batch_cost_and_usage( - _iter_batch_output_entries(file_content), model_name, model_info=model_info + _iter_batch_output_entries(file_content), + model_name, + model_info=model_info, + vertex_location=vertex_location, ) if _uses_native_vertex_output( custom_llm_provider, model_name, next(_iter_batch_output_entries(file_content), None) @@ -165,6 +178,7 @@ async def handle_completed_batch( custom_llm_provider=custom_llm_provider, model_name=model_name, model_info=model_info, + vertex_location=vertex_location, ) ) @@ -203,6 +217,7 @@ def _classify_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> Iterator[_BatchOutputLineStats | _LineOutcome]: """Classify every output line in a single pass, so counting failures never needs a second read of a potentially huge output file. A line the provider reported as @@ -213,7 +228,7 @@ def _classify_output_line_stats( if not _batch_response_was_successful(entry, custom_llm_provider): yield _LineOutcome.PROVIDER_FAILED continue - stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info) + stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location) yield stats if stats is not None else _LineOutcome.UNCOSTABLE @@ -222,13 +237,14 @@ def _safe_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> _BatchOutputLineStats | None: """Return the stats for one provider-successful batch output line, or None when it cannot be costed, so a single bad line never aborts the whole batch's cost accounting.""" custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None try: - return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info) + return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info, vertex_location) except Exception as e: # noqa: BLE001 # any single line's costing failure must not abort the whole batch verbose_logger.warning( "batch output line could not be costed, so it is billed at $0 and the rest of the batch " @@ -244,6 +260,7 @@ def _compute_output_line_stats( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> _BatchOutputLineStats: response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider) usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider) @@ -258,6 +275,7 @@ def _compute_output_line_stats( model_name=model_name, response_model=response_model, model_info=model_info, + vertex_location=vertex_location, ) return _BatchOutputLineStats( prompt_cost=line_prompt_cost, @@ -287,6 +305,7 @@ def _output_line_cost( model_name: str | None, response_model: str | None, model_info: ModelInfo | None, + vertex_location: str | None = None, ) -> tuple[float, float]: """(prompt_cost, completion_cost) for one output line, priced at batch rates.""" from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost @@ -307,6 +326,7 @@ def _output_line_cost( model=cost_model, custom_llm_provider=custom_llm_provider, model_info=model_info, + vertex_location=vertex_location, ) @@ -315,11 +335,14 @@ def _aggregate_batch_cost_usage_models( custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """Aggregate cost, usage, models, and pass/fail counts from batch output entries in a single pass, holding one small stats record per line instead of the parsed file.""" - all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info)) + all_results: Final = tuple( + _classify_output_line_stats(entries, custom_llm_provider, model_name, model_info, vertex_location) + ) line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats)) failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED) successful_requests: Final = len(all_results) - failed_requests @@ -366,6 +389,7 @@ def calculate_vertex_ai_batch_cost_and_usage( vertex_ai_batch_responses: Iterable[dict[str, object]], model_name: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> BatchCostUsageResult: """ Cost and usage of a native Vertex predictions.jsonl, one @@ -384,6 +408,7 @@ def calculate_vertex_ai_batch_cost_and_usage( model_info=model_info, calculate_usage=VertexGeminiConfig.calculate_usage, cost_calculator=batch_cost_calculator, + vertex_location=vertex_location, ) for row in vertex_ai_batch_responses ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8c6e2fead2d..4c6d36c06db 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -45,6 +45,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( get_regional_uplift_multiplier, get_service_tier_cost_key, get_token_type_cost_breakdown, + get_vertex_regional_endpoint_uplift, parse_prompt_tokens_details, select_cost_metric_for_model, ) @@ -676,6 +677,7 @@ def cost_per_token( model=model, custom_llm_provider=custom_llm_provider, data_residency=data_residency, + vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): @@ -2785,6 +2787,8 @@ def batch_cost_calculator( custom_llm_provider: str | None = None, model_info: ModelInfo | None = None, data_residency: str | None = None, + *, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculate the cost of a batch job. @@ -2874,7 +2878,8 @@ def batch_cost_calculator( total_prompt_cost *= uplift total_completion_cost *= uplift - return total_prompt_cost, total_completion_cost + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift def _batch_prompt_cost( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7203a176884..8d5be388eff 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -509,7 +509,7 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None: return None -def _resolve_vertex_location_for_cost( +def resolve_vertex_location_for_cost( custom_llm_provider: str | None, litellm_params: Mapping[str, object] | None, optional_params: Mapping[str, object] | None, @@ -2024,7 +2024,7 @@ class Logging(LiteLLMLoggingBaseClass): if hasattr(self, "litellm_params") and self.litellm_params else None ), - "vertex_location": _resolve_vertex_location_for_cost( + "vertex_location": resolve_vertex_location_for_cost( custom_llm_provider=cost_custom_llm_provider, litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None), optional_params=self.optional_params, @@ -3611,6 +3611,14 @@ class Logging(LiteLLMLoggingBaseClass): model_name=self.get_deployment_model_for_cost(), litellm_params=self.litellm_params, model_info=self.get_router_deployment_model_info(), + vertex_location=resolve_vertex_location_for_cost( + custom_llm_provider=self.custom_llm_provider, + litellm_params=_LITELLM_PARAMS.validate_python( + self.litellm_params # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped + ), + optional_params=None, + model=self.get_deployment_model_for_cost() or self.model, + ), ) set_hidden_param(result, "response_cost", batch_result.cost) diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index 52ea3244cf2..a8d02045740 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -54,6 +54,8 @@ class NativeVertexBatchCostCalculator(Protocol): model: str, custom_llm_provider: str | None = None, model_info: ModelInfo | None = None, + *, + vertex_location: str | None = None, ) -> tuple[float, float]: ... @@ -90,6 +92,7 @@ def native_vertex_batch_row_stats( model_info: ModelInfo | None, calculate_usage: Callable[[GenerateContentResponseBody], Usage], cost_calculator: NativeVertexBatchCostCalculator, + vertex_location: str | None = None, ) -> NativeVertexBatchRowStats | None: """ Usage and cost of one native Vertex predictions.jsonl row, a @@ -121,7 +124,11 @@ def native_vertex_batch_row_stats( ) try: prompt_cost, completion_cost = cost_calculator( - usage=usage, model=model, custom_llm_provider="vertex_ai", model_info=model_info + usage=usage, + model=model, + custom_llm_provider="vertex_ai", + model_info=model_info, + vertex_location=vertex_location, ) except Exception as e: # noqa: BLE001 # one unpriceable row must not abort the batch's cost accounting verbose_logger.warning( diff --git a/tests/unit/batches/test_batch_line_items.py b/tests/unit/batches/test_batch_line_items.py index d181a2faa86..b59802682a6 100644 --- a/tests/unit/batches/test_batch_line_items.py +++ b/tests/unit/batches/test_batch_line_items.py @@ -221,6 +221,31 @@ def test_vertex_lines_pair_with_the_input_row_that_carries_their_custom_id(): ] +def test_vertex_lines_on_a_regional_location_cost_the_uplift_over_global(): + outputs = _jsonl(_success("vx-0", 100, 50)) + model_info = ModelInfo( + input_cost_per_token_batches=INPUT_RATE, + output_cost_per_token_batches=OUTPUT_RATE, + regional_endpoint_uplift_multiplier=1.1, + ) + + def line_cost(vertex_location: str) -> float: + source = _source( + model="gemini-3.5-flash", + custom_llm_provider="vertex_ai", + cost_provider="vertex_ai", + model_info=model_info, + vertex_location=vertex_location, + ) + (line,) = _whole_batch(source, b"", (outputs,))["lines"] + return line["response_cost"] + + assert line_cost("global") == pytest.approx(100 * INPUT_RATE + 50 * OUTPUT_RATE) + assert line_cost("us-central1") == pytest.approx( + line_cost("global") * model_info["regional_endpoint_uplift_multiplier"] + ) + + def test_a_vertex_embeddings_entry_split_into_several_rows_pairs_with_none_of_them(): embeddings = { "custom_id": "emb-1", diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 630b00e172b..f2bd5855d15 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -695,7 +695,7 @@ async def test_calculate_vertex_disable_transform_path(monkeypatch): monkeypatch.setattr( bu, "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model, model_info=None: bu.BatchCostUsageResult( + lambda content, model, model_info=None, vertex_location=None: bu.BatchCostUsageResult( cost=9.9, usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3), models=["gemini-2.0-flash-001"], @@ -726,7 +726,7 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch): monkeypatch.setattr( bu, "calculate_vertex_ai_batch_cost_and_usage", - lambda content, model, model_info=None: pytest.fail("raw vertex path should not run"), + lambda content, model, model_info=None, vertex_location=None: pytest.fail("raw vertex path should not run"), ) result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=[], custom_llm_provider="vertex_ai") @@ -1167,6 +1167,36 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk assert result.failed_requests == 0 +@pytest.mark.asyncio +async def test_handle_completed_vertex_batch_regional_location_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.files.main as files_main + + rows = [_vertex_openai_row("request-1", "gemini-3.6-flash", 10, 5)] + + async def fake_afile_content(**kw): + return type("R", (), {"content": _vertex_jsonl(rows)})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + + global_result = await bu.handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={"vertex_project": "proj-1", "vertex_location": "global"}, + vertex_location="global", + ) + regional_result = await bu.handle_completed_batch( + _batch("gs://litellm-bucket/output/predictions.jsonl"), + custom_llm_provider="vertex_ai", + litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"}, + vertex_location="us-central1", + ) + + uplift = litellm.model_cost["vertex_ai/gemini-3.6-flash"]["regional_endpoint_uplift_multiplier"] + assert global_result.cost > 0 + assert regional_result.cost == pytest.approx(global_result.cost * uplift, rel=1e-9) + assert regional_result.usage == global_result.usage + + @pytest.mark.asyncio async def test_output_file_content_no_output_file_id_raises(): with pytest.raises(ValueError, match="Output file id is None"): @@ -1396,7 +1426,7 @@ async def test_handle_completed_batch_vertex_disable_transform_path(monkeypatch) monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", True, raising=False) seen: dict = {} - def fake_vertex_calc(content, model, model_info=None): + def fake_vertex_calc(content, model, model_info=None, vertex_location=None): seen["content"] = content seen["model"] = model return bu.BatchCostUsageResult( @@ -2734,6 +2764,7 @@ def test_batch_cost_calculator_applies_data_residency_uplift(data_residency, mon finally: litellm.model_cost = prev_model_cost + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_calculate_batch_cost_and_usage_uses_custom_model_info(): diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 041ab2a61b2..00c2b1ad8a8 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -1418,6 +1418,56 @@ class TestRetrieveBatchCostPassesModelIdentity: assert captured["model_info"] is not None assert captured["model_info"]["input_cost_per_token"] == 0.0 + @pytest.mark.asyncio + async def test_forwards_vertex_location_of_the_deployment(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.litellm_core_utils import litellm_logging as logging_module + + captured: dict[str, object] = {} + + from litellm.batches.batch_utils import BatchCostUsageResult + from litellm.types.utils import LiteLLMBatch, Usage + + async def fake_handle_completed_batch(**kwargs: object) -> BatchCostUsageResult: + captured.update(kwargs) + return BatchCostUsageResult( + cost=1.25, + usage=Usage(prompt_tokens=1800, completion_tokens=1000, total_tokens=2800), + models=["gemini-3.5-flash"], + successful_requests=1, + failed_requests=0, + ) + + monkeypatch.setattr(logging_module, "handle_completed_batch", fake_handle_completed_batch) + + obj = LitellmLogging( + model="vertex_ai/gemini-3.5-flash", + messages=[{"role": "user", "content": "Hey"}], + stream=False, + call_type="aretrieve_batch", + start_time=time.time(), + litellm_call_id="batch-call-2", + function_id="f", + ) + obj.custom_llm_provider = "vertex_ai" + obj.litellm_params = {"vertex_project": "proj-1", "vertex_location": "us-central1"} + + batch = LiteLLMBatch( + id="batch_vertex", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-in", + object="batch", + status="completed", + output_file_id="file-out", + ) + + with contextlib.suppress(Exception): + await obj._async_success_handler_body(result=batch, start_time=None, end_time=None) + + assert captured, "handle_completed_batch was never called" + assert captured["vertex_location"] == "us-central1" + class TestRetrieveBatchPricesOnlyFinalBatches: """Regression (LIT-7048): retrieving a provider-id batch priced it on every poll. @@ -8315,10 +8365,10 @@ async def test_supabase_logs_the_request_messages_and_response_for_async_request def _resolve(custom_llm_provider, litellm_params, optional_params, model): from litellm.litellm_core_utils.litellm_logging import ( - _resolve_vertex_location_for_cost, + resolve_vertex_location_for_cost, ) - return _resolve_vertex_location_for_cost( + return resolve_vertex_location_for_cost( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, optional_params=optional_params, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 4e529764229..8b46597b603 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -3424,7 +3424,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati from litellm.litellm_core_utils.litellm_logging import ( Logging, - _resolve_vertex_location_for_cost, + resolve_vertex_location_for_cost, ) monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") @@ -3473,7 +3473,7 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global")) assert global_deployment.litellm_params["vertex_location"] == "global" assert ( - _resolve_vertex_location_for_cost( + resolve_vertex_location_for_cost( custom_llm_provider="vertex_ai", litellm_params=global_deployment.litellm_params, optional_params=global_deployment.optional_params, diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 1253e15676b..e3012dfcd13 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -22,6 +22,7 @@ import pytest import respx from fastapi import HTTPException +from litellm.types.router import LiteLLM_Params from litellm.types.utils import LiteLLMBatch if TYPE_CHECKING: @@ -4194,7 +4195,7 @@ async def test_batch_cost_reaches_configured_success_callbacks( "custom_llm_provider": "openai", } deployment: Final = SimpleNamespace( - litellm_params=SimpleNamespace(model=f"openai/{model}", custom_llm_provider="openai", api_base=None), + litellm_params=LiteLLM_Params(model=f"openai/{model}", custom_llm_provider="openai", api_base=None), model_name="batch-group", ) prisma: Final = MagicMock() diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 2d13829bf33..619d91e478c 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4063,6 +4063,29 @@ def _batch_cache_usage() -> Usage: ) +def test_batch_cost_calculator_vertex_regional_costs_uplift_over_global(monkeypatch: pytest.MonkeyPatch) -> None: + """ + Regression for https://github.com/BerriAI/litellm/issues/46880: a Vertex batch + on a regional location must cost the global batch times the model's + regional_endpoint_uplift_multiplier, the same invariant the online path keeps. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000) + global_prompt, global_completion = batch_cost_calculator( + usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="global" + ) + regional_prompt, regional_completion = batch_cost_calculator( + usage=usage, model="gemini-3.5-flash", custom_llm_provider="vertex_ai", vertex_location="us-central1" + ) + + uplift = litellm.get_model_info("vertex_ai/gemini-3.5-flash")["regional_endpoint_uplift_multiplier"] + assert global_prompt + global_completion > 0 + assert regional_prompt == pytest.approx(global_prompt * uplift, rel=1e-9) + assert regional_completion == pytest.approx(global_completion * uplift, rel=1e-9) + + def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates(): model_info: ModelInfo = { "input_cost_per_token_batches": 1e-7, From 1660c8ba1f7fc5c2a7516d9d4c8dbba1438e7e8a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:53:37 -0700 Subject: [PATCH 048/124] feat(key_management)!: apply default_key_generate_params.allowed_routes on key generation (#41236) * feat(key_management): apply default_key_generate_params.allowed_routes on key generation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(key_management): avoid mutating default routes payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): apply default allowed_routes to key_type presets it covers * fix(proxy): keep management and read_only presets, skip service account keys, and show the default routes to custom_key_policy * fix(proxy): keep Full Access keys out of the default allowed_routes, let owners re-pick a key type, and show custom_key_policy the member key's routes * fix(proxy): keep owners from changing routes a key got from default_key_generate_params * docs(proxy): describe key_type when default_key_generate_params.allowed_routes is set, and test custom_key_policy input with the default off * test(proxy): type the default_key_generate_params allowed_routes tests * fix(proxy): let key owners send unchanged allowed_routes back on /key/update * fix(proxy): ignore a route default with a non-string entry and match resent routes exactly A default_key_generate_params.allowed_routes list with a non-string entry made /key/generate fail at the database, so it is now ignored like a non-list. An owner's resent allowed_routes on /key/update skip the route check only when they equal the stored list in order, so reordered or duplicated lists are checked as a change. Adds integration cells for the default and the owner update paths. * test(proxy): cover a non-string allowed_routes default in the key integration tests * test(proxy): read key rows from the owned proxy's database in the default routes integration tests * fix(proxy): leave allowed_routes out of a key update that resends them unchanged --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../key_management_endpoints.py | 90 ++++- .../test_key_default_allowed_routes.py | 327 ++++++++++++++++++ .../test_key_management_endpoints.py | 302 ++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 709 insertions(+), 12 deletions(-) create mode 100644 tests/integration/management/test_key_default_allowed_routes.py diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 022eba499a4..824b310a1dd 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -471,7 +471,13 @@ def _effective_key_for_generate(data: GenerateKeyRequest, now: datetime) -> Lite ) metadata: Final = data.metadata or MappingProxyType({}) folded_metadata: Final = {**metadata, **metadata_fields} - columns: Final = handle_key_type(data, {**column_fields}) + preset_columns: Final = handle_key_type(data, {**column_fields}) + default_allowed_routes: Final = _default_allowed_routes_for(data, preset_columns.get("allowed_routes")) + columns: Final = ( + preset_columns + if default_allowed_routes is None + else {**preset_columns, "allowed_routes": list(default_allowed_routes)} + ) expires: Final = ( now + timedelta(seconds=duration_in_seconds(duration=data.duration)) if data.duration is not None else None ) @@ -936,7 +942,19 @@ def common_key_access_checks( router: Final = APIRouter() -def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict: +_KEY_TYPE_PRESET_ROUTES: Final = MappingProxyType( + { + LiteLLMKeyType.LLM_API: ("llm_api_routes",), + LiteLLMKeyType.MANAGEMENT: ("management_routes",), + LiteLLMKeyType.READ_ONLY: ("info_routes",), + } +) + + +_KeyJsonT: Final = TypeVar("_KeyJsonT", bound=dict[str, object]) + + +def handle_key_type(data: GenerateKeyRequest, data_json: _KeyJsonT) -> _KeyJsonT: """ Handle the key type. """ @@ -945,15 +963,44 @@ def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict: data_json.pop("key_type", None) return data_json data_json["key_type"] = key_type.value - if key_type == LiteLLMKeyType.LLM_API: - data_json["allowed_routes"] = ["llm_api_routes"] - elif key_type == LiteLLMKeyType.MANAGEMENT: - data_json["allowed_routes"] = ["management_routes"] - elif key_type == LiteLLMKeyType.READ_ONLY: - data_json["allowed_routes"] = ["info_routes"] + preset_routes: Final = _KEY_TYPE_PRESET_ROUTES.get(key_type) + if preset_routes is not None: + data_json["allowed_routes"] = list(preset_routes) return data_json +def _creates_team_service_account(request: GenerateKeyRequest) -> bool: + return ( + request.user_id is None + and request.team_id is not None + and request.metadata is not None + and request.metadata.get("service_account_id") is not None + ) + + +def _configured_default_allowed_routes() -> tuple[str, ...] | None: + default_routes: Final[object] = (litellm.default_key_generate_params or {}).get("allowed_routes") + if not isinstance(default_routes, list) or not default_routes: + return None + routes: Final[tuple[object, ...]] = tuple(default_routes) + string_routes: Final = tuple(route for route in routes if isinstance(route, str)) + return string_routes if len(string_routes) == len(routes) else None + + +def _default_allowed_routes_for(request: GenerateKeyRequest, key_routes: object) -> tuple[str, ...] | None: + configured_routes: Final = _configured_default_allowed_routes() + if configured_routes is None or _creates_team_service_account(request): + return None + if "key_type" not in request.model_fields_set or request.key_type is None: + return None if key_routes else configured_routes + match request.key_type: + case LiteLLMKeyType.LLM_API: + llm_api_routes: Final = _KEY_TYPE_PRESET_ROUTES[LiteLLMKeyType.LLM_API] + return configured_routes if all(route in configured_routes for route in llm_api_routes) else None + case _: + return None + + _NON_ADMIN_SAFE_ALLOWED_ROUTES_PRESETS: Final = frozenset({"llm_api_routes", "info_routes"}) @@ -1068,11 +1115,24 @@ def _is_safe_preset_route_transition( return existing != _READ_ONLY_ALLOWED_ROUTES_PRESET or incoming == existing +def _resends_stored_routes( + data: UpdateKeyRequest | RegenerateKeyRequest, + existing_key_row: LiteLLM_VerificationToken, +) -> bool: + return ( + "allowed_routes" in data.model_fields_set + and data.allowed_routes is not None + and list(data.allowed_routes) == list(existing_key_row.allowed_routes or ()) + ) + + def _enforce_allowed_routes_update_permission( data: UpdateKeyRequest, existing_key_row: LiteLLM_VerificationToken, user_api_key_dict: UserAPIKeyAuth, ) -> None: + if _resends_stored_routes(data, existing_key_row): + return if _is_safe_preset_route_transition( incoming_allowed_routes=data.allowed_routes, existing_allowed_routes=existing_key_row.allowed_routes, @@ -1499,7 +1559,9 @@ async def _common_key_generation_helper( build_policy_request=lambda: CustomKeyPolicyRequest( operation="generate", existing_key=None, - effective_key=_effective_key_for_generate(data=data, now=datetime.now(timezone.utc)), + effective_key=_effective_key_for_generate( + data=_member_key_request(data, acting_as_team_admin), now=datetime.now(timezone.utc) + ), request=data, ), ) @@ -1549,7 +1611,8 @@ async def _common_key_generation_helper( data_json = data.model_dump(exclude_unset=True, exclude_none=True) - data_json = handle_key_type(_member_key_request(data, acting_as_team_admin), data_json) + member_request: Final = _member_key_request(data, acting_as_team_admin) + data_json = handle_key_type(member_request, data_json) # Re-check allowed_routes after handle_key_type, since key_type can derive # an elevated bucket (e.g. ["management_routes"]) that wasn't present in @@ -1561,6 +1624,9 @@ async def _common_key_generation_helper( user_api_key_dict=user_api_key_dict, allow_safe_presets=True, ) + default_allowed_routes: Final = _default_allowed_routes_for(member_request, data_json.get("allowed_routes")) + if default_allowed_routes is not None: + data_json["allowed_routes"] = list(default_allowed_routes) # if we get max_budget passed to /key/generate, then use it as key_max_budget. Since generate_key_helper_fn is used to make new users if "max_budget" in data_json: @@ -2268,7 +2334,7 @@ async def generate_key_fn( - allowed_passthrough_routes: Optional[list] - List of allowed pass through endpoints for the key. Store the actual endpoint or store a wildcard pattern for a set of endpoints. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through endpoints the key can access, without specifying the routes. If allowed_routes is specified, allowed_pass_through_endpoints is ignored. - denied_passthrough_routes: Optional[list] - List of pass through routes the key may not call, even if allowed by `allowed_passthrough_routes` or `allowed_routes`. Matches exact paths, path prefixes, and trailing `*` wildcards. Applies together with the team's `denied_passthrough_routes`. Example - ["/my-custom-endpoint/admin"]. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (uses default allowed routes). Defaults to "default". + - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (Full Access in the Admin UI, skips `default_key_generate_params.allowed_routes`). When omitted, the key gets `default_key_generate_params.allowed_routes` if that is set and the request has no `allowed_routes`, else it works like "default". - prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts. - auto_rotate: Optional[bool] - Whether this key should be automatically rotated (regenerated) - rotation_interval: Optional[str] - How often to auto-rotate this key (e.g., '30s', '30m', '30h', '30d'). Required if auto_rotate=True. @@ -2758,6 +2824,8 @@ async def prepare_key_update_data( data_json.pop("key", None) data_json.pop("new_key", None) data_json.pop("grace_period", None) # Request-only param, not a DB column + if _resends_stored_routes(data, existing_key_row): + data_json.pop("allowed_routes", None) if ( data.metadata is not None and data.metadata.get("service_account_id") is not None diff --git a/tests/integration/management/test_key_default_allowed_routes.py b/tests/integration/management/test_key_default_allowed_routes.py new file mode 100644 index 00000000000..5dfe000a1c5 --- /dev/null +++ b/tests/integration/management/test_key_default_allowed_routes.py @@ -0,0 +1,327 @@ +import uuid +from collections.abc import Iterator, Mapping +from datetime import date +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import JsonValue, TypeAdapter + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, + team_admin_permissions, +) +from tests.integration._support.process import graceful_stop_seconds, owned_proxy + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +DEFAULT_ROUTES: Final = ("llm_api_routes", "/key/spend/report") +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_PROBE_PREFIX: Final = "route-probe-" +_RACE_PREFIX: Final = "route-race-" +_ROUTES_AFTER_ADMIN_REMOVAL: Final = ["llm_api_routes"] +_POLICY_SOURCE: Final = f'''import json + +from litellm.proxy import proxy_server + + +async def route_policy(request): + alias = request.request.key_alias or "" + if request.operation == "update" and alias.startswith("{_RACE_PREFIX}"): + await proxy_server.prisma_client.db.litellm_verificationtoken.update( + where={{"token": request.existing_key.token}}, + data={{"allowed_routes": {_ROUTES_AFTER_ADMIN_REMOVAL!r}}}, + ) + return {{"decision": True}} + if not alias.startswith("{_PROBE_PREFIX}"): + return {{"decision": True}} + return {{"decision": False, "message": "routes=" + json.dumps(request.effective_key.allowed_routes or [])}} +''' + + +def _config_with_default_routes( + directory: Path, default_routes: list[JsonValue], general_settings: Mapping[str, JsonValue] +) -> Path: + raw: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + configured: Final[dict[str, JsonValue]] = { + **raw, + "litellm_settings": { + **object_value(raw.get("litellm_settings") or {}), + "default_key_generate_params": {"allowed_routes": default_routes}, + }, + "general_settings": {**object_value(raw.get("general_settings") or {}), **general_settings}, + } + config: Final = directory / "default-allowed-routes.yaml" + config.write_text(yaml.safe_dump(configured)) + return config + + +@pytest.fixture(scope="module") +def defaults(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("default-allowed-routes") + (directory / "route_policy.py").write_text(_POLICY_SOURCE) + config: Final = _config_with_default_routes( + directory, list(DEFAULT_ROUTES), {"custom_key_policy": "route_policy.route_policy"} + ) + with gateway_from_environment() as gateway, owned_proxy(gateway, directory, {}, config=config) as candidate: + yield candidate + + +@pytest.fixture(scope="module") +def non_string_default(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("non-string-default-allowed-routes") + config: Final = _config_with_default_routes(directory, ["llm_api_routes", 123], {}) + with gateway_from_environment() as gateway, owned_proxy(gateway, directory, {}, config=config) as candidate: + yield candidate + + +def _hash(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _stored_routes(gateway: Gateway, key: str) -> JsonValue: + rows: Final = gateway.rows('SELECT allowed_routes FROM "LiteLLM_VerificationToken" WHERE token = %s', (_hash(key),)) + assert len(rows) == 1, rows + return rows[0]["allowed_routes"] + + +def _created_key(scenario: Scenario, created: dict[str, JsonValue]) -> str: + token: Final = string_value(created["key"]) + scenario.cleanups.callback(scenario.delete_key, token) + return token + + +def _full_access_key(scenario: Scenario, **fields: JsonValue) -> str: + return scenario.key(key_type="default", **fields) + + +@pytest.mark.parametrize( + ("body", "expected"), + ( + ({}, list(DEFAULT_ROUTES)), + ({"key_type": None}, list(DEFAULT_ROUTES)), + ({"allowed_routes": []}, list(DEFAULT_ROUTES)), + ({"key_type": "llm_api"}, list(DEFAULT_ROUTES)), + ({"allowed_routes": ["info_routes"]}, ["info_routes"]), + ({"key_type": "management"}, ["management_routes"]), + ({"key_type": "read_only"}, ["info_routes"]), + ({"key_type": "default"}, []), + ), + ids=("unset", "key_type_null", "empty_list", "llm_api", "explicit_list", "management", "read_only", "full_access"), +) +def test_admin_created_key_routes(defaults: Gateway, body: dict[str, JsonValue], expected: list[str]) -> None: + with defaults.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + created: Final = defaults.post("/key/generate", {"user_id": user, **body}) + token: Final = _created_key(scenario, created) + + assert (created["allowed_routes"], _stored_routes(defaults, token)) == (expected, expected) + + +@pytest.mark.parametrize( + ("body", "expected"), + (({}, []), ({"key_type": "llm_api"}, ["llm_api_routes"])), + ids=("unset", "llm_api"), +) +def test_default_with_a_non_string_route_is_ignored( + non_string_default: Gateway, body: dict[str, JsonValue], expected: list[str] +) -> None: + with non_string_default.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + created: Final = non_string_default.post("/key/generate", {"user_id": user, **body}) + token: Final = _created_key(scenario, created) + + assert (created["allowed_routes"], _stored_routes(non_string_default, token)) == (expected, expected) + + +def test_default_routes_open_own_spend_report_and_nothing_wider(defaults: Gateway) -> None: + with defaults.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + user: Final = scenario.user(user_role="internal_user") + token: Final = _created_key( + scenario, defaults.post("/key/generate", {"user_id": user, "key_type": "llm_api", "models": [model]}) + ) + other: Final = _full_access_key(scenario, user_id=scenario.user(user_role="internal_user")) + defaults.chat(model, key=token, text=f"default routes {uuid.uuid4().hex}") + logged: Final = eventually( + lambda: defaults.rows( + """SELECT to_char("startTime", 'YYYY-MM-DD') AS day FROM "LiteLLM_SpendLogs" WHERE api_key = %s""", + (_hash(token),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + day: Final = date.fromisoformat(string_value(logged[0]["day"])).isoformat() + window: Final = {"start_date": day, "end_date": day} + + own: Final = defaults.request("GET", "/key/spend/report", key=token, params=window) + assert own.status_code == 200, own.text + assert [object_value(row)["api_key"] for row in list_value(_JSON.validate_json(own.content))] == [_hash(token)] + + foreign: Final = defaults.request( + "GET", "/key/spend/report", key=token, params={**window, "api_key": _hash(other)} + ) + assert foreign.status_code == 403, foreign.text + global_report: Final = defaults.request("GET", "/global/spend/report", key=token, params=window) + assert global_report.status_code == 403, global_report.text + minted: Final = defaults.request("POST", "/key/generate", {}, key=token) + assert minted.status_code == 403, minted.text + + +@pytest.mark.parametrize("body", ({}, {"key_type": "llm_api"}), ids=("unset", "llm_api")) +def test_internal_user_own_key_routes(defaults: Gateway, body: dict[str, JsonValue]) -> None: + with defaults.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + own_key: Final = _full_access_key(scenario, user_id=user) + created: Final = defaults.request("POST", "/key/generate", body, key=own_key) + assert created.status_code == 200, created.text + token: Final = _created_key(scenario, JSON_OBJECT.validate_json(created.content)) + + assert defaults.rows( + 'SELECT user_id, allowed_routes FROM "LiteLLM_VerificationToken" WHERE token = %s', (_hash(token),) + ) == [{"user_id": user, "allowed_routes": list(DEFAULT_ROUTES)}] + + +def test_service_account_skips_default_and_its_team_key_gets_it(defaults: Gateway) -> None: + with defaults.scenario() as scenario: + team: Final = scenario.team() + account: Final = _created_key(scenario, defaults.post("/key/service-account/generate", {"team_id": team})) + assert _stored_routes(defaults, account) == [] + + defaults.post("/team/permissions_update", {"team_id": team, "team_member_permissions": ["/key/generate"]}) + team_key: Final = _created_key(scenario, defaults.post("/key/generate", {"team_id": team}, key=account)) + + assert defaults.rows( + 'SELECT user_id, team_id, allowed_routes FROM "LiteLLM_VerificationToken" WHERE token = %s', + (_hash(team_key),), + ) == [{"user_id": None, "team_id": team, "allowed_routes": list(DEFAULT_ROUTES)}] + + +def _policy_seen_routes(response_text: str) -> JsonValue: + message: Final = string_value(object_value(JSON_OBJECT.validate_json(response_text)["error"])["message"]) + assert message.startswith("routes="), message + return _JSON.validate_json(message.removeprefix("routes=")) + + +@pytest.mark.parametrize("team_admin", (False, True), ids=("unset", "team_admin_member_key")) +def test_custom_key_policy_sees_stored_routes(defaults: Gateway, team_admin: bool) -> None: + with defaults.scenario() as scenario: + team: Final = scenario.team() + admin: Final = scenario.member(team, role="admin") + member: Final = scenario.member(team) + caller: Final = _full_access_key(scenario, user_id=admin, team_id=team) if team_admin else defaults.key + body: Final[dict[str, JsonValue]] = {"team_id": team, "user_id": member} + scenario.cleanups.enter_context(team_admin_permissions(defaults, ("create_member_keys",))) + + probe: Final = defaults.request( + "POST", "/key/generate", {**body, "key_alias": f"{_PROBE_PREFIX}{uuid.uuid4().hex}"}, key=caller + ) + assert probe.status_code == 403, probe.text + created: Final = defaults.request( + "POST", "/key/generate", {**body, "key_alias": f"route-twin-{uuid.uuid4().hex}"}, key=caller + ) + assert created.status_code == 200, created.text + token: Final = _created_key(scenario, JSON_OBJECT.validate_json(created.content)) + + assert _policy_seen_routes(probe.text) == _stored_routes(defaults, token) == list(DEFAULT_ROUTES) + + +_ADMIN_SET_ROUTES: Final = ("llm_api_routes", "/key/spend/report") + + +def _owner_key_with_admin_set_routes(gateway: Gateway, scenario: Scenario) -> tuple[str, str]: + owner_key: Final = scenario.key(user_id=scenario.user(user_role="internal_user")) + target: Final = _created_key(scenario, gateway.post("/key/generate", {}, key=owner_key)) + gateway.post("/key/update", {"key": target, "allowed_routes": list(_ADMIN_SET_ROUTES)}) + return owner_key, target + + +def _stored_alias_and_routes(gateway: Gateway, key: str) -> tuple[JsonValue, list[str]]: + rows: Final = gateway.rows( + 'SELECT key_alias, allowed_routes FROM "LiteLLM_VerificationToken" WHERE token = %s', (_hash(key),) + ) + assert len(rows) == 1, rows + return rows[0]["key_alias"], [str(route) for route in list_value(rows[0]["allowed_routes"])] + + +def test_owner_resends_unchanged_routes(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner_key, target = _owner_key_with_admin_set_routes(gateway, scenario) + alias: Final = f"resave-{uuid.uuid4().hex}" + response: Final = gateway.request( + "POST", + "/key/update", + {"key": target, "key_alias": alias, "allowed_routes": list(_ADMIN_SET_ROUTES)}, + key=owner_key, + ) + + assert response.status_code == 200, response.text + assert _stored_alias_and_routes(gateway, target) == (alias, list(_ADMIN_SET_ROUTES)) + + +@pytest.mark.parametrize( + "routes", + ( + [], + ["llm_api_routes"], + [*_ADMIN_SET_ROUTES, "/global/spend/report"], + list(reversed(_ADMIN_SET_ROUTES)), + [*_ADMIN_SET_ROUTES, _ADMIN_SET_ROUTES[-1]], + ), + ids=("clear", "ai_apis", "widen", "reordered", "duplicated"), +) +def test_owner_cannot_change_admin_set_routes(gateway: Gateway, routes: list[JsonValue]) -> None: + with gateway.scenario() as scenario: + owner_key, target = _owner_key_with_admin_set_routes(gateway, scenario) + before: Final = _stored_alias_and_routes(gateway, target) + response: Final = gateway.request( + "POST", "/key/update", {"key": target, "allowed_routes": routes}, key=owner_key + ) + + assert response.status_code == 403, response.text + assert _stored_alias_and_routes(gateway, target) == before + + +def test_non_owner_resending_unchanged_routes_is_refused(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + _owner_key, target = _owner_key_with_admin_set_routes(gateway, scenario) + outsider_key: Final = scenario.key(user_id=scenario.user(user_role="internal_user")) + before: Final = _stored_alias_and_routes(gateway, target) + response: Final = gateway.request( + "POST", + "/key/update", + {"key": target, "key_alias": f"hijack-{uuid.uuid4().hex}", "allowed_routes": list(_ADMIN_SET_ROUTES)}, + key=outsider_key, + ) + + assert response.status_code == 403, response.text + assert _stored_alias_and_routes(gateway, target) == before + + +def test_owner_resend_keeps_an_admin_route_removal_made_mid_update(defaults: Gateway) -> None: + with defaults.scenario() as scenario: + owner_key: Final = _full_access_key(scenario, user_id=scenario.user(user_role="internal_user")) + created: Final = defaults.request("POST", "/key/generate", {}, key=owner_key) + assert created.status_code == 200, created.text + target: Final = _created_key(scenario, JSON_OBJECT.validate_json(created.content)) + assert _stored_routes(defaults, target) == list(DEFAULT_ROUTES) + alias: Final = f"{_RACE_PREFIX}{uuid.uuid4().hex}" + + response: Final = defaults.request( + "POST", + "/key/update", + {"key": target, "key_alias": alias, "allowed_routes": list(DEFAULT_ROUTES)}, + key=owner_key, + ) + + assert response.status_code == 200, response.text + assert _stored_alias_and_routes(defaults, target) == (alias, _ROUTES_AFTER_ADMIN_REMOVAL) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index e451c38a251..d6ee9b1d29e 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -19639,6 +19639,285 @@ async def _generate_key_and_get_persisted_row(data: GenerateKeyRequest, mock_ins return key_call.kwargs["data"] +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_fills_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + response: Final = await generate_key_fn( + data=GenerateKeyRequest(), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-internal", + user_id="internal-user", + ), + litellm_changed_by=None, + ) + + assert response is not None + key_call: Final = next(c for c in mock_insert_data.call_args_list if c.kwargs["table_name"] == "key") + assert key_call.kwargs["data"]["allowed_routes"] == ["llm_api_routes", "/key/spend/report"] + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_keeps_explicit_value(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row( + GenerateKeyRequest(allowed_routes=["info_routes"]), + mock_insert_data, + ) + + assert key_row["allowed_routes"] == ["info_routes"] + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_extends_key_type_preset_it_covers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import LiteLLMKeyType + + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + await generate_key_fn( + data=GenerateKeyRequest(key_type=LiteLLMKeyType.LLM_API), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-internal", + user_id="internal-user", + ), + litellm_changed_by=None, + ) + + key_call: Final = next(c for c in mock_insert_data.call_args_list if c.kwargs["table_name"] == "key") + assert key_call.kwargs["data"]["allowed_routes"] == ["llm_api_routes", "/key/spend/report"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_type", "preset_routes"), + [(LiteLLMKeyType.MANAGEMENT, ["management_routes"]), (LiteLLMKeyType.READ_ONLY, ["info_routes"])], +) +async def test_default_key_generate_params_allowed_routes_keeps_management_and_read_only_presets( + monkeypatch: pytest.MonkeyPatch, key_type: LiteLLMKeyType, preset_routes: list[str] +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "management_routes", "info_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row( + GenerateKeyRequest(key_type=key_type), + mock_insert_data, + ) + + assert key_row["allowed_routes"] == preset_routes + + +@pytest.mark.parametrize( + "data", + [GenerateKeyRequest(), GenerateKeyRequest(key_type=LiteLLMKeyType.LLM_API)], + ids=["unset", "llm_api"], +) +def test_effective_key_for_generate_shows_default_allowed_routes_to_the_policy( + monkeypatch: pytest.MonkeyPatch, data: GenerateKeyRequest +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + + effective_key: Final = _effective_key_for_generate(data=data, now=datetime(2026, 1, 1, tzinfo=timezone.utc)) + + assert effective_key.allowed_routes == ["llm_api_routes", "/key/spend/report"] + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_skips_team_service_account_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row( + GenerateKeyRequest(team_id="team-a", metadata={"service_account_id": "sa-1"}), + mock_insert_data, + ) + + assert not key_row.get("allowed_routes") + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_skips_full_access_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row( + GenerateKeyRequest(key_type=LiteLLMKeyType.DEFAULT), + mock_insert_data, + ) + + assert not key_row.get("allowed_routes") + + +def _owner_route_update_status(existing: list[str], incoming: list[str]) -> int | None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _enforce_allowed_routes_update_permission, + ) + + try: + _enforce_allowed_routes_update_permission( + data=UpdateKeyRequest(key="sk-owned", allowed_routes=incoming), + existing_key_row=LiteLLM_VerificationToken( + token="hashed-owned", user_id="internal-user", allowed_routes=existing + ), + user_api_key_dict=UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER), + ) + except HTTPException as exc: + return exc.status_code + return None + + +@pytest.mark.parametrize( + ("existing", "incoming", "expected_status"), + [ + (["llm_api_routes"], [], None), + (["llm_api_routes", "/key/spend/report"], [], 403), + (["llm_api_routes", "/key/spend/report"], ["llm_api_routes"], 403), + (["llm_api_routes", "/key/spend/report"], ["llm_api_routes", "/key/spend/report"], None), + (["llm_api_routes", "/key/spend/report"], ["/key/spend/report", "llm_api_routes"], 403), + ( + ["llm_api_routes", "/key/spend/report"], + ["llm_api_routes", "/key/spend/report", "/key/spend/report"], + 403, + ), + (["/global/spend/report"], ["/global/spend/report"], None), + ], + ids=[ + "preset_to_full_access", + "default_list_to_full_access", + "default_list_to_ai_apis", + "default_list_sent_back_unchanged", + "default_list_sent_back_reordered", + "default_list_sent_back_with_a_duplicate", + "admin_set_list_sent_back_unchanged", + ], +) +def test_owner_cannot_change_the_routes_a_key_got_from_default_key_generate_params( + monkeypatch: pytest.MonkeyPatch, existing: list[str], incoming: list[str], expected_status: int | None +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + + assert _owner_route_update_status(existing, incoming) == expected_status + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("incoming", "written"), + [ + (["llm_api_routes", "/key/spend/report"], None), + (["/key/spend/report", "llm_api_routes"], ["/key/spend/report", "llm_api_routes"]), + (["llm_api_routes"], ["llm_api_routes"]), + ], + ids=["unchanged_resend", "reordered", "changed"], +) +async def test_key_update_writes_allowed_routes_only_when_they_change( + incoming: list[str], written: list[str] | None +) -> None: + existing_key: Final = LiteLLM_VerificationToken( + token="hashed", allowed_routes=["llm_api_routes", "/key/spend/report"] + ) + + update_values: Final = await prepare_key_update_data( + data=UpdateKeyRequest(key="sk-1", key_alias="renamed", allowed_routes=incoming), existing_key_row=existing_key + ) + + assert (update_values["key_alias"], update_values.get("allowed_routes")) == ("renamed", written) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_data", "expected_routes"), + [(GenerateKeyRequest(), []), (GenerateKeyRequest(key_type=LiteLLMKeyType.LLM_API), ["llm_api_routes"])], + ids=["no_key_type", "ai_apis"], +) +async def test_default_key_generate_params_allowed_routes_with_a_non_string_entry_is_ignored( + monkeypatch: pytest.MonkeyPatch, request_data: GenerateKeyRequest, expected_routes: list[str] +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", 123]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row(request_data, mock_insert_data) + + assert (key_row.get("allowed_routes") or []) == expected_routes + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_empty_list_treated_as_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm, + "default_key_generate_params", + {"allowed_routes": ["llm_api_routes", "/key/spend/report"]}, + ) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row( + GenerateKeyRequest(allowed_routes=[]), + mock_insert_data, + ) + + assert key_row["allowed_routes"] == ["llm_api_routes", "/key/spend/report"] + + +@pytest.mark.asyncio +async def test_default_key_generate_params_allowed_routes_ignores_a_string_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "default_key_generate_params", {"allowed_routes": "llm_api_routes"}) + mock_insert_data: Final = _wire_key_generation_prisma(monkeypatch) + + key_row: Final = await _generate_key_and_get_persisted_row(GenerateKeyRequest(), mock_insert_data) + + assert not key_row.get("allowed_routes") + + @pytest.mark.asyncio async def test_key_generate_explicit_null_budget_duration_beats_default_key_generate_params(monkeypatch): """An explicit `"budget_duration": null` asks for a budget that never resets. @@ -22647,6 +22926,29 @@ async def test_team_admin_creates_an_llm_api_key_owned_by_an_internal_user_teamm assert (saved["key_type"], saved["allowed_routes"]) == ("llm_api", ["llm_api_routes"]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "default_key_generate_params", + [None, {"allowed_routes": ["/chat/completions"]}], + ids=["default_off", "default_without_llm_api_routes"], +) +async def test_custom_key_policy_sees_the_routes_a_member_key_is_saved_with( + monkeypatch: pytest.MonkeyPatch, default_key_generate_params: dict[str, list[str]] | None +) -> None: + monkeypatch.setattr(litellm, "default_key_generate_params", default_key_generate_params) + policy: Final = AsyncMock(return_value={"decision": True}) + generate_key: Final = AsyncMock(return_value={"key": "sk-member", "user_id": "member-1", "team_id": "team-1"}) + + with patch("litellm.proxy.proxy_server.user_custom_key_policy", policy): # test-quality-ok: inject policy hook + await _team_admin_generates_member_key( + AsyncMock(return_value=MagicMock(user_role=LitellmUserRoles.INTERNAL_USER.value)), generate_key + ) + + policy_request: Final = policy.await_args.args[0] + assert policy_request.effective_key.allowed_routes == ["llm_api_routes"] + assert policy_request.effective_key.allowed_routes == generate_key.await_args.kwargs["allowed_routes"] + + @pytest.mark.asyncio async def test_team_admin_cannot_create_a_key_owned_by_a_proxy_admin_teammate(): generate_key: Final = AsyncMock() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 733a093bcac..ac096862b56 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8441,7 +8441,7 @@ export interface paths { * - allowed_passthrough_routes: Optional[list] - List of allowed pass through endpoints for the key. Store the actual endpoint or store a wildcard pattern for a set of endpoints. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through endpoints the key can access, without specifying the routes. If allowed_routes is specified, allowed_pass_through_endpoints is ignored. * - denied_passthrough_routes: Optional[list] - List of pass through routes the key may not call, even if allowed by `allowed_passthrough_routes` or `allowed_routes`. Matches exact paths, path prefixes, and trailing `*` wildcards. Applies together with the team's `denied_passthrough_routes`. Example - ["/my-custom-endpoint/admin"]. * - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "agents": ["agent_1", "agent_2"], "agent_access_groups": ["dev_group"]}. IF null or {} then no object permission. - * - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (uses default allowed routes). Defaults to "default". + * - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (Full Access in the Admin UI, skips `default_key_generate_params.allowed_routes`). When omitted, the key gets `default_key_generate_params.allowed_routes` if that is set and the request has no `allowed_routes`, else it works like "default". * - prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts. * - auto_rotate: Optional[bool] - Whether this key should be automatically rotated (regenerated) * - rotation_interval: Optional[str] - How often to auto-rotate this key (e.g., '30s', '30m', '30h', '30d'). Required if auto_rotate=True. From 0e7c4908bb2269b43e6033ad1ea518cde2369348 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 19:08:30 -0700 Subject: [PATCH 049/124] fix(proxy): price the estimated input tokens on a client-cancelled (499) request's spend row (#47436) * fix(proxy): price the estimated input tokens on a client-cancelled (499) request's spend row * fix(proxy): read the provider handoff before the proxy-only error handler stamps it so a pre-handoff 499 is not priced * fix(proxy): drop the pre-logging handoff read now that pre_call no longer stamps proxy-only failures --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 22 +++++- ..._utils_model_creation_and_error_logging.py | 77 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1bd856803eb..7696a56ca88 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -58,6 +58,7 @@ from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, FILE_USAGE_MAX_TRACKED_COUNTERS, LITELLM_CONFIG_CACHE_TTL_SECONDS, + LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, PROXY_REJECTED_BEFORE_ROUTING_KEY, @@ -1021,8 +1022,8 @@ def _failure_usage_to_lift( cost in model_call_details, so prefer it. Otherwise a request that was dispatched to a provider and failed without upstream usage gets an estimated input-side Usage. When the failure is one the provider bills - anyway (``provider_billed``: a read timeout, where the provider had the - prompt and kept working after LiteLLM hung up) that + anyway (``provider_billed``: a read timeout or a client disconnect, where + the provider had the prompt and kept working after LiteLLM hung up) that usage is priced at the model's input rate; any other post-dispatch failure stays at zero cost. The raw request body backfills the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a @@ -1034,7 +1035,11 @@ def _failure_usage_to_lift( return _billed_across_attempts(failed_attempts, recovered_usage, model_call_details.get("response_cost")) if recovered_usage is not None: return recovered_usage, model_call_details.get("response_cost") - if not dispatched or model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL): + if not dispatched: + return None + if model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL) and not provider_billed: + # The proxy-only error handler stamps this flag on a client disconnect too, even though the + # request had already reached the provider, so a billed failure keeps its estimate. return None if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES: return None @@ -1208,6 +1213,15 @@ def _timed_out_reading_from_provider(exception: Exception) -> bool: ) +def _client_disconnected(exception: Exception) -> bool: + """The 499 the proxy raises when the caller hung up while the provider was still working on the request.""" + return getattr(exception, "status_code", None) == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + + +def _provider_billed_the_failure(exception: Exception) -> bool: + return _timed_out_reading_from_provider(exception) or _client_disconnected(exception) + + def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]: """Failure-path callbacks run after ``litellm_logging_obj`` is popped from request_data (it is not serialisable), so the caller merges these fields @@ -3450,7 +3464,7 @@ class ProxyLogging: request_data.update( await offload_token_count(_failure_fields_to_lift)( - request_data, provider_billed=_timed_out_reading_from_provider(original_exception) + request_data, provider_billed=_provider_billed_the_failure(original_exception) ) ) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 34d6c165182..1b59615876a 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -794,6 +794,83 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: assert expected_input_cost > 0 assert request_data["response_cost"] == pytest.approx(expected_input_cost) + @pytest.mark.asyncio + async def test_client_disconnect_after_dispatch_is_priced_at_input_rate(self): + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True, + } + ), + "metadata": {}, + } + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + expected_input_cost, _ = litellm.cost_per_token(model="gpt-3.5-turbo", prompt_tokens=estimated.prompt_tokens) + assert expected_input_cost > 0 + assert request_data["response_cost"] == pytest.approx(expected_input_cost) + + @pytest.mark.asyncio + async def test_client_disconnect_before_provider_handoff_is_not_priced(self): + """A caller that hung up while the request was still in pre-call hooks never reached the + provider, so there is no handoff instant on the logging object and the 499 row is not priced.""" + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="lit145-pre-handoff", + function_id="lit145", + ) + assert logging_obj.model_call_details.get("first_api_call_start_time") is None + request_data = {"litellm_logging_obj": logging_obj, "model": "gpt-3.5-turbo", "metadata": {}} + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with ( + patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()), + patch.object(proxy_logging_obj, "_dispatch_proxy_only_failure_handlers", new=AsyncMock()), + ): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + ) + + assert request_data.get("first_api_call_start_time") is None + assert "combined_usage_object" not in request_data + assert request_data.get("response_cost", 0.0) == 0.0 + @pytest.mark.asyncio async def test_connect_timeout_never_reached_the_provider_stays_zero_cost(self): from datetime import datetime From 6e90733ffccaf3ce8b0ef0428a11e48f4fdd79cb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 19:40:32 -0700 Subject: [PATCH 050/124] test(perf): gate the gateway's own overhead header, redis latency and postgres load in the perf cases (#47472) * test(perf): gate the gateway's own overhead header, redis latency and postgres load in the perf cases * test(perf): keep the gateway_added_seconds limits next to the header ones --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- tests/e2e/gateway/perf_ci_config.yml | 2 + .../perf/cases/mixed_size_requests_2rps.py | 8 ++ tests/e2e/perf/cases/small_bodies_5rps.py | 8 ++ tests/e2e/perf/setup/datastore.py | 103 ++++++++++++++++-- tests/e2e/perf/setup/locustfile.py | 11 ++ tests/e2e/perf/setup/observe.py | 48 ++++++++ tests/e2e/perf/setup/performance_test_case.py | 6 + tests/e2e/perf/setup/telemetry.py | 65 +++++++++++ tests/e2e/perf/setup/wire.py | 2 + tests/e2e/perf/test_perf_cases.py | 10 +- tests/e2e_harness/perf/setup/test_observe.py | 9 +- 12 files changed, 261 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/perf/setup/telemetry.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index b32cfb25fa6..d976a2b5ee3 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -44,7 +44,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) -- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares) and `Limit`s (one metric at one percentile under one ceiling). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` +- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares) and `Limit`s (one metric at one percentile under one ceiling). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.yml index f5ec2e22693..7ab6637e985 100644 --- a/tests/e2e/gateway/perf_ci_config.yml +++ b/tests/e2e/gateway/perf_ci_config.yml @@ -27,6 +27,8 @@ litellm_settings: callbacks: - prometheus - otel + service_callback: + - prometheus_system require_auth_for_metrics_endpoint: false drop_params: true request_timeout: 120 diff --git a/tests/e2e/perf/cases/mixed_size_requests_2rps.py b/tests/e2e/perf/cases/mixed_size_requests_2rps.py index 89263977fb0..850658a15c5 100644 --- a/tests/e2e/perf/cases/mixed_size_requests_2rps.py +++ b/tests/e2e/perf/cases/mixed_size_requests_2rps.py @@ -64,6 +64,14 @@ CASE: Final = PerformanceTestCase( Limit("unexpected_status_count", max=3), Limit("gateway_added_seconds", percentile=50, max=0.58), Limit("gateway_added_seconds", percentile=99, max=6.1), + Limit("litellm_overhead_seconds", percentile=50, max=0.05), + Limit("litellm_overhead_seconds", percentile=99, max=2.2), + Limit("litellm_response_seconds", percentile=50, max=1.2), + Limit("litellm_response_seconds", percentile=99, max=3.8), + Limit("prometheus_redis_latency_mean_seconds", max=0.003), + Limit("postgres_active_sessions", max=0.004), + Limit("postgres_slowest_statement_seconds", max=0.045), + Limit("postgres_peak_connections", max=30), Limit("time_to_first_token_seconds", percentile=50, max=2.7), Limit("time_to_first_token_seconds", percentile=99, max=8.8), Limit("memory_growth_megabytes", max=600), diff --git a/tests/e2e/perf/cases/small_bodies_5rps.py b/tests/e2e/perf/cases/small_bodies_5rps.py index 73772465492..bc22a9a96c8 100644 --- a/tests/e2e/perf/cases/small_bodies_5rps.py +++ b/tests/e2e/perf/cases/small_bodies_5rps.py @@ -65,6 +65,14 @@ CASE: Final = PerformanceTestCase( limits=( Limit("unexpected_status_count", max=3), Limit("gateway_added_seconds", percentile=50, max=0.29), + Limit("litellm_overhead_seconds", percentile=50, max=0.02), + Limit("litellm_overhead_seconds", percentile=99, max=0.8), + Limit("litellm_response_seconds", percentile=50, max=0.8), + Limit("litellm_response_seconds", percentile=99, max=2.8), + Limit("prometheus_redis_latency_mean_seconds", max=0.006), + Limit("postgres_active_sessions", max=0.006), + Limit("postgres_slowest_statement_seconds", max=0.035), + Limit("postgres_peak_connections", max=30), Limit("time_to_first_token_seconds", percentile=99, max=3.7), Limit("memory_growth_megabytes", max=100), Limit("postgres_calls_per_request", max=1.8), diff --git a/tests/e2e/perf/setup/datastore.py b/tests/e2e/perf/setup/datastore.py index 54c6943a054..89739b6d13e 100644 --- a/tests/e2e/perf/setup/datastore.py +++ b/tests/e2e/perf/setup/datastore.py @@ -1,7 +1,10 @@ """Postgres and Redis work the gateway did during the traffic phase, split into what a request waits on and what runs after the response. Both come from the servers' own accounting, so nothing here depends on how the gateway talks to them: Postgres statement counts from `pg_stat_statements` (read before and after the run) and -Redis commands from a `MONITOR` connection held open for the run. `DATABASE_URL` and `REDIS_HOST` are the +Redis commands from a `MONITOR` connection held open for the run, Postgres execution time per statement from the +same `pg_stat_statements` rows (`postgres_active_sessions`, the average-active-sessions number RDS Performance +Insights calls DB load, and the slowest statement's mean) and the connection count from `pg_stat_activity` sampled +once a second (`postgres_peak_connections`). `DATABASE_URL` and `REDIS_HOST` are the runner's own env, as in the other suites; without them the counters are None and any Limit on them reports as missing. `E2E_DATABASE_ADMIN_URL`, when set, is used once to create the `pg_stat_statements` extension. Only the runtime user's own statements are read: other users' rows come back with a NULL queryid and hidden text.""" @@ -26,13 +29,18 @@ Bucket = Literal["awaited", "background", "plumbing", "unclassified"] _COMMITS_QUERY: Final = "SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()" _STATEMENTS_QUERY: Final = ( - "SELECT queryid, calls, query FROM pg_stat_statements " + "SELECT queryid, calls, query, total_exec_time FROM pg_stat_statements " "WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) " "AND userid = (SELECT usesysid FROM pg_user WHERE usename = current_user) AND queryid IS NOT NULL" ) _CREATE_EXTENSION: Final = "CREATE EXTENSION IF NOT EXISTS pg_stat_statements" +_CONNECTIONS_QUERY: Final = ( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " + "AND backend_type = 'client backend' AND pid <> pg_backend_pid()" +) +_CONNECTIONS_SAMPLE_SECONDS: Final = 1.0 _COMMITS_ROW: Final = TypeAdapter(tuple[int]) -_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str]]) +_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str, float]]) _INFO: Final = TypeAdapter(dict[str, object]) _INT: Final = TypeAdapter(int) _MONITOR_LINE: Final = TypeAdapter(dict[str, object]) @@ -64,6 +72,11 @@ class StatementCalls: queryid: int query: str calls: int + total_exec_ms: float = 0.0 + + @property + def mean_exec_seconds(self) -> float | None: + return self.total_exec_ms / 1000 / self.calls if self.calls > 0 else None @property def bucket(self) -> Bucket: @@ -136,16 +149,85 @@ def statement_deltas(before: DbCounters | None, after: DbCounters | None) -> tup """Calls per statement during the run: `after` minus `before` by queryid, new statements whole.""" if before is None or after is None or before.statements is None or after.statements is None: return None - earlier: Final = {statement.queryid: statement.calls for statement in before.statements} + earlier: Final = {statement.queryid: statement for statement in before.statements} return tuple( - StatementCalls( - queryid=statement.queryid, query=statement.query, calls=statement.calls - earlier.get(statement.queryid, 0) - ) + _statement_delta(statement, earlier.get(statement.queryid)) for statement in after.statements - if statement.calls > earlier.get(statement.queryid, 0) + if statement.calls > (0 if statement.queryid not in earlier else earlier[statement.queryid].calls) ) +def _statement_delta(statement: StatementCalls, earlier: StatementCalls | None) -> StatementCalls: + return StatementCalls( + queryid=statement.queryid, + query=statement.query, + calls=statement.calls - (earlier.calls if earlier is not None else 0), + total_exec_ms=statement.total_exec_ms - (earlier.total_exec_ms if earlier is not None else 0.0), + ) + + +def active_sessions(statements: tuple[StatementCalls, ...] | None, wall_seconds: float) -> float | None: + """Seconds Postgres spent executing the run's statements per second of wall time, the average active + sessions figure RDS Performance Insights reports as DB load.""" + if statements is None or wall_seconds <= 0: + return None + return sum(statement.total_exec_ms for statement in statements) / 1000 / wall_seconds + + +def slowest_statement(statements: tuple[StatementCalls, ...] | None) -> StatementCalls | None: + """The run's statement with the highest mean execution time.""" + timed: Final = tuple(statement for statement in statements or () if statement.mean_exec_seconds is not None) + return max(timed, key=lambda statement: statement.mean_exec_seconds or 0.0, default=None) + + +class ConnectionsSampler: + """`pg_stat_activity` client backends on the gateway's database once a second on a background thread, minus + this sampler's own connection, so the peak is what the gateway (and the test's own reads) held open.""" + + def __init__(self, url: str) -> None: + self._url: Final = url + self._stop: Final = threading.Event() + self._thread: Final = threading.Thread(target=self._run, name="postgres-connections", daemon=True) + self.peak: int | None = None + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stop.set() + self._thread.join(timeout=10) + + def _run(self) -> None: + import psycopg + + with psycopg.connect(self._url, autocommit=True) as conn: + while not self._stop.is_set(): + (count,) = _COMMITS_ROW.validate_python(conn.execute(_CONNECTIONS_QUERY).fetchone()) + self.peak = count if self.peak is None else max(self.peak, count) + _ = self._stop.wait(_CONNECTIONS_SAMPLE_SECONDS) + + +class NoConnectionsSampler: + peak: None = None + + def start(self) -> None: + return None + + def stop(self) -> None: + return None + + +@contextmanager +def connections_sampler() -> Generator[ConnectionsSampler | NoConnectionsSampler]: + url: Final = os.environ.get("DATABASE_URL") + sampler: Final = ConnectionsSampler(url) if url is not None else NoConnectionsSampler() + sampler.start() + try: + yield sampler + finally: + sampler.stop() + + def calls_in(bucket: Bucket, items: tuple[StatementCalls, ...] | tuple[RedisCommand, ...] | None) -> int | None: if items is None: return None @@ -171,7 +253,10 @@ def read_db_counters() -> DbCounters | None: return DbCounters(commits=commits, statements=None) return DbCounters( commits=commits, - statements=tuple(StatementCalls(queryid=queryid, query=query, calls=calls) for queryid, calls, query in rows), + statements=tuple( + StatementCalls(queryid=queryid, query=query, calls=calls, total_exec_ms=total_ms) + for queryid, calls, query, total_ms in rows + ), ) diff --git a/tests/e2e/perf/setup/locustfile.py b/tests/e2e/perf/setup/locustfile.py index ccc7cf698f8..1c38eae22a1 100644 --- a/tests/e2e/perf/setup/locustfile.py +++ b/tests/e2e/perf/setup/locustfile.py @@ -137,6 +137,15 @@ def _header(response: FastResponse, name: str) -> str | None: return value if isinstance(value, str) else None +def _ms_header(response: FastResponse | None, name: str) -> float | None: + """A gateway timing header as a number; the gateway writes the literal `None` when it has no value.""" + value: Final = _header(response, name) if response is not None else None + try: + return float(value) if value is not None else None + except ValueError: + return None + + def _quit(user: FastHttpUser) -> None: """End the run once every record is written. Quitting from a separate greenlet: `runner.quit` kills the user greenlets, and the user calling it would be killed mid-shutdown.""" @@ -309,6 +318,8 @@ class PlanUser(FastHttpUser): call_id=_header(response, "x-litellm-call-id") if response is not None else None, error=error, response_id=identifier, + litellm_overhead_ms=_ms_header(response, "x-litellm-overhead-duration-ms"), + litellm_response_ms=_ms_header(response, "x-litellm-response-duration-ms"), ) ) if _PROGRESS.drained: diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py index f634df7de84..6d207fb0543 100644 --- a/tests/e2e/perf/setup/observe.py +++ b/tests/e2e/perf/setup/observe.py @@ -16,6 +16,8 @@ from memory_readings import RssCapture, RssReading, WorkerKey from pydantic import BaseModel from setup.datastore import ( + active_sessions, + slowest_statement, DbCounters, RedisCommand, RedisCounters, @@ -25,6 +27,7 @@ from setup.datastore import ( ) from setup.mocks.reply_common import Provider from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase +from setup.telemetry import GatewayTelemetry from setup.wire import WireRecord @@ -115,6 +118,16 @@ class Observation: redis_after: RedisCounters | None = None redis_commands: tuple[RedisCommand, ...] | None = None gateway_cpu_seconds: float | None = None + telemetry_before: GatewayTelemetry | None = None + telemetry_after: GatewayTelemetry | None = None + peak_connections: int | None = None + + @property + def telemetry(self) -> GatewayTelemetry | None: + """What the gateway's `/metrics` counted during the run.""" + if self.telemetry_before is None or self.telemetry_after is None: + return None + return self.telemetry_after.minus(self.telemetry_before) def per_request(self, before: int | None, after: int | None) -> float | None: if before is None or after is None or not self.records: @@ -182,6 +195,18 @@ class Observation: ] return non_stream + stream + def _first_attempts(self) -> list[WireRecord]: + return [record for record, _ in self._clean(stream=False)] + [record for record, _ in self._clean(stream=True)] + + def litellm_overhead_seconds(self) -> list[float]: + """`x-litellm-overhead-duration-ms` per first-attempt request, the gateway's own account of what it added + (receive time to response end, minus the provider call), in seconds to sit next to `gateway_added_seconds`.""" + return [r.litellm_overhead_ms / 1000 for r in self._first_attempts() if r.litellm_overhead_ms is not None] + + def litellm_response_seconds(self) -> list[float]: + """`x-litellm-response-duration-ms` per first-attempt request, the gateway's own end-to-end time.""" + return [r.litellm_response_ms / 1000 for r in self._first_attempts() if r.litellm_response_ms is not None] + def time_to_first_token_seconds(self) -> list[float]: """Seconds from the client sending each first-attempt streamed request to its first token, mock delay included, comparable across versions because the mocks generate the same replies after the same wait.""" @@ -217,11 +242,23 @@ def observe( per_request: dict[str, list[float]] = { "gateway_added_seconds": observation.gateway_added_seconds(), "time_to_first_token_seconds": observation.time_to_first_token_seconds(), + "litellm_overhead_seconds": observation.litellm_overhead_seconds(), + "litellm_response_seconds": observation.litellm_response_seconds(), } if metric in per_request: return percentile(per_request[metric], 50 if percentile_rank is None else percentile_rank) if metric == "unexpected_status_count": return float(observation.unexpected_status_count()) + if metric == "prometheus_redis_latency_mean_seconds": + counted: Final = observation.telemetry + return None if counted is None else counted.redis_latency.mean + if metric == "postgres_active_sessions": + return active_sessions(observation.statements, observation.wall_seconds) + if metric == "postgres_slowest_statement_seconds": + slowest: Final = slowest_statement(observation.statements) + return None if slowest is None else slowest.mean_exec_seconds + if metric == "postgres_peak_connections": + return None if observation.peak_connections is None else float(observation.peak_connections) if metric == "postgres_calls_per_request": return observation.count_per_request(calls_in("awaited", observation.statements)) if metric == "postgres_background_calls_per_request": @@ -275,6 +312,14 @@ REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = ( ("unexpected_status_count", None), ("gateway_added_seconds", 50), ("gateway_added_seconds", 99), + ("litellm_overhead_seconds", 50), + ("litellm_overhead_seconds", 99), + ("litellm_response_seconds", 50), + ("litellm_response_seconds", 99), + ("prometheus_redis_latency_mean_seconds", None), + ("postgres_active_sessions", None), + ("postgres_slowest_statement_seconds", None), + ("postgres_peak_connections", None), ("time_to_first_token_seconds", 50), ("time_to_first_token_seconds", 99), ("memory_growth_megabytes", None), @@ -325,6 +370,9 @@ def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[s if metric == "postgres_max_background_statement_per_request": busiest_background: Final = observation.busiest_background_statement(exempt_tables) return "" if busiest_background is None else busiest_background.summary + if metric == "postgres_slowest_statement_seconds": + slowest: Final = slowest_statement(observation.statements) + return "" if slowest is None else slowest.summary return "" diff --git a/tests/e2e/perf/setup/performance_test_case.py b/tests/e2e/perf/setup/performance_test_case.py index 2879866f85b..3f39b4c38c8 100644 --- a/tests/e2e/perf/setup/performance_test_case.py +++ b/tests/e2e/perf/setup/performance_test_case.py @@ -30,6 +30,12 @@ Metric = Literal[ "cpu_seconds_per_request", "redis_calls_per_request", "redis_background_calls_per_request", + "litellm_overhead_seconds", + "litellm_response_seconds", + "prometheus_redis_latency_mean_seconds", + "postgres_active_sessions", + "postgres_slowest_statement_seconds", + "postgres_peak_connections", ] diff --git a/tests/e2e/perf/setup/telemetry.py b/tests/e2e/perf/setup/telemetry.py new file mode 100644 index 00000000000..e52bcabafda --- /dev/null +++ b/tests/e2e/perf/setup/telemetry.py @@ -0,0 +1,65 @@ +"""The gateway's own `/metrics`, read before and after a run for the numbers nothing outside the gateway +can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis +slows the gateway down. The gateway's latency and request counts are read per request from its response +headers instead (`setup/wire.py`), which gives percentiles where a histogram delta only gives a mean. + +RSS and CPU are not here: a multi-worker gateway serves `/metrics` through the prometheus multiprocess +collector, which drops `process_resident_memory_bytes` and `process_cpu_seconds_total`, so those stay +on psutil (`proxy_usage.py`).""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from dataclasses import dataclass +from itertools import chain +from typing import Final + +from e2e_http import NoBody +from prometheus_client.parser import text_string_to_metric_families +from prometheus_client.samples import Sample +from proxy_client import ProxyClient + + +@dataclass(frozen=True, slots=True) +class HistogramTotals: + """`_sum` and `_count` of one histogram across every label set, enough for a mean over a window.""" + + total: float + count: float + + def minus(self, earlier: HistogramTotals) -> HistogramTotals: + return HistogramTotals(total=self.total - earlier.total, count=self.count - earlier.count) + + @property + def mean(self) -> float | None: + return self.total / self.count if self.count > 0 else None + + +@dataclass(frozen=True, slots=True) +class GatewayTelemetry: + redis_latency: HistogramTotals + """`litellm_redis_latency`, seconds per Redis call the gateway made.""" + + def minus(self, earlier: GatewayTelemetry) -> GatewayTelemetry: + return GatewayTelemetry(redis_latency=self.redis_latency.minus(earlier.redis_latency)) + + +def _value(samples: Iterable[Sample], name: str) -> float: + return sum(sample.value for sample in samples if sample.name == name and math.isfinite(sample.value)) + + +def _histogram(samples: tuple[Sample, ...], family: str) -> HistogramTotals: + return HistogramTotals(total=_value(samples, f"{family}_sum"), count=_value(samples, f"{family}_count")) + + +def parse_gateway_telemetry(text: str) -> GatewayTelemetry: + samples: Final = tuple(chain.from_iterable(family.samples for family in text_string_to_metric_families(text))) + return GatewayTelemetry(redis_latency=_histogram(samples, "litellm_redis_latency")) + + +def read_gateway_telemetry(proxy: ProxyClient) -> GatewayTelemetry | None: + """One `/metrics` scrape, or None when the gateway does not serve it (no `prometheus` callback), in which + case every limit on a gateway-reported metric fails as missing rather than reading a silent zero.""" + scrape: Final = proxy.probe("/metrics", params=NoBody()) + return parse_gateway_telemetry(scrape.body) if scrape.status_code == 200 else None diff --git a/tests/e2e/perf/setup/wire.py b/tests/e2e/perf/setup/wire.py index ced79d6970c..18351c18c6b 100644 --- a/tests/e2e/perf/setup/wire.py +++ b/tests/e2e/perf/setup/wire.py @@ -72,6 +72,8 @@ class WireRecord(BaseModel): call_id: str | None error: str | None response_id: str | None = None + litellm_overhead_ms: float | None = None + litellm_response_ms: float | None = None def response_id(payload: Mapping[str, object]) -> str | None: diff --git a/tests/e2e/perf/test_perf_cases.py b/tests/e2e/perf/test_perf_cases.py index 5d0e25b7616..49b23ddfc57 100644 --- a/tests/e2e/perf/test_perf_cases.py +++ b/tests/e2e/perf/test_perf_cases.py @@ -18,7 +18,7 @@ from lifecycle import ResourceManager from memory_readings import read_rss_everywhere from models import SpendLogsParams from proxy_client import ProxyClient -from setup.datastore import read_db_counters, read_redis_counters, redis_monitor +from setup.datastore import connections_sampler, read_db_counters, read_redis_counters, redis_monitor from setup.event_loop import ( LOG_FILE_ENV, OPT_IN_ENV, @@ -44,6 +44,7 @@ from setup.plan import PlannedRequest, build_plan from setup.podcpu import gateway_cpu_sampler from setup.provision import MOCK_PROVIDERS, build_setup, guardrail_url, mock_url from setup.run import run_plan +from setup.telemetry import read_gateway_telemetry pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] _SPEND_SAMPLE: Final = 40 @@ -108,16 +109,18 @@ def test_perf_case( _ = run_plan(case, _warmup_plan(case, plan), setup) _arm_mocks(mock_urls, case.mock_latency) rss_before: Final = read_rss_everywhere(proxy) + telemetry_before: Final = read_gateway_telemetry(proxy) db_before: Final = read_db_counters() redis_before: Final = read_redis_counters() run_started: Final = time.monotonic() traffic_started_at: Final = datetime.now(tz=UTC) - with gateway_cpu_sampler() as cpu, redis_monitor() as monitor: + with gateway_cpu_sampler() as cpu, redis_monitor() as monitor, connections_sampler() as connections: outcome: Final = run_plan(case, plan, setup) run_finished: Final = time.monotonic() db_after: Final = read_db_counters() redis_after: Final = read_redis_counters() rss_after: Final = read_rss_everywhere(proxy) + telemetry_after: Final = read_gateway_telemetry(proxy) rss_read: Final = time.monotonic() assert outcome.records, f"locust wrote no records (exit {outcome.locust_exit}):\n{outcome.locust_output}" served_call_ids: Final = tuple( @@ -144,6 +147,9 @@ def test_perf_case( redis_after=redis_after, redis_commands=monitor.commands, gateway_cpu_seconds=cpu.seconds, + telemetry_before=telemetry_before, + telemetry_after=telemetry_after, + peak_connections=connections.peak, ) rows, exceeded_limits = evaluate(case, observation) stall_rows: Final = _event_loop_rows(case, traffic_started_at) if case.event_loop is not None else () diff --git a/tests/e2e_harness/perf/setup/test_observe.py b/tests/e2e_harness/perf/setup/test_observe.py index ad54c61aed0..281d981fbd9 100644 --- a/tests/e2e_harness/perf/setup/test_observe.py +++ b/tests/e2e_harness/perf/setup/test_observe.py @@ -36,7 +36,12 @@ def served(seq: int, total_s: float) -> WireRecord: def _key_lookup(calls: int) -> StatementCalls: - return StatementCalls(queryid=1, query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1', calls=calls) + return StatementCalls( + queryid=1, + query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1', + calls=calls, + total_exec_ms=calls * 2.0, + ) def test_junit_properties_carry_each_observed_metric_and_each_limit_including_unobserved_ones() -> None: @@ -71,6 +76,8 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un "perf_postgres_background_calls_per_request": 0.0, "perf_postgres_max_statement_per_request": 10.0, "perf_postgres_max_background_statement_per_request": 0.0, + "perf_postgres_active_sessions": 0.02, + "perf_postgres_slowest_statement_seconds": 0.002, "perf_redis_calls_per_request_limit": 35.0, } From edf4097baf91885b443904da5027dd7e883d2c69 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:41:55 +0000 Subject: [PATCH 051/124] fix(databricks): stream Claude json_schema answers as text on every streaming path (#47424) * fix(databricks): stream Claude json_schema answers as text on every streaming path Async and proxy streams on Databricks Claude json_schema returned the internal json_tool_call tool call with empty text, because the async stream factory dropped json_mode. Databricks now passes it through, the json tool's argument fragments stream verbatim as content (a lone {} fragment was blanked before, corrupting nested empty objects), and the stream ends with finish_reason stop like the non-stream answer. * test(databricks): keep the json_schema stream regression tests ahead of the fix Holds the Databricks transformation at the merge base so the first approval covers the regression tests; the fix commit follows. * fix(databricks): stream Claude json_schema answers as text on every streaming path * fix(databricks): keep finish_reason tool_calls when a json_schema stream also calls a user tool * fix(databricks): drop text beside the json tool and renumber user tools in Claude json_schema streams With reasoning_effort set, Claude can write prose before json_tool_call or call it next to a user tool. Streams now hold text until the end and drop it when the json tool was called, renumber user tool deltas past the dropped json tool so indexes start at 0, and non-stream returns the json tool's arguments as content next to the user tools. * fix(databricks): stream JSON-looking text at once and keep prose ahead of user tools in Claude json_schema streams * fix(databricks): hold a user tool call behind held prose so a later json tool answer does not glue to it A json_schema stream with thinking on that wrote prose, called a user tool, then called the json tool sent the prose at the user tool call and appended the JSON answer to it, so content did not parse. User tool deltas behind held prose are now held with it and sent merged per index at the json tool or the finish, and a json tool after text already sent is dropped so the answer is not repeated. * test(databricks): cover json_schema answers on every endpoint in the integration wire cells --- .../llms/databricks/chat/transformation.py | 190 ++++-- .../providers/test_databricks_chat_wire.py | 612 +++++++++++++++++- .../test_databricks_chat_transformation.py | 276 ++++++++ 3 files changed, 1008 insertions(+), 70 deletions(-) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2ecbb2d9c93..d46d04578cc 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -8,7 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -177,6 +177,66 @@ else: LiteLLMLoggingObj = Any +class _ToolCallFunction(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str | None = None + arguments: str | None = None + + +class _StreamedToolCall(BaseModel): + model_config = ConfigDict(frozen=True) + + index: int = 0 + function: _ToolCallFunction | None = None + + @property + def function_name(self) -> str | None: + return None if self.function is None else self.function.name + + @property + def arguments(self) -> str: + return "" if self.function is None or self.function.arguments is None else self.function.arguments + + +_RAW_TOOL_CALLS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_STREAMED_TOOL_CALLS: Final = TypeAdapter(tuple[_StreamedToolCall, ...]) +_JSON_TEXT_STARTS: Final = ("{", "[") + + +def _merged_tool_call_delta( + fragments: Sequence[tuple[Mapping[str, object], _StreamedToolCall]], +) -> Mapping[str, object]: + first_delta, first_call = fragments[0] + if len(fragments) == 1: + return first_delta + arguments: Final = "".join(tool_call.arguments for _, tool_call in fragments) + return {**first_delta, "function": {"name": first_call.function_name, "arguments": arguments}} + + +def _merged_tool_call_deltas(deltas: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + fragments: Final = tuple(zip(deltas, _STREAMED_TOOL_CALLS.validate_python(deltas), strict=True)) + indexes: Final = tuple(dict.fromkeys(tool_call.index for _, tool_call in fragments)) + return tuple( + _merged_tool_call_delta(tuple(fragment for fragment in fragments if fragment[1].index == index)) + for index in indexes + ) + + +def _json_answer_next_to_user_tools( + tool_calls: Sequence[DatabricksTool] | None, json_mode: bool | None +) -> tuple[str, DatabricksTool] | None: + if not json_mode or not tool_calls or len(tool_calls) == 1: + return None + json_tool_calls: Final = tuple( + tool_call for tool_call in tool_calls if tool_call["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME + ) + if len(json_tool_calls) != 1: + return None + arguments: Final = _ToolCallFunction.model_validate(json_tool_calls[0]["function"]).arguments + return None if arguments is None else (arguments, json_tool_calls[0]) + + class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): """ Reference: https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#chat-request @@ -658,13 +718,19 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): ) citations = DatabricksConfig.extract_citations(choice["message"].get("content")) + message_tool_calls = choice["message"].get("tool_calls") + json_answer = _json_answer_next_to_user_tools(message_tool_calls, json_mode) translated_message = Message( role="assistant", - content=content_str, + content=content_str if json_answer is None else json_answer[0], reasoning_content=reasoning_content, thinking_blocks=thinking_blocks, - tool_calls=choice["message"].get("tool_calls"), + tool_calls=( + message_tool_calls + if json_answer is None or message_tool_calls is None + else [tool_call for tool_call in message_tool_calls if tool_call is not json_answer[1]] + ), provider_specific_fields=({"citations": citations} if citations is not None else None), ) @@ -752,6 +818,17 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): json_mode=json_mode, ) + def get_model_response_iterator_for_request( + self, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, + sync_stream: bool, + json_mode: bool | None, + optional_params: Mapping[str, object], + ) -> "DatabricksChatResponseIterator": + return self.get_model_response_iterator( + streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode + ) + class DatabricksChatResponseIterator(BaseModelResponseIterator): def __init__( @@ -763,44 +840,74 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): super().__init__(streaming_response, sync_stream) self.json_mode = json_mode - self._last_function_name = None # Track the last seen function name + self._json_tool_indexes: frozenset[int] = frozenset() + self._called_user_tool = False + self._held_text = "" + self._held_tool_calls: tuple[Mapping[str, object], ...] = () + self._streams_text = False + + def _record_tool_call_names(self, tool_calls: Sequence[_StreamedToolCall]) -> None: + named: Final = tuple((tool_call.index, tool_call.function_name) for tool_call in tool_calls) + self._json_tool_indexes = self._json_tool_indexes | frozenset( + index for index, name in named if name == RESPONSE_FORMAT_TOOL_NAME + ) + self._called_user_tool = self._called_user_tool or any( + name is not None and name != RESPONSE_FORMAT_TOOL_NAME for _, name in named + ) + + def _renumbered_user_tool_delta( + self, raw_tool_call: Mapping[str, object], tool_call: _StreamedToolCall + ) -> Mapping[str, object]: + json_tools_before: Final = sum(1 for json_index in self._json_tool_indexes if json_index < tool_call.index) + return {**raw_tool_call, "index": tool_call.index - json_tools_before} + + def _split_json_mode_tool_calls( + self, raw_tool_calls: Sequence[Mapping[str, object]] + ) -> tuple[str | None, tuple[Mapping[str, object], ...] | None]: + tool_calls: Final = _STREAMED_TOOL_CALLS.validate_python(raw_tool_calls) + self._record_tool_call_names(tool_calls) + json_deltas: Final = tuple(tool_call for tool_call in tool_calls if tool_call.index in self._json_tool_indexes) + user_deltas: Final = tuple( + self._renumbered_user_tool_delta(raw_tool_call, tool_call) + for raw_tool_call, tool_call in zip(raw_tool_calls, tool_calls, strict=True) + if tool_call.index not in self._json_tool_indexes + ) + json_arguments: Final = "".join(tool_call.arguments for tool_call in json_deltas) if json_deltas else None + return json_arguments, user_deltas or None + + def _json_mode_delta( + self, + text: str | None, + json_arguments: str | None, + user_tool_calls: tuple[Mapping[str, object], ...] | None, + finishes: bool, + ) -> tuple[str | None, tuple[Mapping[str, object], ...] | None]: + if self._streams_text: + return text, user_tool_calls + pending: Final = self._held_text + (text or "") + tool_calls: Final = self._held_tool_calls + (user_tool_calls or ()) + if self._json_tool_indexes: + self._held_text, self._held_tool_calls = "", () + return json_arguments, _merged_tool_call_deltas(tool_calls) or None + if finishes or pending.lstrip().startswith(_JSON_TEXT_STARTS): + self._held_text, self._held_tool_calls, self._streams_text = "", (), True + return pending or None, _merged_tool_call_deltas(tool_calls) or None + if pending: + self._held_text, self._held_tool_calls = pending, tool_calls + return None, None + return None, user_tool_calls def chunk_parser(self, chunk: dict) -> ModelResponseStream: try: translated_choices: Final = [] for choice in chunk["choices"]: tool_calls = choice["delta"].get("tool_calls") - if tool_calls and self.json_mode: - # 1. Check if the function name is set and == RESPONSE_FORMAT_TOOL_NAME - # 2. If no function name, just args -> check last function name (saved via state variable) - # 3. Convert args to json - # 4. Convert json to message - # 5. Set content to message.content - # 6. Set tool_calls to None - from litellm.constants import RESPONSE_FORMAT_TOOL_NAME - from litellm.llms.base_llm.base_utils import ( - convert_tool_response_to_message, - ) - - # Check if this chunk has a function name - function_name = tool_calls[0].get("function", {}).get("name") - if function_name is not None: - self._last_function_name = function_name - - # If we have a saved function name that matches RESPONSE_FORMAT_TOOL_NAME - # or this chunk has the matching function name - if ( - self._last_function_name == RESPONSE_FORMAT_TOOL_NAME - or function_name == RESPONSE_FORMAT_TOOL_NAME - ): - # Convert tool calls to message format - message = convert_tool_response_to_message(tool_calls) - if message is not None: - if message.content == "{}": # empty json - message.content = "" - choice["delta"]["content"] = message.content - choice["delta"]["tool_calls"] = None - elif tool_calls: + json_arguments, json_mode_tool_calls = ( + self._split_json_mode_tool_calls(_RAW_TOOL_CALLS.validate_python(tool_calls)) + if tool_calls and self.json_mode + else (None, None) + ) + if tool_calls and not self.json_mode: for _tc in tool_calls: if _tc.get("function", {}).get("arguments") == "{}": _tc["function"]["arguments"] = "" # avoid invalid json @@ -820,7 +927,12 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): thinking_blocks, ) = DatabricksConfig.extract_reasoning_content(choice["delta"].get("content")) - choice["delta"]["content"] = content_str + if self.json_mode: + choice["delta"]["content"], choice["delta"]["tool_calls"] = self._json_mode_delta( + content_str, json_arguments, json_mode_tool_calls, choice.get("finish_reason") is not None + ) + else: + choice["delta"]["content"] = content_str choice["delta"]["reasoning_content"] = ( block_reasoning_content if block_reasoning_content is not None @@ -831,6 +943,12 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): if thinking_blocks else thinking_blocks ) + if ( + len(self._json_tool_indexes) == 1 + and not self._called_user_tool + and choice.get("finish_reason") == "tool_calls" + ): + choice["finish_reason"] = "stop" translated_choices.append(choice) service_tier: Final = chunk.get("service_tier") if isinstance(service_tier, str) and service_tier: diff --git a/tests/integration/providers/test_databricks_chat_wire.py b/tests/integration/providers/test_databricks_chat_wire.py index 016366c040b..b93cebdbc4c 100644 --- a/tests/integration/providers/test_databricks_chat_wire.py +++ b/tests/integration/providers/test_databricks_chat_wire.py @@ -3,6 +3,7 @@ import json import uuid from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Final, Literal, TypeAlias @@ -757,8 +758,6 @@ class _OutputItem(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) type: str content: tuple[_OutputPart, ...] = () - name: str | None = None - arguments: str | None = None class _ResponsesBody(BaseModel): @@ -787,7 +786,6 @@ class _MessagesDelta(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) type: str | None = None text: str | None = None - partial_json: str | None = None class _MessagesEvent(BaseModel): @@ -796,20 +794,9 @@ class _MessagesEvent(BaseModel): delta: _MessagesDelta | None = None -class _FunctionDelta(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - arguments: str | None = None - - -class _ToolCallDelta(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - function: _FunctionDelta | None = None - - class _StreamDelta(BaseModel): model_config = ConfigDict(extra="ignore", frozen=True) content: str | None = None - tool_calls: tuple[_ToolCallDelta, ...] | None = None class _StreamChoice(BaseModel): @@ -910,32 +897,15 @@ def _outage_or_reply(request: Request) -> Reply: def _item_text(item: _OutputItem) -> str: - match item.type: - case "message": - return "".join(part.text or "" for part in item.content) - case "function_call" if item.name == "json_tool_call": - return item.arguments or "" - case _: - return "" + return "".join(part.text or "" for part in item.content) if item.type == "message" else "" def _responses_text(body: _ResponsesBody) -> str: return "".join(_item_text(item) for item in body.output) -def _chat_delta_text(delta: _StreamDelta) -> str: - calls: Final = delta.tool_calls or () - return (delta.content or "") + "".join(call.function.arguments or "" for call in calls if call.function is not None) - - def _messages_delta_text(delta: _MessagesDelta) -> str: - match delta.type: - case "text_delta": - return delta.text or "" - case "input_json_delta": - return delta.partial_json or "" - case _: - return "" + return delta.text or "" if delta.type == "text_delta" else "" def _final_text(endpoint: _Endpoint, content: bytes) -> str: @@ -954,7 +924,7 @@ def _streamed_text(endpoint: _Endpoint, data: tuple[str, ...]) -> str: assert data[-1] == "[DONE]", data chunks: Final = tuple(_StreamChunk.model_validate_json(line) for line in data[:-1]) choices: Final = tuple(itertools.chain.from_iterable(chunk.choices for chunk in chunks)) - return "".join(_chat_delta_text(choice.delta) for choice in choices) + return "".join(choice.delta.content or "" for choice in choices) case "responses": events: Final = tuple(_ResponsesEvent.model_validate_json(line) for line in data if line != "[DONE]") completed: Final = tuple(event.response for event in events if event.type == "response.completed") @@ -1485,3 +1455,577 @@ def test_databricks_non_string_text_block_is_rejected_before_reaching_upstream(g assert response.status_code == 400, response.text assert "'int' object has no attribute 'strip'" in response.text assert received == () + + +_JSON_ANSWER_FRAGMENTS: Final = ('{"p":', '{"name":"Ada"}', "}") +_PROSE_FRAGMENTS: Final = ("I will check ", "the weather first.") +_WEATHER_FRAGMENTS: Final = ('{"city": ', '"Paris"}') +_WEATHER_PARAMETERS: Final[Mapping[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_FORCED_JSON_TOOL_CHOICE: Final[Mapping[str, JsonValue]] = {"type": "function", "function": {"name": "json_tool_call"}} +_JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +@dataclass(frozen=True, slots=True) +class _Reasoning: + text: str + + +@dataclass(frozen=True, slots=True) +class _Text: + fragments: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _ToolUse: + index: int + name: str + fragments: tuple[str, ...] + + +_Part: TypeAlias = _Reasoning | _Text | _ToolUse + + +@dataclass(frozen=True, slots=True) +class _Answer: + text: str + tools: tuple[tuple[str, JsonValue], ...] + finish: str + thought: bool + + +@dataclass(frozen=True, slots=True) +class _Shape: + parts: tuple[_Part, ...] + upstream_finish: str + thinking: bool + user_tools: bool + json_schema: bool + expected: _Answer + + +_THINKING: Final = _Reasoning("The caller wants Ada.") +_PROSE: Final = _Text(_PROSE_FRAGMENTS) +_WEATHER: Final = ("get_weather", _JSON_VALUE.validate_json("".join(_WEATHER_FRAGMENTS))) +_JSON_ONLY: Final = _Answer(_JSON_CONTENT, (), "stop", True) +_JSON_AND_WEATHER: Final = _Answer(_JSON_CONTENT, (_WEATHER,), "tool_calls", True) +_PROSE_AND_WEATHER: Final = _Answer("".join(_PROSE_FRAGMENTS), (_WEATHER,), "tool_calls", True) + + +def _json_tool(index: int) -> _ToolUse: + return _ToolUse(index, "json_tool_call", _JSON_ANSWER_FRAGMENTS) + + +def _weather_tool(index: int) -> _ToolUse: + return _ToolUse(index, "get_weather", _WEATHER_FRAGMENTS) + + +_SHAPES: Final[Mapping[str, _Shape]] = { + "json-only": _Shape((_json_tool(0),), "tool_calls", False, False, True, _Answer(_JSON_CONTENT, (), "stop", False)), + "prose-then-json": _Shape((_THINKING, _PROSE, _json_tool(0)), "tool_calls", True, False, True, _JSON_ONLY), + "json-then-user-tool": _Shape( + (_THINKING, _json_tool(0), _weather_tool(1)), "tool_calls", True, True, True, _JSON_AND_WEATHER + ), + "user-tool-then-json": _Shape( + (_THINKING, _weather_tool(0), _json_tool(1)), "tool_calls", True, True, True, _JSON_AND_WEATHER + ), + "prose-user-tool-json": _Shape( + (_THINKING, _PROSE, _weather_tool(0), _json_tool(1)), "tool_calls", True, True, True, _JSON_AND_WEATHER + ), + "text-answer": _Shape((_THINKING, _Text(_JSON_ANSWER_FRAGMENTS)), "stop", True, False, True, _JSON_ONLY), + "prose-then-user-tool": _Shape( + (_THINKING, _PROSE, _weather_tool(0)), "tool_calls", True, True, True, _PROSE_AND_WEATHER + ), + "no-json-schema-prose-then-user-tool": _Shape( + (_THINKING, _PROSE, _weather_tool(0)), "tool_calls", True, True, False, _PROSE_AND_WEATHER + ), +} + + +def _reasoning_block(text: str) -> JsonValue: + return {"type": "reasoning", "summary": [{"type": "summary_text", "text": text, "signature": "signature"}]} + + +def _text_delta(fragment: str) -> Mapping[str, JsonValue]: + return {"content": fragment} + + +def _part_deltas(part: _Part) -> tuple[Mapping[str, JsonValue], ...]: + match part: + case _Reasoning(text=text): + return ({"content": [_reasoning_block(text)]},) + case _Text(fragments=fragments): + return tuple(_text_delta(fragment) for fragment in fragments) + case _ToolUse(index=index, name=name, fragments=fragments): + opening: Final[Mapping[str, JsonValue]] = { + "tool_calls": [ + { + "index": index, + "id": f"toolu_{name}", + "type": "function", + "function": {"name": name, "arguments": ""}, + } + ] + } + arguments: Final[tuple[Mapping[str, JsonValue], ...]] = tuple( + {"tool_calls": [{"index": index, "function": {"arguments": fragment}}]} for fragment in fragments + ) + return (opening, *arguments) + + +def _answer_message(parts: tuple[_Part, ...]) -> JsonValue: + reasoning: Final = tuple(_reasoning_block(part.text) for part in parts if isinstance(part, _Reasoning)) + texts: Final[tuple[JsonValue, ...]] = tuple( + {"type": "text", "text": "".join(part.fragments)} for part in parts if isinstance(part, _Text) + ) + tool_calls: Final[tuple[JsonValue, ...]] = tuple( + { + "id": f"toolu_{part.name}", + "type": "function", + "function": {"name": part.name, "arguments": "".join(part.fragments)}, + } + for part in parts + if isinstance(part, _ToolUse) + ) + return {"role": "assistant", "content": [*reasoning, *texts], "tool_calls": list(tool_calls)} + + +def _answer_reply(shape: _Shape, stream: bool) -> Reply: + identity: Final = f"databricks-json-answer-{uuid.uuid4().hex}" + if not stream: + completion: Final = { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _CLAUDE_BACKEND, + "choices": [{"index": 0, "message": _answer_message(shape.parts), "finish_reason": shape.upstream_finish}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + return Reply(body=json.dumps(completion).encode()) + deltas: Final = ({"role": "assistant"}, *itertools.chain.from_iterable(_part_deltas(part) for part in shape.parts)) + frames: Final = ( + *( + _frame(identity, [{"index": 0, "delta": delta, "finish_reason": None}], model=_CLAUDE_BACKEND) + for delta in deltas + ), + _frame(identity, [{"index": 0, "delta": {}, "finish_reason": shape.upstream_finish}], model=_CLAUDE_BACKEND), + b"data: [DONE]\n\n", + ) + return Reply(content_type="text/event-stream", chunks=frames) + + +def _answer_request( + endpoint: _Endpoint, model: str, stream: bool, shape: _Shape, marker: str +) -> Mapping[str, JsonValue]: + prompt: Final = f"{_JSON_SCHEMA_PROMPT} {marker}" + match endpoint: + case "chat": + return { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 2048, + "stream": stream, + **({"response_format": _json_schema_format(_PERSON_SCHEMA)} if shape.json_schema else {}), + **( + { + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": dict(_WEATHER_PARAMETERS)}, + } + ] + } + if shape.user_tools + else {} + ), + **({"reasoning_effort": "low"} if shape.thinking else {}), + } + case "responses": + return { + "model": model, + "input": prompt, + "max_output_tokens": 2048, + "stream": stream, + **( + { + "text": { + "format": { + "type": "json_schema", + "name": "P", + "strict": True, + "schema": dict(_PERSON_SCHEMA), + } + } + } + if shape.json_schema + else {} + ), + **( + {"tools": [{"type": "function", "name": "get_weather", "parameters": dict(_WEATHER_PARAMETERS)}]} + if shape.user_tools + else {} + ), + **({"reasoning": {"effort": "low"}} if shape.thinking else {}), + } + case "messages": + return { + "model": model, + "max_tokens": 2048, + "messages": [{"role": "user", "content": prompt}], + "stream": stream, + **( + {"output_format": {"type": "json_schema", "schema": dict(_PERSON_SCHEMA)}} + if shape.json_schema + else {} + ), + **( + {"tools": [{"name": "get_weather", "input_schema": dict(_WEATHER_PARAMETERS)}]} + if shape.user_tools + else {} + ), + **({"thinking": {"type": "enabled", "budget_tokens": 1024}} if shape.thinking else {}), + } + + +class _OutboundFunction(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + name: str + + +class _OutboundTool(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + function: _OutboundFunction + + +class _OutboundBody(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + model: str + stream: bool | None = None + tools: tuple[_OutboundTool, ...] = () + tool_choice: JsonValue = None + + +def _assert_answer_outbound(request: Request, shape: _Shape, stream: bool, marker: str) -> None: + assert (request.method, request.target) == ("POST", "/chat/completions") + assert marker.encode() in request.body + body: Final = _OutboundBody.model_validate_json(request.body) + tool_names: Final = ( + *(("get_weather",) if shape.user_tools else ()), + *(("json_tool_call",) if shape.json_schema else ()), + ) + assert (body.model, body.stream is True, sorted(tool.function.name for tool in body.tools)) == ( + _CLAUDE_BACKEND, + stream, + sorted(tool_names), + ) + assert (body.tool_choice == _FORCED_JSON_TOOL_CHOICE) == (shape.json_schema and not shape.thinking), body + + +class _ChatToolFunction(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + name: str | None = None + arguments: str | None = None + + +class _ChatToolCallDelta(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + index: int + function: _ChatToolFunction | None = None + + +class _ChatAnswerDelta(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + content: str | None = None + reasoning_content: str | None = None + tool_calls: tuple[_ChatToolCallDelta, ...] | None = None + + +class _ChatAnswerStreamChoice(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + delta: _ChatAnswerDelta + finish_reason: str | None = None + + +class _ChatAnswerChunk(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + choices: tuple[_ChatAnswerStreamChoice, ...] + + +class _ChatToolCall(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + function: _ChatToolFunction + + +class _ChatAnswerMessage(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + content: str | None = None + reasoning_content: str | None = None + tool_calls: tuple[_ChatToolCall, ...] | None = None + + +class _ChatAnswerChoice(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + message: _ChatAnswerMessage + finish_reason: str | None = None + + +class _ChatAnswer(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + choices: tuple[_ChatAnswerChoice, ...] + + +class _ResponsesAnswerPart(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str | None = None + text: str | None = None + + +class _ResponsesAnswerItem(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str + name: str | None = None + arguments: str | None = None + content: tuple[_ResponsesAnswerPart, ...] | None = None + + +class _ResponsesAnswer(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + status: str | None = None + output: tuple[_ResponsesAnswerItem, ...] + + +class _ResponsesAnswerEvent(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str + response: _ResponsesAnswer | None = None + + +class _MessagesAnswerBlock(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str + text: str | None = None + name: str | None = None + input: JsonValue = None + + +class _MessagesAnswer(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + content: tuple[_MessagesAnswerBlock, ...] + stop_reason: str | None = None + + +class _MessagesAnswerDelta(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str | None = None + text: str | None = None + partial_json: str | None = None + stop_reason: str | None = None + + +class _MessagesAnswerEvent(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + type: str + index: int | None = None + content_block: _MessagesAnswerBlock | None = None + delta: _MessagesAnswerDelta | None = None + + +def _only(values: tuple[str, ...]) -> str: + assert len(values) == 1, values + return values[0] + + +def _arguments(arguments: str | None) -> JsonValue: + return _JSON_VALUE.validate_json(arguments or "{}") + + +def _chat_answer(body: _ChatAnswer) -> _Answer: + assert len(body.choices) == 1, body + message: Final = body.choices[0].message + return _Answer( + text=message.content or "", + tools=tuple( + (call.function.name or "", _arguments(call.function.arguments)) for call in message.tool_calls or () + ), + finish=body.choices[0].finish_reason or "", + thought=bool(message.reasoning_content), + ) + + +def _chat_stream_tool(fragments: tuple[_ChatToolCallDelta, ...]) -> tuple[str, JsonValue]: + functions: Final = tuple(fragment.function for fragment in fragments if fragment.function is not None) + names: Final = tuple(function.name for function in functions if function.name) + return _only(names), _arguments("".join(function.arguments or "" for function in functions)) + + +def _chat_stream_answer(data: tuple[str, ...]) -> _Answer: + assert data[-1] == "[DONE]", data + choices: Final = tuple( + itertools.chain.from_iterable(_ChatAnswerChunk.model_validate_json(line).choices for line in data[:-1]) + ) + deltas: Final = tuple(choice.delta for choice in choices) + tool_deltas: Final = tuple(itertools.chain.from_iterable(delta.tool_calls or () for delta in deltas)) + indexes: Final = tuple(dict.fromkeys(tool_delta.index for tool_delta in tool_deltas)) + assert indexes == tuple(range(len(indexes))), indexes + return _Answer( + text="".join(delta.content or "" for delta in deltas), + tools=tuple( + _chat_stream_tool(tuple(tool_delta for tool_delta in tool_deltas if tool_delta.index == index)) + for index in indexes + ), + finish=_only(tuple(choice.finish_reason for choice in choices if choice.finish_reason)), + thought=any(delta.reasoning_content for delta in deltas), + ) + + +def _responses_item_text(item: _ResponsesAnswerItem) -> str: + return "".join(part.text or "" for part in item.content or ()) if item.type == "message" else "" + + +def _responses_answer(body: _ResponsesAnswer) -> _Answer: + return _Answer( + text="".join(_responses_item_text(item) for item in body.output), + tools=tuple( + (item.name or "", _arguments(item.arguments)) for item in body.output if item.type == "function_call" + ), + finish=body.status or "", + thought=any(item.type == "reasoning" for item in body.output), + ) + + +def _responses_stream_answer(data: tuple[str, ...]) -> _Answer: + events: Final = tuple(_ResponsesAnswerEvent.model_validate_json(line) for line in data if line != "[DONE]") + completed: Final = tuple(event.response for event in events if event.type == "response.completed") + assert len(completed) == 1 and completed[0] is not None, data + return _responses_answer(completed[0]) + + +def _messages_answer(body: _MessagesAnswer) -> _Answer: + return _Answer( + text="".join(block.text or "" for block in body.content if block.type == "text"), + tools=tuple((block.name or "", block.input) for block in body.content if block.type == "tool_use"), + finish=body.stop_reason or "", + thought=any(block.type == "thinking" for block in body.content), + ) + + +def _messages_streamed_block( + start: _MessagesAnswerEvent, events: tuple[_MessagesAnswerEvent, ...] +) -> _MessagesAnswerBlock: + assert start.content_block is not None, start + block_events: Final = tuple( + event for event in events if event.type == "content_block_delta" and event.index == start.index + ) + deltas: Final = tuple(event.delta for event in block_events if event.delta is not None) + partial_json: Final = "".join(delta.partial_json or "" for delta in deltas) + return start.content_block.model_copy( + update={ + "text": "".join(delta.text or "" for delta in deltas), + "input": _JSON_VALUE.validate_json(partial_json) if partial_json else start.content_block.input, + } + ) + + +def _messages_stream_answer(data: tuple[str, ...]) -> _Answer: + events: Final = tuple(_MessagesAnswerEvent.model_validate_json(line) for line in data) + assert events[-1].type == "message_stop", data + starts: Final = tuple(event for event in events if event.type == "content_block_start") + stop_reasons: Final = tuple( + event.delta.stop_reason for event in events if event.type == "message_delta" and event.delta is not None + ) + return _messages_answer( + _MessagesAnswer( + content=tuple(_messages_streamed_block(start, events) for start in starts), + stop_reason=_only(tuple(reason for reason in stop_reasons if reason)), + ) + ) + + +def _final_answer(endpoint: _Endpoint, payload: bytes) -> _Answer: + match endpoint: + case "chat": + return _chat_answer(_ChatAnswer.model_validate_json(payload)) + case "responses": + return _responses_answer(_ResponsesAnswer.model_validate_json(payload)) + case "messages": + return _messages_answer(_MessagesAnswer.model_validate_json(payload)) + + +def _streamed_answer(endpoint: _Endpoint, data: tuple[str, ...]) -> _Answer: + match endpoint: + case "chat": + return _chat_stream_answer(data) + case "responses": + return _responses_stream_answer(data) + case "messages": + return _messages_stream_answer(data) + + +def _call_for_answer(gateway: Gateway, endpoint: _Endpoint, body: Mapping[str, JsonValue]) -> _Answer: + stream: Final = body["stream"] is True + if not stream: + response: Final = gateway.request("POST", _PATHS[endpoint], body) + assert response.status_code == 200, response.text + return _final_answer(endpoint, response.content) + with gateway.client.stream( + "POST", _PATHS[endpoint], json=dict(body), headers={"Authorization": f"Bearer {gateway.key}"} + ) as streamed: + assert streamed.status_code == 200, streamed.read() + data: Final = tuple(line.removeprefix("data: ") for line in streamed.iter_lines() if line.startswith("data: ")) + return _streamed_answer(endpoint, data) + + +_ANTHROPIC_STOP_REASONS: Final[Mapping[str, str]] = {"stop": "end_turn", "tool_calls": "tool_use"} + + +def _endpoint_answer(endpoint: _Endpoint, answer: _Answer) -> _Answer: + match endpoint: + case "chat": + return answer + case "responses": + return _Answer(answer.text, answer.tools, "completed", answer.thought) + case "messages": + return _Answer(answer.text, answer.tools, _ANTHROPIC_STOP_REASONS[answer.finish], answer.thought) + + +@pytest.mark.parametrize( + ("endpoint", "stream", "shape"), + [ + pytest.param("chat", True, "json-only", id="chat-stream-json-only"), + pytest.param("chat", True, "prose-then-json", id="chat-stream-prose-then-json"), + pytest.param("chat", True, "json-then-user-tool", id="chat-stream-json-then-user-tool"), + pytest.param("chat", True, "user-tool-then-json", id="chat-stream-user-tool-then-json"), + pytest.param("chat", True, "prose-user-tool-json", id="chat-stream-prose-user-tool-json"), + pytest.param("chat", True, "text-answer", id="chat-stream-text-answer"), + pytest.param("chat", True, "prose-then-user-tool", id="chat-stream-prose-then-user-tool"), + pytest.param("chat", True, "no-json-schema-prose-then-user-tool", id="chat-stream-no-json-schema"), + pytest.param("responses", True, "prose-then-json", id="responses-stream-prose-then-json"), + pytest.param("responses", True, "json-then-user-tool", id="responses-stream-json-then-user-tool"), + pytest.param("responses", True, "prose-user-tool-json", id="responses-stream-prose-user-tool-json"), + pytest.param("responses", True, "prose-then-user-tool", id="responses-stream-prose-then-user-tool"), + pytest.param("messages", True, "json-only", id="messages-stream-json-only"), + pytest.param("messages", True, "prose-then-json", id="messages-stream-prose-then-json"), + pytest.param("messages", True, "json-then-user-tool", id="messages-stream-json-then-user-tool"), + pytest.param("messages", True, "prose-user-tool-json", id="messages-stream-prose-user-tool-json"), + pytest.param("messages", True, "prose-then-user-tool", id="messages-stream-prose-then-user-tool"), + pytest.param("messages", True, "no-json-schema-prose-then-user-tool", id="messages-stream-no-json-schema"), + pytest.param("chat", False, "prose-user-tool-json", id="chat-prose-user-tool-json"), + pytest.param("responses", False, "prose-user-tool-json", id="responses-prose-user-tool-json"), + pytest.param("messages", False, "prose-user-tool-json", id="messages-prose-user-tool-json"), + ], +) +def test_databricks_claude_json_schema_answer_reaches_every_endpoint_as_text( + gateway: Gateway, endpoint: _Endpoint, stream: bool, shape: str +) -> None: + scripted: Final = _SHAPES[shape] + marker: Final = uuid.uuid4().hex + with wire_server(lambda _request: _answer_reply(scripted, stream)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"databricks/{_CLAUDE_BACKEND}", api_base=wire.url, api_key=_API_KEY) + answer: Final = _call_for_answer(gateway, endpoint, _answer_request(endpoint, model, stream, scripted, marker)) + received: Final = wire.drain() + assert len(received) == 1, received + _assert_answer_outbound(received[0], scripted, stream, marker) + assert answer == _endpoint_answer(endpoint, scripted.expected) diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 1e353a7a82a..b3a224209e9 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -27,6 +27,7 @@ from litellm.llms.databricks.chat.transformation import ( _sanitize_empty_content, ) from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponseStream, StreamingChoices import asyncio from unittest.mock import Mock from litellm._version import version @@ -2462,3 +2463,278 @@ def test_databricks_claude_drops_the_openai_tool_search_tool_under_drop_params(m assert mapped["tools"] == [ {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}} ] + + +_JSON_TOOL_ARGUMENT_FRAGMENTS: Final = ("", "", '{"name": ', '"Ada"', ', "met', 'a": ', "{}", ', "tags": []}') +_RECORD_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "Record", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "meta": {"type": "object"}, + "tags": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["name", "meta", "tags"], + }, + }, +} + + +def _json_tool_call_frame(delta: dict[str, object], finish_reason: str | None = None) -> bytes: + chunk: Final = { + "id": "msg_1", + "object": "chat.completion.chunk", + "created": 0, + "model": "databricks-claude-haiku-5-5", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + return f"data: {json.dumps(chunk)}\n\n".encode() + + +_GET_WEATHER_ARGUMENTS: Final = '{"city": "Paris"}' +_GET_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, +} + + +def _tool_call_opening(index: int, name: str, arguments: str) -> dict[str, object]: + tool_call: Final = {"index": index, "id": f"toolu_{index}", "type": "function"} + return {"tool_calls": [{**tool_call, "function": {"name": name, "arguments": arguments}}]} + + +def _tool_call_fragment(index: int, fragment: str) -> dict[str, object]: + return {"tool_calls": [{"index": index, "type": "function", "function": {"arguments": fragment}}]} + + +def _json_tool_call_deltas(index: int) -> tuple[dict[str, object], ...]: + fragments: Final = tuple(_tool_call_fragment(index, fragment) for fragment in _JSON_TOOL_ARGUMENT_FRAGMENTS[1:]) + return (_tool_call_opening(index, "json_tool_call", _JSON_TOOL_ARGUMENT_FRAGMENTS[0]), *fragments) + + +def _sse_stream(deltas: tuple[dict[str, object], ...], finish_reason: str) -> bytes: + frames: Final = tuple(_json_tool_call_frame(delta) for delta in deltas) + return b"".join((*frames, _json_tool_call_frame({}, finish_reason=finish_reason), b"data: [DONE]\n\n")) + + +def _completion(content: str | None, tool_calls: tuple[tuple[str, str], ...], finish_reason: str) -> dict[str, object]: + message_tool_calls: Final = [ + {"id": f"toolu_{index}", "type": "function", "function": {"name": name, "arguments": arguments}} + for index, (name, arguments) in enumerate(tool_calls) + ] + return { + "id": "msg_1", + "object": "chat.completion", + "created": 0, + "model": "databricks-claude-haiku-5-5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content, "tool_calls": message_tool_calls or None}, + "finish_reason": finish_reason, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20}, + } + + +_JSON_ANSWER: Final = "".join(_JSON_TOOL_ARGUMENT_FRAGMENTS) +_PROSE_FRAGMENTS: Final = ("I picked ", "Ada with empty ", "meta and tags.") +_PROSE_DELTAS: Final = tuple({"content": fragment} for fragment in _PROSE_FRAGMENTS) +_TEXT_ANSWER_DELTAS: Final = tuple({"content": fragment} for fragment in _JSON_TOOL_ARGUMENT_FRAGMENTS) + + +async def _collect_async_stream(request: dict[str, object]) -> list[ModelResponseStream]: + return [chunk async for chunk in await litellm.acompletion(**request, stream=True)] + + +def _stream_chunks(request: dict[str, object], call: str) -> list[ModelResponseStream]: + if call == "completion": + return list(litellm.completion(**request, stream=True)) + return asyncio.run(_collect_async_stream(request)) + + +@pytest.mark.usefixtures("_databricks_httpx_transport") +@pytest.mark.parametrize("call", ["completion", "acompletion"]) +@pytest.mark.parametrize( + ("completion", "stream"), + [ + pytest.param( + _completion(None, (("json_tool_call", _JSON_ANSWER),), "tool_calls"), + _sse_stream(_json_tool_call_deltas(0), "tool_calls"), + id="json-tool-only", + ), + pytest.param( + _completion("".join(_PROSE_FRAGMENTS), (("json_tool_call", _JSON_ANSWER),), "tool_calls"), + _sse_stream((*_PROSE_DELTAS, *_json_tool_call_deltas(0)), "tool_calls"), + id="prose-before-json-tool", + ), + pytest.param( + _completion(_JSON_ANSWER, (), "stop"), + _sse_stream(_TEXT_ANSWER_DELTAS, "stop"), + id="answer-as-text", + ), + pytest.param( + _completion(_JSON_ANSWER, (("json_tool_call", _JSON_ANSWER),), "tool_calls"), + _sse_stream((*_TEXT_ANSWER_DELTAS, *_json_tool_call_deltas(0)), "tool_calls"), + id="answer-as-text-and-json-tool", + ), + ], +) +def test_claude_json_schema_stream_returns_the_non_stream_answer( + respx_mock: respx.MockRouter, call: str, completion: dict[str, object], stream: bytes +) -> None: + respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=completion), + httpx.Response(200, content=stream, headers={"content-type": "text/event-stream"}), + ] + ) + request: Final[dict[str, object]] = { + "model": "databricks/databricks-claude-haiku-5-5", + "messages": [{"role": "user", "content": "Return name Ada, an empty meta object, and empty tags."}], + "response_format": _RECORD_RESPONSE_FORMAT, + "api_base": DATABRICKS_API_BASE, + "api_key": DATABRICKS_API_KEY, + } + + answer: Final = litellm.completion(**request).choices[0] + choices: Final = tuple(chunk.choices[0] for chunk in _stream_chunks(request, call) if chunk.choices) + + assert answer.message.content == _JSON_ANSWER + assert "".join(choice.delta.content or "" for choice in choices) == answer.message.content + assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason] + assert not any(choice.delta.tool_calls for choice in choices) + + +@pytest.mark.usefixtures("_databricks_httpx_transport") +@pytest.mark.parametrize("call", ["completion", "acompletion"]) +@pytest.mark.parametrize("json_tool_first", [False, True], ids=["user-tool-first", "json-tool-first"]) +@pytest.mark.parametrize("leading_prose", [(), _PROSE_DELTAS], ids=["no-prose", "prose-first"]) +def test_claude_json_schema_stream_keeps_the_user_tool_call_to_run( + respx_mock: respx.MockRouter, call: str, json_tool_first: bool, leading_prose: tuple[dict[str, object], ...] +) -> None: + json_index: Final = 0 if json_tool_first else 1 + user_tool_deltas: Final = ( + _tool_call_opening(1 - json_index, "get_weather", ""), + _tool_call_fragment(1 - json_index, '{"city": '), + _tool_call_fragment(1 - json_index, '"Paris"}'), + ) + tool_calls: Final = (("get_weather", _GET_WEATHER_ARGUMENTS), ("json_tool_call", _JSON_ANSWER)) + respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response( + 200, + json=_completion( + "".join(_PROSE_FRAGMENTS) if leading_prose else None, + tool_calls[::-1] if json_tool_first else tool_calls, + "tool_calls", + ), + ), + httpx.Response( + 200, + content=_sse_stream( + ( + (*leading_prose, *_json_tool_call_deltas(json_index), *user_tool_deltas) + if json_tool_first + else (*leading_prose, *user_tool_deltas, *_json_tool_call_deltas(json_index)) + ), + "tool_calls", + ), + headers={"content-type": "text/event-stream"}, + ), + ] + ) + request: Final[dict[str, object]] = { + "model": "databricks/databricks-claude-haiku-5-5", + "messages": [{"role": "user", "content": "Check the weather in Paris, then return the record."}], + "tools": [_GET_WEATHER_TOOL], + "response_format": _RECORD_RESPONSE_FORMAT, + "api_base": DATABRICKS_API_BASE, + "api_key": DATABRICKS_API_KEY, + } + + answer: Final = litellm.completion(**request).choices[0] + state: Final = ChatCompletionStreamState() + for chunk in _stream_chunks(request, call): + state.handle_chunk(ChatCompletionChunk.model_validate(chunk.model_dump())) + streamed: Final = state.get_final_completion().choices[0] + + assert answer.message.content == _JSON_ANSWER + assert [tool_call.function.name for tool_call in answer.message.tool_calls or ()] == ["get_weather"] + assert answer.finish_reason == "tool_calls" + assert streamed.message.content == answer.message.content + streamed_tool_calls: Final = tuple( + (tool_call.function.name, tool_call.function.arguments) for tool_call in streamed.message.tool_calls or () + ) + assert streamed_tool_calls == (("get_weather", _GET_WEATHER_ARGUMENTS),) + assert streamed.finish_reason == answer.finish_reason + + +def _json_schema_stream_choices(respx_mock: respx.MockRouter, call: str, stream: bytes) -> tuple[StreamingChoices, ...]: + respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, content=stream, headers={"content-type": "text/event-stream"}) + ) + request: Final[dict[str, object]] = { + "model": "databricks/databricks-claude-haiku-5-5", + "messages": [{"role": "user", "content": "Check the weather in Paris, then return the record."}], + "tools": [_GET_WEATHER_TOOL], + "response_format": _RECORD_RESPONSE_FORMAT, + "api_base": DATABRICKS_API_BASE, + "api_key": DATABRICKS_API_KEY, + } + return tuple(chunk.choices[0] for chunk in _stream_chunks(request, call) if chunk.choices) + + +@pytest.mark.usefixtures("_databricks_httpx_transport") +@pytest.mark.parametrize("call", ["completion", "acompletion"]) +def test_claude_json_schema_stream_sends_a_text_answer_as_it_arrives(respx_mock: respx.MockRouter, call: str) -> None: + choices: Final = _json_schema_stream_choices(respx_mock, call, _sse_stream(_TEXT_ANSWER_DELTAS, "stop")) + + assert [choice.delta.content for choice in choices if choice.delta.content] == [ + fragment for fragment in _JSON_TOOL_ARGUMENT_FRAGMENTS if fragment + ] + + +@pytest.mark.usefixtures("_databricks_httpx_transport") +@pytest.mark.parametrize("call", ["completion", "acompletion"]) +def test_claude_json_schema_stream_sends_prose_before_the_user_tool_call(respx_mock: respx.MockRouter, call: str) -> None: + user_tool_opening: Final = _tool_call_opening(0, "get_weather", _GET_WEATHER_ARGUMENTS) + choices: Final = _json_schema_stream_choices( + respx_mock, call, _sse_stream((*_PROSE_DELTAS, user_tool_opening), "tool_calls") + ) + first_tool_call_at: Final = next(position for position, choice in enumerate(choices) if choice.delta.tool_calls) + + assert "".join(choice.delta.content or "" for choice in choices[: first_tool_call_at + 1]) == "".join( + _PROSE_FRAGMENTS + ) + assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["tool_calls"] + + +@pytest.mark.usefixtures("_databricks_httpx_transport") +@pytest.mark.parametrize("call", ["completion", "acompletion"]) +def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_finish_reason( + respx_mock: respx.MockRouter, call: str +) -> None: + halves: Final = ('{"name": "Ada"}', '{"meta": {}, "tags": []}') + respx_mock.post(DATABRICKS_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response( + 200, json=_completion(None, tuple(("json_tool_call", half) for half in halves), "tool_calls") + ) + ) + answer: Final = litellm.completion( + model="databricks/databricks-claude-haiku-5-5", + messages=[{"role": "user", "content": "Return name Ada, an empty meta object, and empty tags."}], + response_format=_RECORD_RESPONSE_FORMAT, + api_base=DATABRICKS_API_BASE, + api_key=DATABRICKS_API_KEY, + ).choices[0] + split_stream: Final = _sse_stream( + tuple(_tool_call_opening(index, "json_tool_call", half) for index, half in enumerate(halves)), "tool_calls" + ) + choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream) + + assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason] From ce4a6cbf5b7aa90d7ea25d1b313142b8b92777ff Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 19:42:32 -0700 Subject: [PATCH 052/124] perf(streaming): cut per-chunk pydantic round trips on openai-compatible streams (#47030) * perf(streaming): skip full-chunk dump and delta rebuild on openai-compatible chunks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(streaming): reuse the built choice and delta on plain openai-compatible chunks Co-authored-by: yassin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(streaming): copy upstream chunk attributes in upstream key order The scalar fast path iterated a frozenset of keys, so the SSE key order of chunk extras like service_tier and obfuscation changed with each worker's hash seed * test(streaming): count chunk_creator calls with a profiler instead of patching classes * test(streaming): check service_tier still passes through chunk_creator * fix(streaming): reuse the built choice for chunks with upstream choice extras * test(streaming): cover ModelResponseStream chunks on the choice reuse path * fix(streaming): keep main's delta rebuild for deltas with nested values Clearing role in place on a delta that holds tool calls left its nested fields unset, so chunk.model_dump() dropped keys like a custom tool call's name on later chunks. Deltas holding only scalars keep the in-place path. * perf(streaming): build each plain OpenAI chunk's delta once and skip pydantic attribute misses Later plain ChatCompletionChunks now build one Delta and one StreamingChoices, with no full-chunk model_dump, and read stored values instead of attribute probes that raise and catch AttributeError. Nested-value chunks no longer pay the dict(model) misses that made them slower than the merge base. * fix(streaming): read hidden params a chunk carries as a pydantic extra * test(streaming): pin logprobs, Azure filter extras, and audio and image deltas on later OpenAI chunks * test(streaming): cover role, reasoning, choice extras and billing on plain OpenAI SDK chunks --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 26 +- litellm/litellm_core_utils/hidden_params.py | 20 +- .../litellm_core_utils/streaming_handler.py | 166 ++++-- .../test_openai_sdk_plain_chunk_fields.py | 142 ++++++ .../litellm_core_utils/test_core_helpers.py | 45 +- .../litellm_core_utils/test_hidden_params.py | 37 ++ .../test_streaming_handler.py | 474 +++++++++++++++++- 7 files changed, 871 insertions(+), 39 deletions(-) create mode 100644 tests/integration/streaming/test_openai_sdk_plain_chunk_fields.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index b7bfd5eeeb3..1e5bc7ecba4 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -612,10 +612,30 @@ def preserve_upstream_non_openai_attributes( Preserve non-OpenAI attributes from the original chunk. """ # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - expected_keys: Final = set(type(model_response).model_fields.keys()).union({"usage"}) - for key, value in original_chunk.model_dump().items(): - if key not in expected_keys: + expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"}) + candidate_keys: Final = tuple( + key + for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) + if key not in expected_keys + ) + if not candidate_keys: + return + candidate_values: Final[tuple[tuple[str, object], ...]] = tuple( + ( + key, + cast( # cast-ok: Pydantic extras are scalar-checked before assignment + object, + getattr(original_chunk, key), + ), + ) + for key in candidate_keys + ) + if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): + for key, value in candidate_values: setattr(model_response, key, value) + return + for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items(): + setattr(model_response, key, value) def safe_deep_copy(data): diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index 4cb9dcd53f1..f8d44d44cdc 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -1,6 +1,8 @@ -from collections.abc import Iterator, MutableMapping +from collections.abc import Iterator, Mapping, MutableMapping from typing import Final, Protocol, cast +from pydantic import BaseModel + from litellm.types.llms.base import HiddenParams _HIDDEN_PARAMS_ATTR: Final = "_hidden_params" @@ -47,8 +49,22 @@ class HiddenParamsModelView(MutableMapping[str, object]): return frozenset(self._hidden_params.model_fields_set) | frozenset(self._hidden_params.model_extra or {}) +def stored_attribute(obj: object, name: str) -> object: + if not isinstance(obj, BaseModel): + return getattr(obj, name, None) + values: Final[Mapping[str, object]] = vars(obj) + if name in values: + return values[name] + extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {} + return extras.get(name) + + def _get_hidden_params_storage(obj: object) -> object | None: - return obj.get(_HIDDEN_PARAMS_ATTR) if isinstance(obj, dict) else getattr(obj, _HIDDEN_PARAMS_ATTR, None) + if isinstance(obj, dict): + return obj.get(_HIDDEN_PARAMS_ATTR) + if isinstance(obj, BaseModel) and _HIDDEN_PARAMS_ATTR in type(obj).__private_attributes__: + return getattr(obj, _HIDDEN_PARAMS_ATTR, None) + return stored_attribute(obj, _HIDDEN_PARAMS_ATTR) def get_hidden_params_storage(obj: object) -> dict[str, object] | HiddenParams | None: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index e0155f88de1..a627b26b9af 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -15,6 +15,8 @@ from typing import Any, Final, NoReturn, Protocol, TypeVar, cast import anyio import httpx from openai.types.chat import ChatCompletionChunk +from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice +from openai.types.chat.chat_completion_chunk import ChoiceDelta from pydantic import BaseModel, ValidationError from typing_extensions import NotRequired, TypedDict @@ -22,7 +24,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.asyncify import asyncify -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, stored_attribute from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) @@ -88,6 +90,30 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o setattr(response, key, value) +def _prepend_upstream_choice_extras( + choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str] +) -> None: + if not extra_keys: + return + upstream_extras: Final[Mapping[str, object]] = upstream_choice.model_dump(include=set(extra_keys)) + choice.__pydantic_extra__ = {**upstream_extras, **(choice.__pydantic_extra__ or {})} # rebind-ok: reuse choice + choice.__pydantic_fields_set__.update(upstream_extras) + + +def _holds_only_scalars(model: BaseModel) -> bool: + fields: Final[Iterator[tuple[str, object]]] = iter(model) + return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) + + +def _delta_from_upstream(upstream_delta: BaseModel | None) -> Delta | None: + if upstream_delta is None: + return Delta() + try: + return Delta(**dict(iter(upstream_delta))) + except Exception: + return None + + def _is_transport_timeout(error: BaseException) -> bool: return isinstance(error, httpx.TimeoutException) or ( isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException) @@ -252,6 +278,7 @@ class CustomStreamWrapper: self.completion_stream = completion_stream self.sent_first_chunk = False self.sent_last_chunk = False + self._openai_handler_receives_chat_chunks = False self._failure_logging: asyncio.Task[None] | None = None self._closed = False self._stream_created_time: float = time.time() @@ -789,7 +816,10 @@ class CustomStreamWrapper: raise e def model_response_creator( - self, chunk: dict | None = None, hidden_params: Mapping[str, object] | None = None + self, + chunk: dict | None = None, + hidden_params: Mapping[str, object] | None = None, + delta: Delta | None = None, ) -> ModelResponseStream: _model: Final = self._cached_model_name _logging_obj_llm_provider: Final = self._cached_logging_llm_provider @@ -802,7 +832,11 @@ class CustomStreamWrapper: if chunk: args.update({k: v for k, v in chunk.items() if k != "stream"}) - model_response: Final = ModelResponseStream(**args) + model_response: Final = ( + ModelResponseStream(**args) + if delta is None + else ModelResponseStream(**args, choices=[StreamingChoices(delta=delta)]) + ) if self.response_id is not None: model_response.id = self.response_id elif model_response.id: @@ -943,6 +977,39 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(**_initial_delta) return model_response + def _prebuilt_plain_delta(self, chunk: object) -> Delta | None: + if ( + not self._openai_handler_receives_chat_chunks + or not isinstance(chunk, ChatCompletionChunk) + or not chunk.choices + ): + return None + upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent + ChoiceDelta | None, + chunk.choices[0].delta, + ) + if ( + upstream_delta is None + or upstream_delta.function_call is not None + or upstream_delta.tool_calls is not None + or self._has_any_special_delta_attributes(upstream_delta) + ): + return None + return _delta_from_upstream(upstream_delta) + + def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None: + delta: Final = model_response.choices[0].delta + if self.sent_first_chunk is False: + delta.role = "assistant" + self.sent_first_chunk = True + return + if not _holds_only_scalars(delta): + self.strip_role_from_delta(model_response) + return + delta.role = None + if "provider_specific_fields" not in delta.model_fields_set: + delta.provider_specific_fields = None + def _has_special_delta_content(self, model_response: ModelResponseStream) -> bool: """ Check if the delta contains special content types (tool_calls, function_call, audio, or image). @@ -975,11 +1042,11 @@ class CustomStreamWrapper: """ return self.strip_role_from_delta(model_response) - def _has_special_delta_attribute(self, delta, attribute_name: str) -> bool: + def _has_special_delta_attribute(self, delta: object, attribute_name: str) -> bool: """ Check if delta has a specific attribute and it's not None. """ - return delta is not None and getattr(delta, attribute_name, None) is not None + return delta is not None and stored_attribute(delta, attribute_name) is not None def _copy_delta_attribute(self, source_delta, target_delta, attribute_name: str) -> None: """ @@ -987,7 +1054,7 @@ class CustomStreamWrapper: """ setattr(target_delta, attribute_name, getattr(source_delta, attribute_name)) - def _has_any_special_delta_attributes(self, delta) -> bool: + def _has_any_special_delta_attributes(self, delta: object) -> bool: """ Check if delta has any special attributes (audio, image). """ @@ -1011,6 +1078,7 @@ class CustomStreamWrapper: completion_obj: dict[str, Any], model_response: ModelResponseStream, response_obj: dict[str, Any], + plain_delta_created: bool = False, ): from litellm.litellm_core_utils.core_helpers import ( preserve_upstream_non_openai_attributes, @@ -1030,32 +1098,66 @@ class CustomStreamWrapper: original_chunk: Final = response_obj.get("original_chunk", None) if original_chunk: if len(original_chunk.choices) > 0: - choices: Final = [] - for choice in original_chunk.choices: - try: - if isinstance(choice, BaseModel): - choice_json = choice.model_dump() - choice_json.pop( - "finish_reason", None - ) # for mistral etc. which return a value in their last chunk (not-openai compatible). - choices.append(StreamingChoices(**choice_json)) - except Exception: - choices.append(StreamingChoices()) - setattr(model_response, "choices", choices) + original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access + object, + original_chunk, + ) + upstream_choice: Final = ( + original_chunk_for_reuse.choices[0] + if plain_delta_created + and isinstance(original_chunk_for_reuse, ChatCompletionChunk) + and len(original_chunk_for_reuse.choices) == 1 + else None + ) + upstream_delta: Final = cast( # cast-ok: [LIT006] runtime delta may be absent + ChoiceDelta | None, + getattr(upstream_choice, "delta", None), + ) + upstream_choice_extra_keys: Final = frozenset( + (upstream_choice.__pydantic_extra__ or {}) if upstream_choice is not None else () + ) + can_reuse_choice: Final = ( + upstream_choice is not None + and upstream_delta is not None + and "enhancements" not in upstream_choice_extra_keys + and upstream_choice.logprobs is None + and _holds_only_scalars(upstream_delta) + ) + if can_reuse_choice and upstream_choice is not None: + model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice + model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice + _prepend_upstream_choice_extras( + model_response.choices[0], + upstream_choice, + upstream_choice_extra_keys - {"message"}, + ) + else: + choices: Final = [] + for choice in original_chunk.choices: + try: + if isinstance(choice, BaseModel): + choice_json = choice.model_dump() + choice_json.pop( + "finish_reason", None + ) # for mistral etc. which return a value in their last chunk (not-openai compatible). + choices.append(StreamingChoices(**choice_json)) + except Exception: + choices.append(StreamingChoices()) + setattr(model_response, "choices", choices) else: return model_response.system_fingerprint = original_chunk.system_fingerprint setattr( model_response, "citations", - getattr(original_chunk, "citations", None), + stored_attribute(original_chunk, "citations"), ) preserve_upstream_non_openai_attributes( model_response=model_response, original_chunk=original_chunk, ) - model_response = self.strip_role_from_delta(model_response) + self._set_role_on_fresh_delta(model_response) if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug( "model_response.choices[0].delta: %s", @@ -1404,6 +1506,8 @@ class CustomStreamWrapper: if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] else: # openai / azure chat model + if isinstance(chunk, ChatCompletionChunk): + self._openai_handler_receives_chat_chunks = True if self.custom_llm_provider in [ LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value, @@ -1468,8 +1572,11 @@ class CustomStreamWrapper: provider_response_model: Final = _provider_response_model(chunk) if provider_response_model is not None: self._provider_response_model = provider_response_model + prebuilt_delta: Final = self._prebuilt_plain_delta(chunk) + plain_delta_created: bool = prebuilt_delta is not None # rebind-ok: the first chunk builds its delta below model_response = self.model_response_creator( - hidden_params=_provider_hidden_params(chunk, self._provider_response_model) + hidden_params=_provider_hidden_params(chunk, self._provider_response_model), + delta=prebuilt_delta, ) response_obj: dict[str, Any] = {} try: @@ -1493,7 +1600,7 @@ class CustomStreamWrapper: # enter this branch when no content has been passed in response if hasattr(original_chunk, "id"): model_response = self.set_model_id(original_chunk.id, model_response) - if hasattr(original_chunk, "provider_specific_fields"): + if stored_attribute(original_chunk, "provider_specific_fields") is not None: model_response = self.copy_model_response_level_provider_specific_fields( original_chunk, model_response ) @@ -1545,16 +1652,10 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta() elif self._has_any_special_delta_attributes(delta): self._handle_special_delta_attributes(delta, model_response) - else: - try: - delta = ( - dict() - if original_chunk.choices[0].delta is None - else dict(original_chunk.choices[0].delta) - ) - model_response.choices[0].delta = Delta(**delta) - except Exception: - model_response.choices[0].delta = Delta() + elif prebuilt_delta is None: + plain_delta: Final = _delta_from_upstream(delta) + model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta() + plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta else: if self.send_stream_usage is True: model_response.choices = [] @@ -1581,6 +1682,7 @@ class CustomStreamWrapper: completion_obj=completion_obj, model_response=model_response, response_obj=response_obj, + plain_delta_created=plain_delta_created, ) return result diff --git a/tests/integration/streaming/test_openai_sdk_plain_chunk_fields.py b/tests/integration/streaming/test_openai_sdk_plain_chunk_fields.py new file mode 100644 index 00000000000..87dd7a16a41 --- /dev/null +++ b/tests/integration/streaming/test_openai_sdk_plain_chunk_fields.py @@ -0,0 +1,142 @@ +import json +import uuid +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually, list_value, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +PROVIDER_FIELDS: Final = MappingProxyType({"tier": "gold"}) + + +def _frame(identity: str, choice: Mapping[str, JsonValue] | None, **chunk_fields: JsonValue) -> bytes: + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [] if choice is None else [{"index": 0, "finish_reason": None, **choice}], + **chunk_fields, + } + return b"data: " + json.dumps(value).encode() + b"\n\n" + + +def _upstream_stream(identity: str) -> tuple[bytes, ...]: + return ( + _frame(identity, {"delta": {"role": "assistant", "content": ""}}, system_fingerprint="fp_audit"), + _frame( + identity, + { + "delta": {"content": "Hel", "reasoning_content": "plan"}, + "content_filter_results": {"hate": {"filtered": False, "severity": "safe"}}, + }, + ), + _frame(identity, {"delta": {"role": "assistant", "content": "lo"}}), + _frame(identity, {"delta": {"content": " there"}}), + _frame(identity, {"delta": {"content": "!", "provider_specific_fields": dict(PROVIDER_FIELDS)}}), + _frame(identity, {"delta": {}, "finish_reason": "stop"}), + _frame(identity, None, usage={"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}), + b"data: [DONE]\n\n", + ) + + +def _deployment(provider: str, api_base: str) -> dict[str, JsonValue]: + if provider == "azure": + return { + "model": "azure/gpt-4.1-mini", + "api_base": api_base, + "api_key": "synthetic-azure-key", + "api_version": "2024-10-21", + } + return {"model": "openai/gpt-4o-mini", "api_base": f"{api_base}/v1"} + + +def _stream_events(gateway: Gateway, model: str, prompt: str) -> tuple[dict[str, JsonValue], ...]: + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + return tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + + +def _deltas(events: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]: + return tuple( + object_value(object_value(list_value(event["choices"])[0])["delta"]) for event in events if event["choices"] + ) + + +def _event_with_content(events: tuple[dict[str, JsonValue], ...], content: str) -> dict[str, JsonValue]: + matches: Final = tuple( + event + for event in events + if event["choices"] + and object_value(object_value(list_value(event["choices"])[0])["delta"]).get("content") == content + ) + assert len(matches) == 1, f"{content!r} arrived in {len(matches)} events" + return matches[0] + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +def test_plain_chunks_keep_text_role_and_provider_fields(gateway: Gateway, provider: str) -> None: + identity: Final = f"plain-chunks-{uuid.uuid4().hex}" + + def upstream(request: Request) -> Reply: + return Reply(content_type="text/event-stream", chunks=_upstream_stream(identity)) + + with wire_server(upstream) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + **_deployment(provider, wire.url), input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + events: Final = _stream_events(gateway, model, identity) + received: Final = wire.drain() + + assert len(received) == 1 + assert json.loads(received[0].body)["stream"] is True + deltas: Final = _deltas(events) + assert "".join(str(delta.get("content") or "") for delta in deltas) == "Hello there!" + assert tuple(delta.get("role") for delta in deltas if delta.get("role") is not None) == ("assistant",) + assert deltas[0].get("role") == "assistant" + assert {event["id"] for event in events} == {identity} + assert events[0].get("system_fingerprint") == "fp_audit" + hel_choice: Final = object_value(list_value(_event_with_content(events, "Hel")["choices"])[0]) + assert object_value(hel_choice["delta"]).get("reasoning_content") == "plan" + assert hel_choice.get("content_filter_results") == {"hate": {"filtered": False, "severity": "safe"}} + bang_delta: Final = object_value(object_value(list_value(_event_with_content(events, "!")["choices"])[0])["delta"]) + assert bang_delta.get("provider_specific_fields") == dict(PROVIDER_FIELDS) + assert tuple( + object_value(list_value(event["choices"])[0]).get("finish_reason") + for event in events + if event["choices"] and object_value(list_value(event["choices"])[0]).get("finish_reason") is not None + ) == ("stop",) + usages: Final = tuple(object_value(event["usage"]) for event in events if event.get("usage")) + assert tuple((usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) for usage in usages) == ( + (11, 4, 15), + ) + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + database_url=gateway.database_url, + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (11, 4) + assert float(str(rows[0]["spend"])) == pytest.approx(11 * 0.001 + 4 * 0.002) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index c3a2f032651..6459aa98e70 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,5 +1,6 @@ """Tests for litellm_core_utils.core_helpers module.""" +import json import logging import threading from collections.abc import Iterator, Mapping @@ -7,6 +8,7 @@ from typing import Final import httpx import pytest +from openai.types.chat import ChatCompletionChunk from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, @@ -22,13 +24,14 @@ from litellm.litellm_core_utils.core_helpers import ( map_finish_reason, normalize_drop_params, process_response_headers, + preserve_upstream_non_openai_attributes, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, unbind_budget_reservation_from_callbacks, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import ImageResponse, TranscriptionResponse +from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse def test_parent_otel_span_reads_dict_like_metadata_by_index(): @@ -57,6 +60,46 @@ def test_native_request_id_survives_stream_header_processing(header: str) -> Non assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native" +def test_preserve_upstream_non_openai_attributes_keeps_scalar_and_structured_values() -> None: + original_chunk: Final = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="hello"))], + scalar_attribute="scalar", + structured_attribute={"items": ["value"]}, + ) + model_response: Final = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="hello"))] + ) + + preserve_upstream_non_openai_attributes(model_response, original_chunk) + + assert model_response.scalar_attribute == "scalar" + assert model_response.structured_attribute == {"items": ["value"]} + + +@pytest.mark.parametrize("last_extra_value", [9, {"nested": 9}], ids=["all_scalar", "one_structured"]) +def test_preserve_upstream_non_openai_attributes_keeps_upstream_key_order(last_extra_value: object) -> None: + extra_names: Final = ("zeta", "alpha", "mid", "beta", "omega", "delta", "kappa", "gamma", "theta", "iota") + extras: Final = { + **{f"extra_{name}": index for index, name in enumerate(extra_names[:-1])}, + f"extra_{extra_names[-1]}": last_extra_value, + } + original_chunk: Final = ChatCompletionChunk( + id="chatcmpl-1", + created=0, + model="gpt-x", + object="chat.completion.chunk", + choices=[], + service_tier="default", + **extras, + ) + model_response: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="hello"))]) + + preserve_upstream_non_openai_attributes(model_response, original_chunk) + + serialized_keys: Final = tuple(json.loads(model_response.model_dump_json(exclude_none=True, exclude_unset=True))) + assert serialized_keys[-len(extras) - 1 :] == ("service_tier", *extras) + + class TestBudgetReservationBinding: """The request-end release skips a reservation a cost callback has claimed, so the claim must land on the one dict auth stamped, through whichever metadata field or auth object diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py index 53b6c0370fa..c0485a07696 100644 --- a/tests/unit/litellm_core_utils/test_hidden_params.py +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -1,6 +1,7 @@ from typing import Final import pytest +from openai.types.chat import ChatCompletionChunk from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, @@ -61,6 +62,42 @@ def test_get_hidden_params_storage_preserves_supported_storage_identity() -> Non assert get_hidden_params_storage(model_response) is model_storage +_UPSTREAM_CHUNK_FIELDS: Final = { + "id": "chunk", + "choices": [], + "created": 1, + "model": "m", + "object": "chat.completion.chunk", +} + + +def _chunk_with_assigned_hidden_params(hidden_params: dict[str, object]) -> ChatCompletionChunk: + chunk: Final = ChatCompletionChunk(**_UPSTREAM_CHUNK_FIELDS) + setattr(chunk, HIDDEN_PARAMS_ATTR, hidden_params) + return chunk + + +@pytest.mark.parametrize( + "build_chunk", + [ + _chunk_with_assigned_hidden_params, + lambda hidden_params: ChatCompletionChunk(**_UPSTREAM_CHUNK_FIELDS, _hidden_params=hidden_params), + lambda hidden_params: ChatCompletionChunk.model_construct( + **_UPSTREAM_CHUNK_FIELDS, _hidden_params=hidden_params + ), + ], + ids=["assigned", "constructor_extra", "model_construct_extra"], +) +def test_get_hidden_params_storage_reads_upstream_chunk_hidden_params(build_chunk) -> None: + hidden_params: Final[dict[str, object]] = {"is_finished": True} + + assert get_hidden_params_storage(build_chunk(hidden_params)) is hidden_params + + +def test_get_hidden_params_storage_is_none_for_upstream_chunk_without_hidden_params() -> None: + assert get_hidden_params_storage(ChatCompletionChunk(**_UPSTREAM_CHUNK_FIELDS)) is None + + def test_get_or_create_hidden_params_sets_empty_dict_on_plain_object() -> None: class PlainResponse: pass diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 50bcc14b681..691442e80d0 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1,12 +1,18 @@ import asyncio import json +import sys import threading import time -from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import CodeType, FrameType, MappingProxyType +from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest +from openai.types.chat import ChatCompletionChunk +from pydantic import BaseModel import litellm from litellm import verbose_logger @@ -5066,6 +5072,472 @@ def test_handle_stream_fallback_error_restores_context_only_after_exception_mapp session_id_var.set("") +@dataclass(frozen=True, slots=True) +class _ProfiledCall: + code: CodeType + arguments: Mapping[str, object] + + +def _chunk_creator_calls( + wrapper: CustomStreamWrapper, + chunks: Sequence[ChatCompletionChunk | ModelResponseStream], + codes: frozenset[CodeType], +) -> tuple[tuple[ModelResponseStream | None, ...], tuple[_ProfiledCall, ...]]: + calls: Final[list[_ProfiledCall]] = [] + + def record(frame: FrameType, event: str, _arg: object) -> None: + if event == "call" and frame.f_code in codes: + arguments: Final = dict(cast(Mapping[str, object], frame.f_locals)) + calls.append(_ProfiledCall(code=frame.f_code, arguments=MappingProxyType(arguments))) + + previous_profiler: Final = sys.getprofile() + sys.setprofile(record) + try: + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) + finally: + sys.setprofile(previous_profiler) + return outputs, tuple(calls) + + +def test_openai_chunk_creator_skips_full_chunk_dump_and_delta_rebuild( + logging_obj: Logging, +) -> None: + chunks: Final[list[ChatCompletionChunk]] = [ + ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"role": "assistant", "content": "hello"}, "finish_reason": None}], + ), + ChatCompletionChunk( + id="chunk-2", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": " world"}, "finish_reason": None}], + service_tier="default", + citations=["https://a"], + custom_chunk_attribute="custom value", + ), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + chunk_model_dump: Final = ChatCompletionChunk.model_dump.__code__ + delta_init: Final = Delta.__init__.__code__ + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls( + wrapper, chunks, frozenset({chunk_model_dump, delta_init, streaming_choices_init}) + ) + emitted_chunks: Final = tuple(output for output in outputs if output is not None) + choice_dumps: Final = tuple( + call + for call in calls + if call.code is chunk_model_dump and not isinstance(call.arguments["self"], ChatCompletionChunk) + ) + full_chunk_dumps: Final = tuple( + call + for call in calls + if call.code is chunk_model_dump + and isinstance(call.arguments["self"], ChatCompletionChunk) + and call.arguments["include"] is None + ) + delta_constructions: Final = tuple(call for call in calls if call.code is delta_init) + streaming_choice_constructions: Final = tuple(call for call in calls if call.code is streaming_choices_init) + + assert len(emitted_chunks) == 2 + assert full_chunk_dumps == () + assert choice_dumps == () + assert len(delta_constructions) <= len(emitted_chunks) * 2 + assert len(streaming_choice_constructions) <= len(emitted_chunks) + assert [chunk.choices[0].delta.content for chunk in emitted_chunks] == ["hello", " world"] + assert emitted_chunks[0].choices[0].delta.role == "assistant" + assert emitted_chunks[1].choices[0].delta.role is None + assert getattr(emitted_chunks[1], "citations") == ["https://a"] + assert getattr(emitted_chunks[1], "custom_chunk_attribute") == "custom value" + + assert getattr(emitted_chunks[1], "service_tier") == "default" + chunk2_delta: Final = emitted_chunks[1].model_dump()["choices"][0]["delta"] + assert "provider_specific_fields" in chunk2_delta + assert chunk2_delta["provider_specific_fields"] is None + + finish_output: Final = wrapper.chunk_creator( + chunk=ChatCompletionChunk( + id="chunk-3", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}], + ) + ) + + assert finish_output is not None + assert finish_output.choices[0].finish_reason == "stop" + assert finish_output.choices[0].delta.content is None + + +def _openai_chunk(chunk_id: str, delta: Mapping[str, object]) -> ChatCompletionChunk: + return ChatCompletionChunk( + id=chunk_id, + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ) + + +def _missed_attributes(wrapper: CustomStreamWrapper, chunks: Sequence[ChatCompletionChunk]) -> tuple[str, ...]: + attribute_lookup: Final = BaseModel.__getattr__.__code__ + misses: Final[list[str]] = [] + + def record_miss(frame: FrameType, event: str, arg: object) -> Callable[[FrameType, str, object], object]: + if event == "exception" and isinstance(arg, tuple) and isinstance(arg[1], AttributeError): + misses.append(str(frame.f_locals["item"])) + return record_miss + + def trace_lookups(frame: FrameType, event: str, _arg: object) -> Callable[[FrameType, str, object], object] | None: + return record_miss if event == "call" and frame.f_code is attribute_lookup else None + + previous_tracer: Final = sys.gettrace() + sys.settrace(trace_lookups) + try: + tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) + finally: + sys.settrace(previous_tracer) + return tuple(misses) + + +def test_openai_chunk_creator_builds_one_delta_and_misses_no_attribute_per_later_chunk( + logging_obj: Logging, +) -> None: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + wrapper.chunk_creator(chunk=_openai_chunk("chunk-1", {"role": "assistant", "content": ""})) + plain_chunks: Final = ( + _openai_chunk("chunk-2", {"content": "hello"}), + _openai_chunk("chunk-3", {"content": " world"}), + ) + nested_chunk: Final = _openai_chunk( + "chunk-4", {"content": "!", "reasoning_details": [{"type": "reasoning.text", "text": "thought"}]} + ) + delta_init: Final = Delta.__init__.__code__ + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, plain_chunks, frozenset({delta_init, streaming_choices_init})) + + assert sum(call.code is delta_init for call in calls) == len(plain_chunks) + assert sum(call.code is streaming_choices_init for call in calls) == len(plain_chunks) + assert [output.choices[0].delta.content if output else None for output in outputs] == ["hello", " world"] + assert _missed_attributes(wrapper, (*plain_chunks, nested_chunk)) == () + + +def test_chunk_creator_skips_full_dump_of_model_response_stream_chunks( + logging_obj: Logging, +) -> None: + chunks: Final = ( + ModelResponseStream( + id="chunk-1", + created=1, + model="claude-model", + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="hello"))], + ), + ModelResponseStream( + id="chunk-2", + created=1, + model="claude-model", + choices=[StreamingChoices(index=0, delta=Delta(content=" world"))], + custom_chunk_attribute="custom value", + ), + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="claude-model", + custom_llm_provider="anthropic", + logging_obj=logging_obj, + ) + model_dump: Final = BaseModel.model_dump.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, chunks, frozenset({model_dump})) + emitted_chunks: Final = tuple(output for output in outputs if output is not None) + full_upstream_dumps: Final = tuple( + call + for call in calls + if any(call.arguments["self"] is chunk for chunk in chunks) and call.arguments["include"] is None + ) + + assert full_upstream_dumps == () + assert [chunk.choices[0].delta.content for chunk in emitted_chunks] == ["hello", " world"] + assert emitted_chunks[1].choices[0].delta.role is None + assert getattr(emitted_chunks[1], "custom_chunk_attribute") == "custom value" + + + +def test_openai_chunk_creator_keeps_multiple_choice_logprobs_and_extras( + logging_obj: Logging, +) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-multiple-choices", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + citations=["https://a"], + choices=[ + { + "index": 3, + "delta": {"content": "first"}, + "finish_reason": None, + "logprobs": {"content": []}, + }, + { + "index": 7, + "delta": {"content": "second"}, + "finish_reason": None, + "custom_choice_attribute": "choice extra", + }, + ], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + result: Final = wrapper.chunk_creator(chunk=chunk) + + assert result is not None + assert [choice.index for choice in result.choices] == [3, 7] + result_logprobs = result.choices[0].logprobs + chunk_logprobs = chunk.choices[0].logprobs + assert result_logprobs is not None + assert chunk_logprobs is not None + assert result_logprobs.model_dump() == chunk_logprobs.model_dump() + assert getattr(result.choices[1], "custom_choice_attribute") == "choice extra" + assert getattr(result, "citations") == ["https://a"] + + +@pytest.mark.parametrize( + ("choice_extras", "expected_choice_keys", "expected_choice_constructions"), + [ + pytest.param( + { + "content_filter_results": {"hate": {"filtered": False, "severity": "safe"}}, + "message": {"role": "assistant", "content": "hello"}, + }, + ["content_filter_results", "index", "delta"], + 1, + id="azure-content-filter-results", + ), + pytest.param( + {"enhancements": {"grounding": {"lines": []}}}, + ["index", "delta", "enhancements"], + 2, + id="enhancements", + ), + ], +) +def test_chunk_creator_keeps_upstream_choice_extras_in_upstream_order( + logging_obj: Logging, + choice_extras: dict[str, object], + expected_choice_keys: list[str], + expected_choice_constructions: int, +) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-choice-extras", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": "hello"}, "finish_reason": None, **choice_extras}], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="azure", + logging_obj=logging_obj, + ) + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, [chunk], frozenset({streaming_choices_init})) + + result: Final = outputs[0] + assert result is not None + choice_dump: Final = result.choices[0].model_dump(exclude_none=True) + assert list(choice_dump) == expected_choice_keys + assert choice_dump["delta"]["content"] == "hello" + assert {key: choice_dump[key] for key in choice_extras if key != "message"} == { + key: value for key, value in choice_extras.items() if key != "message" + } + assert len(calls) == expected_choice_constructions + + +def test_chunk_creator_keeps_upstream_index_on_single_choice_chunk(logging_obj: Logging) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-index-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 1, "delta": {"content": "hello"}, "finish_reason": None}], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + result: Final = wrapper.chunk_creator(chunk=chunk) + + assert result is not None + assert result.choices[0].index == 1 + assert result.choices[0].delta.content == "hello" + + +def _later_chunk_outputs( + custom_llm_provider: str, later_chunks: Sequence[ChatCompletionChunk], logging_obj: Logging +) -> tuple[ModelResponseStream, ...]: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + wrapper.chunk_creator(chunk=_openai_chunk("chunk-0", {"role": "assistant", "content": ""})) + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in later_chunks) + return tuple(output for output in outputs if output is not None) + + +def _wire_dump(chunk: ModelResponseStream) -> dict[str, object]: + return chunk.model_dump(exclude_none=True, exclude_unset=True) + + +def test_openai_chunk_creator_keeps_single_choice_logprobs_on_later_chunks(logging_obj: Logging) -> None: + logprobs: Final = {"content": [{"token": "hello", "logprob": -0.25, "bytes": [104], "top_logprobs": []}]} + chunk: Final = ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": "hello"}, "finish_reason": None, "logprobs": logprobs}], + ) + + outputs: Final = _later_chunk_outputs("openai", [chunk], logging_obj) + + assert [_wire_dump(output)["choices"][0]["logprobs"] for output in outputs] == [logprobs] + + +def test_azure_chunk_creator_streams_choice_content_filter_results_on_later_chunks(logging_obj: Logging) -> None: + filter_results: Final = {"hate": {"filtered": False, "severity": "safe"}} + chunk: Final = ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + {"index": 0, "delta": {"content": "hello"}, "finish_reason": None, "content_filter_results": filter_results} + ], + ) + + outputs: Final = _later_chunk_outputs("azure", [chunk], logging_obj) + + assert [_wire_dump(output)["choices"][0]["content_filter_results"] for output in outputs] == [filter_results] + assert ["content_filter_results" in output.choices[0].model_fields_set for output in outputs] == [True] + + +@pytest.mark.parametrize( + "special_delta", + [ + pytest.param({"audio": {"id": "audio_1", "data": "AAAA", "transcript": "hi"}}, id="audio"), + pytest.param({"images": [{"type": "image_url", "image_url": {"url": "data:,"}, "index": 0}]}, id="images"), + ], +) +def test_openai_chunk_creator_dumps_special_delta_attributes_like_other_deltas( + logging_obj: Logging, special_delta: dict[str, object] +) -> None: + chunk: Final = ChatCompletionChunk.model_construct( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": special_delta, "finish_reason": None}], + ) + + outputs: Final = _later_chunk_outputs("openai", [chunk], logging_obj) + + assert [output.model_dump()["choices"][0]["delta"] for output in outputs] == [ + { + "audio": None, + "content": None, + "function_call": None, + "provider_specific_fields": None, + "role": None, + "tool_calls": None, + **special_delta, + } + ] + + +def test_openai_chunk_creator_keeps_custom_tool_name_key_on_later_chunks( + logging_obj: Logging, +) -> None: + chunks: Final = ( + ChatCompletionChunk.model_construct( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + {"index": 0, "id": "call_1", "type": "custom", "custom": {"name": "run_python", "input": ""}} + ], + }, + "finish_reason": None, + } + ], + ), + ChatCompletionChunk.model_construct( + id="chunk-2", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "custom": {"input": "print"}}]}, + "finish_reason": None, + } + ], + ), + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + emitted_chunks: Final = tuple( + output for output in (wrapper.chunk_creator(chunk=chunk) for chunk in chunks) if output is not None + ) + + assert [chunk.model_dump()["choices"][0]["delta"]["tool_calls"][0]["custom"] for chunk in emitted_chunks] == [ + {"name": "run_python", "input": ""}, + {"name": None, "input": "print"}, + ] + + def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chunk(): wrapper = CustomStreamWrapper( completion_stream=None, From 219694319505c72507a8e409021e5488a83b6089 Mon Sep 17 00:00:00 2001 From: Mrunmayee Daware <165297479+Mrun25@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:19:42 +0530 Subject: [PATCH 053/124] fix(proxy): match partial wildcards against prefix-stripped model ids (#46390) * fix(proxy): match partial wildcards against prefix-stripped model ids (issue #46384) Refactor model checks to improve filtering logic and type hints. * test(proxy): partial wildcard listing regression coverage (issue #46384) * fix(proxy): drop unneeded annotation on suffix list (LIT001) * fix(proxy): annotate suffix list with reasoned mutable-ok (basedpyright) Change suffix_appended_wildcard_models to a typed list. * Add lock timeout before creating telemetry report table Set local lock timeout to 10 seconds before creating the telemetry report table. * fix(proxy): list alias wildcards as routable ids, not raw catalog ids An alias whose suffix collides with a catalog sub-prefix (e.g. public "cloudflare/@cf/*" over deployment "cloudflare/*") took the partial-filter branch and listed raw catalog ids. Routing substitutes the captured suffix into the deployment pattern, so those listed ids routed to targets that do not exist. Only use filter semantics when the deployment repeats the wildcard suffix (or no deployment is available); aliases prepend their prefix to provider-stripped catalog ids so every listed id routes back to a real catalog model. Addresses Greptile review on #46390. * fix(proxy): derive partial wildcard listings from the deployment's routing pattern List an id for a partial wildcard only when routing it through the deployment's pattern lands on a catalog model, so aliases, catch-alls, and key filters over a wildcard deployment list ids that answer when called. /model/info expands a bare * row the same way /v1/models does. * fix(proxy): drop fragment ids for a catch-all over a partial wildcard and route bare aliases through the deployment pattern A * catch-all over a partial deployment now lists no catalog ids, a bare alias like my-cf/* over cloudflare/@cf/* names ids by the deployment pattern, an alias whose wildcard count differs from its deployment lists nothing, and /model/info rows show the model each listed id routes to * fix(proxy): keep listed ids on /model/info alias rows and clean up test-registered models * fix(proxy): list catalog models through routing-prefix wildcard deployments * fix(proxy): keep namespace and bare catch-all wildcards on routable listings A leading deployment segment the model name repeats, or the provider name itself, is no longer read as a routing segment, so same-name namespace wildcards and a key filter over `*` -> `openai/*` stop listing ids that route to no model. A bare `*` deployment under a prefixed name keeps the legacy listing. `*` over a partial deployment now lists the names that route through it, dropping empty ids. * fix(proxy): list nothing for a catch-all over a mid-word wildcard deployment * fix(proxy): list only wildcard ids that route back to their deployment and catalog model A listed id now has to round-trip through the alias pattern to the catalog model it names, and the router's wildcard match for it has to pick this deployment, so an id a more specific sibling wildcard serves is no longer listed under the catch-all on /v1/models or /model/info * fix(proxy): list wildcard aliases that route at a word boundary and ids a sibling serves with the same model --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 175 +++++++++- litellm/proxy/proxy_server.py | 2 +- litellm/router.py | 8 + .../router_utils/pattern_match_deployments.py | 27 +- tests/unit/proxy/auth/test_model_checks.py | 325 ++++++++++++++++++ .../test_pattern_match_deployments.py | 24 ++ tests/unit/test_router/test_router.py | 3 + 7 files changed, 547 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 8a7b6dab419..c4c538083ee 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,16 +1,21 @@ # What is this? ## Common checks for /v1/models and `/model/info` import copy -from collections.abc import Sequence +import functools +import itertools +import re +from collections.abc import Callable, Mapping, Sequence from typing import Final import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models @@ -269,7 +274,136 @@ def _hydrate_litellm_credential_name( return litellm_params -def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_Params | None = None) -> list[str]: +def _is_routing_segment( + segment: str, provider: str, name_segments: frozenset[str], provider_model_ids: tuple[str, ...] +) -> bool: + return ( + "*" not in segment + and segment != provider + and segment not in name_segments + and not any(model_id.startswith(f"{segment}/") for model_id in provider_model_ids) + ) + + +def _without_routing_segments( + model_pattern: str, provider: str, name_pattern: str, provider_model_ids: tuple[str, ...] +) -> str: + segments: Final = model_pattern.split("/") + name_segments: Final = frozenset(name_pattern.split("/")[1:]) + routing_segments: Final = tuple( + itertools.takewhile( + lambda segment: _is_routing_segment(segment, provider, name_segments, provider_model_ids), segments[:-1] + ) + ) + return "/".join(segments[len(routing_segments) :]) + + +def _lists_name_fragments(name_pattern: str, deployment_pattern: str) -> bool: + deployment_partial_word: Final = deployment_pattern.partition("*")[0].rpartition("/")[2] + name_partial_word: Final = name_pattern.partition("*")[0].rpartition("/")[2] + return deployment_partial_word[-1:].isalnum() and not name_partial_word + + +def _routes_back_to( + listed_id: str, + name_regex: re.Pattern[str], + catalog_route: tuple[str, str], + deployment_pattern: str, + routes_to_model: Callable[[str, str], bool], +) -> bool: + name_match: Final = name_regex.match(listed_id) + catalog_pattern, catalog_id = catalog_route + return ( + name_match is not None + and PatternMatchRouter.set_deployment_model_name( + matched_pattern=name_match, litellm_deployment_litellm_model=catalog_pattern + ) + == catalog_id + and routes_to_model( + listed_id, + PatternMatchRouter.set_deployment_model_name( + matched_pattern=name_match, litellm_deployment_litellm_model=deployment_pattern + ), + ) + ) + + +def _ids_routing_to_catalog_models( + name_pattern: str, + deployment_pattern: str, + provider: str, + catalog_ids: Sequence[str], + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: + if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments( + name_pattern, deployment_pattern + ): + return () + provider_id_prefix: Final = f"{provider}/" + provider_model_ids: Final = tuple(catalog_id.removeprefix(provider_id_prefix) for catalog_id in catalog_ids) + catalog_pattern: Final = _without_routing_segments( + deployment_pattern.removeprefix(provider_id_prefix), provider, name_pattern, provider_model_ids + ) + deployment_model_pattern: Final = f"{provider_id_prefix}{catalog_pattern}" + deployment_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(deployment_model_pattern)) + name_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(name_pattern)) + catalog_matches: Final = ( + deployment_regex.fullmatch(f"{provider_id_prefix}{model_id}") for model_id in provider_model_ids + ) + listed_ids: Final = ( + ( + PatternMatchRouter.set_deployment_model_name( + matched_pattern=catalog_match, litellm_deployment_litellm_model=name_pattern + ), + catalog_match.string, + ) + for catalog_match in catalog_matches + if catalog_match is not None + ) + return tuple( + dict.fromkeys( + listed_id + for listed_id, catalog_id in listed_ids + if listed_id + and _routes_back_to( + listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model + ) + ) + ) + + +def _model_info_field(deployment: Mapping[str, object], field: str) -> object: + model_info: Final = deployment.get("model_info") + return model_info.get(field) if is_str_keyed_mapping(model_info) else None + + +def _deployment_team_id(deployment: Mapping[str, object]) -> str | None: + team_id: Final = _model_info_field(deployment, "team_id") + return team_id if isinstance(team_id, str) else None + + +def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None: + return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models) + + +def _routes_to_model( + routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None +) -> Callable[[str, str], bool]: + return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id) + + +def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None: + if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model): + return None + deployment_suffix: Final = litellm_params.model.partition("/")[2] + return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None + + +def get_known_models_from_wildcard( + wildcard_model: str, + litellm_params: LiteLLM_Params | None = None, + routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, +) -> list[str]: wildcard_model_to_expand: Final = ( litellm_params.model if wildcard_model == "*" @@ -282,6 +416,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) except ValueError: # safely fail return [] + deployment_pattern_naming_listed_ids: Final = _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -299,22 +434,29 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ if wildcard_models is None: return [] + if deployment_pattern_naming_listed_ids is not None: + return list( + _ids_routing_to_catalog_models( + name_pattern=wildcard_model, + deployment_pattern=deployment_pattern_naming_listed_ids, + provider=provider, + catalog_ids=wildcard_models, + routes_to_model=routes_to_model, + ) + ) if wildcard_suffix != "*": - ## CHECK IF PARTIAL FILTER e.g. `gemini-*` model_prefix: Final = wildcard_suffix.replace("*", "") - - deployment_repeats_prefix: Final = litellm_params is not None and litellm_params.model.endswith( - f"/{wildcard_suffix}" - ) - is_partial_filter: Final = deployment_repeats_prefix or any( - wc_model.startswith(model_prefix) for wc_model in wildcard_models - ) - if is_partial_filter: - filtered_wildcard_models = [wc_model for wc_model in wildcard_models if wc_model.startswith(model_prefix)] + provider_id_prefix: Final = f"{provider}/" + filtered_wildcard_models: Final = [ + wc_model + for wc_model in wildcard_models + if wc_model.removeprefix(provider_id_prefix).startswith(model_prefix) + ] + if filtered_wildcard_models: wildcard_models = filtered_wildcard_models else: # add model prefix to wildcard models - wildcard_models = [f"{model_prefix}{model}" for model in wildcard_models] + wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] known_providers: Final = {provider.value for provider in LlmProviders} suffix_appended_wildcard_models: Final = [] @@ -337,6 +479,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], + llm_router: Router | None = None, ) -> list[dict[str, object]]: """Expand wildcard deployments into one row per known provider model. @@ -346,6 +489,7 @@ def expand_wildcard_deployments_for_model_info( becomes one entry per known openai model, matching /v1/models behaviour. """ expanded: Final[list[dict[str, object]]] = [] + routed_models: Final = _cached_wildcard_routing(llm_router) for deployment in deployments: model_name = str(deployment.get("model_name") or "") raw_params = deployment.get("litellm_params") @@ -355,7 +499,7 @@ def expand_wildcard_deployments_for_model_info( # Determine the wildcard pattern to expand. # Branch order matters: only fall to litellm_model when model_name is # also a wildcard, so a concrete model_name is never overwritten. - if _check_wildcard_routing(model_name) and "/" in model_name: + if _check_wildcard_routing(model_name) and ("/" in model_name or model_name == "*"): wildcard_pattern = model_name elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model): wildcard_pattern = litellm_model @@ -373,6 +517,7 @@ def expand_wildcard_deployments_for_model_info( expanded_names = get_known_models_from_wildcard( wildcard_model=wildcard_pattern, litellm_params=litellm_params, + routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), ) if not expanded_names: expanded.append(deployment) @@ -396,6 +541,7 @@ def _get_wildcard_models( team_id: str | None = None, ) -> list[str]: models_to_remove: Final = set() + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) all_wildcard_models: Final = [] for model in unique_models: if _check_wildcard_routing(model=model): @@ -411,6 +557,7 @@ def _get_wildcard_models( get_known_models_from_wildcard( wildcard_model=model, litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + routes_to_model=routes_to_model, ) ) else: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 689a5f38416..bbdc996a027 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17596,7 +17596,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models) + all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, diff --git a/litellm/router.py b/litellm/router.py index 5a34a00e204..142b2019e6e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13331,6 +13331,14 @@ class Router: return proxy_wide return team_router.get_deployments_by_pattern(model=model) + def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + """The models wildcard_route_deployments would route ``model`` to, without copying the deployments.""" + proxy_wide: Final = self.pattern_router.get_models_by_pattern(model=model) + team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + if proxy_wide or team_router is None: + return proxy_wide + return team_router.get_models_by_pattern(model=model) + def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, else None, kept until the model list next changes.""" diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 62ba5592f90..f921f907f4c 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -3,6 +3,7 @@ Class to handle llm wildcard routing and regex pattern matching """ import copy +import itertools import re from re import Match from typing import Final @@ -124,7 +125,7 @@ class PatternMatchRouter: _pattern_to_regex = pattern_to_regex - def _return_pattern_matched_deployments(self, matched_pattern: Match, deployments: list[dict]) -> list[dict]: + def _return_pattern_matched_deployments(self, matched_pattern: Match[str], deployments: list[dict]) -> list[dict]: new_deployments: Final = [] for deployment in deployments: new_deployment = copy.deepcopy(deployment) @@ -182,7 +183,7 @@ class PatternMatchRouter: @staticmethod def set_deployment_model_name( - matched_pattern: Match, + matched_pattern: Match[str], litellm_deployment_litellm_model: str, ) -> str: """ @@ -259,6 +260,28 @@ class PatternMatchRouter: return pattern_match return [] + def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: + """The models get_deployments_by_pattern would route ``model`` to, without copying the deployments.""" + provider: Final = custom_llm_provider or inferred_provider(model) + requests_by_pattern: Final = itertools.product((model, f"{provider}/{model}"), self.patterns) + matched: Final = next( + ( + (pattern_match, pattern) + for request, pattern in requests_by_pattern + if (pattern_match := re.match(pattern, request)) is not None + ), + None, + ) + if matched is None: + return frozenset() + pattern_match, pattern = matched + return frozenset( + PatternMatchRouter.set_deployment_model_name( + matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] + ) + for deployment in self.patterns[pattern] + ) + # Example usage: # router = PatternRouter() diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 660cc93874d..83761620cc9 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,3 +1,5 @@ +from collections.abc import Generator, Mapping +from contextlib import contextmanager from typing import Final from unittest.mock import patch @@ -1196,3 +1198,326 @@ def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_m "anthropic/anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic/us.anthropic.claude-haiku-4-5-20251001-v1:0", ] + + +@contextmanager +def _registered_chat_models(provider_by_model: Mapping[str, str]) -> Generator[None, None, None]: + import litellm + + litellm.register_model( + {model: {"litellm_provider": provider, "mode": "chat"} for model, provider in provider_by_model.items()}, + persist_across_reloads=False, + ) + try: + yield + finally: + provider_model_sets: Final = tuple(known for known in vars(litellm).values() if isinstance(known, set)) + for model in provider_by_model: + litellm.model_cost.pop(model, None) + for provider_models in provider_model_sets: + provider_models.discard(model) + litellm.add_known_models(model_cost_map={}) + + +@pytest.mark.parametrize( + ("wildcard", "deployment_model", "catalog", "expected"), + [ + pytest.param( + "cloudflare/@cf/partial-wildcard-org/*", + "cloudflare/@cf/partial-wildcard-org/*", + ("cloudflare/@cf/partial-wildcard-org/model-a", "cloudflare/@hf/partial-wildcard-org/model-b"), + ["cloudflare/@cf/partial-wildcard-org/model-a"], + id="prefixed-catalog-with-deployment", + ), + pytest.param( + "cloudflare/@cf/partial-wildcard-org/*", + None, + ("cloudflare/@cf/partial-wildcard-org/model-a", "cloudflare/@hf/partial-wildcard-org/model-b"), + ["cloudflare/@cf/partial-wildcard-org/model-a"], + id="prefixed-catalog-without-deployment", + ), + pytest.param( + "gemini/gemini-partial-wildcard-9*", + "gemini/gemini-partial-wildcard-9*", + ( + "gemini-partial-wildcard-9-flash", + "gemini/gemini-partial-wildcard-9-pro", + "gemini/gemini-other-partial-wildcard-9", + ), + ["gemini/gemini-partial-wildcard-9-flash", "gemini/gemini-partial-wildcard-9-pro"], + id="mixed-bare-and-prefixed-catalog", + ), + ], +) +def test_partial_wildcard_lists_catalog_ids_stored_with_the_provider_prefix( + wildcard: str, deployment_model: str | None, catalog: tuple[str, ...], expected: list[str] +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + provider: Final = wildcard.split("/", 1)[0] + deployment: Final = None if deployment_model is None else LiteLLM_Params(model=deployment_model) + with _registered_chat_models(dict.fromkeys(catalog, provider)): + listed: Final = get_known_models_from_wildcard(wildcard, deployment) + + assert sorted(listed) == expected + + +_ROUTE_CF_ID: Final = "cloudflare/@cf/partial-route-org/model-a-instruct" +_ROUTE_CF_SUFFIXED_ID: Final = "cloudflare/@cf/partial-route-org/model-b-instruct-v2" +_ROUTE_HF_ID: Final = "cloudflare/@hf/partial-route-org/model-c" +_ROUTE_GEMINI_ID: Final = "gemini-partial-route-9-flash" +_ROUTE_OPENAI_ID: Final = "gpt-partial-route-9-mini" +_ROUTE_BEDROCK_ID: Final = "anthropic.partial-route-9-v1:0" +_ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} + + +@pytest.mark.parametrize( + ("model_name", "deployment_model", "key_model", "reachable"), + [ + pytest.param("cloudflare/@cf/*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="filter"), + pytest.param( + "cloudflare/@cf/*-instruct", "cloudflare/@cf/*-instruct", None, {_ROUTE_CF_ID}, id="filter-with-suffix" + ), + pytest.param( + "cloudflare/@cf/*", + "cloudflare/*", + None, + {*_ROUTE_CF_IDS, _ROUTE_HF_ID}, + id="alias-whose-prefix-matches-catalog-ids", + ), + pytest.param("cloudflare/team-*", "cloudflare/*", None, {*_ROUTE_CF_IDS, _ROUTE_HF_ID}, id="alias"), + pytest.param("cloudflare/cf-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="alias-over-partial-deployment"), + pytest.param("my-cf/*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="bare-alias-over-partial-deployment"), + pytest.param( + "cloudflare/team-*", "cloudflare/@cf/*/*", None, set(), id="alias-with-fewer-wildcards-than-its-deployment" + ), + pytest.param("partial-route-v/*/*", "cloudflare/@cf/*/*", None, _ROUTE_CF_IDS, id="multi-wildcard-alias"), + pytest.param( + "partial-route-v/*-*", "cloudflare/@cf/*/*", None, set(), id="multi-wildcard-alias-routing-splits-elsewhere" + ), + pytest.param( + "partial-route-team/*", "openai/gpt-partial-route-9*", None, set(), id="alias-over-mid-word-deployment" + ), + pytest.param( + "team/gpt-partial-route-9*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="alias-repeating-the-deployment-partial-word", + ), + pytest.param( + "team/gpt-partial-route-9-prod*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="alias-extending-the-deployment-partial-word", + ), + pytest.param( + "partial-route-g/*", + "gemini/gemini-partial-route-*", + None, + {_ROUTE_GEMINI_ID}, + id="alias-over-deployment-cut-at-a-word-boundary", + ), + pytest.param( + "cloudflare/*", "cloudflare/*", "cloudflare/@cf/*", _ROUTE_CF_IDS, id="key-filter-over-provider-wildcard" + ), + pytest.param("gemini/gemini-*", "gemini/*", None, {_ROUTE_GEMINI_ID}, id="alias-over-bare-id"), + pytest.param( + "gemini/*", "gemini/*", "gemini/gemini-partial-route-9*", {_ROUTE_GEMINI_ID}, id="key-filter-over-bare-id" + ), + pytest.param( + "openai/*", "openai/responses/*", None, {f"openai/responses/{_ROUTE_OPENAI_ID}"}, id="responses-route" + ), + pytest.param( + "openai/*", + "openai/responses/*", + "openai/gpt-partial-route-9*", + {f"openai/responses/{_ROUTE_OPENAI_ID}"}, + id="key-filter-over-responses-route", + ), + pytest.param( + "bedrock/*", "bedrock/converse/*", None, {f"bedrock/converse/{_ROUTE_BEDROCK_ID}"}, id="converse-route" + ), + pytest.param( + "openai/partial-route-org/*", + "openai/partial-route-org/*", + None, + set(), + id="same-name-namespace-the-catalog-lacks", + ), + pytest.param( + "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" + ), + pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"), + pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"), + pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"), + pytest.param( + "*", "gemini/gemini-partial-route-9-flash*", None, set(), id="catch-all-whose-only-match-is-its-fixed-text" + ), + ], +) +def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( + model_name: str, deployment_model: str, key_model: str | None, reachable: set[str] +) -> None: + from pydantic import TypeAdapter + + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.types.router import LiteLLM_Params + + routed_params: Final = TypeAdapter(tuple[LiteLLM_Params, ...]) + router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": deployment_model}}]) + + def routed_model(listed_id: str) -> str: + routed: Final = routed_params.validate_python( + tuple(deployment["litellm_params"] for deployment in router.wildcard_route_deployments(listed_id, None)) + ) + return routed[0].model if routed else listed_id + + def routed_catalog_ids(listed: list[str]) -> list[str]: + routed_models: Final = [routed_model(listed_id) for listed_id in listed] + return sorted( + routed.removeprefix("gemini/") if routed.startswith("gemini/gemini-") else routed + for routed in routed_models + if "partial-route" in routed + ) + + with _registered_chat_models( + { + _ROUTE_CF_ID: "cloudflare", + _ROUTE_CF_SUFFIXED_ID: "cloudflare", + _ROUTE_HF_ID: "cloudflare", + _ROUTE_GEMINI_ID: "gemini", + _ROUTE_OPENAI_ID: "openai", + _ROUTE_BEDROCK_ID: "bedrock", + } + ): + v1_models: Final = get_complete_model_list( + key_models=[key_model or model_name], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + if row["model_name"] != model_name + ] + listed_v1_models: Final = routed_catalog_ids(v1_models) + listed_model_info_names: Final = routed_catalog_ids(model_info_names) + + unrouted_ids: Final = [ + listed_id + for listed_id in [*v1_models, *model_info_names] + if not router.wildcard_route_deployments(listed_id, None) + ] + assert unrouted_ids == [] + assert all(v1_models) and all(model_info_names) + if model_name == "*": + assert [listed_id for listed_id in v1_models if not routed_model(listed_id).endswith(f"/{listed_id}")] == [] + assert listed_v1_models == sorted(reachable) + if key_model is None: + assert listed_model_info_names == sorted(reachable) + + +def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere() -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + unshadowed_id: Final = "cloudflare/@cf/unshadowed-route-org/model-d" + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "cloudflare/@cf/*"}}, + {"model_name": "partial-route-org/*", "litellm_params": {"model": "openai/*"}}, + ] + ) + + def routed_models(listed: list[str]) -> dict[str, object]: + return { + listed_id: router.wildcard_route_deployments(listed_id, None)[0]["litellm_params"]["model"] + for listed_id in listed + if "route-org/model-" in listed_id + } + + with _registered_chat_models( + {_ROUTE_CF_ID: "cloudflare", _ROUTE_CF_SUFFIXED_ID: "cloudflare", unshadowed_id: "cloudflare"} + ): + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["*", "partial-route-org/*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert routed_models(v1_models) == {"unshadowed-route-org/model-d": unshadowed_id} + assert routed_models(model_info_names) == {"unshadowed-route-org/model-d": unshadowed_id} + + +def test_wildcard_lists_an_id_a_more_specific_sibling_serves_with_the_same_model() -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + router: Final = Router( + model_list=[ + {"model_name": "openai/gpt-partial-route-9*", "litellm_params": {"model": "openai/gpt-partial-route-9*"}}, + { + "model_name": "openai/gpt-partial-route-9-mini*", + "litellm_params": {"model": "openai/gpt-partial-route-9-mini*"}, + }, + ] + ) + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): + v1_models: Final = get_complete_model_list( + key_models=["openai/gpt-partial-route-9*"], + team_models=[], + proxy_model_list=["openai/gpt-partial-route-9*", "openai/gpt-partial-route-9-mini*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert v1_models == [f"openai/{_ROUTE_OPENAI_ID}"] + assert "openai/gpt-partial-route-9*" not in model_info_names + + +def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setenv("OPENAI_API_KEY", "sk-partial-route-test") + router: Final = Router(model_list=[{"model_name": "team-partial-route/*", "litellm_params": {"model": "*"}}]) + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["team-partial-route/*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + unrouted_ids: Final = [ + listed_id for listed_id in v1_models if not router.wildcard_route_deployments(listed_id, None) + ] + assert f"team-partial-route/{_ROUTE_OPENAI_ID}" in v1_models + assert unrouted_ids == [] diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 01af021112b..0bb33064c68 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -210,6 +210,30 @@ def test_route_with_none(): assert router.route(None) is None +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + pytest.param("openai/gpt-4o", None, id="most-specific-pattern-wins"), + pytest.param("o3", "openai", id="provider-qualified-retry"), + pytest.param("openai/o3", None, id="every-deployment-on-the-pattern"), + pytest.param("mistral/large", "mistral", id="no-pattern-matches"), + ], +) +def test_get_models_by_pattern_names_the_models_get_deployments_by_pattern_routes_to( + model: str, custom_llm_provider: str | None +): + router: Final = PatternMatchRouter() + router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + router.add_pattern("openai/*", {"model_name": "openai/*", "litellm_params": {"model": "azure/*"}}) + router.add_pattern("openai/gpt-*", {"model_name": "openai/gpt-*", "litellm_params": {"model": "azure/gpt-*-prod"}}) + + routed: Final = router.get_deployments_by_pattern(model, custom_llm_provider=custom_llm_provider) + + assert router.get_models_by_pattern(model, custom_llm_provider=custom_llm_provider) == frozenset( + _matched_models(routed) + ) + + def test_route_with_multiple_matching_patterns(): """ Tests that the router returns the first matching pattern when there are multiple matching patterns diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6403f00eb50..9b1bdbf702e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -8618,6 +8618,9 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from( assert tuple(Deployment.model_validate(d).model_info.id for d in served) == expected_ids assert tuple(Deployment.model_validate(d).model_info.id for d in early_deployments) == expected_ids + assert router.wildcard_routed_models(requested, team_id) == frozenset( + Deployment.model_validate(d).litellm_params.model for d in served + ) def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): From fb555b00dc886c88d6986a1bd60415a81d9f875a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 19:55:21 -0700 Subject: [PATCH 054/124] fix(proxy): price a billed failure row at the deployment's own rate like the success path (#47437) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 51 ++++++-- ..._utils_model_creation_and_error_logging.py | 121 ++++++++++++++++++ 2 files changed, 163 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 7696a56ca88..a8bd35644c9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -90,7 +90,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage +from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, CostPerToken, ModelInfo, Usage try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( @@ -157,7 +157,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int -from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -280,6 +280,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionRe from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams from litellm.utils import ( add_configured_callback_to_specific_event, + get_base_model_from_metadata, ) if TYPE_CHECKING: @@ -1016,6 +1017,8 @@ def _failure_usage_to_lift( request_body: Mapping[str, object], dispatched: bool, provider_billed: bool = False, + deployment_id: str | None = None, + base_model: str | None = None, ) -> tuple[object, object] | None: """A stream that broke mid-flight still billed the provider for the chunks already delivered; the streaming handler stashes that recovered usage and @@ -1024,8 +1027,11 @@ def _failure_usage_to_lift( estimated input-side Usage. When the failure is one the provider bills anyway (``provider_billed``: a read timeout or a client disconnect, where the provider had the prompt and kept working after LiteLLM hung up) that - usage is priced at the model's input rate; any other post-dispatch failure - stays at zero cost. The raw request body backfills + usage is priced at the model's input rate, the way the success path prices it: + at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``) + when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure + deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch + failure stays at zero cost. The raw request body backfills the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a chat-completions provider) without filling optional_params. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" @@ -1059,18 +1065,43 @@ def _failure_usage_to_lift( return None if not provider_billed: return estimated_usage, 0.0 - return estimated_usage, _estimated_input_cost(model_call_details, estimated_usage.prompt_tokens) + return estimated_usage, _estimated_input_cost( + model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model + ) -def _estimated_input_cost(model_call_details: Mapping[str, object], prompt_tokens: int) -> float: - """Input price for the estimated prompt at the cost map's rate; an unmapped model keeps the - row at $0 rather than dropping the failure row.""" +def _deployment_cost_per_token(deployment_id: str | None, model: str) -> CostPerToken | None: + """The pricing the router registered for this deployment under its ``model_info.id``, read + through ``deployment_pricing_model_info`` exactly as the success path does; None means the + public cost map. The id is ``Logging.get_router_model_id()`` (the logging object's own + ``litellm_params``), never the flattened ``model_call_details`` copy a request body key can + overwrite.""" + model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) + if model_info is None: + return None + return CostPerToken( + input_cost_per_token=model_info["input_cost_per_token"] or 0.0, + output_cost_per_token=model_info["output_cost_per_token"] or 0.0, + ) + + +def _estimated_input_cost( + model_call_details: Mapping[str, object], + prompt_tokens: int, + deployment_id: str | None, + base_model: str | None, +) -> float: + """Input price for the estimated prompt: the deployment's registered rate when it has one, else + the cost map's rate for ``base_model`` or the deployment model (an Azure deployment name is not + in the cost map, its ``base_model`` is). An unpriceable model keeps the row at $0 rather than + dropping the failure row.""" try: model: Final = str(model_call_details.get("model") or "") estimated_input_cost, _ = litellm.cost_per_token( - model=model, + model=base_model or model, custom_llm_provider=str(model_call_details.get("custom_llm_provider") or "") or None, prompt_tokens=prompt_tokens, + custom_cost_per_token=_deployment_cost_per_token(deployment_id, model), ) except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written verbose_proxy_logger.debug( @@ -1239,6 +1270,8 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: request_body=request_data, dispatched=_first_handoff is not None, provider_billed=provider_billed, + deployment_id=_logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None, + base_model=get_base_model_from_metadata(model_call_details=_model_call_details), ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 1b59615876a..77d17741a18 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2105,6 +2105,127 @@ def test_a_timed_out_request_on_an_unmapped_model_keeps_its_row_at_zero_cost(): assert response_cost == 0.0 +def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the_deployment(monkeypatch): + """Router._create_deployment registers a deployment's own ``input_cost_per_token`` under its + ``model_info.id``; the failure row prices with that entry through the same + ``deployment_pricing_model_info`` the success path uses.""" + from litellm.proxy.utils import _failure_usage_to_lift + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0}, + ) + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + }, + request_body={}, + dispatched=True, + provider_billed=True, + deployment_id="dep-lit145", + ) + + assert lifted is not None + usage, response_cost = lifted + assert usage.prompt_tokens > 0 + assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) + + +def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object: + """A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params() + shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff + instant, so the lift treats the request as dispatched.""" + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=False, + call_type="acompletion", + start_time=real_datetime.datetime.now(), + litellm_call_id="call-lit145", + function_id="fn-lit145", + ) + logging_obj.update_environment_variables(model=model, user="", optional_params={}, litellm_params=litellm_params) + logging_obj.model_call_details["first_api_call_start_time"] = real_datetime.datetime.now() + logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE + logging_obj.model_call_details["call_type"] = "acompletion" + return logging_obj + + +def test_the_deployment_rate_is_found_on_the_logging_objects_real_litellm_params(monkeypatch): + """``get_litellm_params()`` always carries a ``litellm_metadata`` key (None on chat + completions) next to the router-stamped ``metadata.model_info.id``; the failure row must + still price at the deployment's own rate, like ``Logging.get_router_model_id`` on the + success path.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145-real-shape", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 0.25}, + ) + litellm_params = get_litellm_params(metadata={"model_info": {"id": "dep-lit145-real-shape"}}) + assert "litellm_metadata" in litellm_params and litellm_params["litellm_metadata"] is None + logging_obj = _dispatched_failure_logging_obj(FAILURE_USAGE_MODEL, litellm_params) + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + assert lifted["response_cost"] == pytest.approx(lifted["combined_usage_object"].prompt_tokens * 0.25) + + +def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path(): + """An Azure deployment name is not in the cost map; the success path prices it through + ``model_info.base_model`` (``get_base_model_from_metadata``) and so must the failure row.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + "azure/lit145-not-in-cost-map", + get_litellm_params(metadata={"model_info": {"base_model": "azure/gpt-4.1"}}), + ) + logging_obj.model_call_details["custom_llm_provider"] = "azure" + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + expected_cost, _ = litellm.cost_per_token( + model="azure/gpt-4.1", prompt_tokens=lifted["combined_usage_object"].prompt_tokens + ) + assert expected_cost > 0 + assert lifted["response_cost"] == pytest.approx(expected_cost) + + +def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price(): + """``model_call_details`` is flattened with the request's optional_params, so a body key + named ``litellm_params`` can land there; the deployment rate must come from the logging + object's own ``litellm_params`` instead.""" + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + "litellm_params": {"input_cost_per_token": 0}, + }, + request_body={}, + dispatched=True, + provider_billed=True, + deployment_id=None, + ) + + assert lifted is not None + usage, response_cost = lifted + expected_cost, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=usage.prompt_tokens) + assert response_cost == pytest.approx(expected_cost) + assert response_cost > 0 + + def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rate(): from litellm.proxy.utils import _failure_usage_to_lift From d56ff281deeb09e42e6d9e4326e96bc4312c1640 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 19:59:09 -0700 Subject: [PATCH 055/124] fix(router): key prompt caching affinity on tools and the /v1/messages system prompt (#46411) * fix(router): key prompt caching affinity on tools and the /v1/messages system prompt Co-authored-by: nuernber Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): read a 1h tool cache_control nested under function for the pin ttl --------- Co-authored-by: nuernber Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_caching_deployment_check.py | 67 ++++-- litellm/router_utils/prompt_caching_cache.py | 115 +++++++-- .../test_prompt_caching_deployment_check.py | 223 ++++++++++++++++++ .../router_utils/test_prompt_caching_cache.py | 170 ++++++++++++- 4 files changed, 545 insertions(+), 30 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index b2f0c658161..bb3475cdcbe 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -4,7 +4,7 @@ Check if prompt caching is valid for a given deployment Route to previously cached model id, if valid """ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from typing import Final, TypedDict, cast from typing_extensions import ReadOnly @@ -19,7 +19,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import CallTypes, StandardLoggingPayload from litellm.utils import ( get_base_model_from_metadata, @@ -94,10 +94,35 @@ class PromptCachingDeploymentCheck(CustomLogger): if not healthy_deployments[1:]: return healthy_deployments - if messages is not None and await offload_token_count(is_prompt_caching_valid_prompt)( - messages=messages, + request_data: Final = ( + cast( # cast-ok: request_kwargs is a bare dict; this only reads its keys + Mapping[str, object], request_kwargs + ) + if request_kwargs is not None + else None + ) + request_tools: Final = ( + cast( # cast-ok: request_kwargs is untyped; the stand-down scan duck-types every tool it reads + list[AllToolParamValues] | None, request_data.get("tools") + ) + if request_data is not None + else None + ) + prompt_messages: Final = ( + PromptCachingCache.prepend_system_prompt( + messages, request_data.get("system") if request_data is not None else None + ) + if messages is not None + else None + ) + if prompt_messages is not None and await offload_token_count(is_prompt_caching_valid_prompt)( + messages=prompt_messages, model=model, min_token_count=_get_min_token_count_for_deployments(healthy_deployments), + tools=cast( # cast-ok: AllToolParamValues widens the union for Anthropic tool shapes the counter duck-types + list[ChatCompletionToolParam] | None, + request_tools, + ), ): prompt_cache: Final = PromptCachingCache( cache=self.cache, @@ -107,19 +132,13 @@ class PromptCachingDeploymentCheck(CustomLogger): ## `litellm.acompletion`, after a deployment has been picked, so the affinity key has to ## be derived from the messages as they will be sent, not as they arrive here. affinity_messages: Final = AnthropicCacheControlHook.messages_with_default_injections( - messages=cast(list[AllMessageValues], messages), + messages=prompt_messages, models=( deployment["litellm_params"]["model"] for deployment in healthy_deployments if isinstance(deployment.get("litellm_params"), dict) and deployment["litellm_params"].get("model") ), - tools=( - cast( # cast-ok: request_kwargs is untyped; the stand-down scan duck-types every tool it reads - list[AllToolParamValues] | None, request_kwargs.get("tools") - ) - if request_kwargs is not None - else None - ), + tools=request_tools, enable_prompt_caching=( request_kwargs.get("enable_prompt_caching") is True if request_kwargs is not None else None ), @@ -128,7 +147,10 @@ class PromptCachingDeploymentCheck(CustomLogger): model_id_dict: Final = await prompt_cache.async_get_model_id( messages=affinity_messages, - tools=None, + tools=cast( # cast-ok: AllToolParamValues widens the union for Anthropic tool shapes + Sequence[ChatCompletionToolParam] | None, + request_tools, + ), ) if model_id_dict is not None: model_id: Final = model_id_dict["model_id"] @@ -177,11 +199,23 @@ class PromptCachingDeploymentCheck(CustomLogger): ) return - prompt_messages: Final = cast(list[AllMessageValues], messages) + event_data: Final = cast( # cast-ok: success-event kwargs are an untyped dict + Mapping[str, object], kwargs + ) + tools: Final = cast(list[AllToolParamValues] | None, event_data.get("tools")) + prompt_messages: Final = PromptCachingCache.prepend_system_prompt( + cast(list[AllMessageValues], messages), # cast-ok: isinstance(list) checked above + event_data.get("system"), + ) + ## PROMPT CACHING - cache model id, if prompt caching valid prompt + provider if await offload_token_count(is_prompt_caching_valid_prompt)( model=model, messages=prompt_messages, + tools=cast( # cast-ok: AllToolParamValues widens the union for Anthropic tool shapes the counter duck-types + list[ChatCompletionToolParam] | None, + tools, + ), ): cache: Final = PromptCachingCache( cache=self.cache, @@ -193,7 +227,10 @@ class PromptCachingDeploymentCheck(CustomLogger): await cache.async_add_model_id( model_id=model_id, messages=prompt_messages, - tools=None, # [TODO]: add tools once standard_logging_object supports it + tools=cast( # cast-ok: AllToolParamValues widens the union for Anthropic tool shapes + Sequence[ChatCompletionToolParam] | None, + tools, + ), honors_one_hour_ttl=_honors_one_hour_cache_ttl(standard_logging_object, litellm_params), ) diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index 07607f9943c..4ec0bc62e7f 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -85,16 +85,77 @@ def _chain_digest(digest: bytes, unit: bytes) -> bytes: return hashlib.sha256(digest + unit).digest() +def _is_ephemeral_mark(mark: object) -> bool: + if not isinstance(mark, dict): + return False + mark_map: Final = cast( # cast-ok: cache_control marks are plain JSON dicts + Mapping[str, object], mark + ) + return mark_map.get("type") == "ephemeral" + + +def _tool_carries_ephemeral_mark(tool: ChatCompletionToolParam) -> bool: + tool_map: Final = cast( # cast-ok: TypedDict.get rejects keys the type does not declare + Mapping[str, object], tool + ) + if _is_ephemeral_mark(tool_map.get("cache_control")): + return True + function: Final = tool_map.get("function") + if not isinstance(function, dict): + return False + return _is_ephemeral_mark( + cast(Mapping[str, object], function).get("cache_control") # cast-ok: function is a plain JSON dict + ) + + +def _strip_tool_cache_control(tool: JsonValue) -> JsonValue: + if not isinstance(tool, dict): + return tool + function: Final = tool.get("function") + stripped_function: Final = ( + {key: value for key, value in function.items() if key != "cache_control"} + if isinstance(function, dict) and "cache_control" in function + else function + ) + return { + key: (stripped_function if key == "function" else value) + for key, value in tool.items() + if key != "cache_control" + } + + +def _pin_cache_key(digest: bytes) -> str: + return f"deployment:{digest.hex()}:prompt_caching" + + def _seed(tools: Sequence[ChatCompletionToolParam] | None) -> bytes: - if tools is None: + if not tools: return hashlib.sha256(b"").digest() return hashlib.sha256( _canonical_bytes( - _TOOLS_ADAPTER.validate_python(to_jsonable_python(tools, serialize_unknown=True, bytes_mode="base64")) + tuple( + _strip_tool_cache_control(tool) + for tool in _TOOLS_ADAPTER.validate_python( + to_jsonable_python(tools, serialize_unknown=True, bytes_mode="base64") + ) + ) ) ).digest() +def _tool_boundary_position(tools: Sequence[ChatCompletionToolParam] | None) -> tuple[PrefixPosition, ...]: + if not tools: + return () + last_marked: Final = max( + (index for index, tool in enumerate(tools) if _tool_carries_ephemeral_mark(tool)), + default=None, + ) + if last_marked is None: + return () + digest: Final = _seed(tools[: last_marked + 1]) + return (PrefixPosition(cache_key=_pin_cache_key(digest), position=0),) + + def _positions_of( prefix: tuple[Mapping[str, JsonValue], ...], tools: Sequence[ChatCompletionToolParam] | None ) -> tuple[PrefixPosition, ...]: @@ -106,7 +167,7 @@ def _positions_of( for run_type, previous in zip(run_types, (None, *run_types[:-1])) ) return tuple( - PrefixPosition(cache_key=f"deployment:{digest.hex()}:prompt_caching", position=position) + PrefixPosition(cache_key=_pin_cache_key(digest), position=position) for digest, position in zip(digests, positions) ) @@ -118,10 +179,16 @@ def _lookback_keys(positions: tuple[PrefixPosition, ...]) -> tuple[str, ...]: return tuple(entry.cache_key for entry in reversed(positions) if entry.position > oldest_probed_position) +def _carrier_cache_control(carrier: Mapping[str, JsonValue]) -> JsonValue | None: + function: Final = carrier.get("function") + nested: Final = function.get("cache_control") if isinstance(function, dict) else None + return carrier.get("cache_control", nested) + + def _breakpoint_pin_ttl_seconds(carrier: JsonValue | Mapping[str, JsonValue]) -> int | None: if not isinstance(carrier, Mapping): return None - cache_control: Final = carrier.get("cache_control") + cache_control: Final = _carrier_cache_control(carrier) if not isinstance(cache_control, dict) or cache_control.get("type") != "ephemeral": return None return PROMPT_CACHE_ONE_HOUR_PIN_TTL_SECONDS if cache_control.get("ttl") == "1h" else PROMPT_CACHE_PIN_TTL_SECONDS @@ -238,6 +305,22 @@ class PromptCachingCache: return cacheable_prefix + @staticmethod + def prepend_system_prompt( + messages: list[AllMessageValues], # mutable-ok: callers pass the request messages list through + system: object, + ) -> list[AllMessageValues]: # mutable-ok: callers bind the result back into list-typed slots + if not isinstance(system, (str, list)) or not system: + return messages + first: Final = cast( # cast-ok: every AllMessageValues member is a dict + Mapping[str, object], messages[0] if messages else {} + ) + if first.get("role") == "system" and first.get("content") == system: + return messages + return cast( # cast-ok: the prepended system message is a plain dict the AllMessageValues union covers + list[AllMessageValues], [{"role": "system", "content": system}, *messages] + ) + @staticmethod def prefix_positions( messages: list[AllMessageValues] | None, @@ -256,17 +339,21 @@ class PromptCachingCache: already replaced by their size placeholder, so a request carrying the raw image bytes derives the same keys the write side stored. """ + boundary: Final = _tool_boundary_position(tools) if not messages: - return () - return _positions_of( - _PREFIX_ADAPTER.validate_python( - to_jsonable_python( - truncate_base64_in_messages(PromptCachingCache.extract_cacheable_prefix(messages)), - serialize_unknown=True, - bytes_mode="base64", - ) + return boundary + return ( + *boundary, + *_positions_of( + _PREFIX_ADAPTER.validate_python( + to_jsonable_python( + truncate_base64_in_messages(PromptCachingCache.extract_cacheable_prefix(messages)), + serialize_unknown=True, + bytes_mode="base64", + ) + ), + tools, ), - tools, ) @staticmethod @@ -274,7 +361,7 @@ class PromptCachingCache: messages: list[AllMessageValues] | None, tools: Sequence[ChatCompletionToolParam] | None, ) -> tuple[PrefixPosition, ...]: - if not messages: + if not messages and not _tool_boundary_position(tools): return () return await offload_token_count(PromptCachingCache.prefix_positions)(messages, tools) diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 5f018ab5dda..8d893160780 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -13,6 +13,7 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, PROMPT_CACHE_LOOKBACK_POSITIONS from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( PromptCachingDeploymentCheck, _honors_one_hour_cache_ttl, @@ -1236,3 +1237,225 @@ async def test_router_with_prompt_caching(anthropic_messages): print("response=", response) assert response._hidden_params["model_id"] == initial_model_id + + +_OPENAI_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "description": "word " * 3000, + "parameters": {"type": "object", "properties": {}}, + }, + "cache_control": {"type": "ephemeral"}, +} +_ANTHROPIC_TOOL: Final = { + "name": "get_weather", + "description": "word " * 3000, + "input_schema": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, +} + + +def _success_kwargs( + *, + call_type: str, + model: str, + model_id: str, + request_messages: list, + request_kwargs: dict, +) -> dict: + """The success event shape: `messages` is what standard logging produced, `system`/`tools` stay raw.""" + return { + "standard_logging_object": { + "call_type": call_type, + "model": model, + "model_id": model_id, + "custom_llm_provider": "anthropic", + "model_map_information": {"model_map_key": model, "model_map_value": None}, + "messages": StandardLoggingPayloadSetup.append_system_prompt_messages( + kwargs=request_kwargs, messages=copy.deepcopy(request_messages) + ), + }, + "system": request_kwargs.get("system"), + "tools": request_kwargs.get("tools"), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type,request_messages,request_kwargs", + [ + pytest.param( + "acompletion", + [{"role": "user", "content": "weather?"}], + {"tools": [_OPENAI_TOOL]}, + id="chat_tool_breakpoint_only", + ), + pytest.param( + "anthropic_messages", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "word " * 3000, "cache_control": {"type": "ephemeral"}}, + ], + } + ], + {"system": "you are terse"}, + id="anthropic_messages_string_system_and_marked_user_block", + ), + pytest.param( + "anthropic_messages", + [{"role": "user", "content": "hi"}], + { + "system": [ + { + "type": "text", + "text": "word " * 3000, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] + }, + id="anthropic_messages_list_system_breakpoint_only", + ), + pytest.param( + "anthropic_messages", + [{"role": "user", "content": "weather?"}], + {"tools": [_ANTHROPIC_TOOL]}, + id="anthropic_messages_tool_breakpoint_only", + ), + pytest.param( + "acompletion", + [{"role": "user", "content": "weather?"}], + { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "word " * 3000, + "parameters": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + }, + } + ] + }, + id="chat_nested_function_cache_control_only", + ), + pytest.param( + "anthropic_messages", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "word " * 3000, "cache_control": {"type": "ephemeral"}}, + ], + } + ], + {}, + id="control_message_breakpoint_no_system", + ), + ], +) +async def test_affinity_key_covers_the_tools_and_system_prompt_the_provider_sees( + call_type, request_messages, request_kwargs +): + """ + The write side keys the pin on what standard logging logged plus the raw `system`/`tools` + kwargs, and the read side must rebuild the same prefix from the raw request. Anything the + provider caches that the key ignores (tools, the /v1/messages system prompt) either writes no + pin or writes one the lookup can never compute. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments(AUTO_CACHING_MODEL, AUTO_CACHING_MODEL) + + await check.async_log_success_event( + kwargs=_success_kwargs( + call_type=call_type, + model=AUTO_CACHING_MODEL, + model_id="dep-1", + request_messages=request_messages, + request_kwargs=request_kwargs, + ), + response_obj=None, + start_time=None, + end_time=None, + ) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=cast(list[AllMessageValues], request_messages), + request_kwargs=request_kwargs, + ) + + assert filtered == [deployments[0]] + + +@pytest.mark.asyncio +async def test_a_different_tool_set_does_not_share_the_pin(): + """Tools hash into the prefix, so a same-messages request with other tools must not read the pin.""" + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments(AUTO_CACHING_MODEL, AUTO_CACHING_MODEL) + request_messages = [{"role": "user", "content": "weather?"}] + other_tool = {**_OPENAI_TOOL, "function": {**_OPENAI_TOOL["function"], "name": "get_time"}} + + await check.async_log_success_event( + kwargs=_success_kwargs( + call_type="acompletion", + model=AUTO_CACHING_MODEL, + model_id="dep-1", + request_messages=request_messages, + request_kwargs={"tools": [_OPENAI_TOOL]}, + ), + response_obj=None, + start_time=None, + end_time=None, + ) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=cast(list[AllMessageValues], request_messages), + request_kwargs={"tools": [other_tool]}, + ) + + assert filtered == deployments + + +@pytest.mark.asyncio +async def test_a_pin_written_without_tools_is_read_when_tools_is_an_empty_list(): + """`tools=None` on the write and `tools=[]` on the read are the same empty seed.""" + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments(AUTO_CACHING_MODEL, AUTO_CACHING_MODEL) + request_messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "word " * 3000, "cache_control": {"type": "ephemeral"}}], + } + ] + + await check.async_log_success_event( + kwargs=_success_kwargs( + call_type="anthropic_messages", + model=AUTO_CACHING_MODEL, + model_id="dep-1", + request_messages=request_messages, + request_kwargs={}, + ), + response_obj=None, + start_time=None, + end_time=None, + ) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=cast(list[AllMessageValues], request_messages), + request_kwargs={"tools": []}, + ) + + assert filtered == [deployments[0]] diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index b81107b47fa..46af2454dea 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,5 +1,5 @@ import asyncio -from typing import Final +from typing import Final, cast import pytest @@ -65,6 +65,11 @@ def _tool(cache_control: dict[str, str] | None) -> ChatCompletionToolParam: return {**tool, "cache_control": cache_control} if cache_control else tool +def _function_marked_tool(cache_control: dict[str, str]) -> ChatCompletionToolParam: + tool: Final = _tool(None) + return {**tool, "function": {**tool["function"], "cache_control": cache_control}} + + @pytest.mark.parametrize( ("messages", "tools", "expected_ttl"), [ @@ -108,6 +113,12 @@ def _tool(cache_control: dict[str, str] | None) -> ChatCompletionToolParam: PROMPT_CACHE_ONE_HOUR_PIN_TTL_SECONDS, id="1h-on-tools-only", ), + pytest.param( + _system_block_messages({"type": "ephemeral"}), + [_function_marked_tool({"type": "ephemeral", "ttl": "1h"})], + PROMPT_CACHE_ONE_HOUR_PIN_TTL_SECONDS, + id="1h-nested-under-function", + ), pytest.param( _system_block_messages({"type": "ephemeral"}), [_tool(None)], @@ -450,3 +461,160 @@ async def test_same_cacheable_prefix_routes_to_same_deployment() -> None: model_id_3: Final = response3._hidden_params["model_id"] assert model_id_1 == model_id_2 == model_id_3 + + +def _marked_tool( + name: str, cache_control: dict | None = None, *, nested: bool = False +) -> ChatCompletionToolParam: + tool: Final = { + "type": "function", + "function": {"name": name, "description": "word " * 40, "parameters": {"type": "object", "properties": {}}}, + } + if cache_control is not None and nested: + return cast( + ChatCompletionToolParam, + {**tool, "function": {**tool["function"], "cache_control": cache_control}}, + ) + if cache_control is not None: + return cast(ChatCompletionToolParam, {**tool, "cache_control": cache_control}) + return cast(ChatCompletionToolParam, tool) + + +def test_prepend_system_prompt_dedupes_the_string_standard_logging_prepends(): + """ + The success event logs the string system prompt already prepended as a system message, and the + request still carries it in `system`. Prepending unconditionally would hash the prefix twice on + one side, so the copy already sitting at messages[0] must be recognized as the same prompt. + """ + messages = cast( + list[AllMessageValues], + [{"role": "system", "content": "be brief"}, {"role": "user", "content": "hi"}], + ) + + assert PromptCachingCache.prepend_system_prompt(messages, "be brief") is messages + + +def test_prepend_system_prompt_prepends_a_list_system_prompt(): + """ + Standard logging only prepends a string system, so a /v1/messages list system never reaches the + write side's messages. Both sides have to rebuild the same list the provider caches. + """ + system: Final = [{"type": "text", "text": "be brief", "cache_control": {"type": "ephemeral", "ttl": "1h"}}] + messages = cast(list[AllMessageValues], [{"role": "user", "content": "hi"}]) + + result = PromptCachingCache.prepend_system_prompt(messages, system) + + assert result == [{"role": "system", "content": system}, *messages] + + +@pytest.mark.parametrize("system", [None, "", [], 0, {"type": "text"}]) +def test_prepend_system_prompt_leaves_unsystem_values_untouched(system): + messages = cast(list[AllMessageValues], [{"role": "user", "content": "hi"}]) + + assert PromptCachingCache.prepend_system_prompt(messages, system) is messages + + +def test_prepend_system_prompt_never_mutates_the_input_messages(): + system: Final = [{"type": "text", "text": "be brief"}] + user_message: Final = {"role": "user", "content": "hi"} + messages = cast(list[AllMessageValues], [user_message]) + + result = PromptCachingCache.prepend_system_prompt(messages, system) + + assert result is not messages + assert messages == [user_message] + + +def test_a_tool_breakpoint_emits_a_position_zero_key_even_without_messages(): + """ + A request that marks only its tools still hits the provider prompt cache on the tool prefix, so + the pin has to exist at a position before the first message block. + """ + tools: Final = [_marked_tool("read", {"type": "ephemeral"})] + + positions = PromptCachingCache.prefix_positions([], tools) + assert len(positions) == 1 + assert positions[0].position == 0 + + with_messages = PromptCachingCache.prefix_positions( + cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + ), + tools, + ) + assert [position.position for position in with_messages] == [0, 1] + + +def test_a_retimed_or_moved_tool_breakpoint_keeps_the_same_key(): + """ + cache_control on a tool is a boundary marker, not part of the cached content, so a ttl change + or a breakpoint that leaves an earlier tool unmarked hashes to the same prefix key. + """ + marked: Final = {"type": "ephemeral"} + retimed: Final = {"type": "ephemeral", "ttl": "1h"} + tools_5m: Final = [_marked_tool("read", marked)] + tools_1h: Final = [_marked_tool("read", retimed)] + + key_5m: Final = PromptCachingCache.get_prompt_caching_cache_key([], tools_5m) + key_1h: Final = PromptCachingCache.get_prompt_caching_cache_key([], tools_1h) + + assert key_5m is not None + assert key_5m == key_1h + + both_marked: Final = [_marked_tool("read", marked), _marked_tool("write", marked)] + last_marked: Final = [_marked_tool("read"), _marked_tool("write", retimed)] + + key_both: Final = PromptCachingCache.get_prompt_caching_cache_key([], both_marked) + key_last: Final = PromptCachingCache.get_prompt_caching_cache_key([], last_marked) + assert key_both == key_last + + +def test_a_nested_function_cache_control_marks_a_boundary_and_strips_from_the_key(): + """ + LiteLLM also honors the OpenAI-style mark nested inside `function` (the Anthropic transform + maps it to the tool's cache_control), so it has to mark a boundary and hash away exactly like + the top-level form, or a client that marks that way loses affinity to a breakpoint that moved. + """ + marked: Final = {"type": "ephemeral"} + retimed: Final = {"type": "ephemeral", "ttl": "1h"} + + positions = PromptCachingCache.prefix_positions([], [_marked_tool("read", marked, nested=True)]) + assert len(positions) == 1 + assert positions[0].position == 0 + + key_nested: Final = PromptCachingCache.get_prompt_caching_cache_key( + [], [_marked_tool("read", marked, nested=True)] + ) + key_nested_retimed: Final = PromptCachingCache.get_prompt_caching_cache_key( + [], [_marked_tool("read", retimed, nested=True)] + ) + key_toplevel: Final = PromptCachingCache.get_prompt_caching_cache_key([], [_marked_tool("read", marked)]) + + assert key_nested is not None + assert key_nested == key_nested_retimed + assert key_nested == key_toplevel + + +def test_unmarked_tools_change_the_key_marked_tools_do_not_share(): + """Tools are provider-cached content, so a different tool set must never read another request's pin.""" + tools_a: Final = [_marked_tool("read", {"type": "ephemeral"})] + tools_b: Final = [_marked_tool("write", {"type": "ephemeral"})] + + key_a: Final = PromptCachingCache.get_prompt_caching_cache_key([], tools_a) + key_b: Final = PromptCachingCache.get_prompt_caching_cache_key([], tools_b) + + assert key_a is not None + assert key_a != key_b + + +def test_empty_and_missing_tools_seed_the_same_key(): + """`tools=[]` and no tools at all are the same request to the provider, so they must key alike.""" + messages = cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + ) + + key_none: Final = PromptCachingCache.get_prompt_caching_cache_key(messages, None) + key_empty: Final = PromptCachingCache.get_prompt_caching_cache_key(messages, []) + assert key_none == key_empty From 2d181e66337393ed8fb2bbd85e56ced21308366c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:43:35 +0000 Subject: [PATCH 056/124] fix(token_counter): count DeepSeek V4 text with DeepSeek's own tokenizer (#47353) Local token counts for DeepSeek V3 and V4 model names now use DeepSeek's own bundled tokenizer instead of cl100k_base, and tool definitions on V4 names count with V4's chat-template tool framing --- .../prompt_templates/chat_template_tools.py | 4 +- .../tokenizers/deepseek_tokenizer.json | 1 + .../tokenizers/deepseek_tokenizer_LICENSE | 21 ++ litellm/utils.py | 26 +- tests/integration/_support/deepseek_tokens.py | 35 +++ .../test_chat_template_tool_count_wire.py | 275 +++++++++++++++--- ...est_huggingface_tokenizer_downloads_sdk.py | 54 ++++ .../spend/test_failed_dispatch_tokens.py | 41 +++ .../litellm_core_utils/test_token_counter.py | 85 +++++- 9 files changed, 491 insertions(+), 51 deletions(-) create mode 100644 litellm/litellm_core_utils/tokenizers/deepseek_tokenizer.json create mode 100644 litellm/litellm_core_utils/tokenizers/deepseek_tokenizer_LICENSE create mode 100644 tests/integration/_support/deepseek_tokens.py create mode 100644 tests/integration/sdk/test_huggingface_tokenizer_downloads_sdk.py diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 3c437ca7fbb..60134456577 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -24,7 +24,9 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: _TOOL_SECTIONS: Final = ( ( - re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE), + re.compile( + r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE + ), ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas), ), (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), diff --git a/litellm/litellm_core_utils/tokenizers/deepseek_tokenizer.json b/litellm/litellm_core_utils/tokenizers/deepseek_tokenizer.json new file mode 100644 index 00000000000..5648fab136b --- /dev/null +++ b/litellm/litellm_core_utils/tokenizers/deepseek_tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"content":"<|begin▁of▁sentence|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":1,"content":"<|end▁of▁sentence|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":2,"content":"<|▁pad▁|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128000,"content":"<|place▁holder▁no▁0|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128001,"content":"<|place▁holder▁no▁1|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128002,"content":"<|place▁holder▁no▁2|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128003,"content":"<|place▁holder▁no▁3|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128004,"content":"<|place▁holder▁no▁4|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128005,"content":"<|place▁holder▁no▁5|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128006,"content":"<|place▁holder▁no▁6|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128007,"content":"<|place▁holder▁no▁7|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128008,"content":"<|place▁holder▁no▁8|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128009,"content":"<|place▁holder▁no▁9|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128010,"content":"<|place▁holder▁no▁10|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128011,"content":"<|place▁holder▁no▁11|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128012,"content":"<|place▁holder▁no▁12|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128013,"content":"<|place▁holder▁no▁13|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128014,"content":"<|place▁holder▁no▁14|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128015,"content":"<|place▁holder▁no▁15|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128016,"content":"<|place▁holder▁no▁16|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128017,"content":"<|place▁holder▁no▁17|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128018,"content":"<|place▁holder▁no▁18|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128019,"content":"<|place▁holder▁no▁19|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128020,"content":"<|place▁holder▁no▁20|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128021,"content":"<|place▁holder▁no▁21|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128022,"content":"<|place▁holder▁no▁22|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128023,"content":"<|place▁holder▁no▁23|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128024,"content":"<|place▁holder▁no▁24|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128025,"content":"<|place▁holder▁no▁25|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128026,"content":"<|place▁holder▁no▁26|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128027,"content":"<|place▁holder▁no▁27|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128028,"content":"<|place▁holder▁no▁28|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128029,"content":"<|place▁holder▁no▁29|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128030,"content":"<|place▁holder▁no▁30|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128031,"content":"<|place▁holder▁no▁31|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128032,"content":"<|place▁holder▁no▁32|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128033,"content":"<|place▁holder▁no▁33|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128034,"content":"<|place▁holder▁no▁34|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128035,"content":"<|place▁holder▁no▁35|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128036,"content":"<|place▁holder▁no▁36|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128037,"content":"<|place▁holder▁no▁37|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128038,"content":"<|place▁holder▁no▁38|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128039,"content":"<|place▁holder▁no▁39|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128040,"content":"<|place▁holder▁no▁40|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128041,"content":"<|place▁holder▁no▁41|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128042,"content":"<|place▁holder▁no▁42|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128043,"content":"<|place▁holder▁no▁43|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128044,"content":"<|place▁holder▁no▁44|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128045,"content":"<|place▁holder▁no▁45|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128046,"content":"<|place▁holder▁no▁46|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128047,"content":"<|place▁holder▁no▁47|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128048,"content":"<|place▁holder▁no▁48|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128049,"content":"<|place▁holder▁no▁49|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128050,"content":"<|place▁holder▁no▁50|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128051,"content":"<|place▁holder▁no▁51|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128052,"content":"<|place▁holder▁no▁52|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128053,"content":"<|place▁holder▁no▁53|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128054,"content":"<|place▁holder▁no▁54|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128055,"content":"<|place▁holder▁no▁55|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128056,"content":"<|place▁holder▁no▁56|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128057,"content":"<|place▁holder▁no▁57|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128058,"content":"<|place▁holder▁no▁58|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128059,"content":"<|place▁holder▁no▁59|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128060,"content":"<|place▁holder▁no▁60|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128061,"content":"<|place▁holder▁no▁61|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128062,"content":"<|place▁holder▁no▁62|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128063,"content":"<|place▁holder▁no▁63|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128064,"content":"<|place▁holder▁no▁64|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128065,"content":"<|place▁holder▁no▁65|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128066,"content":"<|place▁holder▁no▁66|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128067,"content":"<|place▁holder▁no▁67|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128068,"content":"<|place▁holder▁no▁68|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128069,"content":"<|place▁holder▁no▁69|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128070,"content":"<|place▁holder▁no▁70|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128071,"content":"<|place▁holder▁no▁71|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128072,"content":"<|place▁holder▁no▁72|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128073,"content":"<|place▁holder▁no▁73|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128074,"content":"<|place▁holder▁no▁74|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128075,"content":"<|place▁holder▁no▁75|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128076,"content":"<|place▁holder▁no▁76|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128077,"content":"<|place▁holder▁no▁77|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128078,"content":"<|place▁holder▁no▁78|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128079,"content":"<|place▁holder▁no▁79|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128080,"content":"<|place▁holder▁no▁80|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128081,"content":"<|place▁holder▁no▁81|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128082,"content":"<|place▁holder▁no▁82|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128083,"content":"<|place▁holder▁no▁83|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128084,"content":"<|place▁holder▁no▁84|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128085,"content":"<|place▁holder▁no▁85|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128086,"content":"<|place▁holder▁no▁86|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128087,"content":"<|place▁holder▁no▁87|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128088,"content":"<|place▁holder▁no▁88|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128089,"content":"<|place▁holder▁no▁89|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128090,"content":"<|place▁holder▁no▁90|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128091,"content":"<|place▁holder▁no▁91|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128092,"content":"<|place▁holder▁no▁92|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128093,"content":"<|place▁holder▁no▁93|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128094,"content":"<|place▁holder▁no▁94|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128095,"content":"<|place▁holder▁no▁95|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128096,"content":"<|place▁holder▁no▁96|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128097,"content":"<|place▁holder▁no▁97|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128098,"content":"<|place▁holder▁no▁98|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128099,"content":"<|place▁holder▁no▁99|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128100,"content":"<|place▁holder▁no▁100|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128101,"content":"<|place▁holder▁no▁101|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128102,"content":"<|place▁holder▁no▁102|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128103,"content":"<|place▁holder▁no▁103|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128104,"content":"<|place▁holder▁no▁104|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128105,"content":"<|place▁holder▁no▁105|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128106,"content":"<|place▁holder▁no▁106|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128107,"content":"<|place▁holder▁no▁107|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128108,"content":"<|place▁holder▁no▁108|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128109,"content":"<|place▁holder▁no▁109|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128110,"content":"<|place▁holder▁no▁110|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128111,"content":"<|place▁holder▁no▁111|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128112,"content":"<|place▁holder▁no▁112|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128113,"content":"<|place▁holder▁no▁113|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128114,"content":"<|place▁holder▁no▁114|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128115,"content":"<|place▁holder▁no▁115|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128116,"content":"<|place▁holder▁no▁116|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128117,"content":"<|place▁holder▁no▁117|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128118,"content":"<|place▁holder▁no▁118|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128119,"content":"<|place▁holder▁no▁119|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128120,"content":"<|place▁holder▁no▁120|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128121,"content":"<|place▁holder▁no▁121|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128122,"content":"<|place▁holder▁no▁122|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128123,"content":"<|place▁holder▁no▁123|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128124,"content":"<|place▁holder▁no▁124|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128125,"content":"<|place▁holder▁no▁125|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128126,"content":"<|place▁holder▁no▁126|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128127,"content":"<|place▁holder▁no▁127|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128128,"content":"<|place▁holder▁no▁128|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128129,"content":"<|place▁holder▁no▁129|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128130,"content":"<|place▁holder▁no▁130|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128131,"content":"<|place▁holder▁no▁131|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128132,"content":"<|place▁holder▁no▁132|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128133,"content":"<|place▁holder▁no▁133|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128134,"content":"<|place▁holder▁no▁134|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128135,"content":"<|place▁holder▁no▁135|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128136,"content":"<|place▁holder▁no▁136|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128137,"content":"<|place▁holder▁no▁137|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128138,"content":"<|place▁holder▁no▁138|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128139,"content":"<|place▁holder▁no▁139|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128140,"content":"<|place▁holder▁no▁140|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128141,"content":"<|place▁holder▁no▁141|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128142,"content":"<|place▁holder▁no▁142|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128143,"content":"<|place▁holder▁no▁143|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128144,"content":"<|place▁holder▁no▁144|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128145,"content":"<|place▁holder▁no▁145|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128146,"content":"<|place▁holder▁no▁146|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128147,"content":"<|place▁holder▁no▁147|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128148,"content":"<|place▁holder▁no▁148|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128149,"content":"<|place▁holder▁no▁149|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128150,"content":"<|place▁holder▁no▁150|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128151,"content":"<|place▁holder▁no▁151|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128152,"content":"<|place▁holder▁no▁152|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128153,"content":"<|place▁holder▁no▁153|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128154,"content":"<|place▁holder▁no▁154|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128155,"content":"<|place▁holder▁no▁155|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128156,"content":"<|place▁holder▁no▁156|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128157,"content":"<|place▁holder▁no▁157|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128158,"content":"<|place▁holder▁no▁158|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128159,"content":"<|place▁holder▁no▁159|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128160,"content":"<|place▁holder▁no▁160|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128161,"content":"<|place▁holder▁no▁161|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128162,"content":"<|place▁holder▁no▁162|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128163,"content":"<|place▁holder▁no▁163|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128164,"content":"<|place▁holder▁no▁164|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128165,"content":"<|place▁holder▁no▁165|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128166,"content":"<|place▁holder▁no▁166|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128167,"content":"<|place▁holder▁no▁167|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128168,"content":"<|place▁holder▁no▁168|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128169,"content":"<|place▁holder▁no▁169|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128170,"content":"<|place▁holder▁no▁170|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128171,"content":"<|place▁holder▁no▁171|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128172,"content":"<|place▁holder▁no▁172|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128173,"content":"<|place▁holder▁no▁173|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128174,"content":"<|place▁holder▁no▁174|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128175,"content":"<|place▁holder▁no▁175|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128176,"content":"<|place▁holder▁no▁176|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128177,"content":"<|place▁holder▁no▁177|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128178,"content":"<|place▁holder▁no▁178|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128179,"content":"<|place▁holder▁no▁179|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128180,"content":"<|place▁holder▁no▁180|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128181,"content":"<|place▁holder▁no▁181|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128182,"content":"<|place▁holder▁no▁182|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128183,"content":"<|place▁holder▁no▁183|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128184,"content":"<|place▁holder▁no▁184|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128185,"content":"<|place▁holder▁no▁185|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128186,"content":"<|place▁holder▁no▁186|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128187,"content":"<|place▁holder▁no▁187|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128188,"content":"<|place▁holder▁no▁188|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128189,"content":"<|place▁holder▁no▁189|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128190,"content":"<|place▁holder▁no▁190|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128191,"content":"<|place▁holder▁no▁191|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128192,"content":"<|place▁holder▁no▁192|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128193,"content":"<|place▁holder▁no▁193|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128194,"content":"<|place▁holder▁no▁194|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128195,"content":"<|place▁holder▁no▁195|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128196,"content":"<|place▁holder▁no▁196|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128197,"content":"<|place▁holder▁no▁197|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128198,"content":"<|place▁holder▁no▁198|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128199,"content":"<|place▁holder▁no▁199|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128200,"content":"<|place▁holder▁no▁200|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128201,"content":"<|place▁holder▁no▁201|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128202,"content":"<|place▁holder▁no▁202|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128203,"content":"<|place▁holder▁no▁203|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128204,"content":"<|place▁holder▁no▁204|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128205,"content":"<|place▁holder▁no▁205|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128206,"content":"<|place▁holder▁no▁206|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128207,"content":"<|place▁holder▁no▁207|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128208,"content":"<|place▁holder▁no▁208|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128209,"content":"<|place▁holder▁no▁209|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128210,"content":"<|place▁holder▁no▁210|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128211,"content":"<|place▁holder▁no▁211|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128212,"content":"<|place▁holder▁no▁212|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128213,"content":"<|place▁holder▁no▁213|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128214,"content":"<|place▁holder▁no▁214|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128215,"content":"<|place▁holder▁no▁215|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128216,"content":"<|place▁holder▁no▁216|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128217,"content":"<|place▁holder▁no▁217|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128218,"content":"<|place▁holder▁no▁218|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128219,"content":"<|place▁holder▁no▁219|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128220,"content":"<|place▁holder▁no▁220|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128221,"content":"<|place▁holder▁no▁221|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128222,"content":"<|place▁holder▁no▁222|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128223,"content":"<|place▁holder▁no▁223|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128224,"content":"<|place▁holder▁no▁224|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128225,"content":"<|place▁holder▁no▁225|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128226,"content":"<|place▁holder▁no▁226|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128227,"content":"<|place▁holder▁no▁227|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128228,"content":"<|place▁holder▁no▁228|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128229,"content":"<|place▁holder▁no▁229|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128230,"content":"<|place▁holder▁no▁230|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128231,"content":"<|place▁holder▁no▁231|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128232,"content":"<|place▁holder▁no▁232|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128233,"content":"<|place▁holder▁no▁233|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128234,"content":"<|place▁holder▁no▁234|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128235,"content":"<|place▁holder▁no▁235|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128236,"content":"<|place▁holder▁no▁236|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128237,"content":"<|place▁holder▁no▁237|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128238,"content":"<|place▁holder▁no▁238|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128239,"content":"<|place▁holder▁no▁239|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128240,"content":"<|place▁holder▁no▁240|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128241,"content":"<|place▁holder▁no▁241|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128242,"content":"<|place▁holder▁no▁242|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128243,"content":"<|place▁holder▁no▁243|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128244,"content":"<|place▁holder▁no▁244|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128245,"content":"<|place▁holder▁no▁245|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128246,"content":"<|place▁holder▁no▁246|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128247,"content":"<|place▁holder▁no▁247|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128248,"content":"<|place▁holder▁no▁248|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128249,"content":"<|place▁holder▁no▁249|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128250,"content":"<|place▁holder▁no▁250|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128251,"content":"<|place▁holder▁no▁251|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128252,"content":"<|place▁holder▁no▁252|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128253,"content":"<|place▁holder▁no▁253|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128254,"content":"<|place▁holder▁no▁254|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128255,"content":"<|place▁holder▁no▁255|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128256,"content":"<|place▁holder▁no▁256|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128257,"content":"<|place▁holder▁no▁257|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128258,"content":"<|place▁holder▁no▁258|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128259,"content":"<|place▁holder▁no▁259|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128260,"content":"<|place▁holder▁no▁260|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128261,"content":"<|place▁holder▁no▁261|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128262,"content":"<|place▁holder▁no▁262|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128263,"content":"<|place▁holder▁no▁263|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128264,"content":"<|place▁holder▁no▁264|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128265,"content":"<|place▁holder▁no▁265|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128266,"content":"<|place▁holder▁no▁266|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128267,"content":"<|place▁holder▁no▁267|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128268,"content":"<|place▁holder▁no▁268|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128269,"content":"<|place▁holder▁no▁269|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128270,"content":"<|place▁holder▁no▁270|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128271,"content":"<|place▁holder▁no▁271|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128272,"content":"<|place▁holder▁no▁272|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128273,"content":"<|place▁holder▁no▁273|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128274,"content":"<|place▁holder▁no▁274|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128275,"content":"<|place▁holder▁no▁275|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128276,"content":"<|place▁holder▁no▁276|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128277,"content":"<|place▁holder▁no▁277|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128278,"content":"<|place▁holder▁no▁278|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128279,"content":"<|place▁holder▁no▁279|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128280,"content":"<|place▁holder▁no▁280|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128281,"content":"<|place▁holder▁no▁281|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128282,"content":"<|place▁holder▁no▁282|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128283,"content":"<|place▁holder▁no▁283|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128284,"content":"<|place▁holder▁no▁284|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128285,"content":"<|place▁holder▁no▁285|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128286,"content":"<|place▁holder▁no▁286|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128287,"content":"<|place▁holder▁no▁287|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128288,"content":"<|place▁holder▁no▁288|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128289,"content":"<|place▁holder▁no▁289|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128290,"content":"<|place▁holder▁no▁290|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128291,"content":"<|place▁holder▁no▁291|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128292,"content":"<|place▁holder▁no▁292|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128293,"content":"<|place▁holder▁no▁293|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128294,"content":"<|place▁holder▁no▁294|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128295,"content":"<|place▁holder▁no▁295|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128296,"content":"<|place▁holder▁no▁296|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128297,"content":"<|place▁holder▁no▁297|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128298,"content":"<|place▁holder▁no▁298|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128299,"content":"<|place▁holder▁no▁299|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128300,"content":"<|place▁holder▁no▁300|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128301,"content":"<|place▁holder▁no▁301|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128302,"content":"<|place▁holder▁no▁302|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128303,"content":"<|place▁holder▁no▁303|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128304,"content":"<|place▁holder▁no▁304|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128305,"content":"<|place▁holder▁no▁305|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128306,"content":"<|place▁holder▁no▁306|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128307,"content":"<|place▁holder▁no▁307|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128308,"content":"<|place▁holder▁no▁308|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128309,"content":"<|place▁holder▁no▁309|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128310,"content":"<|place▁holder▁no▁310|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128311,"content":"<|place▁holder▁no▁311|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128312,"content":"<|place▁holder▁no▁312|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128313,"content":"<|place▁holder▁no▁313|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128314,"content":"<|place▁holder▁no▁314|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128315,"content":"<|place▁holder▁no▁315|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128316,"content":"<|place▁holder▁no▁316|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128317,"content":"<|place▁holder▁no▁317|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128318,"content":"<|place▁holder▁no▁318|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128319,"content":"<|place▁holder▁no▁319|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128320,"content":"<|place▁holder▁no▁320|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128321,"content":"<|place▁holder▁no▁321|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128322,"content":"<|place▁holder▁no▁322|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128323,"content":"<|place▁holder▁no▁323|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128324,"content":"<|place▁holder▁no▁324|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128325,"content":"<|place▁holder▁no▁325|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128326,"content":"<|place▁holder▁no▁326|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128327,"content":"<|place▁holder▁no▁327|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128328,"content":"<|place▁holder▁no▁328|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128329,"content":"<|place▁holder▁no▁329|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128330,"content":"<|place▁holder▁no▁330|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128331,"content":"<|place▁holder▁no▁331|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128332,"content":"<|place▁holder▁no▁332|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128333,"content":"<|place▁holder▁no▁333|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128334,"content":"<|place▁holder▁no▁334|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128335,"content":"<|place▁holder▁no▁335|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128336,"content":"<|place▁holder▁no▁336|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128337,"content":"<|place▁holder▁no▁337|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128338,"content":"<|place▁holder▁no▁338|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128339,"content":"<|place▁holder▁no▁339|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128340,"content":"<|place▁holder▁no▁340|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128341,"content":"<|place▁holder▁no▁341|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128342,"content":"<|place▁holder▁no▁342|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128343,"content":"<|place▁holder▁no▁343|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128344,"content":"<|place▁holder▁no▁344|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128345,"content":"<|place▁holder▁no▁345|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128346,"content":"<|place▁holder▁no▁346|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128347,"content":"<|place▁holder▁no▁347|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128348,"content":"<|place▁holder▁no▁348|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128349,"content":"<|place▁holder▁no▁349|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128350,"content":"<|place▁holder▁no▁350|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128351,"content":"<|place▁holder▁no▁351|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128352,"content":"<|place▁holder▁no▁352|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128353,"content":"<|place▁holder▁no▁353|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128354,"content":"<|place▁holder▁no▁354|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128355,"content":"<|place▁holder▁no▁355|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128356,"content":"<|place▁holder▁no▁356|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128357,"content":"<|place▁holder▁no▁357|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128358,"content":"<|place▁holder▁no▁358|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128359,"content":"<|place▁holder▁no▁359|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128360,"content":"<|place▁holder▁no▁360|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128361,"content":"<|place▁holder▁no▁361|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128362,"content":"<|place▁holder▁no▁362|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128363,"content":"<|place▁holder▁no▁363|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128364,"content":"<|place▁holder▁no▁364|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128365,"content":"<|place▁holder▁no▁365|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128366,"content":"<|place▁holder▁no▁366|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128367,"content":"<|place▁holder▁no▁367|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128368,"content":"<|place▁holder▁no▁368|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128369,"content":"<|place▁holder▁no▁369|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128370,"content":"<|place▁holder▁no▁370|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128371,"content":"<|place▁holder▁no▁371|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128372,"content":"<|place▁holder▁no▁372|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128373,"content":"<|place▁holder▁no▁373|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128374,"content":"<|place▁holder▁no▁374|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128375,"content":"<|place▁holder▁no▁375|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128376,"content":"<|place▁holder▁no▁376|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128377,"content":"<|place▁holder▁no▁377|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128378,"content":"<|place▁holder▁no▁378|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128379,"content":"<|place▁holder▁no▁379|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128380,"content":"<|place▁holder▁no▁380|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128381,"content":"<|place▁holder▁no▁381|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128382,"content":"<|place▁holder▁no▁382|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128383,"content":"<|place▁holder▁no▁383|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128384,"content":"<|place▁holder▁no▁384|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128385,"content":"<|place▁holder▁no▁385|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128386,"content":"<|place▁holder▁no▁386|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128387,"content":"<|place▁holder▁no▁387|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128388,"content":"<|place▁holder▁no▁388|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128389,"content":"<|place▁holder▁no▁389|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128390,"content":"<|place▁holder▁no▁390|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128391,"content":"<|place▁holder▁no▁391|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128392,"content":"<|place▁holder▁no▁392|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128393,"content":"<|place▁holder▁no▁393|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128394,"content":"<|place▁holder▁no▁394|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128395,"content":"<|place▁holder▁no▁395|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128396,"content":"<|place▁holder▁no▁396|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128397,"content":"<|place▁holder▁no▁397|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128398,"content":"<|place▁holder▁no▁398|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128399,"content":"<|place▁holder▁no▁399|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128400,"content":"<|place▁holder▁no▁400|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128401,"content":"<|place▁holder▁no▁401|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128402,"content":"<|place▁holder▁no▁402|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128403,"content":"<|place▁holder▁no▁403|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128404,"content":"<|place▁holder▁no▁404|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128405,"content":"<|place▁holder▁no▁405|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128406,"content":"<|place▁holder▁no▁406|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128407,"content":"<|place▁holder▁no▁407|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128408,"content":"<|place▁holder▁no▁408|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128409,"content":"<|place▁holder▁no▁409|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128410,"content":"<|place▁holder▁no▁410|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128411,"content":"<|place▁holder▁no▁411|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128412,"content":"<|place▁holder▁no▁412|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128413,"content":"<|place▁holder▁no▁413|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128414,"content":"<|place▁holder▁no▁414|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128415,"content":"<|place▁holder▁no▁415|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128416,"content":"<|place▁holder▁no▁416|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128417,"content":"<|place▁holder▁no▁417|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128418,"content":"<|place▁holder▁no▁418|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128419,"content":"<|place▁holder▁no▁419|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128420,"content":"<|place▁holder▁no▁420|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128421,"content":"<|place▁holder▁no▁421|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128422,"content":"<|place▁holder▁no▁422|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128423,"content":"<|place▁holder▁no▁423|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128424,"content":"<|place▁holder▁no▁424|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128425,"content":"<|place▁holder▁no▁425|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128426,"content":"<|place▁holder▁no▁426|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128427,"content":"<|place▁holder▁no▁427|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128428,"content":"<|place▁holder▁no▁428|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128429,"content":"<|place▁holder▁no▁429|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128430,"content":"<|place▁holder▁no▁430|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128431,"content":"<|place▁holder▁no▁431|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128432,"content":"<|place▁holder▁no▁432|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128433,"content":"<|place▁holder▁no▁433|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128434,"content":"<|place▁holder▁no▁434|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128435,"content":"<|place▁holder▁no▁435|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128436,"content":"<|place▁holder▁no▁436|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128437,"content":"<|place▁holder▁no▁437|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128438,"content":"<|place▁holder▁no▁438|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128439,"content":"<|place▁holder▁no▁439|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128440,"content":"<|place▁holder▁no▁440|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128441,"content":"<|place▁holder▁no▁441|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128442,"content":"<|place▁holder▁no▁442|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128443,"content":"<|place▁holder▁no▁443|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128444,"content":"<|place▁holder▁no▁444|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128445,"content":"<|place▁holder▁no▁445|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128446,"content":"<|place▁holder▁no▁446|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128447,"content":"<|place▁holder▁no▁447|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128448,"content":"<|place▁holder▁no▁448|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128449,"content":"<|place▁holder▁no▁449|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128450,"content":"<|place▁holder▁no▁450|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128451,"content":"<|place▁holder▁no▁451|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128452,"content":"<|place▁holder▁no▁452|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128453,"content":"<|place▁holder▁no▁453|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128454,"content":"<|place▁holder▁no▁454|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128455,"content":"<|place▁holder▁no▁455|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128456,"content":"<|place▁holder▁no▁456|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128457,"content":"<|place▁holder▁no▁457|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128458,"content":"<|place▁holder▁no▁458|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128459,"content":"<|place▁holder▁no▁459|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128460,"content":"<|place▁holder▁no▁460|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128461,"content":"<|place▁holder▁no▁461|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128462,"content":"<|place▁holder▁no▁462|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128463,"content":"<|place▁holder▁no▁463|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128464,"content":"<|place▁holder▁no▁464|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128465,"content":"<|place▁holder▁no▁465|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128466,"content":"<|place▁holder▁no▁466|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128467,"content":"<|place▁holder▁no▁467|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128468,"content":"<|place▁holder▁no▁468|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128469,"content":"<|place▁holder▁no▁469|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128470,"content":"<|place▁holder▁no▁470|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128471,"content":"<|place▁holder▁no▁471|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128472,"content":"<|place▁holder▁no▁472|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128473,"content":"<|place▁holder▁no▁473|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128474,"content":"<|place▁holder▁no▁474|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128475,"content":"<|place▁holder▁no▁475|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128476,"content":"<|place▁holder▁no▁476|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128477,"content":"<|place▁holder▁no▁477|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128478,"content":"<|place▁holder▁no▁478|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128479,"content":"<|place▁holder▁no▁479|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128480,"content":"<|place▁holder▁no▁480|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128481,"content":"<|place▁holder▁no▁481|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128482,"content":"<|place▁holder▁no▁482|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128483,"content":"<|place▁holder▁no▁483|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128484,"content":"<|place▁holder▁no▁484|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128485,"content":"<|place▁holder▁no▁485|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128486,"content":"<|place▁holder▁no▁486|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128487,"content":"<|place▁holder▁no▁487|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128488,"content":"<|place▁holder▁no▁488|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128489,"content":"<|place▁holder▁no▁489|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128490,"content":"<|place▁holder▁no▁490|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128491,"content":"<|place▁holder▁no▁491|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128492,"content":"<|place▁holder▁no▁492|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128493,"content":"<|place▁holder▁no▁493|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128494,"content":"<|place▁holder▁no▁494|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128495,"content":"<|place▁holder▁no▁495|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128496,"content":"<|place▁holder▁no▁496|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128497,"content":"<|place▁holder▁no▁497|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128498,"content":"<|place▁holder▁no▁498|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128499,"content":"<|place▁holder▁no▁499|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128500,"content":"<|place▁holder▁no▁500|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128501,"content":"<|place▁holder▁no▁501|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128502,"content":"<|place▁holder▁no▁502|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128503,"content":"<|place▁holder▁no▁503|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128504,"content":"<|place▁holder▁no▁504|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128505,"content":"<|place▁holder▁no▁505|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128506,"content":"<|place▁holder▁no▁506|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128507,"content":"<|place▁holder▁no▁507|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128508,"content":"<|place▁holder▁no▁508|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128509,"content":"<|place▁holder▁no▁509|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128510,"content":"<|place▁holder▁no▁510|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128511,"content":"<|place▁holder▁no▁511|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128512,"content":"<|place▁holder▁no▁512|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128513,"content":"<|place▁holder▁no▁513|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128514,"content":"<|place▁holder▁no▁514|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128515,"content":"<|place▁holder▁no▁515|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128516,"content":"<|place▁holder▁no▁516|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128517,"content":"<|place▁holder▁no▁517|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128518,"content":"<|place▁holder▁no▁518|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128519,"content":"<|place▁holder▁no▁519|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128520,"content":"<|place▁holder▁no▁520|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128521,"content":"<|place▁holder▁no▁521|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128522,"content":"<|place▁holder▁no▁522|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128523,"content":"<|place▁holder▁no▁523|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128524,"content":"<|place▁holder▁no▁524|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128525,"content":"<|place▁holder▁no▁525|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128526,"content":"<|place▁holder▁no▁526|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128527,"content":"<|place▁holder▁no▁527|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128528,"content":"<|place▁holder▁no▁528|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128529,"content":"<|place▁holder▁no▁529|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128530,"content":"<|place▁holder▁no▁530|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128531,"content":"<|place▁holder▁no▁531|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128532,"content":"<|place▁holder▁no▁532|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128533,"content":"<|place▁holder▁no▁533|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128534,"content":"<|place▁holder▁no▁534|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128535,"content":"<|place▁holder▁no▁535|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128536,"content":"<|place▁holder▁no▁536|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128537,"content":"<|place▁holder▁no▁537|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128538,"content":"<|place▁holder▁no▁538|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128539,"content":"<|place▁holder▁no▁539|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128540,"content":"<|place▁holder▁no▁540|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128541,"content":"<|place▁holder▁no▁541|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128542,"content":"<|place▁holder▁no▁542|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128543,"content":"<|place▁holder▁no▁543|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128544,"content":"<|place▁holder▁no▁544|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128545,"content":"<|place▁holder▁no▁545|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128546,"content":"<|place▁holder▁no▁546|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128547,"content":"<|place▁holder▁no▁547|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128548,"content":"<|place▁holder▁no▁548|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128549,"content":"<|place▁holder▁no▁549|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128550,"content":"<|place▁holder▁no▁550|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128551,"content":"<|place▁holder▁no▁551|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128552,"content":"<|place▁holder▁no▁552|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128553,"content":"<|place▁holder▁no▁553|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128554,"content":"<|place▁holder▁no▁554|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128555,"content":"<|place▁holder▁no▁555|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128556,"content":"<|place▁holder▁no▁556|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128557,"content":"<|place▁holder▁no▁557|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128558,"content":"<|place▁holder▁no▁558|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128559,"content":"<|place▁holder▁no▁559|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128560,"content":"<|place▁holder▁no▁560|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128561,"content":"<|place▁holder▁no▁561|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128562,"content":"<|place▁holder▁no▁562|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128563,"content":"<|place▁holder▁no▁563|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128564,"content":"<|place▁holder▁no▁564|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128565,"content":"<|place▁holder▁no▁565|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128566,"content":"<|place▁holder▁no▁566|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128567,"content":"<|place▁holder▁no▁567|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128568,"content":"<|place▁holder▁no▁568|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128569,"content":"<|place▁holder▁no▁569|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128570,"content":"<|place▁holder▁no▁570|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128571,"content":"<|place▁holder▁no▁571|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128572,"content":"<|place▁holder▁no▁572|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128573,"content":"<|place▁holder▁no▁573|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128574,"content":"<|place▁holder▁no▁574|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128575,"content":"<|place▁holder▁no▁575|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128576,"content":"<|place▁holder▁no▁576|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128577,"content":"<|place▁holder▁no▁577|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128578,"content":"<|place▁holder▁no▁578|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128579,"content":"<|place▁holder▁no▁579|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128580,"content":"<|place▁holder▁no▁580|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128581,"content":"<|place▁holder▁no▁581|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128582,"content":"<|place▁holder▁no▁582|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128583,"content":"<|place▁holder▁no▁583|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128584,"content":"<|place▁holder▁no▁584|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128585,"content":"<|place▁holder▁no▁585|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128586,"content":"<|place▁holder▁no▁586|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128587,"content":"<|place▁holder▁no▁587|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128588,"content":"<|place▁holder▁no▁588|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128589,"content":"<|place▁holder▁no▁589|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128590,"content":"<|place▁holder▁no▁590|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128591,"content":"<|place▁holder▁no▁591|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128592,"content":"<|place▁holder▁no▁592|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128593,"content":"<|place▁holder▁no▁593|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128594,"content":"<|place▁holder▁no▁594|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128595,"content":"<|place▁holder▁no▁595|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128596,"content":"<|place▁holder▁no▁596|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128597,"content":"<|place▁holder▁no▁597|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128598,"content":"<|place▁holder▁no▁598|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128599,"content":"<|place▁holder▁no▁599|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128600,"content":"<|place▁holder▁no▁600|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128601,"content":"<|place▁holder▁no▁601|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128602,"content":"<|place▁holder▁no▁602|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128603,"content":"<|place▁holder▁no▁603|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128604,"content":"<|place▁holder▁no▁604|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128605,"content":"<|place▁holder▁no▁605|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128606,"content":"<|place▁holder▁no▁606|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128607,"content":"<|place▁holder▁no▁607|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128608,"content":"<|place▁holder▁no▁608|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128609,"content":"<|place▁holder▁no▁609|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128610,"content":"<|place▁holder▁no▁610|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128611,"content":"<|place▁holder▁no▁611|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128612,"content":"<|place▁holder▁no▁612|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128613,"content":"<|place▁holder▁no▁613|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128614,"content":"<|place▁holder▁no▁614|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128615,"content":"<|place▁holder▁no▁615|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128616,"content":"<|place▁holder▁no▁616|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128617,"content":"<|place▁holder▁no▁617|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128618,"content":"<|place▁holder▁no▁618|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128619,"content":"<|place▁holder▁no▁619|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128620,"content":"<|place▁holder▁no▁620|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128621,"content":"<|place▁holder▁no▁621|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128622,"content":"<|place▁holder▁no▁622|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128623,"content":"<|place▁holder▁no▁623|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128624,"content":"<|place▁holder▁no▁624|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128625,"content":"<|place▁holder▁no▁625|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128626,"content":"<|place▁holder▁no▁626|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128627,"content":"<|place▁holder▁no▁627|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128628,"content":"<|place▁holder▁no▁628|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128629,"content":"<|place▁holder▁no▁629|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128630,"content":"<|place▁holder▁no▁630|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128631,"content":"<|place▁holder▁no▁631|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128632,"content":"<|place▁holder▁no▁632|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128633,"content":"<|place▁holder▁no▁633|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128634,"content":"<|place▁holder▁no▁634|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128635,"content":"<|place▁holder▁no▁635|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128636,"content":"<|place▁holder▁no▁636|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128637,"content":"<|place▁holder▁no▁637|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128638,"content":"<|place▁holder▁no▁638|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128639,"content":"<|place▁holder▁no▁639|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128640,"content":"<|place▁holder▁no▁640|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128641,"content":"<|place▁holder▁no▁641|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128642,"content":"<|place▁holder▁no▁642|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128643,"content":"<|place▁holder▁no▁643|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128644,"content":"<|place▁holder▁no▁644|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128645,"content":"<|place▁holder▁no▁645|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128646,"content":"<|place▁holder▁no▁646|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128647,"content":"<|place▁holder▁no▁647|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128648,"content":"<|place▁holder▁no▁648|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128649,"content":"<|place▁holder▁no▁649|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128650,"content":"<|place▁holder▁no▁650|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128651,"content":"<|place▁holder▁no▁651|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128652,"content":"<|place▁holder▁no▁652|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128653,"content":"<|place▁holder▁no▁653|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128654,"content":"<|place▁holder▁no▁654|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128655,"content":"<|place▁holder▁no▁655|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128656,"content":"<|place▁holder▁no▁656|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128657,"content":"<|place▁holder▁no▁657|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128658,"content":"<|place▁holder▁no▁658|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128659,"content":"<|place▁holder▁no▁659|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128660,"content":"<|place▁holder▁no▁660|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128661,"content":"<|place▁holder▁no▁661|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128662,"content":"<|place▁holder▁no▁662|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128663,"content":"<|place▁holder▁no▁663|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128664,"content":"<|place▁holder▁no▁664|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128665,"content":"<|place▁holder▁no▁665|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128666,"content":"<|place▁holder▁no▁666|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128667,"content":"<|place▁holder▁no▁667|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128668,"content":"<|place▁holder▁no▁668|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128669,"content":"<|place▁holder▁no▁669|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128670,"content":"<|place▁holder▁no▁670|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128671,"content":"<|place▁holder▁no▁671|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128672,"content":"<|place▁holder▁no▁672|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128673,"content":"<|place▁holder▁no▁673|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128674,"content":"<|place▁holder▁no▁674|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128675,"content":"<|place▁holder▁no▁675|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128676,"content":"<|place▁holder▁no▁676|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128677,"content":"<|place▁holder▁no▁677|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128678,"content":"<|place▁holder▁no▁678|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128679,"content":"<|place▁holder▁no▁679|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128680,"content":"<|place▁holder▁no▁680|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128681,"content":"<|place▁holder▁no▁681|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128682,"content":"<|place▁holder▁no▁682|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128683,"content":"<|place▁holder▁no▁683|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128684,"content":"<|place▁holder▁no▁684|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128685,"content":"<|place▁holder▁no▁685|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128686,"content":"<|place▁holder▁no▁686|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128687,"content":"<|place▁holder▁no▁687|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128688,"content":"<|place▁holder▁no▁688|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128689,"content":"<|place▁holder▁no▁689|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128690,"content":"<|place▁holder▁no▁690|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128691,"content":"<|place▁holder▁no▁691|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128692,"content":"<|place▁holder▁no▁692|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128693,"content":"<|place▁holder▁no▁693|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128694,"content":"<|place▁holder▁no▁694|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128695,"content":"<|place▁holder▁no▁695|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128696,"content":"<|place▁holder▁no▁696|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128697,"content":"<|place▁holder▁no▁697|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128698,"content":"<|place▁holder▁no▁698|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128699,"content":"<|place▁holder▁no▁699|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128700,"content":"<|place▁holder▁no▁700|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128701,"content":"<|place▁holder▁no▁701|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128702,"content":"<|place▁holder▁no▁702|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128703,"content":"<|place▁holder▁no▁703|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128704,"content":"<|place▁holder▁no▁704|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128705,"content":"<|place▁holder▁no▁705|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128706,"content":"<|place▁holder▁no▁706|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128707,"content":"<|place▁holder▁no▁707|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128708,"content":"<|place▁holder▁no▁708|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128709,"content":"<|place▁holder▁no▁709|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128710,"content":"<|place▁holder▁no▁710|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128711,"content":"<|place▁holder▁no▁711|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128712,"content":"<|place▁holder▁no▁712|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128713,"content":"<|place▁holder▁no▁713|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128714,"content":"<|place▁holder▁no▁714|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128715,"content":"<|place▁holder▁no▁715|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128716,"content":"<|place▁holder▁no▁716|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128717,"content":"<|place▁holder▁no▁717|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128718,"content":"<|place▁holder▁no▁718|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128719,"content":"<|place▁holder▁no▁719|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128720,"content":"<|place▁holder▁no▁720|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128721,"content":"<|place▁holder▁no▁721|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128722,"content":"<|place▁holder▁no▁722|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128723,"content":"<|place▁holder▁no▁723|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128724,"content":"<|place▁holder▁no▁724|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128725,"content":"<|place▁holder▁no▁725|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128726,"content":"<|place▁holder▁no▁726|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128727,"content":"<|place▁holder▁no▁727|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128728,"content":"<|place▁holder▁no▁728|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128729,"content":"<|place▁holder▁no▁729|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128730,"content":"<|place▁holder▁no▁730|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128731,"content":"<|place▁holder▁no▁731|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128732,"content":"<|place▁holder▁no▁732|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128733,"content":"<|place▁holder▁no▁733|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128734,"content":"<|place▁holder▁no▁734|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128735,"content":"<|place▁holder▁no▁735|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128736,"content":"<|place▁holder▁no▁736|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128737,"content":"<|place▁holder▁no▁737|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128738,"content":"<|place▁holder▁no▁738|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128739,"content":"<|place▁holder▁no▁739|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128740,"content":"<|place▁holder▁no▁740|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128741,"content":"<|place▁holder▁no▁741|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128742,"content":"<|place▁holder▁no▁742|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128743,"content":"<|place▁holder▁no▁743|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128744,"content":"<|place▁holder▁no▁744|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128745,"content":"<|place▁holder▁no▁745|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128746,"content":"<|place▁holder▁no▁746|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128747,"content":"<|place▁holder▁no▁747|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128748,"content":"<|place▁holder▁no▁748|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128749,"content":"<|place▁holder▁no▁749|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128750,"content":"<|place▁holder▁no▁750|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128751,"content":"<|place▁holder▁no▁751|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128752,"content":"<|place▁holder▁no▁752|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128753,"content":"<|place▁holder▁no▁753|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128754,"content":"<|place▁holder▁no▁754|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128755,"content":"<|place▁holder▁no▁755|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128756,"content":"<|place▁holder▁no▁756|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128757,"content":"<|place▁holder▁no▁757|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128758,"content":"<|place▁holder▁no▁758|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128759,"content":"<|place▁holder▁no▁759|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128760,"content":"<|place▁holder▁no▁760|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128761,"content":"<|place▁holder▁no▁761|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128762,"content":"<|place▁holder▁no▁762|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128763,"content":"<|place▁holder▁no▁763|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128764,"content":"<|place▁holder▁no▁764|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128765,"content":"<|place▁holder▁no▁765|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128766,"content":"<|place▁holder▁no▁766|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128767,"content":"<|place▁holder▁no▁767|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128768,"content":"<|place▁holder▁no▁768|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128769,"content":"<|place▁holder▁no▁769|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128770,"content":"<|place▁holder▁no▁770|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128771,"content":"<|place▁holder▁no▁771|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128772,"content":"<|place▁holder▁no▁772|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128773,"content":"<|place▁holder▁no▁773|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128774,"content":"<|place▁holder▁no▁774|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128775,"content":"<|place▁holder▁no▁775|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128776,"content":"<|place▁holder▁no▁776|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128777,"content":"<|place▁holder▁no▁777|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128778,"content":"<|place▁holder▁no▁778|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128779,"content":"<|place▁holder▁no▁779|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128780,"content":"<|place▁holder▁no▁780|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128781,"content":"<|place▁holder▁no▁781|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128782,"content":"<|place▁holder▁no▁782|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128783,"content":"<|place▁holder▁no▁783|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128784,"content":"<|place▁holder▁no▁784|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128785,"content":"<|place▁holder▁no▁785|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128786,"content":"<|place▁holder▁no▁786|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128787,"content":"<|place▁holder▁no▁787|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128788,"content":"<|place▁holder▁no▁788|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128789,"content":"<|place▁holder▁no▁789|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128790,"content":"<|place▁holder▁no▁790|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128791,"content":"<|place▁holder▁no▁791|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128792,"content":"<|place▁holder▁no▁792|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128793,"content":"<|place▁holder▁no▁793|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128794,"content":"<|place▁holder▁no▁794|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128795,"content":"<|place▁holder▁no▁795|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128796,"content":"<|place▁holder▁no▁796|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128797,"content":"<|place▁holder▁no▁797|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128798,"content":"<|place▁holder▁no▁798|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128799,"content":"<|place▁holder▁no▁799|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128800,"content":"<|fim▁hole|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128801,"content":"<|fim▁begin|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128802,"content":"<|fim▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128803,"content":"<|User|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128804,"content":"<|Assistant|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128805,"content":"<|EOT|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128806,"content":"<|tool▁calls▁begin|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128807,"content":"<|tool▁calls▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128808,"content":"<|tool▁call▁begin|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128809,"content":"<|tool▁call▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128810,"content":"<|tool▁outputs▁begin|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128811,"content":"<|tool▁outputs▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128812,"content":"<|tool▁output▁begin|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128813,"content":"<|tool▁output▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128814,"content":"<|tool▁sep|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128815,"content":"<|begin▁of▁repo▁name|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128816,"content":"<|end▁of▁repo▁name|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128817,"content":"<|begin▁of▁file▁name|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128818,"content":"<|end▁of▁file▁name|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128819,"content":"<|begin▁of▁file|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128820,"content":"<|end▁of▁file|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128821,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128822,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128823,"content":"<|place▁holder▁for▁copy|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128824,"content":"<|place▁holder▁for▁pointer▁replace|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128825,"content":"|DSML|","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128826,"content":"<|begin▁sys|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128827,"content":"<|end▁sys|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128828,"content":"<|latest_reminder|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128829,"content":"<|action|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128830,"content":"<|query|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128831,"content":"<|authority|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128832,"content":"<|domain|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128833,"content":"<|task|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128834,"content":"<|political|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128835,"content":"<|entity|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128836,"content":"<|title|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128837,"content":"<|safety|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128838,"content":"<|answer|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128839,"content":"<|search|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128840,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128843,"content":"<|search▁end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128844,"content":"<|extracted_url|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128845,"content":"<|read_url|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false},{"id":128846,"content":"<|end_of_query|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128847,"content":"<|place_holder_mm_span_0021|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128848,"content":"<|place_holder_mm_span_0022|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128849,"content":"<|place_holder_mm_span_0023|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128850,"content":"<|place_holder_mm_span_0024|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128851,"content":"<|place_holder_mm_span_0025|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128852,"content":"<|place_holder_mm_span_0026|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128853,"content":"<|place_holder_mm_span_0027|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128854,"content":"<|place_holder_mm_span_0028|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128855,"content":"<|place_holder_mm_span_0029|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128856,"content":"<|place_holder_mm_span_0030|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128857,"content":"<|place_holder_mm_span_0031|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128858,"content":"<|place_holder_mm_span_0032|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128859,"content":"<|place_holder_mm_span_0033|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128860,"content":"<|place_holder_mm_span_0034|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128861,"content":"<|place_holder_mm_span_0035|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128862,"content":"<|place_holder_mm_span_0036|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128863,"content":"<|place_holder_mm_span_0037|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128864,"content":"<|place_holder_mm_span_0038|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128865,"content":"<|place_holder_mm_span_0039|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128866,"content":"<|place_holder_mm_span_0040|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128867,"content":"<|place_holder_mm_span_0041|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128868,"content":"<|place_holder_mm_span_0042|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128869,"content":"<|place_holder_mm_span_0043|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128870,"content":"<|place_holder_mm_span_0044|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128871,"content":"<|place_holder_mm_span_0045|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128872,"content":"<|place_holder_mm_span_0046|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128873,"content":"<|place_holder_mm_span_0047|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128874,"content":"<|place_holder_mm_span_0048|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128875,"content":"<|place_holder_mm_span_0049|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128876,"content":"<|place_holder_mm_span_0050|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128877,"content":"<|place_holder_mm_span_0051|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128878,"content":"<|place_holder_mm_span_0052|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128879,"content":"<|place_holder_mm_span_0053|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128880,"content":"<|place_holder_mm_span_0054|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128881,"content":"<|place_holder_mm_span_0055|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128882,"content":"<|place_holder_mm_span_0056|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128883,"content":"<|place_holder_mm_span_0057|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128884,"content":"<|place_holder_mm_span_0058|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128885,"content":"<|place_holder_mm_span_0059|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128886,"content":"<|place_holder_mm_span_0060|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128887,"content":"<|place_holder_mm_span_0061|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128888,"content":"<|place_holder_mm_span_0062|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128889,"content":"<|place_holder_mm_span_0063|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128890,"content":"<|place_holder_mm_span_0064|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128891,"content":"<|place_holder_mm_span_0065|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128892,"content":"<|place_holder_mm_span_0066|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128893,"content":"<|place_holder_mm_span_0067|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128894,"content":"<|place_holder_mm_span_0068|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128895,"content":"<|place_holder_mm_span_0069|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128896,"content":"<|place_holder_mm_span_0070|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128897,"content":"<|place_holder_mm_span_0071|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128898,"content":"<|place_holder_mm_span_0072|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128899,"content":"<|place_holder_mm_span_0073|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128900,"content":"<|place_holder_mm_span_0074|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128901,"content":"<|place_holder_mm_span_0075|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128902,"content":"<|place_holder_mm_span_0076|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128903,"content":"<|place_holder_mm_span_0077|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128904,"content":"<|place_holder_mm_span_0078|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128905,"content":"<|place_holder_mm_span_0079|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128906,"content":"<|place_holder_mm_span_0080|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128907,"content":"<|place_holder_mm_span_0081|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128908,"content":"<|place_holder_mm_span_0082|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128909,"content":"<|place_holder_mm_span_0083|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128910,"content":"<|place_holder_mm_span_0084|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128911,"content":"<|place_holder_mm_span_0085|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128912,"content":"<|place_holder_mm_span_0086|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128913,"content":"<|place_holder_mm_span_0087|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128914,"content":"<|place_holder_mm_span_0088|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128915,"content":"<|place_holder_mm_span_0089|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128916,"content":"<|place_holder_mm_span_0090|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128917,"content":"<|place_holder_mm_span_0091|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128918,"content":"<|place_holder_mm_span_0092|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128919,"content":"<|place_holder_mm_span_0093|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128920,"content":"<|place_holder_mm_span_0094|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128921,"content":"<|place_holder_mm_span_0095|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128922,"content":"<|place_holder_mm_span_0096|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128923,"content":"<|place_holder_mm_span_0097|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128924,"content":"<|place_holder_mm_span_0098|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128925,"content":"<|place_holder_mm_span_0099|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128926,"content":"<|place_holder_mm_span_0100|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128927,"content":"<|place_holder_mm_span_0101|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128928,"content":"<|place_holder_mm_span_0102|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128929,"content":"<|place_holder_mm_span_0103|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128930,"content":"<|place_holder_mm_span_0104|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128931,"content":"<|place_holder_mm_span_0105|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128932,"content":"<|place_holder_mm_span_0106|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128933,"content":"<|place_holder_mm_span_0107|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128934,"content":"<|place_holder_mm_span_0108|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128935,"content":"<|place_holder_mm_span_0109|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128936,"content":"<|place_holder_mm_span_0110|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128937,"content":"<|place_holder_mm_span_0111|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128938,"content":"<|place_holder_mm_span_0112|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128939,"content":"<|place_holder_mm_span_0113|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128940,"content":"<|place_holder_mm_span_0114|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128941,"content":"<|place_holder_mm_span_0115|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128942,"content":"<|place_holder_mm_span_0116|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128943,"content":"<|place_holder_mm_span_0117|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128944,"content":"<|place_holder_mm_span_0118|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128945,"content":"<|place_holder_mm_span_0119|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128946,"content":"<|place_holder_mm_span_0120|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128947,"content":"<|place_holder_mm_span_0121|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128948,"content":"<|place_holder_mm_span_0122|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128949,"content":"<|place_holder_mm_span_0123|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128950,"content":"<|place_holder_mm_span_0124|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128951,"content":"<|place_holder_mm_span_0125|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128952,"content":"<|place_holder_mm_span_0126|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128953,"content":"<|place_holder_mm_span_0127|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128954,"content":"<|place_holder_mm_span_0128|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128955,"content":"<|place_holder_mm_span_0129|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128956,"content":"<|place_holder_mm_span_0130|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128957,"content":"<|place_holder_mm_span_0131|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128958,"content":"<|place_holder_mm_span_0132|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128959,"content":"<|place_holder_mm_span_0133|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128960,"content":"<|place_holder_mm_span_0134|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128961,"content":"<|place_holder_mm_span_0135|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128962,"content":"<|place_holder_mm_span_0136|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128963,"content":"<|place_holder_mm_span_0137|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128964,"content":"<|place_holder_mm_span_0138|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128965,"content":"<|place_holder_mm_span_0139|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128966,"content":"<|place_holder_mm_span_0140|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128967,"content":"<|place_holder_mm_span_0141|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128968,"content":"<|place_holder_mm_span_0142|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128969,"content":"<|place_holder_mm_span_0143|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128970,"content":"<|place_holder_mm_span_0144|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128971,"content":"<|place_holder_mm_span_0145|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128972,"content":"<|place_holder_mm_span_0146|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128973,"content":"<|place_holder_mm_span_0147|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128974,"content":"<|place_holder_mm_span_0148|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128975,"content":"<|place_holder_mm_span_0149|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128976,"content":"<|place_holder_mm_span_0150|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128977,"content":"<|place_holder_mm_span_0151|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128978,"content":"<|place_holder_mm_span_0152|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128979,"content":"<|place_holder_mm_span_0153|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128980,"content":"<|place_holder_mm_span_0154|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128981,"content":"<|place_holder_mm_span_0155|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128982,"content":"<|place_holder_mm_span_0156|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128983,"content":"<|place_holder_mm_span_0157|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128984,"content":"<|place_holder_mm_span_0158|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128985,"content":"<|place_holder_mm_span_0159|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128986,"content":"<|place_holder_mm_span_0160|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128987,"content":"<|place_holder_mm_span_0161|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128988,"content":"<|place_holder_mm_span_0162|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128989,"content":"<|place_holder_mm_span_0163|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128990,"content":"<|place_holder_mm_span_0164|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128991,"content":"<|place_holder_mm_span_0165|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128992,"content":"<|place_holder_mm_span_0166|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128993,"content":"<|place_holder_mm_span_0167|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128994,"content":"<|place_holder_mm_span_0168|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128995,"content":"<|place_holder_mm_span_0169|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128996,"content":"<|place_holder_mm_span_0170|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128997,"content":"<|place_holder_mm_span_0171|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128998,"content":"<|place_holder_mm_span_0172|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":128999,"content":"<|place_holder_mm_span_0173|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129000,"content":"<|place_holder_mm_span_0174|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129001,"content":"<|place_holder_mm_span_0175|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129002,"content":"<|place_holder_mm_span_0176|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129003,"content":"<|place_holder_mm_span_0177|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129004,"content":"<|place_holder_mm_span_0178|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129005,"content":"<|place_holder_mm_span_0179|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129006,"content":"<|place_holder_mm_span_0180|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129007,"content":"<|place_holder_mm_span_0181|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129008,"content":"<|place_holder_mm_span_0182|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129009,"content":"<|place_holder_mm_span_0183|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129010,"content":"<|place_holder_mm_span_0184|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129011,"content":"<|place_holder_mm_span_0185|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129012,"content":"<|place_holder_mm_span_0186|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129013,"content":"<|place_holder_mm_span_0187|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129014,"content":"<|place_holder_mm_span_0188|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129015,"content":"<|place_holder_mm_span_0189|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129016,"content":"<|place_holder_mm_span_0190|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129017,"content":"<|place_holder_mm_span_0191|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129018,"content":"<|place_holder_mm_span_0192|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129019,"content":"<|place_holder_mm_span_0193|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129020,"content":"<|place_holder_mm_span_0194|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129021,"content":"<|place_holder_mm_span_0195|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129022,"content":"<|place_holder_mm_span_0196|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129023,"content":"<|place_holder_mm_span_0197|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129024,"content":"<|place_holder_mm_span_0198|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129025,"content":"<|place_holder_mm_span_0199|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129026,"content":"<|place_holder_mm_span_0200|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129027,"content":"<|place_holder_mm_span_0201|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129028,"content":"<|place_holder_mm_span_0202|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129029,"content":"<|place_holder_mm_span_0203|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129030,"content":"<|place_holder_mm_span_0204|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129031,"content":"<|place_holder_mm_span_0205|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129032,"content":"<|place_holder_mm_span_0206|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129033,"content":"<|place_holder_mm_span_0207|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129034,"content":"<|place_holder_mm_span_0208|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129035,"content":"<|place_holder_mm_span_0209|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129036,"content":"<|place_holder_mm_span_0210|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129037,"content":"<|place_holder_mm_span_0211|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129038,"content":"<|place_holder_mm_span_0212|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129039,"content":"<|place_holder_mm_span_0213|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129040,"content":"<|place_holder_mm_span_0214|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129041,"content":"<|place_holder_mm_span_0215|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129042,"content":"<|place_holder_mm_span_0216|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129043,"content":"<|place_holder_mm_span_0217|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129044,"content":"<|place_holder_mm_span_0218|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129045,"content":"<|place_holder_mm_span_0219|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129046,"content":"<|place_holder_mm_span_0220|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129047,"content":"<|place_holder_mm_span_0221|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129048,"content":"<|place_holder_mm_span_0222|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129049,"content":"<|place_holder_mm_span_0223|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129050,"content":"<|place_holder_mm_span_0224|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129051,"content":"<|place_holder_mm_span_0225|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129052,"content":"<|place_holder_mm_span_0226|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129053,"content":"<|place_holder_mm_span_0227|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129054,"content":"<|place_holder_mm_span_0228|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129055,"content":"<|place_holder_mm_span_0229|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129056,"content":"<|place_holder_mm_span_0230|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129057,"content":"<|place_holder_mm_span_0231|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129058,"content":"<|place_holder_mm_span_0232|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129059,"content":"<|place_holder_mm_span_0233|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129060,"content":"<|place_holder_mm_span_0234|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129061,"content":"<|place_holder_mm_span_0235|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129062,"content":"<|place_holder_mm_span_0236|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129063,"content":"<|place_holder_mm_span_0237|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129064,"content":"<|place_holder_mm_span_0238|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129065,"content":"<|place_holder_mm_span_0239|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129066,"content":"<|place_holder_mm_span_0240|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129067,"content":"<|place_holder_mm_span_0241|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129068,"content":"<|place_holder_mm_span_0242|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129069,"content":"<|place_holder_mm_span_0243|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129070,"content":"<|place_holder_mm_span_0244|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129071,"content":"<|place_holder_mm_span_0245|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129072,"content":"<|place_holder_mm_span_0246|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129073,"content":"<|place_holder_mm_span_0247|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129074,"content":"<|place_holder_mm_span_0248|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129075,"content":"<|place_holder_mm_span_0249|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129076,"content":"<|place_holder_mm_span_0250|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129077,"content":"<|place_holder_mm_span_0251|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129078,"content":"<|place_holder_mm_span_0252|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129079,"content":"<|place_holder_mm_span_0253|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129080,"content":"<|place_holder_mm_span_0254|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129081,"content":"<|place_holder_mm_span_0255|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129082,"content":"<|place_holder_mm_span_0256|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129083,"content":"<|place_holder_mm_span_0257|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129084,"content":"<|place_holder_mm_span_0258|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129085,"content":"<|place_holder_mm_span_0259|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129086,"content":"<|place_holder_mm_span_0260|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129087,"content":"<|place_holder_mm_span_0261|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129088,"content":"<|place_holder_mm_span_0262|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129089,"content":"<|place_holder_mm_span_0263|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129090,"content":"<|place_holder_mm_span_0264|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129091,"content":"<|place_holder_mm_span_0265|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129092,"content":"<|place_holder_mm_span_0266|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129093,"content":"<|place_holder_mm_span_0267|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129094,"content":"<|place_holder_mm_span_0268|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129095,"content":"<|place_holder_mm_span_0269|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129096,"content":"<|place_holder_mm_span_0270|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129097,"content":"<|place_holder_mm_span_0271|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129098,"content":"<|place_holder_mm_span_0272|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129099,"content":"<|place_holder_mm_span_0273|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129100,"content":"<|place_holder_mm_span_0274|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129101,"content":"<|place_holder_mm_span_0275|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129102,"content":"<|place_holder_mm_span_0276|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129103,"content":"<|place_holder_mm_span_0277|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129104,"content":"<|place_holder_mm_span_0278|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129105,"content":"<|place_holder_mm_span_0279|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129106,"content":"<|place_holder_mm_span_0280|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129107,"content":"<|place_holder_mm_span_0281|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129108,"content":"<|place_holder_mm_span_0282|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129109,"content":"<|place_holder_mm_span_0283|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129110,"content":"<|place_holder_mm_span_0284|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129111,"content":"<|place_holder_mm_span_0285|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129112,"content":"<|place_holder_mm_span_0286|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129113,"content":"<|place_holder_mm_span_0287|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129114,"content":"<|place_holder_mm_span_0288|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129115,"content":"<|place_holder_mm_span_0289|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129116,"content":"<|place_holder_mm_span_0290|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129117,"content":"<|place_holder_mm_span_0291|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129118,"content":"<|place_holder_mm_span_0292|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129119,"content":"<|place_holder_mm_span_0293|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129120,"content":"<|place_holder_mm_span_0294|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129121,"content":"<|place_holder_mm_span_0295|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129122,"content":"<|place_holder_mm_span_0296|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129123,"content":"<|place_holder_mm_span_0297|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129124,"content":"<|place_holder_mm_span_0298|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129125,"content":"<|place_holder_mm_span_0299|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129126,"content":"<|place_holder_mm_span_0300|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129127,"content":"<|place_holder_mm_span_0301|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129128,"content":"<|place_holder_mm_span_0302|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129129,"content":"<|place_holder_mm_span_0303|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129130,"content":"<|place_holder_mm_span_0304|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129131,"content":"<|place_holder_mm_span_0305|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129132,"content":"<|place_holder_mm_span_0306|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129133,"content":"<|place_holder_mm_span_0307|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129134,"content":"<|place_holder_mm_span_0308|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129135,"content":"<|place_holder_mm_span_0309|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129136,"content":"<|place_holder_mm_span_0310|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129137,"content":"<|place_holder_mm_span_0311|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129138,"content":"<|place_holder_mm_span_0312|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129139,"content":"<|place_holder_mm_span_0313|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129140,"content":"<|place_holder_mm_span_0314|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129141,"content":"<|place_holder_mm_span_0315|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129142,"content":"<|place_holder_mm_span_0316|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129143,"content":"<|place_holder_mm_span_0317|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129144,"content":"<|place_holder_mm_span_0318|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129145,"content":"<|place_holder_mm_span_0319|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129146,"content":"<|place_holder_mm_span_0320|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129147,"content":"<|place_holder_mm_span_0321|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129148,"content":"<|place_holder_mm_span_0322|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129149,"content":"<|place_holder_mm_span_0323|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129150,"content":"<|place_holder_mm_span_0324|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129151,"content":"<|place_holder_mm_span_0325|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129152,"content":"<|place_holder_mm_span_0326|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129153,"content":"<|place_holder_mm_span_0327|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129154,"content":"<|place_holder_mm_span_0328|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129155,"content":"<|place_holder_mm_span_0329|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129156,"content":"<|place_holder_mm_span_0330|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129157,"content":"<|place_holder_mm_span_0331|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129158,"content":"<|place_holder_mm_span_0332|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129159,"content":"<|place_holder_mm_span_0333|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129160,"content":"<|place_holder_mm_span_0334|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129161,"content":"<|place_holder_mm_span_0335|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129162,"content":"<|place_holder_mm_span_0336|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129163,"content":"<|place_holder_mm_span_0337|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129164,"content":"<|place_holder_mm_span_0338|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129165,"content":"<|place_holder_mm_span_0339|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129166,"content":"<|place_holder_mm_span_0340|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129167,"content":"<|place_holder_mm_span_0341|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129168,"content":"<|place_holder_mm_span_0342|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129169,"content":"<|place_holder_mm_span_0343|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129170,"content":"<|place_holder_mm_span_0344|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129171,"content":"<|place_holder_mm_span_0345|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129172,"content":"<|place_holder_mm_span_0346|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129173,"content":"<|place_holder_mm_span_0347|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129174,"content":"<|place_holder_mm_span_0348|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129175,"content":"<|place_holder_mm_span_0349|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129176,"content":"<|place_holder_mm_span_0350|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129177,"content":"<|place_holder_mm_span_0351|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129178,"content":"<|place_holder_mm_span_0352|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129179,"content":"<|place_holder_mm_span_0353|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129180,"content":"<|place_holder_mm_span_0354|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129181,"content":"<|place_holder_mm_span_0355|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129182,"content":"<|place_holder_mm_span_0356|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129183,"content":"<|place_holder_mm_span_0357|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129184,"content":"<|place_holder_mm_span_0358|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129185,"content":"<|place_holder_mm_span_0359|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129186,"content":"<|place_holder_mm_span_0360|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129187,"content":"<|place_holder_mm_span_0361|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129188,"content":"<|place_holder_mm_span_0362|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129189,"content":"<|place_holder_mm_span_0363|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129190,"content":"<|place_holder_mm_span_0364|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129191,"content":"<|place_holder_mm_span_0365|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129192,"content":"<|place_holder_mm_span_0366|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129193,"content":"<|place_holder_mm_span_0367|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129194,"content":"<|place_holder_mm_span_0368|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129195,"content":"<|place_holder_mm_span_0369|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129196,"content":"<|place_holder_mm_span_0370|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129197,"content":"<|place_holder_mm_span_0371|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129198,"content":"<|place_holder_mm_span_0372|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129199,"content":"<|place_holder_mm_span_0373|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129200,"content":"<|place_holder_mm_span_0374|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129201,"content":"<|place_holder_mm_span_0375|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129202,"content":"<|place_holder_mm_span_0376|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129203,"content":"<|place_holder_mm_span_0377|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129204,"content":"<|place_holder_mm_span_0378|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129205,"content":"<|place_holder_mm_span_0379|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129206,"content":"<|place_holder_mm_span_0380|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129207,"content":"<|place_holder_mm_span_0381|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129208,"content":"<|place_holder_mm_span_0382|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129209,"content":"<|place_holder_mm_span_0383|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129210,"content":"<|place_holder_mm_span_0384|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129211,"content":"<|place_holder_mm_span_0385|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129212,"content":"<|place_holder_mm_span_0386|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129213,"content":"<|place_holder_mm_span_0387|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129214,"content":"<|place_holder_mm_span_0388|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129215,"content":"<|place_holder_mm_span_0389|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129216,"content":"<|place_holder_mm_span_0390|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129217,"content":"<|place_holder_mm_span_0391|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129218,"content":"<|place_holder_mm_span_0392|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129219,"content":"<|place_holder_mm_span_0393|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129220,"content":"<|place_holder_mm_span_0394|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129221,"content":"<|place_holder_mm_span_0395|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129222,"content":"<|place_holder_mm_span_0396|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129223,"content":"<|place_holder_mm_span_0397|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129224,"content":"<|place_holder_mm_span_0398|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129225,"content":"<|place_holder_mm_span_0399|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129226,"content":"<|place_holder_mm_span_0400|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129227,"content":"<|place_holder_mm_span_0401|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129228,"content":"<|place_holder_mm_span_0402|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129229,"content":"<|place_holder_mm_span_0403|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129230,"content":"<|place_holder_mm_span_0404|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129231,"content":"<|place_holder_mm_span_0405|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129232,"content":"<|place_holder_mm_span_0406|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129233,"content":"<|place_holder_mm_span_0407|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129234,"content":"<|place_holder_mm_span_0408|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129235,"content":"<|place_holder_mm_span_0409|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129236,"content":"<|place_holder_mm_span_0410|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129237,"content":"<|place_holder_mm_span_0411|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129238,"content":"<|place_holder_mm_span_0412|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129239,"content":"<|place_holder_mm_span_0413|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129240,"content":"<|place_holder_mm_span_0414|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129241,"content":"<|place_holder_mm_span_0415|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129242,"content":"<|place_holder_mm_span_0416|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129243,"content":"<|place_holder_mm_span_0417|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129244,"content":"<|place_holder_mm_span_0418|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129245,"content":"<|place_holder_mm_span_0419|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129246,"content":"<|place_holder_mm_span_0420|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129247,"content":"<|place_holder_mm_span_0421|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129248,"content":"<|place_holder_mm_span_0422|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129249,"content":"<|place_holder_mm_span_0423|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129250,"content":"<|place_holder_mm_span_0424|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129251,"content":"<|place_holder_mm_span_0425|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129252,"content":"<|place_holder_mm_span_0426|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129253,"content":"<|place_holder_mm_span_0427|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129254,"content":"<|place_holder_mm_span_0428|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129255,"content":"<|place_holder_mm_span_0429|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129256,"content":"<|place_holder_mm_span_0430|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129257,"content":"<|place_holder_mm_span_0431|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129258,"content":"<|place_holder_mm_span_0432|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129259,"content":"<|place_holder_mm_span_0433|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129260,"content":"<|place_holder_mm_span_0434|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129261,"content":"<|place_holder_mm_span_0435|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129262,"content":"<|rl_image_pad|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129263,"content":"<|rl_image_start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129264,"content":"<|image2|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129265,"content":"<|/table>|","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129266,"content":"<|table|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129267,"content":"<|/td|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129268,"content":"<|td|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129269,"content":"<|/tr|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129270,"content":"<|tr|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":129271,"content":"<|/polygon|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129272,"content":"<|polygon|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129273,"content":"<|/point|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129274,"content":"<|point|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129275,"content":"<|/box|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129276,"content":"<|box|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129277,"content":"<|/ref|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129278,"content":"<|ref|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":129279,"content":"<|image|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":{"type":"Sequence","normalizers":[]},"pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":"\\p{N}{1,3}"},"behavior":"Isolated","invert":false},{"type":"Split","pattern":{"Regex":"[一-龥぀-ゟ゠-ヿ]+"},"behavior":"Isolated","invert":false},{"type":"Split","pattern":{"Regex":"[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true,"use_regex":false}]},"post_processor":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":false,"use_regex":true},"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true,"use_regex":true},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"vocab":{"<|begin▁of▁sentence|>":0,"<|end▁of▁sentence|>":1,"<|▁pad▁|>":2,"!":3,"\"":4,"#":5,"$":6,"%":7,"&":8,"'":9,"(":10,")":11,"*":12,"+":13,",":14,"-":15,".":16,"/":17,"0":18,"1":19,"2":20,"3":21,"4":22,"5":23,"6":24,"7":25,"8":26,"9":27,":":28,";":29,"<":30,"=":31,">":32,"?":33,"@":34,"A":35,"B":36,"C":37,"D":38,"E":39,"F":40,"G":41,"H":42,"I":43,"J":44,"K":45,"L":46,"M":47,"N":48,"O":49,"P":50,"Q":51,"R":52,"S":53,"T":54,"U":55,"V":56,"W":57,"X":58,"Y":59,"Z":60,"[":61,"\\":62,"]":63,"^":64,"_":65,"`":66,"a":67,"b":68,"c":69,"d":70,"e":71,"f":72,"g":73,"h":74,"i":75,"j":76,"k":77,"l":78,"m":79,"n":80,"o":81,"p":82,"q":83,"r":84,"s":85,"t":86,"u":87,"v":88,"w":89,"x":90,"y":91,"z":92,"{":93,"|":94,"}":95,"~":96,"¡":97,"¢":98,"£":99,"¤":100,"¥":101,"¦":102,"§":103,"¨":104,"©":105,"ª":106,"«":107,"¬":108,"®":109,"¯":110,"°":111,"±":112,"²":113,"³":114,"´":115,"µ":116,"¶":117,"·":118,"¸":119,"¹":120,"º":121,"»":122,"¼":123,"½":124,"¾":125,"¿":126,"À":127,"Á":128,"Â":129,"Ã":130,"Ä":131,"Å":132,"Æ":133,"Ç":134,"È":135,"É":136,"Ê":137,"Ë":138,"Ì":139,"Í":140,"Î":141,"Ï":142,"Ð":143,"Ñ":144,"Ò":145,"Ó":146,"Ô":147,"Õ":148,"Ö":149,"×":150,"Ø":151,"Ù":152,"Ú":153,"Û":154,"Ü":155,"Ý":156,"Þ":157,"ß":158,"à":159,"á":160,"â":161,"ã":162,"ä":163,"å":164,"æ":165,"ç":166,"è":167,"é":168,"ê":169,"ë":170,"ì":171,"í":172,"î":173,"ï":174,"ð":175,"ñ":176,"ò":177,"ó":178,"ô":179,"õ":180,"ö":181,"÷":182,"ø":183,"ù":184,"ú":185,"û":186,"ü":187,"ý":188,"þ":189,"ÿ":190,"Ā":191,"ā":192,"Ă":193,"ă":194,"Ą":195,"ą":196,"Ć":197,"ć":198,"Ĉ":199,"ĉ":200,"Ċ":201,"ċ":202,"Č":203,"č":204,"Ď":205,"ď":206,"Đ":207,"đ":208,"Ē":209,"ē":210,"Ĕ":211,"ĕ":212,"Ė":213,"ė":214,"Ę":215,"ę":216,"Ě":217,"ě":218,"Ĝ":219,"ĝ":220,"Ğ":221,"ğ":222,"Ġ":223,"ġ":224,"Ģ":225,"ģ":226,"Ĥ":227,"ĥ":228,"Ħ":229,"ħ":230,"Ĩ":231,"ĩ":232,"Ī":233,"ī":234,"Ĭ":235,"ĭ":236,"Į":237,"į":238,"İ":239,"ı":240,"IJ":241,"ij":242,"Ĵ":243,"ĵ":244,"Ķ":245,"ķ":246,"ĸ":247,"Ĺ":248,"ĺ":249,"Ļ":250,"ļ":251,"Ľ":252,"ľ":253,"Ŀ":254,"ŀ":255,"Ł":256,"ł":257,"Ń":258,"Ġt":259,"Ġa":260,"in":261,"ĠĠ":262,"he":263,"er":264,"on":265,"re":266,"en":267,"at":268,"Ġs":269,"Ġthe":270,"ĊĊ":271,"or":272,"es":273,"Ġc":274,"ä¸":275,"an":276,"Ġo":277,"is":278,"it":279,"Ġp":280,"Ġw":281,"al":282,"Ġd":283,"ed":284,"Ġf":285,"ï¼":286,"ar":287,"ing":288,"nd":289,"ĠĠĠĠ":290,"Ġb":291,"Ġm":292,"ou":293,"Ġof":294,"Ġin":295,"ion":296,"ic":297,"ãĢ":298,"çļ":299,"âĢ":300,"çļĦ":301,"le":302,"ï¼Į":303,"Ġto":304,"Ġand":305,"as":306,"ro":307,"äº":308,"ent":309,"Ġh":310,"ct":311,"Ġe":312,"Ġn":313,"Ġl":314,"Ġth":315,"om":316,"el":317,"st":318,"et":319,"ãĢĤ":320,"il":321,"Ġre":322,"ä»":323,"åı":324,"æľ":325,"à¸":326,"ĠS":327,"im":328,"id":329,"ĠT":330,"ol":331,"ĠÐ":332,"ut":333,"ĠA":334,"åħ":335,"Ġg":336,"ra":337,"å¤":338,".ĊĊ":339,"iv":340,"ation":341,"ĠI":342,"Ġ(":343,"Ġis":344,"ĠC":345,"ur":346,"ot":347,"ch":348,"us":349,"ig":350,"è¿":351,"åĪ":352,"ce":353,"æĺ":354,"о":355,"am":356,"ä½":357,"å®":358,"ow":359,"ad":360,"ĠĠĠ":361,"Ġfor":362,"ul":363,"åIJ":364,"åľ":365,"Ġbe":366,"ly":367,"е":368,"Ġ|":369,"Ġst":370,"un":371,"##":372,"ĠM":373,"Ġv":374,"ä¹":375,"os":376,"Ġon":377,"ä¸Ģ":378,"а":379,"ĠP":380,"em":381,"çĶ":382,"Ġy":383,"æĪ":384,"ĠĠĠĠĠĠĠĠ":385,"ay":386,"ers":387,"ir":388,"æĺ¯":389,"à¦":390,"ا":391,"Ġde":392,"и":393,"if":394,"um":395,"Ġthat":396,"20":397,"å°":398,"Ġcon":399,"ith":400,"od":401,"ter":402,"qu":403,"ç»":404,"åĬ":405,"ĠB":406,"ÑĤ":407,"è¯":408,"ag":409,"ãĢģ":410,"Ġan":411,"Ġas":412,"Ġpro":413,"her":414,"ãģ":415,"est":416,"æĸ":417,"Ġwith":418,"н":419,"ĠD":420,"åŃ":421,"ä¸į":422,"Ġal":423,"åĽ":424,"ab":425,"..":426,"ve":427,"âĢľ":428,"äºĨ":429,"âĢĿ":430,"æĹ":431,"ver":432,"ĠR":433,"ate":434,"ist":435,"Ġit":436,"ĠH":437,"Ġ=":438,"ac":439,"Ġyou":440,"æĿ":441,"âĢĻ":442,"res":443,"Ñģ":444,"åľ¨":445,"ĠE":446,"ĠF":447,"ĠW":448,"ess":449,"æľī":450,"è®":451,"ÑĢ":452,"åį":453,"ect":454,"ĠThe":455,"pp":456,"ä¼":457,"and":458,"Ġwh":459,"ri":460,"æī":461,"ĠL":462,"th":463,"å¹":464,"Ġcom":465,"оÐ":466,"se":467,"Ġhe":468,"Ġor":469,"人":470,"ĠN":471,"Ġex":472,"Ġk":473,"å¾":474,"ill":475,"op":476,"Ġare":477,"ãĢĤĊĊ":478,"ant":479,"ak":480,"ity":481,"ort":482,"å·":483,"oc":484,"éĩ":485,"åĩ":486,"å¼":487,"Ġse":488,"ĠG":489,"ment":490,"ht":491,"ore":492,"èĢ":493,"Ġr":494,"ÙĦ":495,"rom":496,"åº":497,"Ġsu":498,"ain":499,"ie":500,"è¡":501,"æķ":502,"éĢ":503,"00":504,"ive":505,"åĨ":506,"å¸":507,"ĠØ":508,"Ġat":509,";Ċ":510,"19":511,"å¯":512,"Ġby":513,"ld":514,"Ġwas":515,"å¥":516,"ies":517,"ĥ½":518,"ud":519,"og":520,"art":521,"Ġne":522,"end":523,"æĢ":524,"ä¸Ń":525,"çĽ":526,"åĴ":527,"ĠĠĠĠĠĠĠ":528,"pt":529,"è§":530,"æĪij":531,"Ġle":532,"nt":533,"ure":534,"Ġha":535,"ial":536,"Ġch":537,"Ġfrom":538,"ĠĊ":539,"pl":540,"ĠO":541,"åĮ":542,"æł":543,"ĠÙ":544,"为":545,"å¿":546,"大":547,"åĴĮ":548,"Ġu":549,"Ġus":550,"our":551,"ĠJ":552,"10":553,"Ġnot":554,"ang":555,"è¿Ļ":556,"æĬ":557,"个":558,"pe":559,"ine":560,"è¦":561,"èµ":562,"æŃ":563,"ight":564,"Ġ-":565,"Ġthis":566,"per":567,"Ġsh":568,"çİ":569,"åİ":570,"iz":571,"ä¸Ĭ":572,"ç§":573,"ell":574,"л":575,"Ġen":576,"ction":577,"all":578,"Ġwe":579,"以":580,"ber":581,"Ġ\"":582,"ust":583,"çľ":584,"æ°":585,"éĹ":586,"èĩ":587,"Ġcan":588,"è¦ģ":589,"å±":590,"are":591,"te":592,"ard":593,"éĿ":594,"ical":595,"å½":596,"Ġj":597,"æĶ":598,"æ³":599,"è´":600,"ia":601,"ost":602,".Ċ":603,"ub":604,"çī":605,"out":606,"ult":607,"à¹":608,"æ²":609,"--":610,"Ġhave":611,"Ġun":612,"çĶŁ":613,"ue":614,"age":615,"ich":616,"ff":617,"åij":618,"é":619,"rou":620,"åΰ":621,"æĥ":622,"ر":623,"ass":624,"æĹ¶":625,"ä»ĸ":626,"éĻ":627,"ĠU":628,"æŀ":629,"ap":630,"ould":631,"ip":632,"ok":633,"ans":634,"ik":635,"ÙĨ":636,"æĿ¥":637,"ated":638,"Ġab":639,"orm":640,"Ġim":641,"ç͍":642,"201":643,"æİ":644,"Ġqu":645,"Ġpl":646,"Ġwor":647,"ast":648,"Ñĥ":649,"int":650,"act":651,"éģ":652,"åı¯":653,"åĩº":654,"ind":655,"çº":656,"ĠK":657,"к":658,"èĥ½":659,"ĠIn":660,"ome":661,"åѦ":662,"ĠâĢ":663,"du":664,"ãĤ":665,"**":666,"Ġcl":667,"Ġad":668,"Ġ×":669,"cl":670,"The":671,"å°±":672,"ä¼ļ":673,"Ġا":674,"Ġcomp":675,"Ġres":676,"ence":677,"Ġme":678,"able":679,"Ġ{":680,"ide":681,")Ċ":682,"ä¿":683,"ous":684,"ions":685,"ib":686,"ire":687,"Ġint":688,"æµ":689,"hen":690,"ame":691,"cc":692,"对":693,"ä½ľ":694,"å¹´":695,"Ġdo":696,"ÙĬ":697,"port":698,"ary":699,"ong":700,"æĦ":701,"ther":702,"æ¯":703,"é¢":704,"ge":705,"ations":706,"ear":707,"çŃ":708,"è¾":709,"Ġall":710,"å¦":711,"Ġcont":712,"ä¸ĭ":713,"ack":714,"à§":715,"Ġper":716,"ere":717,"åľ°":718,"è¡Į":719,"çIJ":720,"ĠV":721,"ice":722,"ime":723,"av":724,"fer":725,"ase":726,"ru":727,"ä¹Ł":728,"con":729,"ance":730,"æĮ":731,"åī":732,"в":733,"'s":734,"们":735,"12":736,"åĽ½":737,"Ñı":738,"à¤":739,"åıij":740,"æĭ":741,"м":742,"æĪIJ":743,"ry":744,"Ùħ":745,"ä¾":746,"ÙĪ":747,"èĩª":748,"ents":749,"åĵ":750,"åĪĨ":751,"åĢ":752,"ign":753,"),":754,"ep":755,"ach":756,"ov":757,"lic":758,"Ġwill":759,"åŃIJ":760,"æĸ¹":761,"....":762,"ord":763,"Ġ[":764,"Ñĭ":765,"äºİ":766,"ens":767,"ï¼ļ":768,"Ġhas":769,"ç«":770,"ĠTh":771,"gh":772,"è½":773,"ĠSt":774,"ĠĠĠĠĠĠĠĠĠĠĠ":775,"åIJİ":776,"éĺ":777,"Ġwhich":778,"11":779,"çĤ":780,"ress":781,"Ġyour":782,"ت":783,"Ġpr":784,"Ġar":785,"Ġtheir":786,"Ġdis":787,"ç¬":788,"çĿ":789,"Ġbut":790,"one":791,"200":792,"Ġhis":793,"form":794,"###":795,"è¿ĩ":796,").":797,"Ġout":798,"å¤ļ":799,"ä¹ĭ":800,"ÛĮ":801,"Ġapp":802,"ne":803,"ä½ł":804,"ace":805,"ile":806,"Ġgo":807,"ors":808,"ç®":809,"ition":810,"ĠâĢľ":811,"å·¥":812,"ks":813,"ual":814,"Ùĩ":815,"د":816,"å®¶":817,"Ġ<":818,"çIJĨ":819,"éĤ":820,"so":821,"åŁ":822,"Ġte":823,"æ³ķ":824,"ĠاÙĦ":825,"Ġsa":826,"èĤ":827,"д":828,"xt":829,"åĬ¨":830,"Ġà¦":831,"Ġso":832,");Ċ":833,"Ġone":834,"//":835,"Ġman":836,"Ġ}":837,"ä¸ļ":838,"ÑģÑĤ":839,"æ¬":840,"Ġtr":841,"å®ļ":842,"æ±":843,"å°ı":844,"ite":845,"Ġп":846,"Ġla":847,"__":848,"urn":849,"Ġmore":850,"Ġthey":851,"Ġpre":852,"好":853,"ook":854,"Ġif":855,"15":856,"éĿ¢":857,"vel":858,"å¾Ĺ":859,"æ´":860,"çŁ":861,"ll":862,"ose":863,"18":864,"çĦ":865,"ph":866,"èĢĮ":867,")ĊĊ":868,"ory":869,"ount":870,"åģ":871,"说":872,"é¡":873,"Ġ\\":874,"Ġ{Ċ":875,"ãĢĤĊ":876,"ake":877,"Ġsp":878,"ail":879,"å¿ĥ":880,"Ġwere":881,"éĥ½":882,"å¦Ĥ":883,"ç¨":884,"æ¸":885,"ÑĮ":886,"è·":887,"çݰ":888,"é«":889,"Ġup":890,"ely":891,"Ġpart":892,"Ġnum":893,"ĠY":894,"ci":895,"éģĵ":896,"ces":897,"æīĢ":898,"ĠCh":899,"è¿Ľ":900,"ath":901,"çĿĢ":902,"å®ŀ":903,"Ġdes":904,"Ġ'":905,"ree":906,"13":907,"erv":908,"ater":909,"åĬĽ":910,"éĥ":911,"ãĥ":912,"åIJĮ":913,"éķ":914,"Ġother":915,"Ġinter":916,"æľ¬":917,"ata":918,"éĽ":919,"Ġro":920,"Ñĩ":921,"ys":922,"æĽ":923,"ob":924,"ç»ı":925,"16":926,"Ġev":927,"de":928,"14":929,"æĻ":930,"ен":931,"Ġund":932,"ä½ĵ":933,"yst":934,"主":935,"Ġhad":936,"é«ĺ":937,"çľĭ":938,"202":939,"Ġ+":940,"ç½":941,"å¼Ģ":942,"Ġabout":943,"Ġ_":944,"æı":945,"æĢ§":946,"ä¸İ":947,"âĢĿĊĊ":948,"now":949,"åħ¶":950,"è°":951,"ound":952,"ç¤":953,"天":954,"çŃī":955,"çĦ¶":956,"Ġ$":957,"ew":958,"äºĭ":959,"Ġag":960,"Ġz":961,"ple":962,"ĠRe":963,"ject":964,"âĢĶ":965,"ram":966,"oll":967,"com":968,"Ġher":969,"åĮĸ":970,"we":971,"ric":972,"á":973,"åīį":974,"ild":975,"ian":976,"cre":977,"æĸĩ":978,":ĊĊ":979,"Ġem":980,"ring":981,"Ġ*":982,"ĠIt":983,"åħ¨":984,"çĤ¹":985,"ific":986,"åIJĪ":987,"åħ¬":988,",Ċ":989,"Ġalso":990,"éļ":991,"ng":992,"éĤ£":993,"ish":994,"Ġwho":995,"æķ°":996,"ert":997,"ĉĉ":998,"ck":999,"Ġв":1000,"éĥ¨":1001,"17":1002,"erm":1003,"ĠĊĊ":1004,"olog":1005,"×Ļ":1006,"aus":1007,"Ġi":1008,"Ġits":1009,"Ġì":1010,"ä¹Ī":1011,"reat":1012,"ark":1013,"Ġtime":1014,"wo":1015,"ays":1016,"Ġnew":1017,">Ċ":1018,"è¿ĺ":1019,"ÑĢа":1020,"ft":1021,"Ġtra":1022,"å§":1023,"Ġcomm":1024,"ĠÑģ":1025,"Ġmy":1026,"èĢħ":1027,"rit":1028,"度":1029,"Ġam":1030,"Ġthere":1031,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1032,"ÃŃ":1033,"che":1034,"ari":1035,"here":1036,"åĿ":1037,"æį":1038,"ont":1039,"ike":1040,"å»":1041,".,":1042,"æĸ°":1043,"lect":1044,"ings":1045,"çĻ":1046,"Ġbeen":1047,";ĊĊ":1048,"æĥħ":1049,"ystem":1050,"Ġ&":1051,"éĹ´":1052,"ç¾":1053,"ons":1054,"Ġinto":1055,"第":1056,"ä¸Ģ个":1057,"×ķ":1058,"30":1059,"Ġover":1060,"irst":1061,"åħ³":1062,"åİ»":1063,"èī":1064,"pec":1065,"Ġthem":1066,"ĠÃ":1067,"elf":1068,"25":1069,"iew":1070,"é£":1071,"row":1072,"éĩĮ":1073,"ates":1074,"ов":1075,"产":1076,"è¶":1077,"èµ·":1078,"س":1079,"æĹ¥":1080,"éĩį":1081,"Ġwhen":1082,"èģ":1083,"ç©":1084,"æŁ":1085,"Ġacc":1086,"ç§į":1087,"า":1088,"eth":1089,"Ġdif":1090,"з":1091,"åºĶ":1092,"Ġsome":1093,"Ġret":1094,"ss":1095,"çķ":1096,"rib":1097,"Ġpe":1098,"Ġthan":1099,"æĦı":1100,"ally":1101,"èĬ":1102,"æľĢ":1103,"clud":1104,"Ġstud":1105,"Ġbet":1106,"åĬł":1107,"没":1108,"ó":1109,"ond":1110,"und":1111,"ublic":1112,"ç³":1113,"Ġwould":1114,"cess":1115,"Ġwork":1116,"Ġany":1117,"ç´":1118,"Ġno":1119,"Ġcons":1120,"éĩı":1121,"éķ¿":1122,"ĠÙħ":1123,"In":1124,"Ġob":1125,"Ġind":1126,"âĢĵ":1127,"Ġass":1128,"old":1129,"Ġи":1130,"ä¸ī":1131,"Ġour":1132,"get":1133,"æĥ³":1134,"Ġrel":1135,"à®":1136,":Ċ":1137,"ood":1138,"åĽł":1139,"å½ĵ":1140,"Ġyear":1141,"Ġmay":1142,"ink":1143,"éĢļ":1144,"ب":1145,"表":1146,"æľº":1147,"ï¼Ł":1148,"ps":1149,"缸":1150,"iss":1151,"Ñħ":1152,"Ġknow":1153,"les":1154,"----":1155,"ement":1156,"red":1157,"åζ":1158,"Ġpo":1159,"çĪ":1160,"Ġval":1161,"ĠThis":1162,"Ġra":1163,"own":1164,"ah":1165,"ĠHe":1166,"Ġnumber":1167,"å¾Ī":1168,"äºĽ":1169,"æĺİ":1170,"æģ":1171,"æ°´":1172,"24":1173,"าà¸":1174,"ï¼ģ":1175,"åĨħ":1176,"å¢":1177,"Ġget":1178,"Ġform":1179,"ç¥":1180,"ener":1181,"ular":1182,"ع":1183,"ode":1184,"hat":1185,"Ġн":1186,"ced":1187,"ning":1188,"ty":1189,"ä½į":1190,"åħ¥":1191,"Ġhow":1192,"ick":1193,"igh":1194,"å¸Ĥ":1195,"çī©":1196,"Ġpos":1197,"æ¶":1198,"ied":1199,"io":1200,"Ġbl":1201,"Ġunder":1202,"ĠÂ":1203,"Ġ.":1204,"Ġwhat":1205,"Ġexp":1206,"ä½Ĩ":1207,"Ġfl":1208,"ities":1209,"éľ":1210,"èĭ":1211,"ery":1212,"æīĭ":1213,"Ġact":1214,"åķ":1215,"èº":1216,"ating":1217,"Ġco":1218,"ics":1219,"çł":1220,"æıIJ":1221,"èĪ":1222,"è¢":1223,"া":1224,"Ġshe":1225,"ise":1226,"ï¼ī":1227,"æķĻ":1228,"Ġel":1229,"п":1230,"Ġet":1231,"Ġо":1232,"Ġfe":1233,"Ġtwo":1234,"ility":1235,"æŀľ":1236,"ï¼Ī":1237,"ef":1238,"cy":1239,"?ĊĊ":1240,"Ġsub":1241,"fter":1242,"Ġprov":1243,"å¤ĸ":1244,"建":1245,"ative":1246,"ĠÎ":1247,"â̦":1248,"×ķ×":1249,"Ġreg":1250,"ç¨ĭ":1251,"å£":1252,"pr":1253,"çŁ¥":1254,"ä»İ":1255,"ĠâĢĵ":1256,"Ġfirst":1257,"Ġadd":1258,"ract":1259,"oy":1260,"åıĬ":1261,"Ø©":1262,"åı¯ä»¥":1263,"é¢ĺ":1264,"æĹł":1265,"æľĪ":1266,"Ġmod":1267,"她":1268,"ug":1269,"Ġrec":1270,"Ġbec":1271,"ange":1272,"ational":1273,"æŃ¤":1274,"å°Ĩ":1275,"Ġinv":1276,"Ġlike":1277,"Ġcol":1278,"ç͵":1279,"ĠCom":1280,"=\"":1281,"ble":1282,"rough":1283,"èĦ":1284,"ade":1285,"ient":1286,"æŃ£":1287,"å·±":1288,"ex":1289,"als":1290,"å±ķ":1291,"ç³»":1292,"次":1293,"ĠUn":1294,"æł·":1295,"èIJ":1296,"pro":1297,"Ġdi":1298,"åıª":1299,"æĪij们":1300,"æľŁ":1301,"22":1302,"its":1303,"äºĮ":1304,"Ġthese":1305,"Ġeff":1306,"×Ļ×":1307,"ause":1308,"Ġneed":1309,"ments":1310,"eng":1311,"Ġclass":1312,"Ġ:":1313,"tern":1314,"缮":1315,"æĪĸ":1316,"ĠPro":1317,"æ¯Ķ":1318,"Ġph":1319,"000":1320,"管":1321,"èĥ":1322,"æ·":1323,"身":1324,"ax":1325,"设":1326,"Ġâ":1327,"50":1328,"éħ":1329,"èĩªå·±":1330,"Ġtrans":1331,"ution":1332,"éĶ":1333,"使":1334,"è§£":1335,"mer":1336,"Ġsc":1337,"ãĢĭ":1338,"ç±":1339,"ç¡":1340,"Ġset":1341,"ãĢĬ":1342,"æ¡":1343,"ower":1344,"Ġsuch":1345,"Ġdiffer":1346,"Ġuse":1347,"æ°ij":1348,"23":1349,"ĠWe":1350,"Ġdef":1351,"å¹³":1352,"Ġonly":1353,"Ġreturn":1354,"ock":1355,"å¼ı":1356,"199":1357,"çģ":1358,"Ġsaid":1359,"æ´»":1360,"çĹ":1361,"常":1362,"ople":1363,"_{":1364,"ä¿Ŀ":1365,"ec":1366,"à¥":1367,"åĵģ":1368,"åĮº":1369,"ove":1370,"ĠÙĪ":1371,"Ġب":1372,"round":1373,"ier":1374,"Ġoff":1375,"åĸ":1376,"cept":1377,"à¸Ļ":1378,"ç¼":1379,"å¹¶":1380,"ased":1381,"ren":1382,"Ġpar":1383,"ни":1384,"çĸ":1385,"计":1386,"ize":1387,"itt":1388,"Ġinclud":1389,"èµĦ":1390,"å®ī":1391,"Ġprodu":1392,"()":1393,"oth":1394,"æĽ´":1395,"Ġac":1396,"Ġк":1397,"ÑĢе":1398,"ures":1399,"St":1400,"ç²":1401,"çĥ":1402,"б":1403,"Ġfun":1404,"Ġatt":1405,"very":1406,"Ġthrough":1407,"put":1408,"åĬ¡":1409,"ç»ĵ":1410,"eg":1411,"两":1412,"éĹ®":1413,"æİ¥":1414,"被":1415,"velop":1416,"ĠAn":1417,"ä¿¡":1418,"ä":1419,"Ġest":1420,"ween":1421,"ull":1422,"ix":1423,"ten":1424,"up":1425,"........":1426,"åIJij":1427,"代":1428,"ible":1429,"Ġpres":1430,"ey":1431,"Ġsur":1432,"é»":1433,"iel":1434,"ict":1435,"åĪ©":1436,"æĦŁ":1437,"Ġjust":1438,"éĴ":1439,"Ġhim":1440,"==":1441,"র":1442,"й":1443,"uch":1444,"ĠĠĠĠĠ":1445,"ough":1446,"ues":1447,"ork":1448,"28":1449,"26":1450,"Ġresp":1451,"Ġdet":1452,"çī¹":1453,"åĽŀ":1454,"ident":1455,"Ġrem":1456,"100":1457,"ool":1458,"ivers":1459,"åģļ":1460,"wn":1461,"hed":1462,"åľº":1463,"}\\":1464,"ek":1465,"èį":1466,"Ġpol":1467,"åijĺ":1468,"Ġbetween":1469,"Ġent":1470,"åįģ":1471,"å·¥ä½ľ":1472,"Ġmost":1473,"Ġpers":1474,"åŁº":1475,"è£":1476,"27":1477,"ism":1478,"Ġwhere":1479,"ym":1480,"å·²":1481,"Ġpeople":1482,"sp":1483,"40":1484,"Ġspec":1485,"fore":1486,"Ġsystem":1487,"ìĿ":1488,"Ġд":1489,"г":1490,"ä»¶":1491,"Ġ/":1492,"ife":1493,"Ġcould":1494,"to":1495,"еÑĤ":1496,"uc":1497,"Ġsupp":1498,"Ġdata":1499,"èĢģ":1500,"arch":1501,"uring":1502,"åIJį":1503,"ollow":1504,"Ġused":1505,"Ġhel":1506,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1507,"gan":1508,"ins":1509,"éĩij":1510,"cond":1511,"Ġ\\(":1512,"æĶ¿":1513,"der":1514,"éª":1515,"çͱ":1516,"ç¦":1517,"ful":1518,"à§ĩ":1519,"Ġsign":1520,"az":1521,"Ġend":1522,"ол":1523,"é¦":1524,"Ġë":1525,"Ġque":1526,"Ġx":1527,"Ġpublic":1528,"Ġresult":1529,"没æľī":1530,"Ġshould":1531,"åİŁ":1532,"ç¾İ":1533,"inal":1534,"ection":1535,"####":1536,"Ġ,":1537,"erg":1538,"Ġthen":1539,"Ùģ":1540,"ÑĪ":1541,"åıĺ":1542,"Ġdel":1543,"ĠAr":1544,"å½¢":1545,"Ġinst":1546,"æ°Ķ":1547,"èĻ":1548,"à§į":1549,"Ġmin":1550,"æº":1551,"tt":1552,"æ»":1553,"Ġ}Ċ":1554,"æĬĬ":1555,"ä»Ģ":1556,"29":1557,"ÑĢи":1558,"Ġback":1559,"ious":1560,"Ġafter":1561,"alth":1562,"ÙĤ":1563,"头":1564,"åIJĦ":1565,"Ġsim":1566,"Ġsm":1567,"à¹Ī":1568,"ruct":1569,"è¨":1570,"од":1571,"ages":1572,"åı£":1573,"Ġty":1574,"iqu":1575,"ãģ®":1576,"Ġfact":1577,"Ġrequ":1578,"ited":1579,"formation":1580,"ÑĨ":1581,"ĠAl":1582,"ĠSe":1583,"羣":1584,"Ġwell":1585,"ily":1586,"aj":1587,"æ¨":1588,"pos":1589,"ale":1590,"cent":1591,"ann":1592,"ices":1593,"ien":1594,"chn":1595,"æ®":1596,"缴":1597,"ร":1598,"ctions":1599,"Ġconst":1600,"vent":1601,"21":1602,"à¸Ń":1603,"Ġexper":1604,"Ġfollow":1605,"Ġlong":1606,"åŀ":1607,"ç»Ħ":1608,"æĤ":1609,"led":1610,"اØ":1611,"èī²":1612,"...":1613,"ural":1614,"ven":1615,"æį®":1616,"ung":1617,"ature":1618,"ran":1619,"åĽ¾":1620,"ç»Ļ":1621,"tic":1622,"Ġmany":1623,"Ġvari":1624,"ward":1625,"æĮĩ":1626,"Ġdevelop":1627,"²":1628,"è·¯":1629,"Ġequ":1630,"any":1631,"Ġdist":1632,"Ġcur":1633,"Ġcor":1634,"Ġmake":1635,"Th":1636,"ä»»":1637,"ider":1638,"Ġche":1639,"Ġed":1640,"社":1641,"Ġdec":1642,"Ġpat":1643,"åįķ":1644,"æ±Ĥ":1645,"ĠQ":1646,"ER":1647,"ts":1648,"aw":1649,"Ġes":1650,"å¤Ħ":1651,"нÑĭ":1652,"ĠZ":1653,"å°ij":1654,"é©":1655,"ines":1656,"на":1657,"oci":1658,"è¯Ŀ":1659,"Ġeach":1660,"ç«ĭ":1661,"Ġimport":1662,"Ġsol":1663,"'t":1664,"uth":1665,"Ġcar":1666,"но":1667,"fl":1668,"Ġhigh":1669,"强":1670,"33":1671,"other":1672,"åħĥ":1673,"hip":1674,"ĠDe":1675,"ern":1676,"ology":1677,"Ñİ":1678,"Ġchar":1679,"åıĪ":1680,"éĵ":1681,"à¸ģ":1682,"60":1683,"æµģ":1684,"Ġmed":1685,"____":1686,"Ġdid":1687,"Ġdifferent":1688,"Ġbu":1689,"aking":1690,"Ġstr":1691,"co":1692,"Ġext":1693,"Ġhelp":1694,"imes":1695,"Ġgener":1696,"ets":1697,"(\"":1698,"Ġprocess":1699,"让":1700,"æłĩ":1701,"æīĵ":1702,"è´¨":1703,"ом":1704,"Ùĥ":1705,"Ġmem":1706,"ÑĤе":1707,"Ġexam":1708,"ants":1709,"ä»Ģä¹Ī":1710,"交":1711,"اÙĦ":1712,"els":1713,"éľĢ":1714,"-s":1715,"ting":1716,"è¥":1717,"":1955,"Ġown":1956,"Ġsecond":1957,"åıijå±ķ":1958,"è¿Ļ个":1959,"Ġmade":1960,"éĻ¢":1961,"ross":1962,"ั":1963,"ON":1964,"çłĶ":1965,"èİ":1966,"è®°":1967,"举":1968,"ision":1969,"Ġwant":1970,"ÑĤа":1971,"çħ":1972,"Ġexpl":1973,"ä½ķ":1974,"Ġsame":1975,"hes":1976,"99":1977,"å²":1978,"à¸ĩ":1979,",âĢĿ":1980,"ank":1981,"ä»·":1982,"éŁ":1983,"ä¸ĸ":1984,"ral":1985,"ases":1986,"æĬ¥":1987,"Ġlife":1988,"oad":1989,"Ġvalue":1990,"èij":1991,"åĶ":1992,"Ġins":1993,"èµ°":1994,"æŃ¥":1995,"çľ¼":1996,"ince":1997,"Ġrep":1998,"ĠWhat":1999,"该":2000,"iven":2001,"Ķ×":2002,"ody":2003,"Ġword":2004,"stand":2005,"Ġfound":2006,"ected":2007,").ĊĊ":2008,"ĠSh":2009,"ĠNew":2010,"ton":2011,"34":2012,"Ġfam":2013,"anc":2014,"iness":2015,"ember":2016,"áĥ":2017,"íķ":2018,"Ġfunction":2019,"oh":2020,"^{":2021,"å̼":2022,"gg":2023,"åŃĹ":2024,"à¯":2025,"Ġت":2026,"éĿŀ":2027,"ĠCl":2028,"级":2029,"amp":2030,"ired":2031,"åĪĻ":2032,"ĠLe":2033,"Ġconf":2034,"æ¼":2035,"èĤ²":2036,"Ġcommun":2037,"Ġthree":2038,"ä¼ģ":2039,"åĩł":2040,"Ġreal":2041,"ĠYou":2042,"åĦ¿":2043,"èħ":2044,"èĬĤ":2045,"ght":2046,"ij":2047,"ES":2048,"ateg":2049,"带":2050,"ç½ij":2051,"ĠIf":2052,"æĶ¹":2053,"{Ċ":2054,"éĢł":2055,"éĹ®é¢ĺ":2056,"Ġquest":2057,"Ġworld":2058,"Ġtem":2059,"Ġanal":2060,"ö":2061,"æ¢":2062,"AT":2063,"èª":2064,"ole":2065,"åķĨ":2066,"text":2067,"Ġfin":2068,"ä¸ŃåĽ½":2069,"Ġlead":2070,"ĠInd":2071,"Ġele":2072,"åύ":2073,"Ġdep":2074,"åħ±":2075,"å¢ŀ":2076,"way":2077,"ä¹ł":2078,"лÑĮ":2079,"38":2080,"太":2081,"转":2082,"me":2083,"空":2084,"Ġsom":2085,"åħ¬åı¸":2086,"åŁİ":2087,"ม":2088,"éĽĨ":2089,"Ġdon":2090,"ina":2091,"Ġder":2092,"urs":2093,"æģ¯":2094,"ой":2095,"ĠâĢĺ":2096,"æŁ¥":2097,"ĠÑ":2098,"ä¸įæĺ¯":2099,"ŀ×":2100,"ret":2101,"æħ":2102,"æł¹":2103,"uk":2104,"->":2105,"ι":2106,"It":2107,"çĹħ":2108,"è¯ģ":2109,"ames":2110,"32":2111,"Ġterm":2112,"--------":2113,"Ġtechn":2114,"ä¸ĩ":2115,"39":2116,"alk":2117,"Ġthink":2118,"ually":2119,"æ¥":2120,"Ġmark":2121,"70":2122,"Ġsupport":2123,"Ġke":2124,"ç²¾":2125,"åĩĨ":2126,"ĠRes":2127,"ving":2128,"ior":2129,"æĹ¶éĹ´":2130,"Ġdem":2131,"Ġcour":2132,"ists":2133,"ü":2134,"ãĢĤâĢĿ":2135,"ĠâĢĶ":2136,"ĠX":2137,"arly":2138,"注":2139,"åĢĻ":2140,"åŃĺ":2141,"Ġmethod":2142,"ĠĠĊ":2143,"è¯Ĩ":2144,"Ġprovid":2145,"Ġposs":2146,"ва":2147,".\"":2148,"æºIJ":2149,"ences":2150,"Ġimp":2151,"vern":2152,"äºĶ":2153,"of":2154,"Ġhere":2155,"çª":2156,"ä»Ĭ":2157,"çİĭ":2158,"Ġgr":2159,"man":2160,"self":2161,"Âł":2162,"å¿«":2163,"ason":2164,"Ø´":2165,"ãĢĤâĢĿĊĊ":2166,"Re":2167,"æĪĺ":2168,"åĮħ":2169,"48":2170,"Ġà¤":2171,"åįĹ":2172,"Ġday":2173,"Ġhum":2174,"raph":2175,"ration":2176,"è¾ĥ":2177,"ability":2178,"ี":2179,"å¿ħ":2180,"31":2181,"acter":2182,"Ġد":2183,"Ġduring":2184,"Ġproble":2185,"åį³":2186,"ense":2187,"Ġtake":2188,"æł¡":2189,"âĪ":2190,"æŀĦ":2191,"Ġlevel":2192,".com":2193,"ĠÙģ":2194,"Ġhealth":2195,"ify":2196,"ç½®":2197,"li":2198,"ла":2199,"æ·±":2200,"\\)":2201,"è§Ĥ":2202,"ocial":2203,"å¦Ĥæŀľ":2204,"iron":2205,"âĢĶâĢĶ":2206,"Ġactiv":2207,"åĪĽ":2208,"便":2209,"ÅĤ":2210,"meric":2211,"ØŃ":2212,"æİ¨":2213,"èĬ±":2214,"ished":2215,"ä¸ĵ":2216,"æij":2217,"声":2218,"à¹Ģà¸":2219,"è¾¹":2220,"ار":2221,"Ġorgan":2222,"ν":2223,"åĨ³":2224,"90":2225,"ز":2226,"å¤ĩ":2227,"è¾¾":2228,"ä¼ģä¸ļ":2229,"à²":2230,"Ġmust":2231,"à°":2232,"è¯Ń":2233,"çķĮ":2234,"æĸĻ":2235,"Ġpresent":2236,"Ġwater":2237,"ai":2238,"Ġimportant":2239,"44":2240,"æĿĥ":2241,"ĠTe":2242,"å¤į":2243,"Ġlos":2244,"oot":2245,"ãģĦ":2246,"ience":2247,"ee":2248,"å§ĭ":2249,"åł":2250,"for":2251,"ĠÑĥ":2252,"Ġcell":2253,"197":2254,"Ġconsider":2255,"æĶ¯":2256,"ç©¶":2257,"ma":2258,"Ġcontin":2259,"tain":2260,"Ġmult":2261,"ج":2262,"Ġdr":2263,"çļĦ人":2264,".[":2265,"管çIJĨ":2266,"è¿ij":2267,"ĠSp":2268,"åĬŁ":2269,"еÑĢ":2270,"AR":2271,"://":2272,"æ¡Ī":2273,"Ġfil":2274,"ĠBut":2275,"cript":2276,"Ùī":2277,"dition":2278,"Ġoper":2279,"Ġself":2280,"Ġpass":2281,"ĠWh":2282,"/s":2283,"ï¼ļâĢľ":2284,"ר":2285,"Ġstudy":2286,"lex":2287,"itive":2288,"ĠPh":2289,"äºĨä¸Ģ":2290,"imal":2291,"('":2292,"Ġcal":2293,"åıĤ":2294,"çİĩ":2295,"]Ċ":2296,"Ġê":2297,"ãģ«":2298,"åĥı":2299,"èģĶ":2300,"iversity":2301,"åij¨":2302,"Ġbus":2303,"be":2304,"Ġprogram":2305,"Ġprof":2306,".âĢĿ":2307,"ÑĤи":2308,"åħļ":2309,"Ġlist":2310,"模":2311,"Ġcare":2312,"att":2313,"ote":2314,"55":2315,"uro":2316,"ze":2317,"itions":2318,"ource":2319,"ón":2320,"ÑĤÑĮ":2321,"ouse":2322,"Ġб":2323,"ĠPl":2324,"Ġperform":2325,"åij½":2326,"ium":2327,"׾":2328,"Ġname":2329,"è§ī":2330,"iving":2331,"Ġvis":2332,"Ġpower":2333,"Ġgrow":2334,"ccess":2335,"Ġlast":2336,"This":2337,"éļ¾":2338,"Ġbook":2339,"ients":2340,"ç":2341,"Ġcap":2342,"lish":2343,"è¨Ģ":2344,"duct":2345,"ves":2346,"swer":2347,"æ¶Ī":2348,".s":2349,"ality":2350,"Ġwrit":2351,"Ġcase":2352,"å¼ł":2353,"oint":2354,"ĠIs":2355,"ww":2356,"akes":2357,"ene":2358,"ĠThey":2359,"å¨":2360,"åĨµ":2361,"Ġref":2362,"é¢Ĩ":2363,"-t":2364,"}ĊĊ":2365,"::":2366,"ining":2367,"Ġopt":2368,"ä¸Ķ":2369,"å¹²":2370,"æļ":2371,"46":2372,"Ġmov":2373,"ĠEng":2374,"Ġbre":2375,"Ġ}ĊĊ":2376,"社ä¼ļ":2377,"çݯ":2378,"rid":2379,"iver":2380,"ument":2381,"bs":2382,"Ġpot":2383,"ว":2384,"eter":2385,"éľĢè¦ģ":2386,"å¼ķ":2387,"éĸ":2388,"EN":2389,"Ġ@":2390,"ively":2391,"Ġз":2392,"ç´ł":2393,"yl":2394,"Ġsmall":2395,".S":2396,"ĠâĢ¢":2397,"åĮĹ":2398,"ified":2399,"Ġcle":2400,"è¯ķ":2401,"75":2402,"ploy":2403,"vert":2404,"Ġgreat":2405,"Ġdisc":2406,"atic":2407,"Ġnon":2408,"িà¦":2409,"-f":2410,"Ġpost":2411,"é¥":2412,"Ġstill":2413,"åĬŀ":2414,"çα":2415,"ä½ı":2416,"-d":2417,"Ñī":2418,"ived":2419,"Ġmen":2420,"象":2421,"Ġг":2422,"nal":2423,"éĢĻ":2424,"Ch":2425,"åı°":2426,"è§Ĩ":2427,"son":2428,"ĠAmeric":2429,"Ġdesign":2430,"åı¯èĥ½":2431,"æĺĵ":2432,"ulation":2433,"ã":2434,"éĺ²":2435,"Ġprot":2436,"ØĮ":2437,"ke":2438,"ination":2439,"æĢģ":2440,"Ġadv":2441,"ä¾ĭ":2442,"Ġproper":2443,"æĸ½":2444,"Ġplace":2445,"Ġreport":2446,"Ġع":2447,"Ġaround":2448,"åı·":2449,"orn":2450,"å¸ĥ":2451,"less":2452,"ST":2453,"ĠThere":2454,"Ġbest":2455,"è´¹":2456,"ÑĤо":2457,"ç¢":2458,"right":2459,"Ġelect":2460,"ĠEn":2461,"ä¾Ľ":2462,"ada":2463,"Ġdie":2464,"viron":2465,"Ġstand":2466,"ä½İ":2467,"****":2468,"irc":2469,"Ġrese":2470,"atch":2471,"Ġinf":2472,"æĵ":2473,"Ġhist":2474,"ÑģÑı":2475,"uthor":2476,"Ġless":2477,"éªĮ":2478,"ãĤĭ":2479,"åĨĽ":2480,"conom":2481,"Ġpop":2482,"ĠOn":2483,"段":2484,"éĺŁ":2485,"Ġmil":2486,"竳":2487,"Ġident":2488,"Ġbeh":2489,"éĢļè¿ĩ":2490,"47":2491,"ror":2492,"ought":2493,"æµİ":2494,"ãģ¨":2495,"Ġorder":2496,"Pro":2497,"ем":2498,"Ġproduct":2499,"aterial":2500,"Ġstate":2501,"Ġfollowing":2502,"Ġwithout":2503,"med":2504,"49":2505,"resent":2506,"Ġsay":2507,"OR":2508,"离":2509,"èı":2510,"Ġexample":2511,"div":2512,"Ġlet":2513,"å¢ĥ":2514,"æĸŃ":2515,"çŁ¥éģĵ":2516,"ament":2517,"ID":2518,"æĬķ":2519,"ε":2520,"ends":2521,"æĴ":2522,"ird":2523,"åĽłä¸º":2524,"ка":2525,"Ġopen":2526,"åĮ»":2527,"ล":2528,"éĢŁ":2529,"omen":2530,"ĠComm":2531,"è¶Ĭ":2532,"str":2533,"Ġallow":2534,"ão":2535,"gen":2536,"å±Ģ":2537,"Ġvol":2538,"ãģ§":2539,"åijĬ":2540,"使ç͍":2541,"))":2542,"ä¸ŃçļĦ":2543,"æŀĹ":2544,"angu":2545,"Ġpract":2546,"ique":2547,"Ġspe":2548,"Ġwithin":2549,"è¡Ģ":2550,"AN":2551,"ĠTr":2552,"ย":2553,"â̦â̦":2554,"è£ħ":2555,"æľª":2556,"Ġtri":2557,"agn":2558,"çĮ":2559,"çīĩ":2560,"ane":2561,"Ġline":2562,".âĢĿĊĊ":2563,"è®®":2564,"Ġinterest":2565,"ĠShe":2566,"Ġ×Ķ×":2567,"ta":2568,"éº":2569,"AL":2570,"rist":2571,"Ġunderstand":2572,"Ġcurrent":2573,"66":2574,"éϤ":2575,"................":2576,"æŀģ":2577,"Ġhead":2578,"åѦçĶŁ":2579,"Ġinvest":2580,"We":2581,"arge":2582,"ÑĨи":2583,"apt":2584,"ission":2585,"undred":2586,"por":2587,"æĹ¶åĢĻ":2588,"rac":2589,"Ġbas":2590,"Ġrest":2591,"Ġdev":2592,"ãģĹ":2593,"ertain":2594,"Ġsum":2595,"!ĊĊ":2596,"78":2597,"çĥŃ":2598,"ger":2599,"ĠTo":2600,"åĤ":2601,"Ġiss":2602,"çłĶç©¶":2603,"Ġstudents":2604,"):":2605,"Ġ==":2606,"Ġmill":2607,"æİ§":2608,"马":2609,"ention":2610,"ات":2611,"áĢ":2612,"Ġtype":2613,"°":2614,"Ġris":2615,"01":2616,"ysis":2617,"åŃ©":2618,"Ġ**":2619,"æĢİ":2620,"æĪ¿":2621,"Ġincluding":2622,"ÑĢо":2623,"Ġdirect":2624,"å§Ķ":2625,"Ġaff":2626,"ways":2627,"yd":2628,"èIJ¥":2629,"ength":2630,"Ġbo":2631,"Ġrun":2632,"Ġocc":2633,"iter":2634,"æĮī":2635,"æīĢ以":2636,"ividual":2637,"ris":2638,"Ġmeas":2639,"ains":2640,"-m":2641,"05":2642,"å·²ç»ı":2643,"igure":2644,"Ġmodel":2645,"Ġdiv":2646,"Ġredu":2647,"éħį":2648,"Ïģ":2649,"çħ§":2650,"人çļĦ":2651,"arent":2652,"ately":2653,"ç¬ij":2654,"].":2655,"Ġtop":2656,"广":2657,"Ġanother":2658,"ิ":2659,"æľĽ":2660,"失":2661,"Ġschool":2662,"æIJ":2663,"æĺ¾":2664,"à¸Ķ":2665,"è":2666,"Ġaut":2667,"amb":2668,"Ġopp":2669,"åIJĥ":2670,"è¿ŀ":2671,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2672,"rad":2673,"Ġide":2674,"ittle":2675,"umber":2676,"An":2677,"Ġé":2678,"æµĭ":2679,"Ġhome":2680,"æĬ¤":2681,"ĠÙĦ":2682,"æĸ¯":2683,"è¿Ļæł·":2684,"èIJ½":2685,"roll":2686,"ples":2687,"çļĦä¸Ģ":2688,"Ġfour":2689,"rop":2690,"ç»Ń":2691,"Ġmanag":2692,"åĪĩ":2693,"Ġchang":2694,"é£Ł":2695,"Ġsignific":2696,"å¾Ģ":2697,"ĠPr":2698,"face":2699,"Ex":2700,"\"Ċ":2701,"åIJ¬":2702,"Ġcontrol":2703,"cur":2704,"Ġ=>":2705,"ãģ¦":2706,"åĵį":2707,"ç»ıæµİ":2708,"ĠOr":2709,"go":2710,"çĬ¶":2711,"åĪĹ":2712,"iment":2713,"ëĭ":2714,"é¾":2715,"Ġmean":2716,"åİĭ":2717,"Ġmus":2718,"ression":2719,"na":2720,"åħĭ":2721,"196":2722,"çϾ":2723,"å¡":2724,"ott":2725,"AS":2726,"Ġtoo":2727,"Ġ":3320,"clus":3321,"建设":3322,"èİ·":3323,"åı¤":3324,"çŃĸ":3325,"æĺŁ":3326,"add":3327,"اÙħ":3328,"åŁŁ":3329,"Ġlo":3330,"que":3331,"ka":3332,"Ġpress":3333,"Ġpatients":3334,"\\.":3335,"ledge":3336,"osed":3337,"Ġpossible":3338,"rie":3339,"arget":3340,"Ġang":3341,"Ġenergy":3342,"éĥ¨åĪĨ":3343,"Ġfood":3344,"Ġwords":3345,"Cl":3346,"ç»Ī":3347,"åıĮ":3348,"cient":3349,"ा":3350,"57":3351,"oor":3352,"Ġpay":3353,"43":3354,"ç»Ħç»ĩ":3355,"aster":3356,"大çļĦ":3357,"Ġmot":3358,"ĠInt":3359,"åħħ":3360,"Ġ·":3361,"\":":3362,"Ġcomb":3363,"Ġfri":3364,"emb":3365,"çĶŁäº§":3366,"Ġmar":3367,"æ¿":3368,"ż":3369,"à´":3370,"Ġphys":3371,"Id":3372,"za":3373,"æķ°æį®":3374,"Ġhard":3375,"он":3376,"åħŃ":3377,"çĶ·":3378,"ilar":3379,"rodu":3380,"ĠCont":3381,"Ġarg":3382,"ither":3383,"comm":3384,"æĿ¿":3385,"Ġport":3386,"ows":3387,"ued":3388,"alse":3389,"()Ċ":3390,"æĢİä¹Ī":3391,"ĠĠĠĠĠĠĠĠĠ":3392,"ana":3393,"Ġinclude":3394,"):Ċ":3395,"Ġleast":3396,"Ġcorre":3397,"06":3398,"ortun":3399,"Ġrelations":3400,"ĠGo":3401,"ET":3402,"ки":3403,"Ġut":3404,"ext":3405,"arl":3406,"ousand":3407,"ones":3408,"äºī":3409,"utions":3410,"çīĪ":3411,"å½±åĵį":3412,"æī¿":3413,"èµ·æĿ¥":3414,"æĸĩåĮĸ":3415,"Ġpercent":3416,"Ġquestion":3417,"Ġstring":3418,"verage":3419,".m":3420,"ili":3421,"λ":3422,"Ġfr":3423,"earch":3424,"02":3425,"ift":3426,"ä¸ĸçķĮ":3427,"ĠMe":3428,"ley":3429,"ÑģÑĤи":3430,"Ġenvironment":3431,"II":3432,"enn":3433,"æ²¹":3434,"é»Ħ":3435,"ĠChrist":3436,"åIJĮæĹ¶":3437,"Ġens":3438,"Ġenc":3439,"ä»ħ":3440,"col":3441,"åħ¶ä»ĸ":3442,"çªģ":3443,"ara":3444,"Ġcontent":3445,"iet":3446,"Ġinit":3447,"æł¸":3448,"ç®Ģ":3449,"ured":3450,"åĿĩ":3451,"Ġtotal":3452,"ÑĦ":3453,"à¹ģ":3454,"Ġprim":3455,"äºij":3456,"ľ×":3457,"Ġsam":3458,"Ġknown":3459,"ĠMay":3460,"éĢīæĭ©":3461,"à¥įà¤":3462,"èŀ":3463,"194":3464,"def":3465,"çķĻ":3466,"åIJĹ":3467,"Ġcrit":3468,"Ġweek":3469,"uture":3470,"aps":3471,"yt":3472,"ault":3473,"lete":3474,"Ġgive":3475,"You":3476,"Ġoffer":3477,"κ":3478,"ç¼ĸ":3479,"Ġcertain":3480,"è¿°":3481,"Ġdescrib":3482,"室":3483,"Ïħ":3484,"æĹı":3485,"讲":3486,"isc":3487,"ä¸Ģå®ļ":3488,"ites":3489,"Ġmaking":3490,"åĩ»":3491,"严":3492,"Ġil":3493,"份":3494,"Ġable":3495,"é»ij":3496,"æŁIJ":3497,"serv":3498,"Ġanalysis":3499,"é¡¹çĽ®":3500,"Ġey":3501,"Ġdiscuss":3502,"rict":3503,"Ġdue":3504,"âĢĺ":3505,"Ġrequire":3506,"ered":3507,"âĢ¢":3508,"Al":3509,"Ġavailable":3510,"η":3511,"Ġindust":3512,"Ġaccount":3513,"Ġuntil":3514,"以åıĬ":3515,"æ¯į":3516,"Ġ\\(\\":3517,"Ġlove":3518,"Ġsym":3519,"åħ³ç³»":3520,"Ġprob":3521,"Ġarr":3522,"è¿ĩç¨ĭ":3523,"String":3524,"Ġair":3525,"Äį":3526,"omet":3527,"Ġindic":3528,"Ġbenef":3529,"Ġfull":3530,"è´Ł":3531,"è©":3532,".C":3533,"æ¦":3534,"iple":3535,"List":3536,"rand":3537,"ournal":3538,"Ġcalcul":3539,"ais":3540,"bo":3541,"èĥ½åĬĽ":3542,"Ġaway":3543,"Ġhtt":3544,"Ġpolit":3545,"Ġlik":3546,"iol":3547,"pre":3548,"Ġspecific":3549,"cont":3550,"Ġcreate":3551,"ĠPol":3552,"ĠDes":3553,"Ġabove":3554,"back":3555,"ма":3556,"Ġgot":3557,"Ú¯":3558,"sel":3559,"ĠÙħÙĨ":3560,"×Ļ×Ŀ":3561,"ett":3562,"åįĥ":3563,"Ġcirc":3564,"98":3565,"Ġcr":3566,"no":3567,"Ġfocus":3568,"imate":3569,"arr":3570,"ored":3571,"aring":3572,"Ġcreat":3573,"ðŁ":3574,"If":3575,"Ġkind":3576,"æ¼Ķ":3577,"ival":3578,"ION":3579,"obal":3580,"ivity":3581,"ibility":3582,"Ġpara":3583,"Ġcourse":3584,"è¾ĵ":3585,"Ġseveral":3586,"ho":3587,".g":3588,"ĠÑį":3589,"Ġge":3590,"ĠSc":3591,"ä½ľä¸º":3592,"Ġоб":3593,"âĢĿ,":3594,"icy":3595,"etic":3596,"åĪ»":3597,"ениÑı":3598,"æīĢæľī":3599,"03":3600,"åħ«":3601,"ava":3602,"inter":3603,"ĠCent":3604,"Ġcolor":3605,"æĸ¹å¼ı":3606,"Ġlearning":3607,"Ġ`":3608,"Ġposition":3609,"é¸":3610,"Ġamong":3611,"害":3612,"产åĵģ":3613,"htt":3614,"Ġrole":3615,"zy":3616,"istic":3617,"Ġpath":3618,"ç¯":3619,"inary":3620,"________":3621,"çĽij":3622,"ector":3623,"Ġvarious":3624,"/h":3625,"abel":3626,"大家":3627,"Ġothers":3628,"èĹ":3629,"ä¼¼":3630,"Ġmajor":3631,"Ġ«":3632,"Ġر":3633,"ž":3634,"Ġgovernment":3635,"åIJ¦":3636,"å±ħ":3637,"Ġhaving":3638,"è¿Ļä¸Ģ":3639,"ож":3640,"人æ°ij":3641,"aken":3642,"åĵª":3643,"Ġbecome":3644,"Ġsure":3645,"Ġmillion":3646,"欢":3647,"好çļĦ":3648,"Ġí":3649,"åįı":3650,"ĠEuro":3651,"alf":3652,"ators":3653,"cle":3654,"æł¹æį®":3655,"å¯Ĩ":3656,"éĢģ":3657,"àª":3658,"ained":3659,"对äºİ":3660,"56":3661,"lement":3662,"04":3663,"ĉĉĉĉ":3664,"gether":3665,"ок":3666,"Ġsent":3667,"å®Ŀ":3668,"Ġpast":3669,"stit":3670,"à§ĩà¦":3671,"Ġtogether":3672,"Ġexist":3673,"RO":3674,"pped":3675,"Ġrecord":3676,"çıŃ":3677,"Ġrespect":3678,"ĠPer":3679,"Ġann":3680,"ĠCal":3681,"_t":3682,"Ġimpact":3683,"æŃ¢":3684,"ude":3685,"ĠпÑĢи":3686,"Ġfactors":3687,"Ġum":3688,"Ġpor":3689,"oney":3690,"kt":3691,"ober":3692,"ato":3693,"lev":3694,"éĻį":3695,"Ġdoc":3696,"ians":3697,"é¡»":3698,"LL":3699,"ste":3700,"Ġsize":3701,"ĠUnited":3702,"令":3703,"Ġsens":3704,"Ġcaus":3705,"Ġfar":3706,"ĠAmerican":3707,"arth":3708,"Res":3709,"ĠWith":3710,"Ġrate":3711,"500":3712,"br":3713,"ĠEm":3714,"ĠBy":3715,"åı¥":3716,"Ġthousand":3717,"ĠFl":3718,"Ġmom":3719,"ights":3720,"ĠCan":3721,"èĭ¥":3722,"å¾Īå¤ļ":3723,"©×":3724,"Ġauthor":3725,"oss":3726,"ilities":3727,"æĪijçļĦ":3728,"Ġprivate":3729,"._":3730,"ĠGu":3731,"Ġdom":3732,"ores":3733,"Ġbig":3734,"æ²³":3735,"cription":3736,"Ġnumbers":3737,"opy":3738,"atory":3739,"çĶ»":3740,"де":3741,"Ġneg":3742,"leg":3743,"æĩ":3744,"ä¸ĢäºĽ":3745,"ä½ľç͍":3746,"ĠGe":3747,"hers":3748,"Ġ;":3749,"å®ĺ":3750,"ĠStud":3751,"asing":3752,"ĠCo":3753,"åij³":3754,"اد":3755,"Ġaddress":3756,"{\\":3757,"Ġalong":3758,"Ġrespons":3759,"ails":3760,"ìĿ´":3761,"è²":3762,"Ġsugg":3763,"×ķת":3764,"æį¢":3765,"çļĦæĹ¶åĢĻ":3766,"class":3767,"าร":3768,"ÙĪØ±":3769,"Ġsaf":3770,"ÅĽ":3771,"Ġamount":3772,"Ġfund":3773,"设计":3774,"åĩı":3775,"Ġmeet":3776,"Ġsuper":3777,"bl":3778,"this":3779,"Ġfurther":3780,"ĠÙĬ":3781,"Ġdise":3782,"Ġarticle":3783,"Õ¡Õ":3784,"Ġvalues":3785,"oms":3786,"ĠÙģÙĬ":3787,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":3788,"ney":3789,"ication":3790,"åħ´":3791,"Ġnecess":3792,"ready":3793,"NA":3794,"å°½":3795,"æıIJä¾Ľ":3796,"Ġsuggest":3797,"ump":3798,"Ġap":3799,"éĶĻ":3800,"ä¹Łä¸į":3801,"éĨ":3802,"åĪĨæŀIJ":3803,"èµĽ":3804,"elt":3805,"åģ¥":3806,"»":3807,"Ġearly":3808,"ption":3809,"Ġgeneral":3810,"Ġbase":3811,"rem":3812,"odel":3813,"ae":3814,"Ġvoid":3815,"\">Ċ":3816,"Ġcompany":3817,"Ġfive":3818,"人åijĺ":3819,"Ġ--":3820,"ëĬ":3821,"Ñĩи":3822,"aces":3823,"京":3824,"Ġsat":3825,"çĸĹ":3826,"RE":3827,"Ġstrong":3828,"Ġnorm":3829,"ception":3830,"}}":3831,"åŁ¹":3832,"çł´":3833,"æľĥ":3834,"ä¸Ģèά":3835,"bre":3836,":âĢľ":3837,"å¾ħ":3838,"ç´§":3839,"ĠSte":3840,"ĠThat":3841,"èϽ":3842,"Ġwomen":3843,"Ġdat":3844,".d":3845,"head":3846,"apter":3847,"åĿļ":3848,"atures":3849,"urch":3850,"Ġrisk":3851,"Ġchall":3852,"Ġtw":3853,"第äºĮ":3854,"èīº":3855,"elling":3856,"ä¹°":3857,"ĠAct":3858,"Ġlater":3859,"ements":3860,"Ġpain":3861,"Ġreview":3862,"Ġsubject":3863,"åĪļ":3864,"US":3865,"é¾Ļ":3866,"ĠС":3867,"éĶĢ":3868,"ties":3869,"For":3870,"aff":3871,"),Ċ":3872,"ino":3873,"ëĭ¤":3874,"Ġstrateg":3875,"Ġstre":3876,"æ¤":3877,"两个":3878,"ih":3879,"ĠJohn":3880,"Ġacross":3881,"Ġл":3882,"åĽŃ":3883,"ero":3884,"193":3885,"اÛĮ":3886,"Ġemb":3887,"ĠÐŁ":3888,"åı¦":3889,"æĿ¥çļĦ":3890,"Ġje":3891,"essage":3892,"ï¼ĮâĢľ":3893,"Ġpred":3894,"wards":3895,"åĸĦ":3896,"ĠIntern":3897,"Ġconc":3898,"word":3899,"ogle":3900,"Cont":3901,"ĠMin":3902,"æĭ¬":3903,"Ġprofess":3904,"éĴ±":3905,"epend":3906,"éĿŀ常":3907,"Ġexc":3908,"ĠEl":3909,"åIJ«":3910,"Ġdeg":3911,"uly":3912,"(n":3913,"æĻ®":3914,"夫":3915,"return":3916,"Ġgame":3917,"èĨ":3918,"Ġuna":3919,"Ġwhether":3920,"plement":3921,"ĠRet":3922,"éħĴ":3923,"Ġcountry":3924,"çݯå¢ĥ":3925,"acy":3926,"chie":3927,"Ġmind":3928,"Ġlot":3929,"è´¢":3930,"though":3931,"load":3932,"Ġcustom":3933,"æ¿Ģ":3934,"Ġmor":3935,"μ":3936,"atter":3937,"å¯Ł":3938,"Ġwhy":3939,"Ġcontrib":3940,"Ġabs":3941,"åĢij":3942,"康":3943,"Ġcomput":3944,"äºĴ":3945,"Ġworking":3946,"ĠEnglish":3947,"ĠоÑĤ":3948,"const":3949,"帮":3950,"uck":3951,"çĤº":3952,"Ġspecial":3953,"Å¡":3954,"\",Ċ":3955,"举":3956,"ml":3957,"ä¸ĥ":3958,"Ġhold":3959,"ĠCO":3960,"eral":3961,"ï¼ģĊĊ":3962,"Ã¥":3963,".org":3964,"erson":3965,"idd":3966,"Ġuser":3967,"为äºĨ":3968,"Ġinteg":3969,"umn":3970,"ĠNe":3971,"ê":3972,"AP":3973,"åıijçĶŁ":3974,"åĨħ容":3975,"ison":3976,"Ġsystems":3977,"public":3978,"Ġmax":3979,"Ġhistory":3980,"79":3981,"(s":3982,"伤":3983,"Ġclaim":3984,"ĠØ´":3985,"Ġreason":3986,"Ġspace":3987,"Ġfuture":3988,"Ġdone":3989,"Ġtemper":3990,"chan":3991,"unt":3992,"ÅĻ":3993,"oman":3994,"ĠâĨij":3995,"éĤ£ä¹Ī":3996,"Ġlay":3997,"Ġrelationship":3998,"Ġterms":3999,"AD":4000,".c":4001,"Ġdidn":4002,"åĩºçݰ":4003,"é¦Ļ":4004,"Ġdou":4005,"Ġalready":4006,"åıijçݰ":4007,"Ġservice":4008,"åĽłæŃ¤":4009,"order":4010,"Ġcells":4011,"ÙĪÙĨ":4012,"ĠJan":4013,"side":4014,"frac":4015,"do":4016,"èĽ":4017,"Ġevent":4018,"åı«":4019,"Ġpri":4020,"ç¶":4021,"Ġcommunity":4022,"âĪĴ":4023,"æŃ¦":4024,"è¿ĺæľī":4025,"Ġobserv":4026,"ÙĬÙĨ":4027,"ales":4028,"æĪĸèĢħ":4029,"ви":4030,"Ġsingle":4031,"Ġsimilar":4032,"Ġselect":4033,"Ġlarg":4034,"å¼Ĥ":4035,"缴æİ¥":4036,"çļ®":4037,"олÑĮ":4038,"Ġur":4039,"æĺ¥":4040,"ç¦ı":4041,"(x":4042,"å½ķ":4043,"ï¼İ":4044,"ãĥ¼":4045,"atal":4046,"Ġyoung":4047,"ê³":4048,"iam":4049,"Ġ!":4050,"ä¹Ŀ":4051,"Ġbehav":4052,"Ġson":4053,"Ġ?":4054,"è¯į":4055,"sk":4056,"Ġlas":4057,"æĶ¿åºľ":4058,"ha":4059,"ruction":4060,"cing":4061,"ried":4062,"Ġlanguage":4063,"Ġintern":4064,"ëĬĶ":4065,"ди":4066,"Ġappear":4067,"åĨ·":4068,"³³":4069,"most":4070,"审":4071,"Ġrequired":4072,"ibr":4073,"åħį":4074,"alu":4075,"à¸Ī":4076,"Ġhigher":4077,"åIJ¸":4078,"using":4079,"}^{":4080,"Ġsitu":4081,"Ġheart":4082,"é£ŀ":4083,"_d":4084,"Ġcheck":4085,"Ġneeds":4086,"Ġfinal":4087,"é¼":4088,"On":4089,"Ġapproach":4090,"ios":4091,"æ½":4092,"Ġstory":4093,"å¸Į":4094,"Ġnatural":4095,"Ġgrowth":4096,"ção":4097,"ض":4098,"Ġid":4099,"Qu":4100,"端":4101,"æĻļ":4102,"ä»ĭ":4103,"Ġsix":4104,"Ġtool":4105,"ĠStates":4106,"åŁºæľ¬":4107,"à¸Ľ":4108,"Ġage":4109,"èĴ":4110,"izing":4111,"çķ¥":4112,"å¯Į":4113,"ĠOne":4114,"æĭ¿":4115,"Ġ×ŀ×":4116,"How":4117,"ìĿĺ":4118,"ç§»":4119,"my":4120,"Ġste":4121,"èī¯":4122,"Ġchanges":4123,"ening":4124,"Ġjob":4125,"pper":4126,"Ġcame":4127,"Ġenough":4128,"оÑģ":4129,"AM":4130,"iction":4131,"Ġconditions":4132,"à¹ĥ":4133,"Ġknowledge":4134,"Ġtreatment":4135,"не":4136,"onse":4137,"Ġincrease":4138,"uff":4139,"çζ":4140,"ros":4141,"ising":4142,"èŀį":4143,"_,":4144,"à§ģ":4145,"But":4146,"ĠDo":4147,"ìł":4148,"View":4149,"era":4150,"ÙĪÙĦ":4151,"çĵ":4152,"åħ·æľī":4153,"hold":4154,"éĿ©":4155,"å·¦":4156,"52":4157,"Ġка":4158,"Ġvir":4159,"Ġill":4160,"Ġdistrib":4161,"åŃĺåľ¨":4162,"çĭ¬":4163,"å¦Ī":4164,"ĠNot":4165,"ender":4166,"Ġbelow":4167,"Ġ׾×":4168,"ä½Ļ":4169,"Ġbegin":4170,"ula":4171,"說":4172,"å¾ģ":4173,"ĠEurope":4174,"Ġoffic":4175,"ĠLa":4176,"Ġparticip":4177,"ione":4178,"ges":4179,"ú":4180,"Ġcode":4181,"æıIJé«ĺ":4182,"Ġservices":4183,"Ġtable":4184,"artment":4185,"çģµ":4186,"ç»§":4187,"æī§":4188,"Ġbreak":4189,"Ġcomplex":4190,"ä¹İ":4191,"Ġà®":4192,"é²":4193,"ervice":4194,"And":4195,"ãģĭ":4196,"Ġperformance":4197,"ips":4198,"ìĦ":4199,"Ġhouse":4200,"ounds":4201,"uit":4202,"opt":4203,"Ġthough":4204,"place":4205,"ĠIN":4206,"ĠMy":4207,"èĦ¸":4208,"à¹Ħ":4209,"ancial":4210,"表示":4211,"Ġimm":4212,"Ġwind":4213,"fect":4214,"192":4215,"Ġtarget":4216,"----------------":4217,"Ġquestions":4218,"Ġface":4219,"ãĤĮ":4220,"ficult":4221,"Ġparent":4222,"ĠÕ":4223,"åı¶":4224,"èĢĮä¸Ķ":4225,".get":4226,"Ġmanagement":4227,"ĠPar":4228,"æ³¢":4229,"Ġsays":4230,"âĢĿï¼Į":4231,"ç»Ŀ":4232,".;":4233,"ĠReg":4234,"ाà¤":4235,"roups":4236,"unction":4237,"èĭı":4238,"æľ¨":4239,"åı³":4240,"ires":4241,"éĵ¶":4242,"нÑĭÑħ":4243,"Ġestab":4244,"Ġfile":4245,"There":4246,"hel":4247,"hib":4248,"ules":4249,"åĮħæĭ¬":4250,"è³":4251,"ä¹ĭåIJİ":4252,"/Ċ":4253,"list":4254,"åĿIJ":4255,"Ġresponse":4256,"Ñĩа":4257,"Ùİ":4258,"sych":4259,"iber":4260,"Ġinflu":4261,"duc":4262,"Ġlower":4263,"Ġ×ij×":4264,"çĩ":4265,"ý":4266,"ื":4267,"ii":4268,"po":4269,"æ¯Ķè¾ĥ":4270,"Ġaction":4271,"çĦ¶åIJİ":4272,"Ġmass":4273,"];Ċ":4274,"ĠPart":4275,"ĠNov":4276,"нов":4277,"ĠAf":4278,"ä¸Ńå¿ĥ":4279,"iff":4280,"rug":4281,"olar":4282,"Ġcou":4283,"оз":4284,"Ġce":4285,"wh":4286,"51":4287,"aching":4288,"åį¡":4289,"iting":4290,"Wh":4291,"Ġyang":4292,"纪":4293,"Ġدر":4294,"ged":4295,"æĢ¥":4296,"ublished":4297,"ension":4298,"æĭħ":4299,"clude":4300,"De":4301,"æĦ¿":4302,"Ġdan":4303,"empt":4304,"ä¹ĭéĹ´":4305,"-g":4306,"76":4307,"â":4308,"Ġprime":4309,"Ġtook":4310,"éģİ":4311,"Ġmembers":4312,"Ñģки":4313,"300":4314,"imum":4315,"Ġtoday":4316,"ico":4317,"æİ§åζ":4318,"arc":4319,"éĻĪ":4320,"Ġconvert":4321,"ç¨İ":4322,"Ġpersonal":4323,"ÙĨد":4324,"Ġespec":4325,"Ġseen":4326,"çŁŃ":4327,"æľĿ":4328,"åĿĹ":4329,"å¿ħé¡»":4330,".\"ĊĊ":4331,"Ġred":4332,"Ġinde":4333,"è§Ħå®ļ":4334,"ĠComp":4335,"arb":4336,"Ġpositive":4337,"Ġeither":4338,"计ç®Ĺ":4339,"è´§":4340,"��":4341,"Ġeducation":4342,"Ġhours":4343,"æłij":4344,"istics":4345,"Ġinput":4346,"Ġachie":4347,"Ġopportun":4348,"ìĹIJ":4349,"åįķä½į":4350,"é¢Ŀ":4351,"åģľ":4352,"aining":4353,"å®ŀçݰ":4354,"ç£":4355,"è¡¥":4356,"ί":4357,"èĥĮ":4358,"Ġе":4359,"è®Ń":4360,"।":4361,"89":4362,"èĥ½å¤Ł":4363,"54":4364,"Ġpoints":4365,"Ġpage":4366,"date":4367,"Ġthing":4368,"HE":4369,"ilt":4370,"ened":4371,"è¿Ļæĺ¯":4372,"nder":4373,"Se":4374,"çľĭåΰ":4375,"åį°":4376,"èĩªçĦ¶":4377,"En":4378,"Ġbring":4379,"ной":4380,"大åѦ":4381,"å¦Ĥä½ķ":4382,"æī¹":4383,"ার":4384,"缸åħ³":4385,"æĻº":4386,"Ġonce":4387,"Ġphot":4388,"çͱäºİ":4389,"éķĩ":4390,"ithm":4391,"LE":4392,"å¾Į":4393,".t":4394,"ación":4395,"ĠAg":4396,"Ġcompet":4397,"å¤ĦçIJĨ":4398,"宣":4399,"åºĹ":4400,"Ġdifficult":4401,"Ġcompon":4402,"ades":4403,"ĠYork":4404,"ĠÐĴ":4405,"æĪIJ为":4406,"è¿ĻéĩĮ":4407,"anced":4408,"irm":4409,"Ġà¦ķ":4410,"ground":4411,"Ġprevious":4412,"å·¥ç¨ĭ":4413,"53":4414,"ç¡Ģ":4415,"во":4416,"åıªæĺ¯":4417,"æĵį":4418,"rel":4419,"raft":4420,"uj":4421,"amm":4422,"Ġdeb":4423,"主ä¹ī":4424,"åºĶ该":4425,"ĠState":4426,"个人":4427,"ĠÏĦ":4428,"åŃ¦æł¡":4429,"оди":4430,"ç²¾ç¥ŀ":4431,"Ġfrequ":4432,"Ġsurface":4433,"ĠØŃ":4434,"åŁºç¡Ģ":4435,"Ñģи":4436,"Un":4437,"itted":4438,"æĽ¾":4439,"Ġein":4440,"eters":4441,"Ġfail":4442,"Ġblood":4443,"ение":4444,"Ġwhole":4445,"ä»ĺ":4446,"Ġmonths":4447,"gress":4448,"Ġtalk":4449,"oud":4450,"è¯ī":4451,"çŀ":4452,"atus":4453,"Ġproblems":4454,"ç½Ĺ":4455,"Ġstructure":4456,"ĠHis":4457,"itation":4458,"за":4459,"101":4460,"sequ":4461,"Ġdire":4462,"string":4463,"ĠAng":4464,"Ġupon":4465,"ĠOct":4466,"äºļ":4467,"Ġpaper":4468,"æĬĹ":4469,"191":4470,"ĠPa":4471,"Ġmeasure":4472,"Ġvo":4473,"Ġquality":4474,"Ġsem":4475,"Ġshown":4476,"åľ°æĸ¹":4477,"ä¹ħ":4478,"éĺ¶":4479,"ĠLet":4480,"å¥ĩ":4481,"æĤ¨":4482,"åī¯":4483,"No":4484,"Ġcomplet":4485,"Ġobtain":4486,"éĺ¿":4487,"-h":4488,"********":4489,"ken":4490,"Ġmakes":4491,"æĬķèµĦ":4492,"IP":4493,"Ġtax":4494,"ĠWorld":4495,"Ġprovided":4496,"ï¼ŁĊĊ":4497,"ç»ĵæŀľ":4498,"Ġhowever":4499,"Ġsoft":4500,"Ġareas":4501,"Ġonline":4502,"乡":4503,"éĿĻ":4504,"ociety":4505,"AB":4506,"å¤ľ":4507,"Ġcover":4508,"Ġaccording":4509,"Äĩ":4510,"Ġassess":4511,"ä¸Ģèµ·":4512,"çĸ«":4513,"Ġplant":4514,"Ġassociated":4515,"medi":4516,"ony":4517,"çĶļ":4518,"çŁ¥è¯Ĩ":4519,"åĢĴ":4520,"Ġclear":4521,"ese":4522,"Ġcoll":4523,"Ġrelated":4524,"itch":4525,"UR":4526,"ĠRep":4527,"ذ":4528,"Ġdivis":4529,"æ¶²":4530,"ÛĮÙĨ":4531,"Data":4532,"?âĢĿ":4533,"ä¸įè¿ĩ":4534,"pose":4535,"åĬ³":4536,"ella":4537,"认为":4538,"çļĦäºĭ":4539,"Ġshall":4540,"Ġever":4541,"ÙĦÙī":4542,"à¯įà®":4543,"å·´":4544,"ĠNational":4545,"By":4546,"Ġeffic":4547,"åį«":4548,"ãģĵ":4549,"ĠTra":4550,"èĸ":4551,"bol":4552,"ãĤĬ":4553,"}_{":4554,"ä¸įä¼ļ":4555,"Ġseem":4556,"/(":4557,"la":4558,"Ġwar":4559,"ĠEd":4560,"Ñĺ":4561,"Ġrather":4562,"Ġlevels":4563,"ĉĉĉ":4564,"mit":4565,"à¸Ńà¸ĩ":4566,"íķĺ":4567,"ç´¢":4568,"åħ¶ä¸Ń":4569,"Ġstudies":4570,"'m":4571,"ç»ĵæŀĦ":4572,"åΤ":4573,"临":4574,"Ġtell":4575,"-st":4576,"Ġactivity":4577,"Ġparam":4578,"istance":4579,"bb":4580,"Ø«":4581,"Ġcy":4582,"asc":4583,".A":4584,"illed":4585,"-w":4586,"Ġconsum":4587,"Ġ...":4588,"left":4589,"ils":4590,"ãģĨ":4591,"è´Ń":4592,"Ġcity":4593,"åģĩ":4594,"Type":4595,"符":4596,"ä½łçļĦ":4597,"ê°":4598,"Ġcases":4599,"Ġrefer":4600,"Ġmoney":4601,"åıªæľī":4602,"ĠString":4603,"åĽº":4604,"Ġmicro":4605,"Ġproduction":4606,"cem":4607,"Ġfall":4608,"Ġimage":4609,"72":4610,"type":4611,"avor":4612,"åĨ²":4613,"Sh":4614,"Ġsy":4615,"atur":4616,"Val":4617,"æłĩåĩĨ":4618,"Ġcut":4619,"ĠNumber":4620,"èϽçĦ¶":4621,"èįī":4622,"da":4623,"ief":4624,"Ġdefin":4625,"zen":4626,"Ġvar":4627,"eds":4628,"Ġflow":4629,"Ġsolution":4630,"æľīäºĽ":4631,"Ġoriginal":4632,"ç»ĥ":4633,"æ¯Ĵ":4634,"selves":4635,"æĢķ":4636,"Ġfactor":4637,"Ġprote":4638,"ĠAug":4639,"Ġdam":4640,"Ġdeath":4641,"æĹħ":4642,"minist":4643,"竣":4644,"ุ":4645,"Ġpractice":4646,"Ġunderstanding":4647,"ĠApr":4648,"ĠMon":4649,"ida":4650,"Ġexpress":4651,"pri":4652,"Ġissues":4653,"Ġsimple":4654,"ĠвÑĭ":4655,"æħ¢":4656,"Ġrecogn":4657,".f":4658,"EL":4659,"ë¡":4660,"Ġwent":4661,"ãĤĤ":4662,"ï¼ŁâĢĿĊĊ":4663,"Ġconvers":4664,"(t":4665,"Ġnight":4666,"ius":4667,"éħ¸":4668,"Ġbit":4669,"ת":4670,"Ġyet":4671,"çİ©":4672,"è½½":4673,"æĿ¡ä»¶":4674,"raction":4675,"off":4676,"çīĮ":4677,"Ġimplement":4678,"uted":4679,"Ġeffects":4680,"Ġconduct":4681,"Ġground":4682,"座":4683,"çĹĽ":4684,"å°į":4685,"Ġиз":4686,"OS":4687,"Ġsource":4688,"Ġways":4689,"At":4690,"Ġgroups":4691,"ises":4692,"è¡£":4693,"ĠĠĠĠĠĠ":4694,"ีà¹Ī":4695,"alt":4696,">ĊĊ":4697,"Ġmo":4698,"ama":4699,"èĦij":4700,"ĠPre":4701,"inn":4702,"æĽ²":4703,"comes":4704,".M":4705,"åĩºæĿ¥":4706,"\"ĊĊ":4707,"idth":4708,"pping":4709,"rt":4710,"Ġkg":4711,"Ġmoment":4712,"UT":4713,"Ġdest":4714,"å®ĮæĪIJ":4715,"kan":4716,"ental":4717,"ä¸Ģæł·":4718,"ÑģÑĤÑĮ":4719,"å³":4720,"Ġloss":4721,"uel":4722,"èѦ":4723,"ĠGl":4724,"aging":4725,"æĤ£":4726,"اب":4727,"ano":4728,"ynam":4729,"IG":4730,"_s":4731,"оп":4732,"èĤī":4733,"Ġod":4734,"Ġlooking":4735,"ĠTrans":4736,"Ġtaken":4737,"Ġconcept":4738,"61":4739,"é½":4740,"ç§ijåѦ":4741,"åħµ":4742,"æ¦Ĥ":4743,"Ġ×ķ":4744,"Ġdisease":4745,"cover":4746,"Ġhalf":4747,"ĠPM":4748,"Ġevalu":4749,"'re":4750,"iques":4751,"ĠVal":4752,"ĠCor":4753,"注æĦı":4754,"æľĢåIJİ":4755,"ĠIS":4756,"',Ċ":4757,"Ġbar":4758,"ops":4759,"ĠÙĥ":4760,"Ġunit":4761,"Ġapplication":4762,"ÑĤелÑĮ":4763,"ĠпÑĢо":4764,"Ġexpect":4765,"Ġinvestig":4766,"-in":4767,"Ġactivities":4768,"Ġsepar":4769,"è´¨éĩı":4770,"www":4771,"Ġroom":4772,"æķ£":4773,"63":4774,"rown":4775,"Ġcause":4776,"ĠDis":4777,"ideo":4778,"where":4779,"èĵ":4780,"cret":4781,"Ġprovides":4782,"Ġস":4783,"Ġج":4784,"ত":4785,"Ġminutes":4786,"Ġquick":4787,"aling":4788,"ya":4789,"é¤":4790,"ĠDep":4791,"ÃĹ":4792,"åĬŁèĥ½":4793,"lying":4794,"uary":4795,"Int":4796,"ante":4797,"Ġroot":4798,"MM":4799,"Ġcannot":4800,"å°Ħ":4801,"Tr":4802,",\\":4803,"Ġmechan":4804,"osis":4805,"hing":4806,"Ġtechnology":4807,"Ġintrodu":4808,"Ġpropos":4809,"è§īå¾Ĺ":4810,"pen":4811,"à¦ķ":4812,"Ġcorrect":4813,"nÃŃ":4814,"Ġtypes":4815,"Col":4816,"ÙĪØ¯":4817,"ного":4818,"Ġprevent":4819,"ĠInter":4820,"icht":4821,"èĭ¦":4822,"When":4823,"Ġthird":4824,"ãĥ»":4825,"Ġliter":4826,"Ġteac":4827,"èıľ":4828,"Ġconcern":4829,"oper":4830,"ç¾İåĽ½":4831,"缺":4832,"ĠÙĤ":4833,"iation":4834,"book":4835,"Ġmethods":4836,"ĠCar":4837,"ĠÑĦ":4838,"ki":4839,"ás":4840,"éģĩ":4841,"ĠHealth":4842,"Ġdoing":4843,"Ġneeded":4844,"ðĿij":4845,"EM":4846,"Ġregard":4847,"otal":4848,"Ġshows":4849,"å²ģ":4850,"Ġnear":4851,"à¸Ĥ":4852,"aves":4853,"Ġnetwork":4854,"è¡Į为":4855,"ìŀ":4856,"åįł":4857,"62":4858,"稳":4859,"å®Ī":4860,"Ġespecially":4861,"ita":4862,"Ġshare":4863,"adem":4864,"ĠDr":4865,"Ġwritten":4866,"顾":4867,"Ġsection":4868,"Ġasked":4869,"120":4870,"ĠSouth":4871,"éĢĢ":4872,"нÑĭе":4873,"Ġaud":4874,"log":4875,"ĠÑĢе":4876,"ä¸ĢæŃ¥":4877,"ĠFran":4878,"æŁĵ":4879,"Ġsense":4880,"æľĭ":4881,"è°¢":4882,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":4883,"Ġexperien":4884,"Ġelement":4885,"顺":4886,"éĵģ":4887,"Ġlikely":4888,"å½¢æĪIJ":4889,"-based":4890,"çłģ":4891,"Ġlength":4892,"umb":4893,"OT":4894,"ĠMore":4895,"åĶ®":4896,"ĠPe":4897,"éĥ¨éŨ":4898,"å®ŀéĻħ":4899,"åĵ¥":4900,"vis":4901,"Ġপ":4902,"irl":4903,"Ġaccept":4904,"ìĿĦ":4905,"Ġsus":4906,"楼":4907,"pond":4908,"Ġoccur":4909,"Is":4910,"Ġstatic":4911,"Ġlink":4912,"\");Ċ":4913,"-S":4914,"ä¸Ģä¸ĭ":4915,"ä¸įè¦ģ":4916,"Ġhab":4917,"ounter":4918,"ãĥ³":4919,"iddle":4920,"ster":4921,"Ġfram":4922,"Ġseries":4923,"Ġmi":4924,"met":4925,"ç©¿":4926,"æĭĽ":4927,"ND":4928,"代表":4929,"Ġessential":4930,"Ġclos":4931,"ï¼ģâĢĿĊĊ":4932,"Ġevidence":4933,"men":4934,"Ġpressure":4935,"Ġnature":4936,"绾":4937,"æķĻåѦ":4938,"Ġequal":4939,"Ġdocument":4940,"_c":4941,"come":4942,"æįŁ":4943,"Ġeight":4944,"ĠãĢģ":4945,"Ġinform":4946,"åĩºäºĨ":4947,"çij":4948,"éļľ":4949,"ects":4950,"ING":4951,"éĤĦ":4952,"åĪ©ç͍":4953,"ÏĮ":4954,"åŃ£":4955,"Ġব":4956,"è¿Ļä¹Ī":4957,"ä¸ĵä¸ļ":4958,"ĠSim":4959,"ู":4960,"Ġwalk":4961,"Ġtold":4962,"Ġskills":4963,"ãģ£":4964,"ership":4965,"uild":4966,"è¯Ĺ":4967,"æİ¢":4968,"aced":4969,"ãĤī":4970,"Ġcamp":4971,"鼨":4972,"çĽĺ":4973,"ackage":4974,"Ġalmost":4975,"ä¸Ģ缴":4976,"æŃĮ":4977,"Ñĸ":4978,"lo":4979,"150":4980,"ini":4981,"ей":4982,"ĠBo":4983,"ocal":4984,"Ġwriting":4985,"itude":4986,".e":4987,"åŁİå¸Ĥ":4988,"key":4989,"Ġmultiple":4990,"Ġ\\\\":4991,"ä¸Ģç§į":4992,"ÑİÑĤ":4993,"rieved":4994,"Ġlive":4995,"Ġhom":4996,"Ġmaintain":4997,"è§£åĨ³":4998,"âĢĿãĢĤ":4999,"à¨":5000,"åĹ":5001,"Ġmove":5002,"åĵĪ":5003,"Ġaverage":5004,"ì§":5005,"áº":5006,"ando":5007,"èĢģå¸Ī":5008,"ply":5009,"Ġclose":5010,"Ġbal":5011,"ĠìĿ":5012,"---":5013,"Ġeffort":5014,"estion":5015,"ui":5016,"iles":5017,"Ġmit":5018,"Le":5019,"Ø£":5020,"ĠBrit":5021,"oring":5022,"ditional":5023,"Ġfinancial":5024,"ĠبÙĩ":5025,"Ġtraining":5026,"ĠØ®":5027,"è°Ī":5028,"Ġseason":5029,"Ġpattern":5030,"享":5031,"ptember":5032,"Ġnecessary":5033,"ĠEr":5034,"Ġfa":5035,"Ġmatter":5036,"Ġsite":5037,"åĪĺ":5038,"Ġproducts":5039,"rix":5040,"rim":5041,"Ġhttps":5042,"Ĥ¬":5043,"ref":5044,"Ġdate":5045,"Ġcross":5046,"(int":5047,"æ±ī":5048,"uration":5049,"å¼Ģå±ķ":5050,"joy":5051,"à¸ģาร":5052,"ixed":5053,"urg":5054,"CO":5055,"Ġindustry":5056,"ĠAfter":5057,"é¢ij":5058,"Ġquant":5059,"å§IJ":5060,"Ġseg":5061,"Ġfalse":5062,"ille":5063,"ĠAfric":5064,"aint":5065,"ĠAust":5066,"骨":5067,"ĠCons":5068,"uments":5069,"ĠWill":5070,"[]":5071,"åºŃ":5072,"çİī":5073,"æ²Ĵ":5074,"ä¸Ģ次":5075,"Ġresources":5076,"ĠMarch":5077,"缮æłĩ":5078,"Ġmer":5079,"Ġsquare":5080,"Ġreading":5081,"æĿĢ":5082,"Ġconsidered":5083,"Ġago":5084,"Ġwrite":5085,"Ġchalleng":5086,"CH":5087,"mon":5088,"Ġep":5089,"Ġidea":5090,"æ¯Ľ":5091,"票":5092,"产çĶŁ":5093,"========":5094,"Ġcomes":5095,"cal":5096,"Ġdemand":5097,"ided":5098,".j":5099,"Ġnull":5100,"ĠMark":5101,"ĠMr":5102,"Ġstandard":5103,"æľĭåıĭ":5104,"æĪı":5105,"97":5106,"纳":5107,"ĠAc":5108,"å¸ĮæľĽ":5109,".#":5110,"ctor":5111,"Ġ<<":5112,"ENT":5113,"æĿ¾":5114,"Ġprior":5115,"ল":5116,"Ġeyes":5117,"Ġrequest":5118,"ñ":5119,"缮åīį":5120,"åª":5121,"nes":5122,"(f":5123,"Or":5124,"äºĨè§£":5125,"400":5126,"ĠEduc":5127,"lege":5128,"追":5129,"åºĶç͍":5130,"Ġphysical":5131,"ĠFeb":5132,"å¢ŀåĬł":5133,"just":5134,"å¸Ŀ":5135,".)":5136,"éĿł":5137,"å¤ı":5138,"Ġrat":5139,"Ġdraw":5140,"curity":5141,"fully":5142,"à¸ŀ":5143,"Ġadded":5144,"Ġinn":5145,"Ġmusic":5146,"Ġten":5147,"Ġcontext":5148,"åħ³äºİ":5149,"ä½łä»¬":5150,"190":5151,"Ġsil":5152,"cember":5153,"ja":5154,"Ġhet":5155,"Ġnov":5156,"Ġmiles":5157,"mod":5158,"Ġblack":5159,"astic":5160,"Ġfront":5161,"Im":5162,"åİĨåı²":5163,"Ġma":5164,"_id":5165,"ĠMc":5166,"Ġincreased":5167,"gor":5168,"Ġtemperature":5169,"éĴĪ":5170,"ras":5171,"Ġspecies":5172,"ר×":5173,"æ©":5174,"fo":5175,"го":5176,"Ġtaking":5177,"ĠØ¢":5178,"åįļ":5179,"#####":5180,"Ġregion":5181,"Ġcollect":5182,"ĠSome":5183,"Ġconsist":5184,"Ġpopulation":5185,"lied":5186,"æĹ¢":5187,"Go":5188,"ï¼ļĊĊ":5189,"亮":5190,"ĠĠĠĠĊ":5191,"çͳ":5192,"va":5193,"Ġpie":5194,"à¹ĩ":5195,".h":5196,"value":5197,"87":5198,"à§įর":5199,"-y":5200,"èµĦæºIJ":5201,"Ñģп":5202,"))Ċ":5203,"agement":5204,"ব":5205,"ко":5206,"ua":5207,"اع":5208,"çļĦ大":5209,"ĠJune":5210,"include":5211,"Ġdeep":5212,"æ¥ļ":5213,"Ġnational":5214,"ĠGerm":5215,"æ²ī":5216,"Ġglobal":5217,"Ġpolitical":5218,"ĠGener":5219,"Ġprice":5220,"Ġentire":5221,"åī§":5222,"){Ċ":5223,"_p":5224,"\")Ċ":5225,"ensive":5226,"Ġdecision":5227,"mar":5228,"à±":5229,"``":5230,"Ġitself":5231,"ceed":5232,"iat":5233,"110":5234,"åłĤ":5235,"责任":5236,"ä»Ĭ天":5237,"Ġoutput":5238,"çı¾":5239,"à³":5240,"var":5241,"Ġpolicy":5242,"æĢĿæĥ³":5243,"Ġindividuals":5244,"ĠPost":5245,"ached":5246,"ÙĬر":5247,"rast":5248,".D":5249,"Ġsequ":5250,"Ph":5251,"/c":5252,"IM":5253,"æĶ»":5254,"ydro":5255,"æĸĹ":5256,"éĸĵ":5257,"odes":5258,"mary":5259,"Ġincludes":5260,"Ġensure":5261,"æľīåħ³":5262,"data":5263,"ĠNorth":5264,"user":5265,"çļĩ":5266,"Ġìŀ":5267,"anks":5268,"Ġchem":5269,"'ve":5270,"Ġstarted":5271,"CT":5272,"acc":5273,"stract":5274,"æ´¾":5275,"ÑĢед":5276,"ĠCount":5277,"Ġworkshe":5278,"åĸľæ¬¢":5279,"ĠPress":5280,"Ġmeaning":5281,"ÖĢ":5282,"oved":5283,"ä¸įæĸŃ":5284,"ĠRetrieved":5285,"Ġ*/Ċ":5286,"ira":5287,"Ġweight":5288,"å¾Ĺåΰ":5289,"ĠFin":5290,"Ġdevice":5291,"Ġusually":5292,"ä»»ä½ķ":5293,"羣çļĦ":5294,"ç·":5295,"light":5296,"eks":5297,"Ġrecomm":5298,"Ġvon":5299,"Ġactually":5300,"æĿĤ":5301,"æ°´å¹³":5302,"ä¿ĥ":5303,"Ġability":5304,"Ġscre":5305,"ĠAcc":5306,"Ġdemon":5307,"ำ":5308,"ĠãĢĤ":5309,"Ġcontact":5310,"éĸĭ":5311,"ming":5312,"ager":5313,"Ind":5314,"IV":5315,"Ġune":5316,"ports":5317,"Ġjud":5318,"çļĦè¯Ŀ":5319,"ĠSub":5320,"Ġpour":5321,"éĩĬ":5322,"aving":5323,"çĹĩ":5324,"åħ¸":5325,"é¢Ĩ导":5326,"æ¹ĸ":5327,"So":5328,"-M":5329,"Ġrev":5330,"reed":5331,"çĸij":5332,"td":5333,"ĠHer":5334,"Ġdifference":5335,"ĠUs":5336,"vey":5337,"ography":5338,"bject":5339,"ĠCour":5340,"hr":5341,"igen":5342,"æĴŃ":5343,"ĠÐļ":5344,"Ġbuilding":5345,"apan":5346,"Ġstudent":5347,"ections":5348,"................................":5349,"clusion":5350,"Ġsearch":5351,"è°ģ":5352,"éĶ®":5353,"Ġdoesn":5354,"actions":5355,"from":5356,"页":5357,"åį·":5358,"ë¡ľ":5359,"lin":5360,"85":5361,"ĠResearch":5362,"Ġmodels":5363,"={":5364,"å¥Ĺ":5365,"ł×":5366,"**ĊĊ":5367,"åĭķ":5368,"mission":5369,"ĠSchool":5370,"ares":5371,"åIJĦç§į":5372,"æĭį":5373,"Ġnormal":5374,"ĠYour":5375,"èĹı":5376,"Ġcompan":5377,"andom":5378,"Ġeffective":5379,"Ġmedia":5380,"Ġfeatures":5381,"åĽ°":5382,"ĠApril":5383,"å½Ĵ":5384,"ĠHist":5385,"åĪ¶åº¦":5386,"à§ĩর":5387,"èģļ":5388,"Ġeconomic":5389,"Ġbill":5390,"计åĪĴ":5391,"Ġstyle":5392,"Ġdecl":5393,"çĶļèĩ³":5394,"https":5395,"ë¥":5396,"åĽ½éĻħ":5397,"ĠÐľ":5398,"Sub":5399,"Ġdé":5400,"Ġexec":5401,"æĺ¯åIJ¦":5402,"Ġwhite":5403,"ample":5404,"Ġeen":5405,"oke":5406,"Ġcountries":5407,"ç¬Ķ":5408,"ï¼ŁâĢĿ":5409,"lim":5410,"irit":5411,"éľ²":5412,"ln":5413,"åħ°":5414,"Ġhor":5415,"uz":5416,"Ġdivid":5417,"Ġeasy":5418,"ession":5419,"ĠâĪĴ":5420,"ür":5421,"åıĺåĮĸ":5422,"Ġhistor":5423,"Ġregul":5424,"ĠII":5425,"ging":5426,"ĠChe":5427,"($":5428,"Ġbelieve":5429,"ech":5430,"æİĮ":5431,"ĠPat":5432,"çī¹åĪ«":5433,"Ġstay":5434,"ĠاÙĦت":5435,"еÑĤÑģÑı":5436,"产ä¸ļ":5437,"px":5438,"cz":5439,"ham":5440,"ан":5441,"æĿIJæĸĻ":5442,"ĠSm":5443,"(m":5444,"éĻĦ":5445,"Ġfriends":5446,"ults":5447,"Ġcontinue":5448,"ilit":5449,"第ä¸ī":5450,"><":5451,"Ġissue":5452,"ĠRead":5453,"åij¼":5454,"With":5455,"hern":5456,"-e":5457,"è¼":5458,"upp":5459,"itting":5460,"æĮ¥":5461,"swers":5462,"Ġunique":5463,"æ¯ķ":5464,"Ġenjoy":5465,"Ġengine":5466,"å®Įåħ¨":5467,"ä¸ľè¥¿":5468,"建ç«ĭ":5469,"Ġproperties":5470,"Ġflu":5471,"ä»į":5472,"æ³ķå¾ĭ":5473,"Ġcard":5474,"ĠØ¥":5475,"Ġcourt":5476,"Ġpen":5477,"Ġforce":5478,"Ġmiss":5479,"All":5480,"ball":5481,"Ġprec":5482,"ko":5483,"osition":5484,"Ġfilm":5485,"Ġelements":5486,"âĸ":5487,"å®Ĺ":5488,"è¸":5489,"åľĭ":5490,"Ġsch":5491,"çͲ":5492,"\";Ċ":5493,"param":5494,"ĠSer":5495,"ÑģÑĤо":5496,".P":5497,"Ġstreng":5498,"Ġgetting":5499,"å¼Ł":5500,"ĠاÙĦØ£":5501,"罪":5502,"åģ¥åº·":5503,"à¸Ĭ":5504,"Ġstress":5505,"èĦļ":5506,"leep":5507,"Ġindex":5508,"Ġconcent":5509,"ĠÙĪØ§ÙĦ":5510,"èĮ¶":5511,"ĠList":5512,"ĠJanuary":5513,"Ġdisplay":5514,"éĴŁ":5515,"Ġevents":5516,"éĹ»":5517,"Ñīи":5518,"istry":5519,"lation":5520,"Ġsett":5521,"¯à¦":5522,"inate":5523,"Ġsomeone":5524,"oles":5525,"Ġmach":5526,"æķij":5527,"亿":5528,"enty":5529,"Ġliving":5530,"Ñģе":5531,"Ġâ̦":5532,"çĨŁ":5533,"ÃŃa":5534,"ĠHar":5535,"ronic":5536,"éĽª":5537,"Ġproperty":5538,"Ġfoot":5539,"ĠScience":5540,"ĠпÑĢ":5541,"çͰ":5542,"Ġ×IJ×":5543,"lig":5544,"ĠÑģÑĤа":5545,"ught":5546,"à¹Į":5547,"ĠAugust":5548,"çϼ":5549,"rial":5550,"ä¿ĿæĬ¤":5551,"ĠTechn":5552,"Ġcomplete":5553,"Ġarray":5554,"ç½ij绾":5555,"Ġqual":5556,"å®ģ":5557,"åľ°åĮº":5558,"Ġwebs":5559,"=\\":5560,"Ġmagn":5561,"ĉreturn":5562,"ά":5563,"CC":5564,"ĠÑĤе":5565,"Ġvia":5566,"ding":5567,"eta":5568,"Ġpan":5569,"ĠGra":5570,"respond":5571,"Ġcreated":5572,"Ġbehind":5573,"误":5574,"ĠWeb":5575,"Ġdrug":5576,"dom":5577,"软":5578,"×ij":5579,"roy":5580,"å¹´çļĦ":5581,"Ġplus":5582,"Ïī":5583,"çĻ»":5584,"ä½įç½®":5585,"ло":5586,"ä¸Ŀ":5587,"Ġcred":5588,"å°ģ":5589,"Ġjo":5590,"γ":5591,"ĠHere":5592,"Ġsec":5593,"Ġrecent":5594,"æĶ¿çŃĸ":5595,"Ġenh":5596,"Ġsecurity":5597,"_f":5598,"ä»·å̼":5599,"inical":5600,"мен":5601,"ĠWar":5602,"Ġblock":5603,"Ġexpected":5604,"ĠPres":5605,"Ġgas":5606,"Ġheld":5607,"è»":5608,"Ġsubst":5609,"Ġerror":5610,"-n":5611,"åºĵ":5612,"Ġarch":5613,"Ġviol":5614,"hic":5615,"åİŁåĽł":5616,"cial":5617,"Ġq":5618,"Ġhimself":5619,"Ġdas":5620,"ä¹ĭåīį":5621,"ĠJuly":5622,"åĢŁ":5623,"ogen":5624,"ĠInternational":5625,"ĠFr":5626,"ucle":5627,"ba":5628,"ĠDec":5629,"ependent":5630,"ĠÙħÛĮ":5631,"ï¼ģĊ":5632,"è¿IJåĬ¨":5633,"íķľ":5634,"sub":5635,"je":5636,"Ġmember":5637,"room":5638,"change":5639,"æ£ĢæŁ¥":5640,"åľĨ":5641,"Ġmother":5642,"Ġrights":5643,"Ġcru":5644,"Ġwin":5645,"Ġwon":5646,"aries":5647,"çīĽ":5648,"åºĬ":5649,"æĬĵ":5650,"ĠData":5651,"彩":5652,"æ°¸":5653,"Ġdescribed":5654,"Ġetc":5655,"Ġpod":5656,"Ġanything":5657,"å©ļ":5658,"è·ij":5659,"æľīæķĪ":5660,"çŃij":5661,"Ġdim":5662,"xim":5663,"Ġcondition":5664,"Ġmulti":5665,"Ġdivisors":5666,"æĵįä½ľ":5667,"éĢIJ":5668,"Ġcm":5669,"oe":5670,"Ġattention":5671,"¸°":5672,"ĠOf":5673,"sum":5674,"Ġeverything":5675,"æĸ°çļĦ":5676,"(Ċ":5677,"éĩĩç͍":5678,"Ġnumer":5679,"ç¯ĩ":5680,"æĥĬ":5681,"Ġattack":5682,"è½®":5683,"æľºæŀĦ":5684,"ि":5685,"Ġquite":5686,"ম":5687,"Ġos":5688,"à¸Ĺีà¹Ī":5689,"Ġproced":5690,"Ġж":5691,"SC":5692,"га":5693,"åį±":5694,"hens":5695,"ï¼ģâĢĿ":5696,"æĦŁè§ī":5697,"aily":5698,"ÑĪе":5699,"οÏħ":5700,"ky":5701,"Ġprinc":5702,"oul":5703,"Ġcontinu":5704,"rated":5705,"Ġsound":5706,"ĠAnt":5707,"积æŀģ":5708,"Ġload":5709,"rapy":5710,"Ġvisit":5711,"ban":5712,"à§ĭ":5713,"åİĤ":5714,"ĠThen":5715,"åħ¶å®ŀ":5716,"Pl":5717,"Let":5718,"ç»§ç»Ń":5719,"Ġbehavior":5720,"æīĺ":5721,"isf":5722,"æĢĢ":5723,"Ġcompared":5724,"ç²ī":5725,"ĠJournal":5726,"Ġpp":5727,"verse":5728,"æ²Ļ":5729,"Ġreceived":5730,"ĠRel":5731,"Ġperfect":5732,"ĉif":5733,"å¨ģ":5734,"eful":5735,"fficient":5736,"App":5737,"ÑĥÑİ":5738,"以ä¸Ĭ":5739,"ãģı":5740,"è·Ŀ":5741,"æı¡":5742,"AG":5743,"Ġthemselves":5744,"gl":5745,"ç§Ģ":5746,"rong":5747,"orks":5748,"}(":5749,"Ġbecame":5750,"Ġzu":5751,"С":5752,"Ġox":5753,"Ġaspect":5754,"ç¹":5755,"Ġstates":5756,"atively":5757,"Ġcapac":5758,"Ġaccom":5759,"Ġnothing":5760,"综":5761,"æĸ¼":5762,"Ġbad":5763,"æ··":5764,"ategory":5765,"Ġir":5766,"Ġincreasing":5767,"Ġreported":5768,"102":5769,"-C":5770,"èİ·å¾Ĺ":5771,").Ċ":5772,"ĠSeptember":5773,"éģ¿":5774,"';Ċ":5775,"æĢ§çļĦ":5776,"ternal":5777,"abase":5778,"Ġbr":5779,"hemat":5780,"ĠDav":5781,"Ġfem":5782,"è´µ":5783,"Ġidentify":5784,"Ġculture":5785,"Ġdans":5786,"ni":5787,"å¥ĸ":5788,"Ġavoid":5789,"pace":5790,"æĸĩä»¶":5791,"æŀ¶":5792,"Ġexpression":5793,"ĠChina":5794,"ĠFrom":5795,")(":5796,"æ¤į":5797,"Ġgreater":5798,"ĠVol":5799,"yth":5800,"mp":5801,"éĤ£äºĽ":5802,"Ġsort":5803,"Ġbeaut":5804,"onal":5805,"Ġpublished":5806,"Ġcapt":5807,"uh":5808,"\\n":5809,"æĿŁ":5810,"ensity":5811,"èĻij":5812,"aches":5813,"اس":5814,"Ġsche":5815,"è¿Ľåħ¥":5816,"73":5817,"æľīä¸Ģ":5818,")\\":5819,"ä»·æł¼":5820,"part":5821,"111":5822,"Ġdoor":5823,"Ġtou":5824,"â̲":5825,"³":5826,"Ġlonger":5827,"Ġpatient":5828,"Ġwanted":5829,"äºĪ":5830,"Ġimprove":5831,"'Ċ":5832,"Ġincluded":5833,"ruary":5834,"éĢı":5835,"Ġregular":5836,"ĠIndia":5837,"fig":5838,"åįĪ":5839,"New":5840,"æĿ¥è¯´":5841,"æĶ¿æ²»":5842,"表çݰ":5843,"84":5844,"atform":5845,"Õ¸":5846,"Ġsaw":5847,"ве":5848,"bt":5849,"ĠRuss":5850,"airs":5851,"æĶ¯æĮģ":5852,"Ġheav":5853,"Ġoutside":5854,"çĺ":5855,"Ġinstit":5856,"ĠOctober":5857,"Ġful":5858,"çĿ£":5859,"é±":5860,"ora":5861,"ridge":5862,"71":5863,"_m":5864,"Ġfeed":5865,"**Ċ":5866,"ĠBi":5867,"IL":5868,"讨":5869,"å½¢å¼ı":5870,"-B":5871,"net":5872,"Ġdeveloped":5873,"Ġpict":5874,"æľīçļĦ":5875,"ube":5876,"绿":5877,"pan":5878,"ï½":5879,"Ġsimply":5880,"æ´Ĺ":5881,"Ġinsp":5882,"ead":5883,"梦":5884,"Ġpopular":5885,"礼":5886,"but":5887,"æķ¢":5888,"çľĭçĿĢ":5889,"ÐŁ":5890,"inks":5891,"æĪijåĽ½":5892,"éĿ¢çļĦ":5893,"Ġutil":5894,"180":5895,"OL":5896,"Ġprep":5897,"-se":5898,"ĠDecember":5899,"Ġsex":5900,"Ġtravel":5901,"Ġfire":5902,".T":5903,"Med":5904,"quest":5905,"~~":5906,"isions":5907,"Ġhost":5908,"Ġmaterials":5909,"åı¯æĺ¯":5910,"ãģ¾ãģĻ":5911,"Ġexerc":5912,"ivil":5913,"ufact":5914,"Ġscient":5915,"gy":5916,"æķĻå¸Ī":5917,"fort":5918,"åĨ³å®ļ":5919,"df":5920,"Ġhy":5921,"'ll":5922,"Ġestim":5923,"script":5924,"\")":5925,"86":5926,"SS":5927,"Ġaltern":5928,"81":5929,"ĠSpec":5930,"Ġmedical":5931,"ĠÑĢаз":5932,"åIJĪä½ľ":5933,"Ġappropri":5934,"rat":5935,"96":5936,"ĠWest":5937,"ìĭ":5938,"Ġcancer":5939,"人们":5940,"ondon":5941,"ĠCity":5942,".âĢĿĊ":5943,"å´":5944,"empl":5945,"wa":5946,"ãģĤ":5947,"ĠVer":5948,"Ġparts":5949,"Ġemerg":5950,"ä½Ľ":5951,"ocation":5952,"å«":5953,"幸":5954,"Ġcomo":5955,"Ġanim":5956,">>":5957,"Ġtrying":5958,"Ġemot":5959,"çIJĨ论":5960,"åĩĨå¤ĩ":5961,"à§įয":5962,"sole":5963,"Ġש":5964,"Ġaim":5965,"غ":5966,"ĠÐĿ":5967,"çļĦå°ı":5968,"Ġwond":5969,"Ġmodern":5970,"æİī":5971,"Ġrelig":5972,"Node":5973,"Ġadditional":5974,"vest":5975,"pi":5976,"ados":5977,"ĠFirst":5978,"以ä¸ĭ":5979,"_n":5980,"è«":5981,"cos":5982,"anger":5983,"Ġinvolved":5984,"Ġorganiz":5985,"equ":5986,"ĠSw":5987,"éĺµ":5988,"\\({":5989,"unch":5990,"Ġfigure":5991,"ĠAmerica":5992,"ees":5993,"é±¼":5994,"iency":5995,"Ġprefer":5996,"ĠNovember":5997,"åħ·ä½ĵ":5998,"Ġdemonstr":5999,"bit":6000,"ĠWhile":6001,"mm":6002,"Ġdecre":6003,"Ġpsych":6004,"-to":6005,"Ġbegan":6006,"åīĤ":6007,"Ġappe":6008,"Ġpick":6009,"ĠOF":6010,"à¥ĩ":6011,"ponse":6012,"Ġversion":6013,"ÑĪи":6014,"ç¡®å®ļ":6015,"ĠDef":6016,"нÑĭй":6017,"input":6018,"span":6019,"é¡¶":6020,"è¡Įä¸ļ":6021,"ìľ":6022,"Ġµ":6023,"OD":6024,"Ġhot":6025,"Ġtakes":6026,"åѸ":6027,"Ġcarb":6028,"Ġsun":6029,"zi":6030,"omin":6031,"æĪIJåĬŁ":6032,"Ġknew":6033,"设å¤ĩ":6034,"å¿Ļ":6035,"(a":6036,"ford":6037,"èĥľ":6038,"çĬ¯":6039,"ÙĦÙĬ":6040,"Ġattempt":6041,"Ġdouble":6042,"ä¼ļè®®":6043,"ĠNet":6044,"ĠMat":6045,"lick":6046,"dis":6047,"74":6048,"ée":6049,"Ġاست":6050,"rought":6051,"ç»į":6052,"æ²»çĸĹ":6053,"丰":6054,"aker":6055,"Ġα":6056,"igned":6057,"Ġforward":6058,"Ġlat":6059,"pat":6060,"è¡Ĺ":6061,"ework":6062,"æĺ¯åľ¨":6063,"ç¨ĭåºı":6064,"och":6065,"ĠÑģÑĤ":6066,"Ġist":6067,"ме":6068,"Ġinitial":6069,"urt":6070,"é¦Ĩ":6071,"Ġwall":6072,"Ġgraph":6073,"Ġprimary":6074,"Ġcorrespond":6075,"ido":6076,"å¦ĤæŃ¤":6077,"82":6078,"ĠÑĤа":6079,"ä¾§":6080,"ç¶ĵ":6081,"æĸ¹åIJij":6082,"amin":6083,"omy":6084,"Ġcontract":6085,"Ġhon":6086,"Ġcu":6087,"bf":6088,"['":6089,"Ġfelt":6090,"oma":6091,"ниÑı":6092,"Ġreflect":6093,"ç»ĵåIJĪ":6094,"èĽĭ":6095,"åĶIJ":6096,"Ġbuy":6097,"ä¸įå¾Ĺ":6098,"Ġhands":6099,"cast":6100,"Ġqui":6101,"Ġcapital":6102,"Ġoil":6103,"Ġtowards":6104,"Ġ]":6105,"rench":6106,"Ġcommand":6107,"ospital":6108,"Ġpas":6109,"æĿ¨":6110,"One":6111,"arter":6112,"ĠÄ":6113,"éĻĨ":6114,"DF":6115,"çĦ¡":6116,"Ġfriend":6117,"ĠĠĠĠĠĠĠĠĠĠĠĠĠ":6118,"Ġdetermine":6119,"pha":6120,"ĠForm":6121,"roller":6122,"amed":6123,"鼷":6124,"ĠBook":6125,"ระ":6126,"åĮĹ京":6127,"Ġinternational":6128,"Ġtheory":6129,"丽":6130,"83":6131,"ias":6132,"å»¶":6133,"Ġاز":6134,"with":6135,"ör":6136,"Ġcompanies":6137,"Ġber":6138,"OC":6139,"arily":6140,"ìļ":6141,"../":6142,"鼶":6143,"Ġautom":6144,"Time":6145,"ario":6146,"åĩºçļĦ":6147,"ĠArch":6148,"inct":6149,"itte":6150,"é¹":6151,"92":6152,"ĠClass":6153,"æĮīçħ§":6154,"ãģį":6155,"Ġlives":6156,"Ġni":6157,"åIJ¯":6158,"ederal":6159,"ously":6160,"æıIJåįĩ":6161,"Ġprobably":6162,"ç§ijæĬĢ":6163,"иÑģ":6164,"ĠÑģо":6165,"Ġ×¢":6166,"Ġweeks":6167,"Ġlack":6168,"她çļĦ":6169,"Ġseven":6170,"æ±½":6171,".E":6172,"à¹Īาà¸":6173,"index":6174,"urance":6175,"resh":6176,"Ġfunctions":6177,"elle":6178,"Ġseems":6179,"BN":6180,"à¯ģ":6181,"容æĺĵ":6182,"ika":6183,"çĮ®":6184,"cil":6185,"æİª":6186,"rate":6187,"irth":6188,"ä¸ĭæĿ¥":6189,"æķĪæŀľ":6190,"ibrary":6191,"Ġled":6192,"Ġnews":6193,"èĦ±":6194,"ĠHigh":6195,"AA":6196,"æ¸IJ":6197,"oz":6198,"ective":6199,"¯à¦¼":6200,"#include":6201,"æİĪ":6202,"More":6203,"SE":6204,"ĠTest":6205,"æīįèĥ½":6206,"91":6207,"اÙĩ":6208,"à¹Ĥ":6209,"èľ":6210,"Ġprop":6211,"Ġheat":6212,"åįĸ":6213,"çĥĪ":6214,"åĬłå¼º":6215,"ç§ĭ":6216,"Ġvideo":6217,"Ġlate":6218,"Ġclean":6219,"æĽ´å¤ļ":6220,"æī§è¡Į":6221,"еÑĢе":6222,"ικ":6223,"coming":6224,"å·¨":6225,"?âĢĿĊĊ":6226,"åħ¨åĽ½":6227,"Ġcompl":6228,".b":6229,"éĩİ":6230,"ä¸įä»ħ":6231,"ç«ŀ":6232,"å¼Ģåıij":6233,"ĠCommun":6234,"Ġpredict":6235,"Ġsust":6236,"gn":6237,"Ġmag":6238,"ader":6239,"Ġinstead":6240,"ĠOther":6241,"Ġcontain":6242,"Ġlines":6243,"éĺ´":6244,"Ġconstant":6245,"Ľ×":6246,"ĠRed":6247,"Ġ->":6248,"èĻļ":6249,"以åIJİ":6250,"Ġ×ķ×":6251,"Ġreduce":6252,"Ġtherefore":6253,"ç¿»":6254,"ĠSupp":6255,"pa":6256,"vement":6257,"Ġcommunication":6258,"ĠST":6259,"Ġreve":6260,"atives":6261,"Ġscience":6262,"宫":6263,"ĠGre":6264,"aged":6265,"ĠWorks":6266,"ĠاÙĦع":6267,"è¶£":6268,"Ġ!=":6269,"åĩ½":6270,"Ġdetail":6271,"ĠKing":6272,"Ġlooked":6273,"_{\\":6274,"egin":6275,"Ġspeed":6276,"////":6277,"Ġìł":6278,"usion":6279,"ä¸įåı¯":6280,"åĬª":6281,"åıĤåĬł":6282,"term":6283,"ä¼ij":6284,"èĤ¯":6285,"Ġbenefits":6286,"Get":6287,"uman":6288,"Ġcompar":6289,"IR":6290,"æļĹ":6291,"Ġfast":6292,"idad":6293,"Ġgrand":6294,"é¥Ń":6295,"ãģķ":6296,"ĠEducation":6297,"å¾Ħ":6298,"Ġsituation":6299,"orage":6300,"Ġacid":6301,"Ġfeet":6302,"éĤ£ä¸ª":6303,"Ġmessage":6304,"ĠDevelop":6305,"lt":6306,"Ġstra":6307,"ø":6308,"ria":6309,"ĠJapan":6310,"Des":6311,"ĠAnal":6312,"ĠSum":6313,"Ġма":6314,"Ġdirection":6315,"Ġpack":6316,"Ġstatus":6317,"Ġbott":6318,"Ġexact":6319,"Ġom":6320,"len":6321,"空éĹ´":6322,"Ġsignal":6323,"des":6324,"ĠAustral":6325,"Ġaw":6326,"èĥŀ":6327,"ĊĊĊ":6328,"Ġcosts":6329,"ç»ıèIJ¥":6330,"Ġexperiment":6331,"ĠEst":6332,"rest":6333,"çĬ¶æĢģ":6334,"é¬":6335,"rab":6336,"Ġroad":6337,"94":6338,"án":6339,"Add":6340,"Ġcomputer":6341,"çĿ¡":6342,"Not":6343,"lor":6344,"Ġhope":6345,"麼":6346,"ек":6347,"ë¦":6348,"åĪĽæĸ°":6349,"Ùħا":6350,"-P":6351,"Ġinside":6352,"Ġبر":6353,"Ġcenter":6354,"èĶ":6355,"pite":6356,"oly":6357,"ette":6358,"Ġcry":6359,"Ġremember":6360,"Ġwait":6361,"Ġnames":6362,"ı":6363,"ி":6364,"许å¤ļ":6365,"hi":6366,"ĠTHE":6367,"Ġpal":6368,"Ġfather":6369,"èĮĥåĽ´":6370,"æĬĺ":6371,"èĪª":6372,"身ä½ĵ":6373,"')Ċ":6374,"秦":6375,"åģı":6376,"Ġau":6377,"ãģ¦ãģĦ":6378,"oo":6379,"Ġdistribution":6380,"ä¼°":6381,"Ġallows":6382,"ów":6383,"ev":6384,"én":6385,"ĠSol":6386,"ĠÐŀ":6387,"Ġdeal":6388,"é»ŀ":6389,"Ġparents":6390,"600":6391,"æĹ¥æľ¬":6392,"roid":6393,"Ġbooks":6394,"ĠMus":6395,"ĠFebruary":6396,"Ġdog":6397,"Ġimmedi":6398,"帮åĬ©":6399,"Ġsn":6400,"icon":6401,"ÑĤÑĥ":6402,"Ġmap":6403,"à§Ģ":6404,"Ġvalid":6405,"Ġdark":6406,"Ġtitle":6407,"ÐĴ":6408,"Ġstop":6409,"è¿Ľä¸ĢæŃ¥":6410,"ala":6411,"ï¼īĊĊ":6412,"esus":6413,"|ĊĊ":6414,"Ġsoon":6415,"Ġmut":6416,"Ġmole":6417,"Ġtransfer":6418,"çĤİ":6419,"93":6420,"ãĤ¹":6421,"Ġbed":6422,"Ġnut":6423,"mat":6424,"Ġpurpose":6425,"ç¼ĵ":6426,"ĠScholar":6427,"Ġdefined":6428,"Ġinj":6429,"âij":6430,"Ġmid":6431,"189":6432,"aur":6433,"(c":6434,"haps":6435,"éĺ»":6436,"Ġdiagn":6437,"omb":6438,"more":6439,"Ġparticularly":6440,"ç͍äºİ":6441,"Ġadminist":6442,"Ġthroughout":6443,"è¿İ":6444,"ä¹±":6445,"PS":6446,"å¯Ĵ":6447,"rig":6448,"æ±ĩ":6449,"Ġdepend":6450,"ä½ľèĢħ":6451,"pret":6452,"Ðŀ":6453,"è¿ĩåİ»":6454,"001":6455,"nown":6456,"ali":6457,"å®ŀæĸ½":6458,"FF":6459,"Ġsoftware":6460,"Ġlimit":6461,"alle":6462,"ĠLear":6463,"Ġmemory":6464,"ĠArt":6465,"àµ":6466,"%,":6467,"ä¸įçŁ¥éģĵ":6468,"硬":6469,"Ġslow":6470,"='":6471,"è·µ":6472,"Ġthus":6473,"ů":6474,"asons":6475,"Ġrespond":6476,"çĽĸ":6477,"Ġwer":6478,"æĹ§":6479,"Ġwebsite":6480,"éķ·":6481,"Ġgirl":6482,"void":6483,"Form":6484,"Ġbox":6485,"Ġprogress":6486,"éĵ¶è¡Į":6487,"Ġca":6488,"Ġsuff":6489,"Ġcritical":6490,".R":6491,"Ġoverall":6492,"港":6493,"ä¸ļåĬ¡":6494,"Ġfavor":6495,"Ġnm":6496,"used":6497,"ani":6498,"iverse":6499,"Ġple":6500,"Ġwhose":6501,"å°ļ":6502,"Ġnga":6503,"ĠPhys":6504,"Ġeth":6505,"ç«¥":6506,"edia":6507,"模å¼ı":6508,"èīºæľ¯":6509,"Ġemp":6510,"å¼±":6511,"æī©":6512,"board":6513,"ponent":6514,"bar":6515,"ĠOur":6516,"Ġdigital":6517,"Ġtas":6518,"çѾ":6519,"Ġdistance":6520,"Ġmis":6521,"åĨ°":6522,"Ñīе":6523,"×ķר":6524,"æĹłæ³ķ":6525,"-A":6526,".âĢĻ":6527,"')":6528,"ĠAre":6529,"izes":6530,"Ġsteps":6531,"Ġapplications":6532,"ä»»åĬ¡":6533,"see":6534,"绩":6535,"Ġoptions":6536,"Ġlegal":6537,"quare":6538,"cia":6539,"Ġcoming":6540,"html":6541,"å¼¹":6542,"Ġlimited":6543,"ca":6544,"Ġweb":6545,"Ġ&&":6546,".L":6547,"erve":6548,"æ·¡":6549,"Ġequation":6550,"å°¼":6551,"ULL":6552,"Ġax":6553,"ÉĻ":6554,"Ġsafety":6555,"Ġbound":6556,"Ġsurv":6557,"Ġdesigned":6558,"Ġleave":6559,"aren":6560,"éľĢæ±Ĥ":6561,"å®ľ":6562,"رÙĬ":6563,"Ġinstall":6564,"าม":6565,"央":6566,"Ġdro":6567,"Ġvehic":6568,"Ġbasic":6569,"Ġog":6570,"Äģ":6571,"éĩįçĤ¹":6572,"ç®Ģåįķ":6573,"Ġdynam":6574,"ĠSee":6575,"综åIJĪ":6576,"Ġtal":6577,"Ġdirectly":6578,"Ġprocesses":6579,"exp":6580,"िà¤":6581,"oon":6582,"ĠNow":6583,"Ġbasis":6584,"Ġspect":6585,"ĠInstit":6586,"vention":6587,"BC":6588,"vo":6589,"ocks":6590,"Ġsociety":6591,"åĬªåĬĽ":6592,"vol":6593,"Ùij":6594,"éļIJ":6595,"ords":6596,"Õ«":6597,"éĺħ":6598,"Ġwatch":6599,"(i":6600,"Ġenter":6601,"çĴ":6602,"itable":6603,"æ¾":6604,"ĠCourt":6605,"åIJĽ":6606,"ĠDel":6607,"rew":6608,"Ġsin":6609,"Ġlic":6610,"ĠFe":6611,"Ġalgor":6612,"ĠMod":6613,"Ġmatch":6614,"active":6615,"Ġball":6616,"çIJĨè§£":6617,"oken":6618,"触":6619,"ĠÐIJ":6620,"cd":6621,"Ġou":6622,"Ġuses":6623,"ç§ģ":6624,"ĠDi":6625,"Ġwoman":6626,"Ġê°":6627,"Ġusers":6628,"ellow":6629,"Ġstru":6630,"è¾ij":6631,"Ġstage":6632,"اء":6633,"ache":6634,"Google":6635,"ĠDet":6636,"ashing":6637,"Ġemail":6638,"Ġacqu":6639,"ç͍æĪ·":6640,"é»ĺ":6641,"å¨ĺ":6642,"åĸĿ":6643,"Ġlost":6644,"Ġremain":6645,"Ġleading":6646,"å±ĭ":6647,"ku":6648,"Ġdiscover":6649,"103":6650,"Ġvolume":6651,"Ġpret":6652,"Ġadvant":6653,"Ġassum":6654,"ĠÐł":6655,"为ä»Ģä¹Ī":6656,"ä»ĭç»į":6657,"欧":6658,"iers":6659,"Ġeveryone":6660,"âĨ":6661,"çΏ":6662,"è¡ĮæĶ¿":6663,"èι":6664,"Ġпол":6665,"(self":6666,"Ġmm":6667,"è´Łè´£":6668,"Ġdos":6669,"é¤IJ":6670,"Ġrates":6671,"Ġcentral":6672,"ags":6673,"Ġcook":6674,"Ġcit":6675,"ì§Ģ":6676,"ĠEarth":6677,"访":6678,"dated":6679,"-year":6680,"lands":6681,"ĠÙ¾":6682,"Ġcentury":6683,"Ġones":6684,"Value":6685,"Ġש×":6686,"Ġprofessional":6687,"ĠJust":6688,"Pr":6689,"Ġactive":6690,"æĢª":6691,"ï¼ŁĊ":6692,"ĠLondon":6693,"iring":6694,"next":6695,"åĪº":6696,"ĠDepartment":6697,"Ġcas":6698,"Ġrelev":6699,"gar":6700,"å·Ŀ":6701,"æĦıè¯Ĩ":6702,"Ġanaly":6703,"Ġtools":6704,"pid":6705,"Ġproviding":6706,"Ġstri":6707,"Ġadapt":6708,"ĠÑħ":6709,"ä¿ĿæĮģ":6710,"ÎŃ":6711,"Ġextra":6712,"']":6713,"ĠÑģа":6714,"æĺ¯ä¸Ģ个":6715,"èĦī":6716,"......":6717,"Ġgave":6718,"Ġrandom":6719,"åĺ´":6720,"Ġparty":6721,"éĢłæĪIJ":6722,"Ġdefault":6723,"ÑĤÑĭ":6724,"ÙĪÙħ":6725,"Ġgreen":6726,"åĩĢ":6727,"ancy":6728,"EE":6729,"éŃĶ":6730,"Ġmention":6731,"ĠعÙĦÙī":6732,"Ġmob":6733,"Ġerr":6734,"é½IJ":6735,"ĠFact":6736,"She":6737,"åĬ³åĬ¨":6738,"è¿ĩç¨ĭä¸Ń":6739,"Ġnegative":6740,"Figure":6741,"éĢŁåº¦":6742,"uge":6743,"Ġdetails":6744,"ê³ł":6745,"ati":6746,"ĠÙħع":6747,"Ġsomet":6748,"Ġchoice":6749,"æ¶Īè´¹":6750,"ê°Ģ":6751,"Ġstaff":6752,"åĿı":6753,"Ab":6754,"Ġà¦ı":6755,"User":6756,"ĠGet":6757,"Ġnode":6758,"My":6759,"iful":6760,"/d":6761,"Ġband":6762,"ä¼´":6763,"Ġcos":6764,"奶":6765,"кÑĥ":6766,"Ġseek":6767,"Īëĭ¤":6768,"Ġimag":6769,"æ´²":6770,"estern":6771,"è¾¾åΰ":6772,"Ġbrain":6773,"inder":6774,"æŃ£ç¡®":6775,"ĠĠĊĊ":6776,"mitted":6777,"æĪIJæľ¬":6778,"Ġtransform":6779,"().":6780,"Ġtrack":6781,"Ġord":6782,"Ġprograms":6783,"ĠMor":6784,"æľ«":6785,"建çŃij":6786,"ĠâĨĴ":6787,"ä¸ĢçĤ¹":6788,"Õ¥":6789,"ĠJe":6790,"Ġboard":6791,"TO":6792,"250":6793,"ague":6794,"éļıçĿĢ":6795,"ÃŁ":6796,"isation":6797,"Ġappropriate":6798,"Ġbur":6799,"ĠдлÑı":6800,"ëı":6801,"ses":6802,"Ġapplied":6803,"缮çļĦ":6804,"Ġofficial":6805,"MP":6806,"èĪĩ":6807,"Ġorigin":6808,"Ġstatement":6809,"Ġsample":6810,"åĬĽéĩı":6811,"дÑĥ":6812,"item":6813,"Ġtour":6814,"olic":6815,"Ġexcept":6816,"θ":6817,"Ġturned":6818,"Ġencou":6819,"ĠReview":6820,"ä¼łç»Ł":6821,"Ġmechanism":6822,"Ġforms":6823,"Ġplatform":6824,"Ġsatisf":6825,"ecause":6826,"麻":6827,"åİļ":6828,"attle":6829,"Ġlocation":6830,"åĪĨåĪ«":6831,"ocol":6832,"Ġtim":6833,"angle":6834,"ĠDay":6835,"па":6836,"åŃĶ":6837,"ĠбÑĭ":6838,"rote":6839,"Ġentre":6840,"aled":6841,"Ġhyp":6842,"uy":6843,"Ġtransport":6844,"Ġtrust":6845,"ente":6846,"åıªè¦ģ":6847,"ena":6848,"Ġstd":6849,"éĨĴ":6850,".W":6851,"èĪŀ":6852,"Ġinfluence":6853,"ç³ĸ":6854,"Ġtree":6855,"è¿Ļæł·çļĦ":6856,"Ġhour":6857,"è̳":6858,"enge":6859,"188":6860,"ç´¯":6861,"Ġcand":6862,"å¤ļå°ij":6863,"ĠPublic":6864,"Ġpresence":6865,"Ġing":6866,"itary":6867,"ums":6868,"à§Ł":6869,"berg":6870,"认è¯Ĩ":6871,"æĦıä¹ī":6872,"Ġplants":6873,"Ġbud":6874,"Ġpet":6875,"Ġult":6876,"Ġround":6877,"State":6878,"Ġjour":6879,"ona":6880,"ä¹ĭä¸Ģ":6881,"Ġpurch":6882,"Ġ~":6883,"800":6884,"illing":6885,"Ġprotein":6886,"åѦéĻ¢":6887,"æĹ¶ä»£":6888,"Ġquickly":6889,"Ġvariety":6890,"ĠProgram":6891,"Ġthinking":6892,"é²ľ":6893,"를":6894,"123":6895,"Ġmanufact":6896,"-D":6897,"ç¦ģ":6898,"群ä¼Ĺ":6899,"http":6900,"UN":6901,"ĠLaw":6902,"Ġft":6903,"ĠOver":6904,"ouncil":6905,"ìĦľ":6906,"ª×":6907,"reement":6908,"!!":6909,"ĠJesus":6910,"导èĩ´":6911,")=":6912,"ÛĮد":6913,"æĽ´åĬł":6914,"Ġreference":6915,"ams":6916,"186":6917,"rief":6918,"ĠEuropean":6919,"স":6920,"rc":6921,"wise":6922,"Ġuseful":6923,"108":6924,"mes":6925,"Ġstrength":6926,"æĤ£èĢħ":6927,"ائ":6928,"æ§":6929,"çĸ¾":6930,"ĠÏĥ":6931,"Of":6932,"æľī人":6933,"Ġrunning":6934,"ĠSan":6935,"hood":6936,"çĥŁ":6937,"ĠPark":6938,"Ġbank":6939,"agram":6940,"plit":6941,"å¸Ń":6942,"Ġdoi":6943,"åĩ¡":6944,"isher":6945,"Ġrow":6946,"åıªèĥ½":6947,"ĠUse":6948,"Ġtown":6949,"Īĺ":6950,"Ġbackground":6951,"ĠOut":6952,"ĠGovern":6953,"Ġdegree":6954,"çĪ·":6955,"лÑĥ":6956,"ÑĢÑĭ":6957,"Ġ×IJ":6958,"å½ĵçĦ¶":6959,"æĭ¥":6960,"Ġcam":6961,"Em":6962,"çݰ代":6963,"éłŃ":6964,"iding":6965,"åĪĢ":6966,"é̲":6967,"Ġideas":6968,"Õ¶":6969,"160":6970,"bor":6971,"×ĵ":6972,"Er":6973,"éϤäºĨ":6974,"_ĊĊ":6975,"æģ¶":6976,"说æĺİ":6977,"Ġpow":6978,"åIJĮåѦ":6979,"((":6980,"cho":6981,"ĠTime":6982,"ĠBar":6983,"./":6984,"ä¸ĭçļĦ":6985,"å°±ä¼ļ":6986,"annel":6987,"position":6988,"},":6989,"Ġaffect":6990,"å®¶åºŃ":6991,"105":6992,"ĠAssoci":6993,"inese":6994,"å¢ŀéķ¿":6995,"ét":6996,"é£İéĻ©":6997,"ä¸įåΰ":6998,"纸":6999,"æĺ¾ç¤º":7000,"Ġworth":7001,"ears":7002,"ilos":7003,"Ġ+=":7004,"ĠProf":7005,"Ġcomment":7006,"é¡¿":7007,"Ġopportunity":7008,"Ġproduce":7009,"Ġletter":7010,"(b":7011,"åįģåĪĨ":7012,"130":7013,"ĠÏĢ":7014,"ή":7015,"毫":7016,"Ġformer":7017,"æĬ¥åijĬ":7018,"fe":7019,"Ġwasn":7020,"));Ċ":7021,"Ġcompre":7022,"Ġhydro":7023,"å³°":7024,"init":7025,"ECT":7026,"Ġrules":7027,"ĉĊ":7028,"ãĤĪ":7029,"Ġпод":7030,"ç¨ĭ度":7031,"Ġoffice":7032,"å¹³åı°":7033,"lished":7034,"rack":7035,"ிà®":7036,"ÑĨии":7037,"pert":7038,"Ġheight":7039,"chen":7040,"éĵ¾":7041,"CE":7042,"ĠAdd":7043,"åľĪ":7044,"å®ļçļĦ":7045,"éĺ¶æ®µ":7046,"Ġgives":7047,"unk":7048,"Ġvirt":7049,"Ġwide":7050,"çģ¯":7051,"uthors":7052,"Ġsleep":7053,"From":7054,"缩":7055,"rage":7056,"ান":7057,"Ġaware":7058,"Ġswe":7059,"force":7060,"ìķ":7061,"нÑı":7062,"ji":7063,"å·¥ä¸ļ":7064,"Ġspeak":7065,"Ġpoor":7066,"æĶ¹éĿ©":7067,"Ġbrought":7068,"ĠÑĩÑĤо":7069,"Ġoffers":7070,"Ġдо":7071,"æĹģ":7072,"Ġconstruct":7073,"æľīéĻIJ":7074,"Ġtraditional":7075,"Ġgoal":7076,"æķ´ä¸ª":7077,"iments":7078,"jo":7079,"Ġfeature":7080,"ĠInc":7081,"unc":7082,"Ġobtained":7083,"eria":7084,"å½ĵæĹ¶":7085,"åĩłä¸ª":7086,"åĿļæĮģ":7087,"Ġhar":7088,"ĠAP":7089,"è·³":7090,"å®ĭ":7091,"omas":7092,"(p":7093,"åĩ½æķ°":7094,"åĤ¨":7095,"Ġfight":7096,"Ġsometimes":7097,"寻":7098,"Ġaf":7099,"Ġmovement":7100,"ologies":7101,"è¦ĭ":7102,"åħ±åIJĮ":7103,"Ġlayer":7104,"éĴ¢":7105,"Ġkil":7106,"ellig":7107,"Ġmys":7108,"æ´ĭ":7109,"Ġphase":7110,"Ġshowed":7111,"设置":7112,"éģį":7113,"cles":7114,"Key":7115,"Ġfamil":7116,"amente":7117,"éģĹ":7118,"æŃ£å¸¸":7119,"ĠGeneral":7120,"åħĪçĶŁ":7121,"ĠSci":7122,"è¡¡":7123,"à¹ģล":7124,"usiness":7125,"åıĹåΰ":7126,"alam":7127,"Ġfollowed":7128,"mo":7129,"Ġunits":7130,"ĠOff":7131,"asi":7132,"Ġcolumn":7133,"Ġlabor":7134,"Ġgames":7135,"ao":7136,"åĽłç´ł":7137,"èģĶç³»":7138,"Ġwrong":7139,"Ġvoice":7140,".print":7141,"Ġchallenges":7142,"å°¤":7143,"æ¥Ń":7144,"æĪij们çļĦ":7145,"èĥ¡":7146,"Ġskin":7147,"ĠUp":7148,"å¿ĥçIJĨ":7149,"urb":7150,"ivo":7151,"ĠDem":7152,"Be":7153,"cks":7154,"Ġnote":7155,"ãĢĤâĢľ":7156,"èĢĥèĻij":7157,"ingu":7158,"Ġhighly":7159,"Ġlett":7160,"å¸ģ":7161,"Ġserious":7162,"104":7163,"(d":7164,"Ġexamples":7165,"iny":7166,"ĠAM":7167,"實":7168,"Ġpages":7169,"iance":7170,"ìĬ":7171,"Ġrequirements":7172,"ĠBen":7173,"å®ĥ们":7174,"Ġgenerally":7175,"Ġimportance":7176,"Ġbey":7177,"Ġimages":7178,"ĠTable":7179,"ìĿĢ":7180,"ady":7181,"ãĥĪ":7182,"otes":7183,"Ġtend":7184,"æĸ¹æ¡Ī":7185,"Ġeasily":7186,"çķ¶":7187,"Ġrot":7188,"Ġtechniques":7189,"SP":7190,"å¹ħ":7191,".st":7192,"body":7193,"ĠWork":7194,"Ġorganization":7195,"èİ«":7196,"iva":7197,"inding":7198,"Ġobserved":7199,"樣":7200,"187":7201,"Ġré":7202,"Ġdivided":7203,"Ġvict":7204,"ç§ĺ":7205,"ç»ıè¿ĩ":7206,"125":7207,"Ġbeyond":7208,"arlier":7209,"éĩĮçļĦ":7210,"Ġsympt":7211,"Ġtoward":7212,"ĠFrench":7213,"ĠMet":7214,"تر":7215,"mi":7216,"Ġthreat":7217,"ĠBritish":7218,"çĶŁåij½":7219,"ãģĻãĤĭ":7220,"ito":7221,"ĠDate":7222,"Ġsong":7223,"XX":7224,"宽":7225,"Ġfollows":7226,"avig":7227,"nie":7228,"Ġpull":7229,"That":7230,"Ġtask":7231,"ĠWilliam":7232,"Text":7233,"xx":7234,"åıįåºĶ":7235,"Ġsources":7236,"询":7237,"Ġchoose":7238,"OM":7239,"ulated":7240,"ä¸Ĭæµ·":7241,"å¹¶ä¸Ķ":7242,"uan":7243,"Ġfra":7244,"ãĤĵ":7245,"Ġillust":7246,"ноÑģÑĤи":7247,"ĠDist":7248,"package":7249,"ĠPaul":7250,"ĠChar":7251,"Ġinnov":7252,"׾×":7253,"Ġalthough":7254,"Ġamb":7255,"าà¸ģ":7256,"Ġcomponents":7257,"ĠPort":7258,"åķı":7259,"è°ĵ":7260,"åħ¨éĿ¢":7261,"ги":7262,"Ġapply":7263,"åIJĪåIJĮ":7264,"Ùĭ":7265,"Ġspirit":7266,"Ġcultural":7267,"çīĻ":7268,"Ġè":7269,"Ġম":7270,"ä¹Łæľī":7271,"Ġcontains":7272,"å®ŀè·µ":7273,"Ġdaily":7274,"itory":7275,"ĠMich":7276,"亦":7277,"utive":7278,"itect":7279,"ĠMicro":7280,"缼":7281,"èµµ":7282,"Ġhttp":7283,"Ġoption":7284,"ние":7285,"Ġkids":7286,"arrow":7287,"ĠView":7288,"cular":7289,"ĠItal":7290,"æĶ¶åħ¥":7291,"δ":7292,"оÑĤоÑĢ":7293,"Ġlarger":7294,"ĠÐĶ":7295,"Rep":7296,"ä½³":7297,"çļĦéĹ®é¢ĺ":7298,"Ġdead":7299,"ructure":7300,"zer":7301,"_P":7302,"اÙģ":7303,"ç»Ĩèĥŀ":7304,"èµĦéĩij":7305,"Ġreceive":7306,"abet":7307,"ÑģÑģ":7308,"ittee":7309,"çªģçĦ¶":7310,"å¹¼":7311,"Ġorg":7312,"åĢº":7313,"Ġlab":7314,"æĺİæĺ¾":7315,"Ġitems":7316,"orge":7317,"仪":7318,"Service":7319,"ĠPal":7320,"encies":7321,"');Ċ":7322,"Ġpi":7323,"è¿Ŀ":7324,"ึ":7325,"mathrm":7326,"Ġfif":7327,"ipal":7328,"Ġpoly":7329,"Ġdeliver":7330,"140":7331,"Ġmal":7332,"ests":7333,"润":7334,"109":7335,"106":7336,"Ġnucle":7337,"iones":7338,">Ċ":10292,"å°ĸ":10293,"ä¹łæĥ¯":10294,"%ï¼Į":10295,"Ġauthority":10296,"ptions":10297,"è§Ĩé¢ij":10298,"Ġhusband":10299,"Ġahead":10300,"ç»Łè®¡":10301,"Ġideal":10302,"Ġframework":10303,"Ġevolution":10304,"åĪ¶ä½ľ":10305,"/min":10306,"ESS":10307,"Ġreb":10308,"çļĦå½±åĵį":10309,"ĠAst":10310,"HT":10311,"rence":10312,"Ġdespite":10313,"åģ¶":10314,"Ġwish":10315,"ĠHall":10316,"Ġminute":10317,"First":10318,"æķ´ä½ĵ":10319,"ĠاÙĦب":10320,"éĢļçŁ¥":10321,"ĠGermany":10322,"color":10323,"valu":10324,"object":10325,"åī©":10326,"ĠHand":10327,"èµı":10328,"''":10329,"软件":10330,"Ðľ":10331,"ĠTur":10332,"cohol":10333,"139":10334,"éĿ¢åīį":10335,"å¼·":10336,"Ġnavig":10337,"hand":10338,"Field":10339,"Ġreturns":10340,"isp":10341,"Ġpurposes":10342,"ethyl":10343,"icit":10344,"æĿ±":10345,"Ġchannel":10346,"Ġbaby":10347,"Ġlinks":10348,"ĠWord":10349,"\\({}^{":10350,"пи":10351,"ä¸ĭéĿ¢":10352,"Cal":10353,"itative":10354,"Ġarticles":10355,"rav":10356,"ĠпÑĢе":10357,"othing":10358,"iana":10359,"ĠServices":10360,"èħ°":10361,"æģ¢å¤į":10362,"138":10363,"ĠOffice":10364,"oster":10365,"Ġvacc":10366,"Ġserved":10367,"ĠYear":10368,"åĪĽå»º":10369,"Ġtissue":10370,"eph":10371,"Ġpel":10372,"åζå®ļ":10373,"ä¸įä½ı":10374,"Question":10375,"stitution":10376,"æĿĨ":10377,"uls":10378,"缸äºĴ":10379,"EO":10380,"Ġcertainly":10381,"-de":10382,"tenance":10383,"ĠPeter":10384,"riend":10385,"-con":10386,"è¨ĺ":10387,"dule":10388,"ictionary":10389,"ÃŃn":10390,"ĠLu":10391,"Ġcontribute":10392,"Ùħر":10393,"лениÑı":10394,"åĨħçļĦ":10395,"Ġaudience":10396,"Ġfunctional":10397,"Supp":10398,"ĠMont":10399,"Ġwherein":10400,"Ġleads":10401,"Ġacademic":10402,"ĠNon":10403,"Ġclub":10404,"ï¸":10405,"że":10406,"à³į":10407,"ä¸Ńåįİ":10408,"ĠFood":10409,"161":10410,"Ġframe":10411,"ĠBet":10412,"è¿Ľè¡ĮäºĨ":10413,"()ĊĊ":10414,"enth":10415,"Ġáĥ":10416,"anner":10417,"yles":10418,"âĦ":10419,"ogy":10420,"èĩ£":10421,"¨×":10422,"Ġloved":10423,"亿åħĥ":10424,"([":10425,"å¾Ģå¾Ģ":10426,"ugh":10427,"Ġped":10428,"æļĤ":10429,"imens":10430,"ez":10431,"Ġcircumst":10432,"ĠAuthor":10433,"Ġsit":10434,"Ġ(-":10435,"Ġinteraction":10436,"176":10437,"Ġsubsequ":10438,"Ñĭй":10439,"æIJľ":10440,"rend":10441,"æĺ¯ä¸įæĺ¯":10442,"Ġcro":10443,"ĠDO":10444,"Ġdiverse":10445,"eler":10446,"éĢĤåIJĪ":10447,"Ġequivalent":10448,"Ġleadership":10449,"Ġantib":10450,"Ġentry":10451,"Ġprinciples":10452,"raf":10453,"ĠAbout":10454,"ĠÑĩа":10455,"iffer":10456,"æľīä»Ģä¹Ī":10457,"eless":10458,"ometric":10459,"Ġinfection":10460,"Ġmarked":10461,"Ġmale":10462,"Ġnorth":10463,"irk":10464,"Ġalb":10465,"Ġeasier":10466,"ĠSpe":10467,"ĠSpring":10468,"åıij表":10469,"002":10470,"æĭľ":10471,"çľĭäºĨ":10472,"oir":10473,"(),":10474,"168":10475,"âĶĢ":10476,"åļ":10477,"See":10478,"Ġsexual":10479,"iler":10480,"æı´":10481,"åºĶçļĦ":10482,"ishes":10483,"Ġtid":10484,"Ġmouth":10485,"ÑĤÑĮÑģÑı":10486,"ĠTex":10487,"165":10488,"asa":10489,"ë¶":10490,"æĪIJåijĺ":10491,"éģŃ":10492,"让人":10493,"ERS":10494,"ÙĨا":10495,"ĠCommon":10496,"ibilities":10497,"ĠRobert":10498,"count":10499,"enu":10500,"Ġcomprehensive":10501,"Ġagree":10502,"ä¸¥æł¼":10503,"Ġinvention":10504,"ĠDescription":10505,"Ġlandsc":10506,"è®°èĢħ":10507,"æĬĢèĥ½":10508,"æī£":10509,"Ġ/**Ċ":10510,"eding":10511,"ono":10512,"start":10513,"ä¹Į":10514,"ç»ıåİĨ":10515,"ublish":10516,"ç͵è§Ĩ":10517,"ads":10518,"Ġ:ĊĊ":10519,"Ġevaluation":10520,"عد":10521,"encia":10522,"ĠвÑģе":10523,"Ġcash":10524,"DS":10525,"Ġslightly":10526,"Ġfluid":10527,"ĠSquare":10528,"çķª":10529,"Ġcombined":10530,"-by":10531,"Ġextract":10532,"Ġtemp":10533,"ĠCatal":10534,"èĿ":10535,"phone":10536,"Ġexposure":10537,"çĪ¶äº²":10538,"让æĪij":10539,"æĿľ":10540,"æĸĩåŃĹ":10541,"å°ıçļĦ":10542,"143":10543,"å¦Ļ":10544,"ç»Īäºİ":10545,"alled":10546,"æĬ¬":10547,"pling":10548,"Ġbutton":10549,"ĠSil":10550,"Ġfab":10551,"Ġexperienced":10552,"ĠStreet":10553,"request":10554,"Ġlanguages":10555,"ĠProt":10556,"Ġlie":10557,".N":10558,"Ġgenerate":10559,"ĠCommittee":10560,"Ġsie":10561,"Ġchain":10562,")*":10563,"QL":10564,"-sh":10565,"lock":10566,"ัà¸ģ":10567,"Ge":10568,"éĢĻåĢĭ":10569,"Ġexcellent":10570,"uracy":10571,"kg":10572,"ĠPDF":10573,"iene":10574,"Ġinstitutions":10575,"Ġapproaches":10576,"Ġthirty":10577,"ĠCongress":10578,"ĠBa":10579,"é£Łåĵģ":10580,"è·¨":10581,"åľ¨ä¸Ģèµ·":10582,"æĦŁåΰ":10583,"師":10584,"Ġtele":10585,"æīĢè°ĵ":10586,"Ñĥд":10587,"Ġopinion":10588,"æľĢé«ĺ":10589,"åľ¨è¿Ļ":10590,"è·Į":10591,"ipped":10592,"ĠInvest":10593,"147":10594,"words":10595,"串":10596,"ä¹Łå°±æĺ¯":10597,"æŁ³":10598,"atu":10599,"Ġradio":10600,"Ġjudg":10601,"åıĤæķ°":10602,"Ġneut":10603,"amma":10604,"ulum":10605,"Ġ[Ċ":10606,"Ġdrop":10607,"ĠPop":10608,"Ġinsert":10609,"Result":10610,"izz":10611,"çĽ¾":10612,"Ġemotional":10613,"è¦Ĩ":10614,"Ġtang":10615,"orter":10616,"js":10617,"ĠProfess":10618,"岸":10619,"Ġhypot":10620,"uto":10621,"è»Ĭ":10622,"Ġinterests":10623,"icated":10624,"æĽ°":10625,"è¾ĵåĩº":10626,"Ñģли":10627,"stream":10628,"Ġrecept":10629,"ä¹ĥ":10630,"çļĦæīĭ":10631,"Ġjed":10632,"/cm":10633,"True":10634,"Ġaux":10635,"Ġextent":10636,"ijk":10637,"æľ¬èº«":10638,"Ġice":10639,"Ġreact":10640,"Ġindustrial":10641,"è´Ńä¹°":10642,"Ġdynamic":10643,"PE":10644,"èģĮå·¥":10645,"Ġsont":10646,"å¹¾":10647,"би":10648,"ĠAny":10649,"156":10650,"ĠEnvironment":10651,"ég":10652,"Ġ>>":10653,"Ġdriving":10654,"æĢĿèĢĥ":10655,"},Ċ":10656,"×Ļ×Ķ":10657,"çļĨ":10658,"Ġdetailed":10659,"åĩŃ":10660,"Ġsand":10661,"Ġplays":10662,"Ġ±":10663,"å¼Ħ":10664,"寸":10665,"isms":10666,"ĠìĪĺ":10667,"Ġпа":10668,"162":10669,"-ch":10670,"Ġexcess":10671,"xture":10672,"竹":10673,"缸å½ĵ":10674,"adow":10675,".in":10676,"Ġfell":10677,"à§įব":10678,"èĺ":10679,"Ġthoughts":10680,"Ġrank":10681,"çijŀ":10682,"ĠLes":10683,"æ³ī":10684,"åıįæĺł":10685,"èĥ¶":10686,"çĮª":10687,"PC":10688,"vector":10689,"Ġcalculated":10690,"kip":10691,"Ġresist":10692,"Ġvac":10693,"Ġalter":10694,"Ġï¼Į":10695,"ç¾Ĭ":10696,"宿":10697,"pet":10698,"Ġcalls":10699,"ĠBay":10700,"Some":10701,"é©¶":10702,"Ġsobre":10703,"Ġdegrees":10704,"æĻ¨":10705,"éģĩåΰ":10706,"ÙĦÛĮ":10707,"ön":10708,"éĻª":10709,"á½":10710,"ĠObs":10711,"Ġini":10712,"Ġnarr":10713,"ìĬ¤":10714,"Ġtak":10715,"çī¹èī²":10716,"å°±åĥı":10717,"covery":10718,"blem":10719,"Input":10720,"çļĦéĤ£":10721,"Ġnice":10722,"Ser":10723,"Ġmetab":10724,"Ġopened":10725,"Author":10726,"åħ¼":10727,"æĢ»ç»ĵ":10728,"ãģ£ãģ¦":10729,"æķĪçİĩ":10730,"çħ¤":10731,"stead":10732,"æĮĩåĩº":10733,"ĠWestern":10734,"Ġterrit":10735,"],Ċ":10736,"Ġcin":10737,"åĦ¿ç«¥":10738,"166":10739,"羣æĺ¯":10740,"Ġtrip":10741,"iento":10742,"Ġble":10743,"Ġinjury":10744,"Info":10745,"Ġfacilit":10746,"Ġκα":10747,"меÑĢ":10748,"Pa":10749,"PR":10750,"149":10751,"à¸ł":10752,"IST":10753,"ĠOl":10754,"íĻ":10755,"ä¸ĵå®¶":10756,"æľĢè¿ij":10757,"360":10758,"ï¸ı":10759,"åĩī":10760,"154":10761,"ãĢĭï¼Į":10762,",âĢĻ":10763,"Ġjoin":10764,"146":10765,"Ġì§":10766,"Ġparameter":10767,"æ¯Ķä¾ĭ":10768,"ĠNor":10769,"Ñħоди":10770,"ectors":10771,"Sim":10772,"ï¬":10773,".x":10774,"Ġdalam":10775,"年代":10776,"Ġdepending":10777,"ä¸ĭéĻį":10778,"ìľ¼ë¡ľ":10779,"åĪĨå¸ĥ":10780,"ounce":10781,"Ġleader":10782,"times":10783,"ismo":10784,"Ġexplained":10785,"åĬłå·¥":10786,"身份":10787,"Ġindicate":10788,"eren":10789,"ĠCa":10790,"å°±è¦ģ":10791,"Ġfocused":10792,"ĠIntroduction":10793,"ĠاÙĦÙĥ":10794,"å¾Īå¿«":10795,"Ġmaybe":10796,"å°Ŀ":10797,"áĥIJáĥ":10798,"Ġtechnologies":10799,"akt":10800,"rastructure":10801,"åį³ä½¿":10802,"Ġnone":10803,"åįģäºĮ":10804,"æĪijæĺ¯":10805,"à¸Ńย":10806,"ĠÑįÑĤо":10807,"}{\\":10808,"竣çĦ¶":10809,"ĠاÙĦج":10810,"iano":10811,"Ġvul":10812,"並":10813,"Ġvirtual":10814,"Ġfailed":10815,"ĠPage":10816,"Ġdoctor":10817,"Ġcatal":10818,"ä¹ı":10819,"ìĭľ":10820,"æĿ¥çľĭ":10821,"風":10822,"iated":10823,"amento":10824,"Ġho":10825,"Ñļ":10826,"Ġwerden":10827,"Ġsouth":10828,"anto":10829,"Ġvoc":10830,"оли":10831,"ät":10832,"à¸Ľà¸£à¸°":10833,"æ´¥":10834,"Ġoch":10835,"Ġteach":10836,"Ġcogn":10837,"ока":10838,"å¿ĥä¸Ń":10839,"ygen":10840,"Ġdiseases":10841,"çļĦä¸Ģ个":10842,"èĥĨ":10843,"Ġstated":10844,"Ġsevere":10845,"æ¯į亲":10846,"alah":10847,"å¹´æĿ¥":10848,"欣":10849,"åij½ä»¤":10850,"åĵ²":10851,"Ġзна":10852,"Ġfeedback":10853,"ĠEnergy":10854,"å±ħæ°ij":10855,"æĥľ":10856,"171":10857,"aten":10858,"åħģ":10859,"ĠEss":10860,"æĢ§èĥ½":10861,"157":10862,"ĠclassName":10863,"åĺī":10864,"map":10865,"ĠпеÑĢе":10866,"ioni":10867,"å¡ij":10868,"Ġconcepts":10869,"Ġcomparison":10870,"át":10871,"akers":10872,"覺":10873,"owa":10874,"Ġment":10875,"Ġ);Ċ":10876,"çŃīçŃī":10877,"����":10878,"Ġva":10879,"Book":10880,"Ġfamiliar":10881,"ĠFund":10882,"Ġproperly":10883,"ä½ĵéªĮ":10884,"åIJĮæł·":10885,"ĠBud":10886,"åªĴä½ĵ":10887,"Ġdocuments":10888,"rows":10889,"Ġnos":10890,"ĠاÙĦØ¥":10891,"amples":10892,"é¢Ĺ":10893,"éģĵè·¯":10894,"åıĤèĢĥ":10895,"Ġlock":10896,"äºĭå®ŀ":10897,"ìĤ¬":10898,"ĠMA":10899,"ĠLanguage":10900,"olds":10901,"Ġadults":10902,"两ç§į":10903,"Ġdirector":10904,"Ġsuggests":10905,"Ġiron":10906,"istan":10907,"Ġcommonly":10908,"æĮĩæłĩ":10909,"ì":10910,"button":10911,"ĠìĤ¬":10912,"Ġtip":10913,"Foot":10914,"sin":10915,"åĵ¡":10916,"agen":10917,"Ġprocedures":10918,"Point":10919,"Ġsecure":10920,"Ġvoltage":10921,"rency":10922,"Ġdigits":10923,"æŀª":10924,"172":10925,"Ġrom":10926,"ĠEp":10927,"Ġprobability":10928,".next":10929,"second":10930,"[\"":10931,"Ġapparent":10932,"ĠJun":10933,"å¼ĢæĶ¾":10934,"erse":10935,"è¿IJç͍":10936,"Ġmovie":10937,"Ġatmosp":10938,"faces":10939,"Ġurban":10940,"Ġshot":10941,"åĨĴ":10942,"Ġü":10943,"ĠValue":10944,"ĠPhil":10945,"å¿ł":10946,"owe":10947,"ĠZh":10948,"ï¼ĽĊĊ":10949,"ĠãĢĬ":10950,"åĩºåı£":10951,"Ġadministration":10952,"Ġপà§įর":10953,"æ":10954,"irmed":10955,"Ġformal":10956,"Ġsuggested":10957,"è¿ħéĢŁ":10958,"Ġindu":10959,"åģ·":10960,"Ġgod":10961,"িত":10962,"æ¹¾":10963,"hips":10964,"Ġspend":10965,"åħĴ":10966,"èªŀ":10967,"ĠSpanish":10968,"159":10969,"Ġï¼Ī":10970,"ä¸Ģ天":10971,"namespace":10972,"LO":10973,"ĠBur":10974,"ls":10975,"ει":10976,"Ġresearchers":10977,"åıĺå¾Ĺ":10978,"ĠKore":10979,"Event":10980,"Ġleaving":10981,"Ġà¦ľ":10982,"ĠMean":10983,"Ġintegr":10984,"Ġê³":10985,"PT":10986,"ĠBill":10987,"ĠMax":10988,"Ġfort":10989,"Ġsudden":10990,"Ġul":10991,"åŀĤ":10992,"_S":10993,"Last":10994,"ĠCamp":10995,"350":10996,"Ġsession":10997,"Ġbecoming":10998,"ĠJapanese":10999,"Ġconclusion":11000,"ĠProduct":11001,"Ġell":11002,"Ġcourses":11003,"Ġmarketing":11004,"Ġdoubt":11005,"define":11006,"Ġvaluable":11007,"éĩį大":11008,"Ġec":11009,"第ä¸Ģ次":11010,"ëĿ¼":11011,"ansion":11012,"HA":11013,"Ġagent":11014,"Ġëĭ":11015,"æĦŁæŁĵ":11016,"Det":11017,"à¤ķ":11018,"ĠÑĺ":11019,"ä¿Ħ":11020,"Ġadult":11021,"æ´ª":11022,"骤":11023,"Ġnetworks":11024,"-related":11025,"weight":11026,"Ġrisks":11027,"ία":11028,"angan":11029,"output":11030,"emy":11031,"åľ°ä½į":11032,"ãģ£ãģŁ":11033,"éĥ½åľ¨":11034,"è¡ĮçļĦ":11035,"Ġalle":11036,"SN":11037,"çĵ¶":11038,"ĠKnow":11039,"Ðķ":11040,"ĉint":11041,"éĶģ":11042,"èµ°äºĨ":11043,"ÙĦا":11044,"fa":11045,"Ġtrad":11046,"ĠBest":11047,"ĠSign":11048,"æĪIJæŀľ":11049,"Ġnumerous":11050,"Ġfrag":11051,"Ġwouldn":11052,"度çļĦ":11053,"åıĶ":11054,"Ġconsistent":11055,"Ġconsole":11056,"Ġdivision":11057,"lu":11058,"LC":11059,"ĠIr":11060,"麦":11061,"Ġdisp":11062,"à§ĩন":11063,"Ġsigns":11064,"148":11065,"ä¼ı":11066,"Ne":11067,"身边":11068,"Att":11069,"ĠSolution":11070,"åIJ¬åΰ":11071,"à¥įय":11072,"ĠOh":11073,"çĬ¯ç½ª":11074,"ipedia":11075,"åŃķ":11076,"Ù¾":11077,"Ġot":11078,"spring":11079,"æĭ¼":11080,"Ġми":11081,"${":11082,"edy":11083,"pes":11084,"æľīæīĢ":11085,"设æĸ½":11086,"зÑĥ":11087,"Ġimmun":11088,"ä»Ģ麼":11089,"ĠStudies":11090,"να":11091,"ç²Ĺ":11092,"etch":11093,"æĿ°":11094,"Ġorient":11095,"Ġtables":11096,"æĢ»æĺ¯":11097,"ĠObject":11098,"ĠDev":11099,"Ġcomposition":11100,"////////":11101,"Ġ׼×":11102,"çļĦæĸ°":11103,"Ġtail":11104,"æĦŁåıĹ":11105,"ffee":11106,"abetes":11107,"cluded":11108,"Ġveloc":11109,"ifies":11110,"ĠParis":11111,"åĪĨç±»":11112,"éĻIJåζ":11113,"Ġoblig":11114,"æ»´":11115,"ĠWell":11116,"Ġtrib":11117,"ĠTor":11118,"umbers":11119,"Ġdelivery":11120,"ĠAR":11121,"×ķ׾":11122,"åĽ¢éĺŁ":11123,"Ag":11124,"ĠOR":11125,"rics":11126,"Ġfinished":11127,"Ġsalt":11128,"Ġ׼":11129,"Ġremoved":11130,"ç¨į":11131,"å°ĺ":11132,"ĠCard":11133,"inating":11134,"Ġreducing":11135,"éĥij":11136,"Ġrepresentation":11137,"Stud":11138,"mic":11139,"ãģĤãĤĭ":11140,"hline":11141,"ĠColl":11142,"æĬ¢":11143,"ÑĨиÑı":11144,"Ġfavorite":11145,"ĠOnce":11146,"Ġconflict":11147,"Ġkan":11148,"ascular":11149,"Ġancient":11150,"àµį":11151,"ĠWhich":11152,"æĶ¯ä»ĺ":11153,"204":11154,"ĠControl":11155,"ithub":11156,"è¨Ń":11157,"Ġ£":11158,"ÛĮÙħ":11159,"ttp":11160,"TR":11161,"othe":11162,"æŃ£å¼ı":11163,"ĠMil":11164,"Ġfunds":11165,"æĸ¹ä¾¿":11166,"Ġwire":11167,"Ġmixed":11168,"Response":11169,"asp":11170,"Ġú":11171,"éĶħ":11172,"DE":11173,"ÑİÑĤÑģÑı":11174,"à³įà²":11175,"§×":11176,"âĢĿãĢĤĊĊ":11177,"èħIJ":11178,"Ġdari":11179,"Ġauto":11180,"Ġempty":11181,"ç«Ļåľ¨":11182,"ova":11183,"rec":11184,"167":11185,"iration":11186,"ĠBack":11187,"åĩºä¸Ģ":11188,"Ġtruly":11189,"å®¶éķ¿":11190,"Ġsist":11191,"ÅĤa":11192,"Ġjobs":11193,"åĪĬ":11194,"æĥħ绪":11195,"ç¼ĸè¾ij":11196,"Ġconsumption":11197,"Ġconfidence":11198,"åĬ¨ä½ľ":11199,"Ġreferred":11200,"Ġont":11201,"Ġlibrary":11202,"éĹ²":11203,"ĠTim":11204,"Ġpregn":11205,"èĥİ":11206,"Ġauch":11207,"CI":11208,"ëIJ":11209,"ç²®":11210,"ĠProcess":11211,"Ġhumans":11212,"ษ":11213,"hether":11214,"Fe":11215,"è´¡":11216,"Ġhighlight":11217,"Ġdiagram":11218,"Ġscene":11219,"(l":11220,"Ġkid":11221,"ä½ĵèĤ²":11222,"éļĨ":11223,"çİĦ":11224,"Ġfav":11225,"Ġmeasurement":11226,"ç»Ŀ对":11227,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":11228,"ä¸Ĭè¿°":11229,"ĠSi":11230,"Ġverb":11231,"ĠRussian":11232,"ĠÄį":11233,"Ġcities":11234,"summary":11235,"ä¸į管":11236,"æĻļä¸Ĭ":11237,"orph":11238,"Ġdiscovered":11239,"Ġavec":11240,"ë":11241,"Ġrh":11242,"rant":11243,"Ġappeared":11244,"ções":11245,"ashion":11246,"Ġinvent":11247,"ceived":11248,"158":11249,"Ġsolar":11250,"174":11251,"ĠاÙĦÙģ":11252,"ä¹Łæ²¡æľī":11253,"ĠGrade":11254,"Ġrevealed":11255,"Ġrecip":11256,"ä¼ļ计":11257,"åĮĸåѦ":11258,"anie":11259,"Ġrepresented":11260,"å½¼":11261,"ал":11262,"Ġsoul":11263,"Ġfundamental":11264,"Ġresponsibility":11265,"Ġderiv":11266,"纹":11267,"为主":11268,"-cent":11269,"енÑĮ":11270,"Dis":11271,"表éĿ¢":11272,"Ġcolors":11273,"ç§Ĵ":11274,"Ġfreedom":11275,"race":11276,"æĽ´æĸ°":11277,"ĠCON":11278,"æĿĥåĪ©":11279,"Ġpromote":11280,"çĤ¸":11281,"ĠMinister":11282,"adecimal":11283,"Ġfix":11284,"豪":11285,"Ġans":11286,"ĠØ¥ÙĦÙī":11287,"Ġsugar":11288,"ĠExt":11289,"clusive":11290,"Ġsho":11291,"Ġgoods":11292,"[j":11293,"cin":11294,"Ġpieces":11295,"Ġvirus":11296,"Ġtroy":11297,"ĠPR":11298,"Ġattend":11299,"Ġfilled":11300,"èĤ©":11301,"éĶĭ":11302,"Ġsan":11303,"Ġarms":11304,"Ġsuitable":11305,"Ġjest":11306,"inations":11307,"arian":11308,"ĠBoth":11309,"éĢIJæ¸IJ":11310,"æĤ²":11311,"ÙĪØ³":11312,"..ĊĊ":11313,"Ġscript":11314,"ĠShow":11315,"Ġsmooth":11316,"åı¬å¼Ģ":11317,"LA":11318,"Ġship":11319,"ĠArticle":11320,"stein":11321,"强è°ĥ":11322,"è¾°":11323,"è¿ģ":11324,"Ġinstitution":11325,"Ġzijn":11326,"å°ıåѦ":11327,"Web":11328,"æŃ£æĺ¯":11329,"/**Ċ":11330,"اط":11331,"Ġkit":11332,"ĠOx":11333,"ÃŃt":11334,"éĤ®":11335,"Ġupdated":11336,"ĠStart":11337,"ĠMedical":11338,"Ġgender":11339,"Co":11340,"Ġ׾":11341,"230":11342,"ĠWhere":11343,"Ġstuff":11344,"çĮĽ":11345,"Ġë³":11346,"Ġdetection":11347,"Ġdefine":11348,"Ġestimated":11349,"Ġsweet":11350,"erd":11351,"sm":11352,"Ġintended":11353,"icol":11354,"¬":11355,"valid":11356,"说è¯Ŀ":11357,"ĠAN":11358,"Ġspecifically":11359,"åIJIJ":11360,"inity":11361,"Ġminim":11362,"ĠFrank":11363,"à¥įर":11364,"ĠاÙĦØ´":11365,"磩":11366,"Ġpure":11367,"Ġdrugs":11368,"Ġwinter":11369,"/S":11370,"ç¥ŀç»ı":11371,"Ġadj":11372,"ĠاÙĦد":11373,"Button":11374,"web":11375,"gency":11376,"è¯ķéªĮ":11377,"ometry":11378,"ĠAbstract":11379,"oses":11380,"Ġherself":11381,"-the":11382,"ĠPoint":11383,"稿":11384,"Ġlived":11385,"tex":11386,"\">=":11620,"]]":11621,"Ġphenomen":11622,"Ġcoe":11623,"Ġmargin":11624,"Ġapart":11625,"igu":11626,"Ġoxygen":11627,"Vol":11628,"éħįåIJĪ":11629,"Ġholding":11630,"ĠMount":11631,"Ġresponses":11632,"çı¾åľ¨":11633,"à¥ģ":11634,"Ġcontinuous":11635,"大ä¼ļ":11636,"èĻķ":11637,"ĠÐķ":11638,"ä¸Ģè¾¹":11639,"Ġcategory":11640,"Ġaz":11641,"ç¡®å®ŀ":11642,"æĶ¹åĸĦ":11643,"Ġextremely":11644,"å̼å¾Ĺ":11645,"su":11646,"ç»´æĬ¤":11647,"di":11648,"ĠInf":11649,"Click":11650,"Ġreco":11651,"ìĹIJìĦľ":11652,"çĵ¦":11653,"Ġwitness":11654,"######":11655,"ĠDisc":11656,"host":11657,"!âĢĿĊĊ":11658,"cules":11659,"Ġnit":11660,"cluding":11661,"èĥ½æºIJ":11662,"lements":11663,"ÙħÙĪ":11664,"unte":11665,"gypt":11666,"uke":11667,"ĠNatural":11668,"åħ´è¶£":11669,"Ġfruit":11670,"à¦Ĥ":11671,"abe":11672,"ployment":11673,"zing":11674,"case":11675,"010":11676,"ĠDu":11677,"ãĢģãĢĬ":11678,"åIJĮå¿Ĺ":11679,"Ġbinary":11680,"Ġforg":11681,"rich":11682,"èªį":11683,"plete":11684,"è¤":11685,"/f":11686,"Why":11687,"ĠCamb":11688,"antly":11689,"ourse":11690,"åĨľæ°ij":11691,"ĠУ":11692,"Ġran":11693,"Ġbattle":11694,"Ġkill":11695,"ÑĪа":11696,"ĠHy":11697,"UE":11698,"(":13461,"Su":13462,"ç¼´":13463,"Ġ×Ĵ":13464,"боÑĤ":13465,"åĤħ":13466,"_M":13467,"ä¸īåįģ":13468,"Ġamounts":13469,"æĹ¢çĦ¶":13470,"-J":13471,"æĮĸ":13472,"Ġvelocity":13473,"Ġmarriage":13474,"ĠThree":13475,"206":13476,"ĠStrateg":13477,"Ġclin":13478,"æŁIJäºĽ":13479,"Ġfinish":13480,"ifferent":13481,"Ġthanks":13482,"emia":13483,"ican":13484,"æ²ŁéĢļ":13485,"hered":13486,"umes":13487,"è§ĤçĤ¹":13488,"208":13489,"amil":13490,"çļĦåŁºæľ¬":13491,"annels":13492,"ĠComments":13493,"ä»ĸåľ¨":13494,"ĠÑģÑĤановниÑĪÑĤво":13495,"ĠFIG":13496,"è¡°":13497,"ribute":13498,"ostic":13499,"rij":13500,"ometimes":13501,"åIJ¯åĬ¨":13502,"AI":13503,"eps":13504,"imensional":13505,"339":13506,"Ġsitting":13507,"SO":13508,"éĽĨä½ĵ":13509,"**,":13510,"Dec":13511,"Ġspending":13512,"azine":13513,"ä¸Ģ声":13514,"-res":13515,"Ġpy":13516,"Ġgift":13517,"å·®å¼Ĥ":13518,"Ġacts":13519,"Ñĺа":13520,"ĠReal":13521,"èŀº":13522,"Ġexperts":13523,"hu":13524,"Ġmeters":13525,"Ġanxiety":13526,"Ġpresentation":13527,"Ġincreasingly":13528,"大éĥ¨åĪĨ":13529,"van":13530,"¡°":13531,"aze":13532,"Ġdefinitely":13533,"IPT":13534,"åIJįç§°":13535,"Ġauthent":13536,"-qu":13537,"天ä¸ĭ":13538,"Ġinflamm":13539,"Ġplanet":13540,"Ġdress":13541,"Fin":13542,"}=\\":13543,"åĪļåĪļ":13544,"unci":13545,"çµĦ":13546,"ĠHill":13547,"ĠInstead":13548,"Well":13549,"aki":13550,"æİ¥çĿĢ":13551,"æĥħåĨµä¸ĭ":13552,"Ġmoder":13553,"åıĺæĪIJ":13554,"260":13555,"Ġfashion":13556,"\\[\\":13557,"ĠAL":13558,"ĠFile":13559,"Integer":13560,"çŀ¬éĹ´":13561,"DO":13562,"Ġfest":13563,"Ġspaces":13564,"ailability":13565,"Ġroots":13566,"åľ¨æŃ¤":13567,"éĥİ":13568,"imp":13569,"æ¸ł":13570,"ĠRelated":13571,"riter":13572,"ekt":13573,"quir":13574,"ìĨ":13575,"ä»¶äºĭ":13576,"Ġwalked":13577,"åΰçļĦ":13578,"ĠChicago":13579,"/w":13580,"Ġsight":13581,"-V":13582,"Ġentirely":13583,"ĠÐŃ":13584,"Ġsuc":13585,"ĠCase":13586,"270":13587,"({":13588,"é©»":13589,"opp":13590,"ĠApplication":13591,"creen":13592,"enza":13593,"åħ¨ä½ĵ":13594,"اض":13595,"ży":13596,"åįıä¼ļ":13597,"Ġnach":13598,"Ġneighbor":13599,"èī¯å¥½çļĦ":13600,"ÅĤy":13601,"ĠFre":13602,"eper":13603,"馬":13604,"ĠTerm":13605,"Ġimpossible":13606,"Ġquery":13607,"ĠRights":13608,"หà¸Ļ":13609,"ĠOffic":13610,"Ġmir":13611,"âĪļ":13612,"Spec":13613,"ç¨Ģ":13614,"Ġstructural":13615,"Ġfert":13616,"onald":13617,"è¶ĭåĬ¿":13618,"ãĥŃ":13619,"éģĭ":13620,"Ġreligion":13621,"Ġtoler":13622,"å½¹":13623,"Ġadvoc":13624,"æīĢè¿°":13625,"lik":13626,"aset":13627,"upy":13628,"Ġremote":13629,"idos":13630,"Ġemployed":13631,"Ġjudgment":13632,"Ġdivide":13633,"eties":13634,"224":13635,"213":13636,"idents":13637,"(k":13638,"Ġya":13639,"iar":13640,"ëŁ":13641,"Ġjoy":13642,"ĠMov":13643,"normal":13644,"/M":13645,"Ġoccasion":13646,"åŁºå±Ĥ":13647,"real":13648,"Ġcontrols":13649,"æķ´çIJĨ":13650,"Ġabsence":13651,"plant":13652,"ä¸Ĭä¸ĭ":13653,"Ġwalls":13654,"arest":13655,"Ġstages":13656,"ĠCP":13657,"ĠAnswers":13658,"æĪijåľ¨":13659,"论æĸĩ":13660,"åı¯è§ģ":13661,".y":13662,"CN":13663,"Ġtalent":13664,"ajÄħ":13665,"ูà¹ī":13666,"inu":13667,"æ¾³":13668,"ä¹Łè¦ģ":13669,"Hz":13670,"Ġvillage":13671,"Ġsummary":13672,"æµĻ":13673,"ĠVirgin":13674,"èĩĤ":13675,"èĤ¡ç¥¨":13676,"Ġmedicine":13677,"Ġcycl":13678,"bur":13679,"Ġscope":13680,"车è¾Ĩ":13681,"æīįæĺ¯":13682,"Ġfulf":13683,"enta":13684,"ĠWal":13685,"Ġarrest":13686,"Ġwonderful":13687,"Ġintervention":13688,"Ġbehaviour":13689,"ĠFamily":13690,"Ġstake":13691,"(\"%":13692,"ship":13693,"Ġsurrounding":13694,"ahr":13695,"Ġmaintaining":13696,"-free":13697,"icken":13698,"Ġwird":13699,"ĠMur":13700,"åŃŁ":13701,"rowser":13702,"onom":13703,"Ġhur":13704,"Ġtradition":13705,"াম":13706,"Ġcognitive":13707,"åĽĽä¸ª":13708,"åı¯èĥ½ä¼ļ":13709,"è£ħç½®":13710,"çŃĭ":13711,"Ġreviews":13712,"Ġcompounds":13713,"ĠÂł":13714,"ĠJohnson":13715,"Ġunderlying":13716,"Ġfan":13717,"Another":13718,"oin":13719,"主è¦ģæĺ¯":13720,"Ġarrived":13721,"è¿ĩçļĦ":13722,"221":13723,"ĠDoes":13724,"ç´łè´¨":13725,"è¡ĵ":13726,"aman":13727,"ĠJac":13728,"riers":13729,"欢è¿İ":13730,"ĠEX":13731,"Ġreasonable":13732,"ograf":13733,"Ġgran":13734,"Ġshares":13735,"ometer":13736,"ĠGeorg":13737,"Ġturns":13738,"ĠChange":13739,"apped":13740,"注åĨĮ":13741,"Ġconfirm":13742,"cosystem":13743,"Ġfabric":13744,"-cont":13745,"Ġsensor":13746,"\"]":13747,"Prov":13748,"amber":13749,"Ġsections":13750,"ogene":13751,"ÑĤÑģÑı":13752,"çķħ":13753,"Ġstone":13754,"ALL":13755,"zn":13756,"Ġdogs":13757,"обÑĭ":13758,"Ġcategories":13759,"æĬ¥éģĵ":13760,"ãģ§ãģ¯":13761,"áĥĶáĥ":13762,"â̦â̦ĊĊ":13763,"çIJĨ念":13764,"Ġfasc":13765,"åħ¬å®ī":13766,"çĽij管":13767,"ele":13768,"Ġbrown":13769,"ptr":13770,"Ġ['":13771,"entes":13772,"hematical":13773,"ĠRock":13774,"fit":13775,"å§¿":13776,"Ġny":13777,"Ġinterval":13778,"æĥ³æ³ķ":13779,"ê²Į":13780,"ĠLink":13781,"Ġguidelines":13782,"ä¸įæķ¢":13783,"liament":13784,"Ġüber":13785,"è½ī":13786,"åIJ¸æĶ¶":13787,"ĠSoft":13788,"绿èī²":13789,"нÑĭми":13790,"Ġcomposite":13791,"Ġcommitted":13792,"Ġjun":13793,"cs":13794,"ĠCreative":13795,"Ġtwice":13796,"ëª":13797,"ä¸Ī":13798,"ĠHenry":13799,"Ġbuildings":13800,"Ġatmosphere":13801,"åĬ¨æľº":13802,"ĠболÑĮ":13803,"ä¸Ģæĺ¯":13804,"ÑĩеÑĤ":13805,"Ġpump":13806,"ivated":13807,"ç³»ç»ŁçļĦ":13808,"Ġstands":13809,"ĠLabor":13810,"Ġbinding":13811,"Content":13812,"ä¹Ķ":13813,"سÛĮ":13814,"Ġfant":13815,"Ġadalah":13816,"ibli":13817,"Ġjoined":13818,"å®ħ":13819,"大æ¦Ĥ":13820,"Ġhun":13821,"216":13822,"ĠWay":13823,"åħĦå¼Ł":13824,"åħģ许":13825,"ç¾İçļĦ":13826,"å²Ĺä½į":13827,"Ġnam":13828,"ĠFacebook":13829,"ãĢĭãĢĬ":13830,"Ġfootball":13831,"å±Ī":13832,"Ġtaught":13833,"Client":13834,"quad":13835,"AY":13836,"______":13837,"Ġvertical":13838,"ĠClub":13839,"Ġ?ĊĊ":13840,"Ġmarried":13841,"贯彻":13842,"++;Ċ":13843,"åħ¬å¼ı":13844,"ĠFollow":13845,"Ġgrav":13846,"Rem":13847,"Ġtemperatures":13848,"è§£æŀIJ":13849,"æ·»åĬł":13850,"ĠLatin":13851,"Ġdiscrim":13852,"éĤĢ":13853,"å§ĭç»Ī":13854,"éͦ":13855,"ĠÙĩاÛĮ":13856,"ĠChen":13857,"Ġsuit":13858,"Ġvehicles":13859,"ĠFlorida":13860,"Ġtod":13861,"ника":13862,"ник":13863,"ĠFort":13864,"Ġtur":13865,"ĠÙģÙī":13866,"MC":13867,"Ġofficer":13868,"é̼":13869,"cie":13870,"utch":13871,"Ġeditor":13872,"ĠTax":13873,"Ġhex":13874,"å°ıç»Ħ":13875,"elen":13876,"ĠEmp":13877,"ucky":13878,"Ġradiation":13879,"æľºåύ":13880,"æĸ¹ç¨ĭ":13881,"Ġvariation":13882,"اÙĭ":13883,"ár":13884,"Ġentertain":13885,"Ġpolitics":13886,"ĠProm":13887,"許":13888,"çݯèĬĤ":13889,"ĠItalian":13890,"Ġmanifest":13891,"æ¯ıä¸Ģ":13892,"Ġaxis":13893,"apa":13894,"让ä»ĸ":13895,"ĠWords":13896,"广大":13897,"/g":13898,"Ġspl":13899,"Ġlargely":13900,"Select":13901,"Pe":13902,"utely":13903,"ventions":13904,"åĸ®":13905,"ÙĪØ¬":13906,"çİ»":13907,"æĹ¥åŃIJ":13908,"NO":13909,"cipe":13910,"çĤī":13911,"*x":13912,"aga":13913,"تب":13914,"Ġpicked":13915,"ĠWikipedia":13916,"çīµ":13917,"Chapter":13918,"Ġencourage":13919,"End":13920,"à¸ĭ":13921,"Ġcausing":13922,"215":13923,"ĠMuseum":13924,"Ġspin":13925,"RS":13926,"Ġenerg":13927,"ĠSup":13928,"Ġfro":13929,"hist":13930,"ĠMS":13931,"à¦ľ":13932,"select":13933,"rium":13934,"oro":13935,"ĠSar":13936,"é¡ŀ":13937,"è´§å¸ģ":13938,"Ġsister":13939,"ĠAnnual":13940,"ĠFactor":13941,"Since":13942,"ä¸īè§Ĵ":13943,"ологи":13944,"imeters":13945,"Ġnel":13946,"ipher":13947,"tery":13948,"ĉĉĉĉĉ":13949,"åģļäºĨ":13950,"ĠÏĦοÏħ":13951,"çĤ®":13952,"ups":13953,"ç©·":13954,"ĠOxford":13955,"Ġlies":13956,"Ġdisorder":13957,"çľ¼åīį":13958,"209":13959,"Ġ'./":13960,"Ġsixty":13961,"%ãĢĤ":13962,"rapeut":13963,"çĨŁæĤī":13964,"Ġye":13965,"ĠOrder":13966,"irus":13967,"ï¼īãĢģ":13968,"KE":13969,"æĸ¤":13970,"rod":13971,"ĠAtl":13972,"Ġarchitecture":13973,"Ġstrategic":13974,"rary":13975,"Ġillness":13976,"Ġimmune":13977,"Expl":13978,"vertisement":13979,"Ġà¦Ĺ":13980,"Ġexception":13981,"tical":13982,"ĠRom":13983,"èĢĮä¸į":13984,"åĽŀå®¶":13985,"ç¼Ŀ":13986,"Ġflood":13987,"ande":13988,"":14620,"Ġgun":14621,"ĠоÑĢ":14622,"Module":14623,"upyter":14624,"×Ļ×ĵ":14625,"Ġcandidate":14626,"Ġmga":14627,"Ġposts":14628,"Ġusage":14629,"ä»ĸ说":14630,"åľ¨äºĨ":14631,"217":14632,"éĴ»":14633,"tan":14634,"Ġimplications":14635,"Ġholid":14636,"chem":14637,"iko":14638,"252":14639,"ĠKen":14640,"ERE":14641,"ĠBritain":14642,"æµģåĬ¨":14643,"RNA":14644,"ĠStandard":14645,"holder":14646,"Ġrapidly":14647,"çļĦæĥħ":14648,"Ġvolunte":14649,"ĠWhether":14650,"aylor":14651,"åIJĮä¸Ģ":14652,"Ġorders":14653,"åĤ¬":14654,"Ġво":14655,"åıĺéĩı":14656,"Ġseventy":14657,"ĠCancer":14658,"Ġsono":14659,"vere":14660,"Ġmechanical":14661,"ãģ°":14662,"ìĥģ":14663,"Ġbanks":14664,"ĠLearn":14665,"310":14666,"ĠLicense":14667,"Link":14668,"Ġweekend":14669,"åĿĽ":14670,"OU":14671,"Ġcyt":14672,"ĠFac":14673,"Ġmurder":14674,"ĠHim":14675,"ĠRest":14676,"Ġassembly":14677,"éĢī项":14678,"æľ¬æĸĩ":14679,"udi":14680,"Ġdangerous":14681,"éĴ®":14682,"èĦ¾":14683,"æĪĴ":14684,"女åŃ©":14685,"ologist":14686,"ighter":14687,"etics":14688,"ĠCurrent":14689,"éĨī":14690,"ellen":14691,"Ġflowers":14692,"éģĶ":14693,"Ġpulled":14694,"лÑĭ":14695,"yer":14696,"Ġfractions":14697,"শ":14698,"æĢ¨":14699,"Ġquantity":14700,"Ġinvestors":14701,"ĠWorksheet":14702,"Ġinterpretation":14703,"ãĢĤ#":14704,"Ġmodified":14705,"\\in":14706,"base":14707,"è°ĥèĬĤ":14708,"ĠVe":14709,"war":14710,"Ġvor":14711,"Ġmoves":14712,"åĩ¯":14713,"esse":14714,"chron":14715,"conds":14716,"Ġdirected":14717,"éĢĻæ¨£":14718,"зова":14719,"çĤ¹åĩ»":14720,"çļĦåĨħ容":14721,"ĠSep":14722,"ĠSeries":14723,"(v":14724,"Ġtall":14725,".'":14726,"å±Ĥ次":14727,"expected":14728,"Ġao":14729,"emos":14730,"rine":14731,"æ¯ģ":14732,"ĠComputer":14733,"lib":14734,"<":15073,"Ġsubstance":15074,"Ġextreme":15075,"ottom":15076,"bro":15077,"åŃ©åŃIJ们":15078,"Group":15079,"Ġliver":15080,"è´Łè´£äºº":15081,"charge":15082,"çİ»çĴĥ":15083,"Ġnecessarily":15084,"äºı":15085,"æĤ¦":15086,"对è¯Ŀ":15087,"Ġtout":15088,"Ġabstract":15089,"Ġ...Ċ":15090,"ä¸ĵéŨ":15091,"Ġfing":15092,"_to":15093,"оÑĢо":15094,"æĪIJçĨŁ":15095,"Ġlang":15096,"ĠStar":15097,"750":15098,"ĠSaint":15099,"Category":15100,".K":15101,"è¿IJèIJ¥":15102,"乡æĿij":15103,"Ġgent":15104,"ä¸Ĭåįĩ":15105,"Ġwest":15106,"Ġο":15107,"nis":15108,"åĵ²åѦ":15109,"Ġwaves":15110,"Ġexamine":15111,"Ġfurn":15112,"idi":15113,"Ġclot":15114,".pro":15115,")/(":15116,"mediate":15117,"Ġëı":15118,"à¸ķร":15119,"çŃīäºİ":15120,"çļĦæĥħåĨµä¸ĭ":15121,"undo":15122,"prene":15123,"ãĤ¸":15124,"Ang":15125,"Ġдол":15126,"Ġgeb":15127,"ä»¿ä½Ľ":15128,"Ġartists":15129,"bed":15130,"Ġtea":15131,"Ġsuperior":15132,"éĥ¨ç½²":15133,"æĹ¨":15134,".js":15135,"_type":15136,"æĹ¶åĪ»":15137,"(y":15138,"ĠPak":15139,"azz":15140,"Ġecosystem":15141,"Ġchest":15142,"ĠBell":15143,"Ġsul":15144,"ÙĦاÙħ":15145,"Ġthorough":15146,"å·ŀå¸Ĥ":15147,"({Ċ":15148,"è¡¥åħħ":15149,"æıIJ示":15150,"Ġowners":15151,"Ġpes":15152,"iatric":15153,"饱":15154,"Ġ×§":15155,"Ġurl":15156,"Ġspr":15157,"é¢Ħæµĭ":15158,"å±±ä¸ľ":15159,"åĩºåıij":15160,"èŀįåIJĪ":15161,"Ġtent":15162,"Ġdelivered":15163,"ÙĦØ©":15164,"unit":15165,"ä¸ĢåIJį":15166,"Est":15167,"ĠCup":15168,"ĠEt":15169,"Ġcreates":15170,"-two":15171,"Dep":15172,"ahl":15173,"Ġsentences":15174,"Ġbare":15175,"Ġlift":15176,"ĠAk":15177,"说æĺ¯":15178,"Ġhung":15179,"Ġlung":15180,"è·ŁçĿĢ":15181,"寻æī¾":15182,"ãģ¨ãģĦãģĨ":15183,"ĠSource":15184,"ĠâĹ":15185,"éĽ¾":15186,"Ġstrongly":15187,"ĠPL":15188,"èijī":15189,"();Ċ":18798,"help":18799,"Ġprediction":18800,"æķħéļľ":18801,"éĽĻ":18802,"à¹ģà¸ķ":18803,"æĸĩæľ¬":18804,"Ġassessed":18805,"èµĶåģ¿":18806,"Ġleur":18807,"Command":18808,"εν":18809,"ĠMechan":18810,"Ġske":18811,"íļĮ":18812,"大åĬĽ":18813,"Ġstreets":18814,"bul":18815,"Ġej":18816,"UTC":18817,"åĴĸåķ¡":18818,"å¸Ĥæ°ij":18819,"失åİ»":18820,"éĵģè·¯":18821,"332":18822,"amination":18823,"ĠBrad":18824,"ĠExerc":18825,"||":18826,"ãĤīãĤĮ":18827,"admin":18828,"sey":18829,"×ķ×ŀ×":18830,"é»Ħéĩij":18831,"\\begin":18832,"antee":18833,"dir":18834,"Ġresc":18835,"Ġmedication":18836,"ĠMaria":18837,"è£ħå¤ĩ":18838,"åĽ¾å½¢":18839,"é¢Ħ计":18840,"Ġapparently":18841,"Ġcompat":18842,"echo":18843,"ĠJacob":18844,"Ġexempl":18845,"Ġgenu":18846,"ç»ĨèĬĤ":18847,"Ġsurvive":18848,"åĢ¡":18849,"Because":18850,"äºŃ":18851,"ĠÙĨÛĮ":18852,"ä¼Ĺ人":18853,"Ġmile":18854,"åľ°ä¸Ĭ":18855,"åĨ¯":18856,"Ġmild":18857,"}}\\)":18858,"éļĻ":18859,"mitt":18860,"ĠPakistan":18861,"319":18862,"Car":18863,"GS":18864,"вÑĥ":18865,"ষ":18866,"Ġuncertainty":18867,"æŃ¦åύ":18868,"ä¸Ļ":18869,"çļĦ身":18870,"Ġempower":18871,"emo":18872,"ìĨĮ":18873,"åįĥä¸ĩ":18874,"ĠFair":18875,"娱ä¹IJ":18876,"Ġmodeling":18877,"à¹Īà¸Ńà¸ĩ":18878,"ĠبÙĬÙĨ":18879,"athered":18880,"ĠHarry":18881,"Ġcleaning":18882,"åĪĨ离":18883,"ãĤµ":18884,"Ġadjacent":18885,"å®ĹæķĻ":18886,"åħĶ":18887,"Ġpredicted":18888,"ĠобÑĬ":18889,"à§ģর":18890,"ById":18891,"bn":18892,"丸":18893,"294":18894,"Sw":18895,"Ġstakeholders":18896,"Ġ׾×IJ":18897,"Ġremarkable":18898,"Ġdrivers":18899,"æ£ĭ":18900,"ä¿ĿçķĻ":18901,"Ġidentical":18902,"è´¾":18903,"ĠCapital":18904,"çŃīæĸ¹éĿ¢":18905,"Ö°":18906,"achers":18907,"Ġtriangle":18908,"bial":18909,".Collections":18910,"About":18911,"æĪijä¼ļ":18912,"ĠMess":18913,"ovascular":18914,"åºĶ该æĺ¯":18915,"NE":18916,"åį¿":18917,"quiry":18918,"Ġdenominator":18919,"Ġpitch":18920,"Types":18921,"Ġsubmit":18922,"Ġanche":18923,"ĠÑĨе":18924,"Ġlingu":18925,"ä¸ĭè½½":18926,"׼":18927,"ï¼IJ":18928,"è½°":18929,"336":18930,"Ġannounce":18931,"/ml":18932,"åıĥ":18933,"Ġintense":18934,"~~~~":18935,"anol":18936,"说æ³ķ":18937,"æ°´çļĦ":18938,"åħ³èĬĤ":18939,"ế":18940,"ĠNig":18941,"æ°¢":18942,"Ġpreferences":18943,"åĪłéϤ":18944,"ĠKong":18945,"å§Ķæīĺ":18946,"ifting":18947,"Look":18948,"ÛĴ":18949,"FO":18950,"Ġinstances":18951,"ĠRepresent":18952,"Ġpadding":18953,"Ġkeeps":18954,"Ġshadow":18955,"é»ijèī²":18956,"ĠExperience":18957,"Ġdeux":18958,"ĠSud":18959,"-dr":18960,"ého":18961,"ä¹Łä¸įä¼ļ":18962,"Ġcot":18963,"æĬķèµĦèĢħ":18964,"àº":18965,"444":18966,"Ġcompletion":18967,"å¯ĨåĪĩ":18968,"asm":18969,"auc":18970,"纽":18971,"ĠSyn":18972,"éħ·":18973,"Ġбез":18974,"æĶ¶èİ·":18975,"Ġflexibility":18976,"cr":18977,"Ġprofound":18978,"оÑģоб":18979,"ÄŁ":18980,"究竣":18981,"ĠPerformance":18982,"é¡¿æĹ¶":18983,"adores":18984,"berry":18985,"Card":18986,"lined":18987,"had":18988,"***":18989,"+,":18990,".Text":18991,"Ġbeach":18992,"Ġtalked":18993,"å®¡æł¸":18994,"Ġterritory":18995,"ãģ¦ãģĦãģ¾ãģĻ":18996,"ĠбÑĥд":18997,"Ġowned":18998,"Ġzm":18999,"Ġfruits":19000,"Ġconform":19001,"对æīĭ":19002,"Si":19003,"ÙĬع":19004,"Ġ×ij×ŀ×":19005,"{ĊĊ":19006,"éĮ¢":19007,"Ġhay":19008,"erts":19009,"TE":19010,"Ġдан":19011,"éĢĶå¾Ħ":19012,"queue":19013,"_R":19014,"éĹ·":19015,"èľĤ":19016,"æķĪåºĶ":19017,"ĠKeep":19018,"Ġmamm":19019,"Ġпоз":19020,"iop":19021,"Ġbalanced":19022,"ĠInterest":19023,"羸":19024,"åIJķ":19025,"Ġnumerical":19026,"status":19027,"ĠVideo":19028,"éŃħ":19029,"isl":19030,"nego":19031,"ÙĪØ´":19032,"ceedings":19033,"essa":19034,"isons":19035,"ĠRog":19036,"510":19037,"åīįçļĦ":19038,"åľŁå£¤":19039,"ĠOhio":19040,"Ġprecise":19041,"=-":19042,"Ġdoors":19043,"Ġliterary":19044,"/pro":19045,"Ġlabour":19046,"ĠSW":19047,"rz":19048,"Ġvie":19049,"оми":19050,"èĩ³ä»Ĭ":19051,"Ident":19052,"_text":19053,"Ġsorry":19054,"ĠChemistry":19055,"Ġearned":19056,"à´¿":19057,"Ġsampling":19058,"chers":19059,"local":19060,"æ°ĶçļĦ":19061,"éĸ¢":19062,"áĥĿ":19063,"åį³å°Ĩ":19064,"Factory":19065,"random":19066,"Ġdocumentation":19067,"Ùħع":19068,"Ġzo":19069,"ĠSever":19070,"Ġprecip":19071,"Av":19072,"Ġimprovements":19073,"åĭ¢":19074,"Ġtrick":19075,"Ġsees":19076,"ĠArk":19077,"(w":19078,"Ġë³´":19079,"Ġillustrated":19080,"ä¸ĢæĬĬ":19081,"å®ł":19082,"318":19083,"thew":19084,"ึà¸ĩ":19085,"_file":19086,"带é¢Ĩ":19087,"Ġindependence":19088,"aban":19089,"Ġتر":19090,"apit":19091,"acon":19092,"009":19093,"Math":19094,"ĠGram":19095,"ĠEffect":19096,"501":19097,"æķĻæĿIJ":19098,"Ġlag":19099,"onymous":19100,"å®¶ä¼Ļ":19101,"Ġconfident":19102,"çĭ±":19103,"Ġnerve":19104,"ĠRub":19105,"Ġpoetry":19106,"Ġoverride":19107,"çķ°":19108,"ĠOrganization":19109,"kk":19110,"人ä½ĵ":19111,"Ġkiss":19112,"Ġembodiments":19113,"ushing":19114,"Ġcosine":19115,"-class":19116,"ĠCourse":19117,"Ġduties":19118,"Ġhosp":19119,"ĠAsh":19120,"åĬ«":19121,"angang":19122,"ζ":19123,"390":19124,"Ġlegit":19125,"çĢ":19126,"åĦĴ":19127,"å®Ł":19128,"using":19129,"Ġдолж":19130,"Ġseriously":19131,"Ġelif":19132,"atura":19133,"split":19134,"çĸ¼çĹĽ":19135,"ĠCountry":19136,"advant":19137,"ä¸ĵé¢ĺ":19138,"hort":19139,"Ġtact":19140,"è¿Ļå°±":19141,"Ġconfigured":19142,"åĬŁçİĩ":19143,"Ġlearners":19144,"ìļ°":19145,"esome":19146,"Ġsab":19147,"ishop":19148,"Ġrestrictions":19149,"Ġreflected":19150,"ĠÑĩелов":19151,"Ġbroke":19152,"æĿĥåĬĽ":19153,"ç¿ł":19154,"Ġби":19155,"BP":19156,"Ġimplementing":19157,"Ġunw":19158,"Ġpup":19159,"ĠFinal":19160,"å½¼æŃ¤":19161,"ĠHR":19162,"梦æĥ³":19163,"енÑĤа":19164,"Ġig":19165,"Ġbelieves":19166,"anes":19167,"äºĨåIJ§":19168,"Ġrapp":19169,"ĠPriv":19170,"Ġtap":19171,"ูà¹Ī":19172,"req":19173,"ĠSin":19174,"PDF":19175,"ثر":19176,"(h":19177,"妻åŃIJ":19178,"åIJĪæ³ķ":19179,"缴æĴŃ":19180,"Ġscreening":19181,"æĬ¥åIJį":19182,"UC":19183,"å¾Īå¤ļ人":19184,"ĠEC":19185,"Row":19186,"ä½łè¯´":19187,"ĠHong":19188,"åζçļĦ":19189,"äst":19190,"ога":19191,"èģ¯":19192,"缸åħ³çļĦ":19193,"Ġheating":19194,"Ġownership":19195,"Ġads":19196,"Ġrelating":19197,"iously":19198,"thur":19199,"æŀľçĦ¶":19200,"ä¸ĺ":19201,"Ġitalic":19202,"ogl":19203,"335":19204,"letter":19205,"åĩºçİ°åľ¨":19206,"ĠCos":19207,"æĿĥçĽĬ":19208,"Ġsustainability":19209,"ortion":19210,"åīµ":19211,"Ġâī¤":19212,"Ġutility":19213,"Ġequilibrium":19214,"éĮ¯":19215,"Ġvý":19216,"ĠÎķ":19217,"ĠкоÑĢ":19218,"Ġthemes":19219,"èĻŁ":19220,"æĭħä»»":19221,"328":19222,"Ġdot":19223,"æĢİ麼":19224,"Ġendl":19225,"æµ·æ´ĭ":19226,"ück":19227,"ê±":19228,"ĠEsp":19229,"ĠNorm":19230,"Ġperceived":19231,"å¤ĸåĽ½":19232,"Ġloading":19233,"Ġheaven":19234,"üh":19235,"¤×¨":19236,"ĠPersonal":19237,"ĠÐĹа":19238,"ĠVi":19239,"ettings":19240,"imi":19241,"Ġagriculture":19242,"缸åıį":19243,"çŃĴ":19244,"à¹Ĥà¸Ķ":19245,"оÑĩ":19246,"Ġud":19247,"Using":19248,"çłĸ":19249,"×ķ׾×":19250,"å¤©åľ°":19251,"毫æĹł":19252,"è¿Ļä¸ĢçĤ¹":19253,"ĠElizabeth":19254,"ள":19255,"éģ©":19256,"ften":19257,"û":19258,"Ġgut":19259,"Ġsatell":19260,"èĢĥçĶŁ":19261,"mentation":19262,"éĻĮ":19263,"Ġholiday":19264,"Ġdevelopers":19265,"ureau":19266,"about":19267,"Ġdrain":19268,"ĠEL":19269,"Ġsão":19270,"_back":19271,"åĸ»":19272,"çļĦåŁºç¡Ģä¸Ĭ":19273,"ILL":19274,"Ġ________":19275,"Ġtomorrow":19276,"åĽ½æľī":19277,"ebut":19278,"ä¼ĺè´¨":19279,"Ġnad":19280,"ija":19281,"ĠTa":19282,"indi":19283,"à¦¾à§Ł":19284,"Ġpathway":19285,"Ġartik":19286,"ï¼ħ":19287,"åīįå¾Ģ":19288,"Ġuno":19289,"Ġincent":19290,"ĠWithout":19291,"æļij":19292,"Ġvictims":19293,"ĠProp":19294,"Ġpad":19295,"åłµ":19296,"åıijçļĦ":19297,"Ġwithdraw":19298,"Ġfait":19299,"Ùħد":19300,"Ġvotes":19301,"ĠCloud":19302,"eneration":19303,"èĩªä¿¡":19304,"Ġsomewhere":19305,",n":19306,"Ġleague":19307,"Ġvocabulary":19308,"Instance":19309,"Ġdisrupt":19310,"·ĊĊ":19311,"åĮĨ":19312,"Loading":19313,"(_":19314,"使åij½":19315,"Ġstomach":19316,"ç·Ĭ":19317,"BM":19318,"eny":19319,"ĠCC":19320,"Ġinterventions":19321,"ĠArm":19322,"Unit":19323,"æĺĤ":19324,"èĬ½":19325,"ĠSales":19326,"Ġregime":19327,"Ġwider":19328,"ril":19329,"ĠUnd":19330,"ä½ĵä¼ļ":19331,"Ø£ÙĨ":19332,"VC":19333,"ĠInput":19334,"asarangang":19335,"cyclop":19336,"åħ±äº§åħļ":19337,"Ġapplies":19338,"ĠGames":19339,"åĴĮå¹³":19340,"巨大":19341,"Section":19342,"ĠIraq":19343,"ä¸ĸçķĮä¸Ĭ":19344,"Ġdeploy":19345,"Ġhi":19346,"aly":19347,"Ġtodo":19348,"ĠÑģлÑĥÑĩа":19349,"ĠThird":19350,"ĠMoh":19351,"327":19352,"Ġtherapeutic":19353,"ĠGa":19354,"403":19355,"ĠPolice":19356,"352":19357,"精彩":19358,"èĶ¡":19359,"Ġgradually":19360,"Nov":19361,"Ġadoption":19362,"Ġprosec":19363,"å·¥åİĤ":19364,"ibration":19365,"æ¯ı个人":19366,"Ġpushed":19367,"çļĦæł·åŃIJ":19368,"Ġspan":19369,"Ut":19370,"åĭĥ":19371,"Ġheavily":19372,"Ġfactorization":19373,"Ġestud":19374,"Ġà®ķ":19375,"åľ°ä¸ĭ":19376,"utors":19377,"rosc":19378,"Ġpetition":19379,"åĬłä»¥":19380,"ĠìĿ¸":19381,"碱":19382,"ĠDaily":19383,"é¢ĺ缮":19384,"GL":19385,"ĠSong":19386,"ä¸Ģä½ĵ":19387,"forward":19388,"ĠкоÑĤоÑĢÑĭе":19389,"ellular":19390,"дина":19391,"Ġnavigate":19392,"ĠBinary":19393,"ATH":19394,"缸åºĶ":19395,"ĠBow":19396,"Ġν":19397,"Ġsurely":19398,"æĺ¯è¦ģ":19399,"å¿įä¸įä½ı":19400,"Ġalgebra":19401,"ĠGeorgia":19402,"ĠQuality":19403,"обÑħоди":19404,"大éĩıçļĦ":19405,"åĽ¢ç»ĵ":19406,"人æķ°":19407,"顾客":19408,"Ġaddresses":19409,"Height":19410,"Eff":19411,"Ġмо":19412,"Ġথ":19413,"ĠReply":19414,"ĠCI":19415,"èĭ¹æŀľ":19416,"Review":19417,"ìĬµëĭĪëĭ¤":19418,"Ġensures":19419,"æĺ¨å¤©":19420,"åħ¨èº«":19421,"ĠBlog":19422,"å¥ĭæĸĹ":19423,"Ġspons":19424,"ingle":19425,"å°ijæķ°":19426,"Ġ+Ċ":19427,"Ġר":19428,"æ³Į":19429,"åͤ":19430,"ĠShould":19431,"Ġexpanded":19432,"ĠвоÑģ":19433,"SR":19434,"Ġsymbols":19435,"ĠHash":19436,"æī¿è®¤":19437,"ós":19438,"Ġhol":19439,"对æŃ¤":19440,"Ġspoken":19441,"ä½łå°±":19442,"åİŁå§ĭ":19443,"ä½İäºİ":19444,"ĠBio":19445,"DI":19446,"æĸ·":19447,"ilty":19448,"ĠChris":19449,"ĠPalest":19450,"));ĊĊ":19451,"大äºĨ":19452,"415":19453,"VA":19454,"Ġ},":19455,"column":19456,"Ġchamber":19457,"Ġloans":19458,"books":19459,"ĠGro":19460,"osta":19461,"Show":19462,"(str":19463,"ethe":19464,"ĠÑĥп":19465,"remove":19466,"geb":19467,"atching":19468,"Ġoffset":19469,"æ¡£æ¡Ī":19470,".edu":19471,"æ¯ıæĹ¥":19472,"ä½łåĢij":19473,"ubble":19474,"ĠDocument":19475,"è·¯å¾Ħ":19476,"-j":19477,"Ġmayor":19478,"å¥ĩæĢª":19479,"æĪ¿åľ°äº§":19480,"ĠOd":19481,"Ġedit":19482,"æĬ¤çIJĨ":19483,"ä¸įéľĢè¦ģ":19484,"elcome":19485,"é¤Ĭ":19486,"ĠComb":19487,"ĠIndeed":19488,"inator":19489,"Ġoppon":19490,"asks":19491,"ĠGree":19492,"Ġcurs":19493,"ĠPercent":19494,"Ġfm":19495,"Ġprominent":19496,"åı¯éĿł":19497,"agg":19498,"íķł":19499,"ạ":19500,"ĠThousand":19501,"å±Ģéĥ¨":19502,"ìĤ":19503,"Ġmineral":19504,"åıįå¤į":19505,"Ġdual":19506,"å¹¶åľ¨":19507,"=>":19508,"寶":19509,"èµ¶ç´§":19510,"اث":19511,"odd":19512,"hell":19513,"ĠCoast":19514,"ĠBenef":19515,"Ġvulnerable":19516,"è¿ijå¹³":19517,"åħ±æľī":19518,"ĠìľĦ":19519,".jpg":19520,"Ġprecision":19521,"Previous":19522,"äºĨä»ĸ":19523,"æĵĬ":19524,"_v":19525,"Ġmovies":19526,"è¾ħåĬ©":19527,"éĩįåºĨ":19528,"lar":19529,"æ¶ĪèĢĹ":19530,"Ġfounded":19531,"Ġforever":19532,"344":19533,"Ġorientation":19534,"Ĥà°":19535,"week":19536,"æįŁå®³":19537,"uis":19538,"ĠNE":19539,"积累":19540,"Ġslope":19541,"ä¸Ĭ涨":19542,"麻çĥ¦":19543,"Ġinstallation":19544,"cycle":19545,"ĠTransport":19546,"êµIJ":19547,"craft":19548,"åħ±åĴĮåĽ½":19549,"013":19550,"ĠDif":19551,"=True":19552,"ĠScientific":19553,"Ġnombre":19554,"Ġseu":19555,"ĠPi":19556,"Ġproc":19557,"adium":19558,"PER":19559,"Ġalg":19560,"rada":19561,"Ġintra":19562,"Ġdevelopments":19563,"á»ĩ":19564,"Ġspelling":19565,"wen":19566,"å¨ĩ":19567,"Full":19568,"Ġepisode":19569,"åľ¨å¤§":19570,"Ġglucose":19571,"Ġemerged":19572,"infty":19573,"éħ¬":19574,"343":19575,"Ġcontracts":19576,"low":19577,"Ġcomputers":19578,"èIJĮ":19579,"Ġbranches":19580,"Ġvend":19581,"visor":19582,"Ġmotivation":19583,"Ġdeaths":19584,"å®ĥæĺ¯":19585,"SH":19586,"ĠпÑĥ":19587,"Ġpic":19588,"é£ŀæľº":19589,"åĤ·":19590,"帽":19591,"ĠSurvey":19592,"åįģåħŃ":19593,"Ġappointed":19594,"Ġfoss":19595,"Ġarranged":19596,"ardo":19597,"å²ģçļĦ":19598,"éį":19599,"Ġunexpected":19600,"keys":19601,"æĢİä¹Īæł·":19602,"Ġmaj":19603,"Ġsuffered":19604,"çľ¾":19605,"ĠMom":19606,"Ġmemor":19607,"å᱿ľº":19608,"Ġentering":19609,"Today":19610,"(z":19611,"Ġstations":19612,"æī®":19613,"éĤĢ请":19614,"351":19615,"chain":19616,"Ġreveals":19617,"ĠUm":19618,"ĠEdward":19619,"Ġnose":19620,"éĩĬæĶ¾":19621,"ĠKa":19622,"äºĨåĩł":19623,"λλ":19624,"Ġзада":19625,"Ġmatching":19626,"Ġmapping":19627,"âĢĿ(":19628,"merce":19629,"Ġêtre":19630,"Ġencouraged":19631,"Ġvaccine":19632,"ĠRun":19633,"饼":19634,"Ġwisdom":19635,"å±ħä½ı":19636,"plements":19637,"Ġtissues":19638,"PO":19639,"users":19640,"溢":19641,"342":19642,"ĠGrowth":19643,"Ġthrows":19644,"Ġadmitted":19645,"Ġsoph":19646,"以ä¸ĬçļĦ":19647,"åIJĮæ¯Ķ":19648,"Ġsimultaneously":19649,"Ġyard":19650,".ann":19651,"reements":19652,"pack":19653,"ÙĬØ«":19654,"ĠRES":19655,"ĠCert":19656,"Ġhash":19657,"æľ¬äºº":19658,"âĤ¬":19659,"tra":19660,"çļĦæķ°æį®":19661,"ĠSQL":19662,"355":19663,"btn":19664,"Ġtargeted":19665,"ãģ«ãĤĪ":19666,"ilton":19667,"Access":19668,"334":19669,"á̝":19670,"è¿ijæĹ¥":19671,"æµ´":19672,"æīĢéľĢ":19673,"rov":19674,"ĠбÑĭÑĤÑĮ":19675,"ĠìĨ":19676,"Ġaddressing":19677,"alan":19678,"tu":19679,"符åı·":19680,"æ¼Ĩ":19681,"çļĦä¿¡æģ¯":19682,"ায়":19683,"matrix":19684,"Ġtrace":19685,"ented":19686,"_count":19687,"ĠMichigan":19688,"主人":19689,"å¸Ĥå§Ķ":19690,"ouch":19691,"Ġsavings":19692,"çļĦç¥ŀ":19693,"yll":19694,"æµij":19695,"å®ĩå®Ļ":19696,"ä¸įä¹ħ":19697,"601":19698,"Christ":19699,"341":19700,"Ġger":19701,".pdf":19702,"ÛĮÚ©":19703,"_of":19704,".create":19705,"ĠNight":19706,"Ġpok":19707,"icul":19708,"å¯Ĩ度":19709,"ĠClark":19710,"äºĪ以":19711,"Ġfaculty":19712,"Ġvitamin":19713,"Ġov":19714,"åĽ½åĬ¡éĻ¢":19715,"/D":19716,"grad":19717,"ĉSystem":19718,"æ¬Ĭ":19719,"ç»§æī¿":19720,"群ä½ĵ":19721,"被åijĬ":19722,"Ġnitrogen":19723,"绳":19724,"à¥Ĥ":19725,"Ġsein":19726,"Ġresolve":19727,"ÈĽ":19728,"ĠUnt":19729,"Ġgrat":19730,"Pages":19731,"ĠеÑģли":19732,"Ġnursing":19733,"Ġbot":19734,"ivos":19735,"ĠÃĥ":19736,"ĠAlexander":19737,"ĠEmpire":19738,"Ġelected":19739,"Ġconvenient":19740,"Ġjustify":19741,"è¯ĨåĪ«":19742,"Ġbomb":19743,"aska":19744,"ัย":19745,"////////////////":19746,"Ġ(âĢľ":19747,"éĶ»çĤ¼":19748,"560":19749,"ë£":19750,"æ§ĭ":19751,"asting":19752,"bet":19753,"èĬĿ":19754,"ãĤ£":19755,"ãģŁãĤģ":19756,"ĠÑįк":19757,"519":19758,"Ġintake":19759,"æĭĮ":19760,"ä¸Ģæĸ¹éĿ¢":19761,"()->":19762,"ĠвÑĢемÑı":19763,"æĮĩ示":19764,"gged":19765,"缮å½ķ":19766,"osion":19767,"Ġmeth":19768,"社ä¼ļçļĦ":19769,"heric":19770,"ĠYears":19771,"length":19772,"Ġ;Ċ":19773,"ãĤ¦":19774,"çݲ":19775,"hend":19776,"BR":19777,"èĪªç©º":19778,"Ġancest":19779,"opter":19780,"ãģĵãģ¨ãģĮ":19781,"Ġpossibilities":19782,"éģ®":19783,"Ġweakness":19784,"æł¡éķ¿":19785,"Ġcellular":19786,"ê³Ħ":19787,"Ġtort":19788,"unct":19789,"ãĤį":19790,"ä¸įè¡Į":19791,"Ñĩен":19792,"pc":19793,"Ġshopping":19794,"vet":19795,"Ġpdf":19796,"ynomial":19797,"Ġrarely":19798,"Ġव":19799,"Ġsuspect":19800,"Ġpriorit":19801,"Ġprod":19802,"éĻķ":19803,":\\":19804,"Ġpleasure":19805,"Ġdoctors":19806,"ì¹ĺ":19807,"æĪijä»¬åľ¨":19808,"å¤Ħç½ļ":19809,"Ġraising":19810,"ĠCas":19811,"Ġtestim":19812,"ä¿Ħç½Ĺæĸ¯":19813,"Ġgig":19814,"许åı¯":19815,"Ġsauce":19816,"ĠPerhaps":19817,"ĠIR":19818,"337":19819,"Ġض":19820,"oust":19821,"åħ¬ä¸»":19822,"Ġfinite":19823,"å¯Ĥ":19824,"ĠGive":19825,"Ġlips":19826,"姨":19827,"Ġtroops":19828,"avelength":19829,"zw":19830,"ĠKenn":19831,"oln":19832,"acion":19833,"pread":19834,"æľ´":19835,"false":19836,"contin":19837,"ä¸Ģåı¥":19838,"405":19839,"ĠThough":19840,"ÑģÑĤавлÑı":19841,"Ġarrangement":19842,"Ġcrop":19843,"Ġcomparing":19844,"èĮĥåĽ´åĨħ":19845,"Ġglob":19846,"irable":19847,"ç쵿´»":19848,"Ġsending":19849,"Ġaver":19850,"chema":19851,"ï¼ĭ":19852,"326":19853,"Ġintr":19854,"cut":19855,"ĠDown":19856,"ĠMiller":19857,"Ġregistration":19858,"Ġfertil":19859,"Ġtons":19860,"Ġoptimization":19861,"ĠSize":19862,"Non":19863,"460":19864,"ĠBoy":19865,"çļĦåħī":19866,"Ġactively":19867,"Ġtrail":19868,"isors":19869,"ĠSports":19870,"Ġputs":19871,"ĠLewis":19872,"Ġtracking":19873,"æīĵåį°":19874,"æĬĵä½ı":19875,"Ġcontributing":19876,"çļĦè¡Į为":19877,"Ġamino":19878,"arry":19879,"éĽĸçĦ¶":19880,"ATA":19881,"ç§ĺå¯Ĩ":19882,"Ġmoderate":19883,"Did":19884,"Ġpré":19885,"formed":19886,"Ġreputation":19887,"Ġcaptured":19888,"ĠÙĩذا":19889,"etts":19890,"é«ĺæķĪ":19891,"个ä½ĵ":19892,"eah":19893,"plicit":19894,"ĠCatalogue":19895,"à¸Ļีà¹ī":19896,"读èĢħ":19897,"ì¡°":19898,"ĠChemical":19899,"Ġbowl":19900,"ĠChecklist":19901,"æĽ´å¥½çļĦ":19902,"uty":19903,"Ġsmallest":19904,"Reply":19905,"None":19906,"Ġprinted":19907,"ĠØ®ÙĪØ¯":19908,"èŃ¦å¯Ł":19909,"istically":19910,"æľīæķĪçļĦ":19911,"Ġassemb":19912,"640":19913,"SM":19914,"Ġtransferred":19915,"ä¸įåıĺ":19916,"ĠAnton":19917,"à¹Īม":19918,"Ġeigen":19919,"éij":19920,"æĻĥ":19921,"ĠKi":19922,"Hello":19923,"ä¹Łåı¯":19924,"Ġenorm":19925,"ĠFactorization":19926,"Ġsteady":19927,"OUT":19928,"дов":19929,"Ġalongside":19930,"Ġanger":19931,"pend":19932,"å°ıåĮº":19933,"arty":19934,"chant":19935,"åĴ½":19936,"æ¤Ĵ":19937,"åĨ²çªģ":19938,"è¿Ľç¨ĭ":19939,"Ġenforcement":19940,"Ġ]ĊĊ":19941,"åĢºåĬ¡":19942,"Direct":19943,"Ġcooperation":19944,"igue":19945,"ĠPoly":19946,"Ġincom":19947,"ĠоÑģнов":19948,"ç§©åºı":19949,"ĠPy":19950,"Ġcouncil":19951,"Ľ×ľ":19952,"Square":19953,"inois":19954,"ĠTreatment":19955,"Ġبد":19956,"ussi":19957,"ĠRole":19958,"ĠкоÑĺе":19959,"apper":19960,"ĠConsult":19961,"omo":19962,"产éĩı":19963,"ÙĨس":19964,"ĠJoh":19965,"353":19966,"oco":19967,"Ġsettlement":19968,"覺å¾Ĺ":19969,"Ġopinions":19970,"åĬ£":19971,"Ġreflects":19972,"aceut":19973,"Ġinflammation":19974,"imen":19975,"Ġtorn":19976,"夸":19977,"329":19978,"Ġmarine":19979,"å·¥ä½ľä¸Ń":19980,"Ġtens":19981,"arma":19982,"大大":19983,"ĠDetails":19984,"Ġbid":19985,"æİ¥ä¸ĭæĿ¥":19986,"Ġcord":19987,"Ġrecall":19988,"æ·Ģ":19989,"Ġsek":19990,"Ġhang":19991,"ĠÑĢав":19992,"Ġcalcium":19993,"Ġosc":19994,"answer":19995,"ä¸Ńåľĭ":19996,"ä¸Ī夫":19997,"Ġmyth":19998,"ĠExpress":19999,"uru":20000,"ĠYouTube":20001,"ĠÑģÑĤÑĢа":20002,"ÑĤелÑı":20003,"------":20004,"æŃ¦æ±ī":20005,"_pro":20006,"Ġtodos":20007,"碧":20008,"Ùĩد":20009,"tein":20010,"Ġrespiratory":20011,"Ġsodium":20012,"åĿĬ":20013,"Ġcrew":20014,"æľĢä½İ":20015,"Ġল":20016,"Ġfeeding":20017,"stack":20018,"ĠStatistics":20019,"ĠOcean":20020,"364":20021,"UK":20022,"erver":20023,"Const":20024,"heet":20025,"åĴĮåħ¶ä»ĸ":20026,"æĻĴ":20027,"å¼ĺ":20028,"Ġchances":20029,".Gener":20030,"021":20031,"jen":20032,"à¦¾à¦ľ":20033,"ÙĤÙĩ":20034,"_time":20035,"Ġcombine":20036,"Ġoh":20037,"module":20038,"èĺĩ":20039,".Data":20040,"they":20041,"ecting":20042,"Ġaged":20043,"yy":20044,"Ġalphabet":20045,"ĠStill":20046,"ĠRoot":20047,"äng":20048,"ÑĢом":20049,"PH":20050,"Ġmining":20051,"Ġcorn":20052,"Ġмог":20053,"Ġperspectives":20054,"ĠTeaching":20055,"åı¦ä¸Ģæĸ¹éĿ¢":20056,"Ġregarded":20057,"æ¹ĸåįĹ":20058,"بÙĩ":20059,"ĠWis":20060,"å¦ĤåIJĮ":20061,"Ġelder":20062,"าว":20063,"ĠÑĢаÑģп":20064,"rá":20065,"ĠNan":20066,"ĠоÑĢгани":20067,"endix":20068,"âī¤":20069,"Ġloyal":20070,"Ġdataset":20071,"question":20072,"Ġmant":20073,"ĠâĢĺâĢĺ":20074,"åĵ¼":20075,"Ġsurge":20076,"ĠIntellig":20077,"Ġhighlights":20078,"Ġcher":20079,"Õ¡Õµ":20080,"Ġpilot":20081,"Ġpill":20082,"ÑģÑģи":20083,"ç͵æºIJ":20084,"dam":20085,"Ġdeleg":20086,"Ġcoat":20087,"425":20088,"ãĤĤãģ®":20089,":_":20090,"?.":20091,"Ġresponsibilities":20092,"enders":20093,"posed":20094,"Ġcampus":20095,"eness":20096,"ê¸":20097,"æĢİä¹ĪåĬŀ":20098,"ÙĥاÙĨ":20099,"å®ļäºĨ":20100,"Ġtechnological":20101,"Ġpasses":20102,"inners":20103,"绵":20104,"Ġcreativity":20105,"渴":20106,"|\\":20107,"level":20108,"anny":20109,"Ġrobot":20110,"ĠÙħس":20111,"ĠгÑĢÑĥ":20112,"ÑĩеÑģкие":20113,"æĢĸ":20114,"çļĦ社ä¼ļ":20115,"åŁºæľ¬ä¸Ĭ":20116,"çξ":20117,"Ġhide":20118,"ä¹łè¿ijå¹³":20119,"ĠSant":20120,"superscript":20121,"Ġlibr":20122,"ĠRat":20123,"æīĭä¸Ń":20124,"åħ¥äºĨ":20125,"Ġtong":20126,"'un":20127,"è¬Ŀ":20128,"ĠÑĢазви":20129,"ä½łåı¯ä»¥":20130,"à¹Ģรียà¸Ļ":20131,"æ³µ":20132,"fulness":20133,"eros":20134,"Ġai":20135,"å¹¼åĦ¿åĽŃ":20136,"åĨĽéĺŁ":20137,"Ġvom":20138,"à¹īว":20139,"ĠParliament":20140,"erving":20141,"Ġscored":20142,"åĽŀå¤į":20143,"armaceut":20144,"jar":20145,"é«ĺè´¨éĩı":20146,"ĠMes":20147,"DL":20148,"Ġpreparing":20149,"ctic":20150,"Long":20151,"اÛĮÛĮ":20152,"é«ĺçŃī":20153,"Also":20154,"å®ĺæĸ¹":20155,"å¸ĤåľºçļĦ":20156,"uras":20157,"æĪijçŁ¥éģĵ":20158,"åģĩ设":20159,"Ġbuck":20160,"æ°Ĺ":20161,"ĠIEEE":20162,"ĠEsc":20163,"×Ļ×ķ":20164,"Ġgear":20165,"zu":20166,"ĠJane":20167,"ritis":20168,"bow":20169,"rett":20170,"Ġconce":20171,"lier":20172,"à¸Ł":20173,"ĠÏĦηÏĤ":20174,"Ġconsisting":20175,"鸿":20176,"isters":20177,"operator":20178,"Ġadvertising":20179,"丼":20180,"ĠMB":20181,"ĠContin":20182,"Ġvaried":20183,"Ġinterviews":20184,"ĠNaz":20185,"ĠRout":20186,"Ġleb":20187,"çι":20188,"Ġqualified":20189,"说çĿĢ":20190,"-min":20191,"361":20192,"Ġclothing":20193,"Ġtsp":20194,"è¡Ģåİĭ":20195,"çŀ§":20196,",t":20197,"ä¸į对":20198,"FP":20199,"éľŀ":20200,"Ġaffecting":20201,"scription":20202,"Ġ\"\\":20203,"Ĺר":20204,"itos":20205,"ĠÙĪØ¬":20206,"ULT":20207,"432":20208,"æī¹è¯Ħ":20209,"Ġgrate":20210,"Ġdiagnostic":20211,"Ġworker":20212,"away":20213,"Ġmirror":20214,"çĶ»éĿ¢":20215,"ĠThom":20216,"idel":20217,"å¿ĹæĦ¿èĢħ":20218,"Ġbab":20219,"åŀ«":20220,"iations":20221,"ãĤĩ":20222,"æĶ¯åĩº":20223,"Ġtun":20224,"Ġfost":20225,"è¡į":20226,"Ġsilence":20227,"tz":20228,"space":20229,"éģĬ":20230,"Ġradians":20231,"ĠDifferent":20232,"Ġay":20233,"Ġcontrolling":20234,"Ġbreathing":20235,"ĠMars":20236,"Valid":20237,"åįľ":20238,"åıijèĤ²":20239,"Ġгода":20240,"Ġsuggesting":20241,"åħ¬å¹³":20242,"çĨĻ":20243,"ptic":20244,"{d":20245,"Ġkont":20246,"á»Ļ":20247,"International":20248,"çµķ":20249,"Ġη":20250,"è¿IJç®Ĺ":20251,"emed":20252,"ÑĩеÑģкой":20253,"κε":20254,"Ġcoding":20255,"ĠĠĠĠĠĠĠĠĠĠĠĠĊ":20256,"}_{\\":20257,"Ġelectrons":20258,"æ¸Ĭ":20259,"363":20260,"路线":20261,"ĠDom":20262,"ĠعÙĦ":20263,"Copyright":20264,"child":20265,"åĩºæīĭ":20266,"èĤª":20267,"Ġ::":20268,"Ġdefinitions":20269,"IAL":20270,"è®¤çŁ¥":20271,"chor":20272,"Ġà¸Ħ":20273,"Ġê²½":20274,"ä¿¡ä»»":20275,"ĠRegion":20276,"仪å¼ı":20277,"à¸ģัà¸Ļ":20278,"ujÄħ":20279,"ĠChair":20280,"adata":20281,"409":20282,"Provider":20283,"çŃī人":20284,"ãĢĭãĢĤ":20285,"åĵªä¸ª":20286,"Ġhip":20287,"utable":20288,"Ġdirectory":20289,"Û³":20290,"Ġadverse":20291,"vare":20292,"åŃĺåľ¨çļĦ":20293,"idespread":20294,"555":20295,"åıįé¦Ī":20296,"Ġchat":20297,"ĠAssembly":20298,"æıIJ交":20299,"loc":20300,"ним":20301,"Cell":20302,"ĠRisk":20303,"":21858,"ĠGR":21859,"tor":21860,"ÙijÙİ":21861,"ĠPapers":21862,"人åĬĽ":21863,"Ġdental":21864,"Ġì¶":21865,"çļĦæľī":21866,"×Ļ×ĺ":21867,"浦":21868,"Finally":21869,"Ġdesert":21870,"achusetts":21871,"ĠBetween":21872,"441":21873,"éĶ¡":21874,"çļĦåĬĽéĩı":21875,"ĠRespons":21876,"\\/":21877,"Core":21878,"Ġgrande":21879,"Ġpropose":21880,")?":21881,"Ġкла":21882,"ĠFem":21883,"ĠRepublican":21884,"åħ³éĹŃ":21885,"Ġcompact":21886,"æµģè¡Į":21887,"妹妹":21888,"æ¼Ķåijĺ":21889,"ĠKo":21890,"Ġreceives":21891,"ĠGil":21892,"Ġcual":21893,"Ġuniversities":21894,"388":21895,"Ġminimize":21896,"Ġtransf":21897,"éĤ£å°±æĺ¯":21898,"382":21899,"头åıij":21900,"Ġtempo":21901,"charg":21902,"Ġpulse":21903,"finder":21904,"Ġprogression":21905,"Ġspecialized":21906,"æīĭæĮĩ":21907,"there":21908,"Micro":21909,"best":21910,"verter":21911,"seud":21912,"Ġchains":21913,"ĠMeg":21914,"Ġболее":21915,"SQL":21916,"Ġbull":21917,"Ġpathways":21918,"yk":21919,"Ġmomentum":21920,"بت":21921,"থ":21922,"é»ı":21923,"Feb":21924,"Ġentities":21925,"uum":21926,"Api":21927,"Ġwound":21928,"Ġצ":21929,"ytic":21930,"iego":21931,"424":21932,"ILE":21933,"Ġmá":21934,"{array":21935,"Ġstaying":21936,"Ġalarm":21937,"Ġpersu":21938,"onds":21939,"èĭ¥å¹²":21940,"roc":21941,"Ùĥر":21942,"Ġorange":21943,"Ġwavelength":21944,"}+\\":21945,"ej":21946,"ĠиÑģÑĤо":21947,"Ġcoordinate":21948,"åĪĨæķ°":21949,"Ġbeings":21950,"-dependent":21951,"040":21952,"Ġnurse":21953,"onth":21954,"æĥ¹":21955,"Õ¿":21956,"åĪĿå§ĭ":21957,"úblic":21958,"Account":21959,".)Ċ":21960,"Ùĥات":21961,"Ġdesignated":21962,"è´¢å¯Į":21963,"Ġphases":21964,"Ġboxes":21965,"à¯įà®Ł":21966,"×ķ×¢":21967,"stdio":21968,"ีà¹Īย":21969,"Mc":21970,"Ġdownt":21971,"National":21972,"Ġbottle":21973,"Ġcopies":21974,"/H":21975,"Ġguilty":21976,"Ġlin":21977,"åı¯ç͍":21978,"rez":21979,"ĠHop":21980,"oning":21981,"å·¥åķĨ":21982,"home":21983,"ĠPlus":21984,"áĥĿáĥ":21985,"Ġconclude":21986,".Generic":21987,"ä¸Ģè¡Į":21988,"èģ·":21989,"认å®ļ":21990,"ĠFra":21991,"APP":21992,"ä¸ĢçĶŁ":21993,"Ġbreaks":21994,"Ġexpense":21995,"οÏį":21996,"ición":21997,"......ĊĊ":21998,"....ĊĊ":21999,"who":22000,"éĺ²æĬ¤":22001,"ĠKan":22002,"Ġfitness":22003,"ĠLie":22004,"åIJĮäºĭ":22005,"å¾Ī容æĺĵ":22006,"Ġmehr":22007,"íķ©":22008,"Ġsuggestions":22009,"æĪIJå¹´":22010,"ĠBan":22011,"âĢĿ)":22012,"æī©å±ķ":22013,"nab":22014,"ĠParent":22015,"Ġconstru":22016,"efully":22017,"connect":22018,"ĠRoss":22019,"ĠRelig":22020,"å§ļ":22021,"Ġî":22022,"kar":22023,"025":22024,"ÑĩиÑĤа":22025,"ÑıÑħ":22026,"æ¸IJæ¸IJ":22027,"лав":22028,"è¿Ļç±»":22029,"Ġfunctionality":22030,"ĠEnc":22031,"Ġenthusi":22032,"olesterol":22033,"Ġtrauma":22034,"ãģĹãģ¾ãģĻ":22035,"ç»Łæ²»":22036,"Filter":22037,"强大":22038,"ÙģÙĬ":22039,"æķħæĦı":22040,"Ġsearching":22041,"Ġdisability":22042,"åľĴ":22043,"ĠBail":22044,"Ġremoving":22045,"Ġrepet":22046,"rer":22047,"Publication":22048,"Ġë²":22049,"Ġdeviation":22050,"ĠRate":22051,"à¹īาà¸ĩ":22052,"羣çļĦæĺ¯":22053,"Ġshouldn":22054,"amen":22055,"ĠبÙĪØ¯":22056,"俱":22057,"chnology":22058,"Ġimpression":22059,"Ġdisplays":22060,"Ġبعد":22061,"Ġsomehow":22062,"-control":22063,"ĠFord":22064,"æºĥ":22065,"801":22066,"çļĦæķħäºĭ":22067,"Ġgains":22068,"ĠSolutions":22069,"(value":22070,"Ùħس":22071,"Ġsmoking":22072,"808":22073,"alis":22074,"หม":22075,"Ġtambién":22076,"ibles":22077,"ç½IJ":22078,"ä¸Ńåįİ人æ°ij":22079,"å¥ĸåĬ±":22080,"IND":22081,"ĠÙĨظ":22082,"blog":22083,"å¾Ī好çļĦ":22084,"NC":22085,"è¯Ī":22086,"çĸ«èĭĹ":22087,"oples":22088,"åºıåĪĹ":22089,"ochond":22090,"åħ¶ä»ĸçļĦ":22091,"TV":22092,"æIJı":22093,"ĠLive":22094,"ĠUI":22095,"ĠTu":22096,"çī²":22097,"erta":22098,"éķ¿æĹ¶éĹ´":22099,"è£ķ":22100,"åŃ¦æľŁ":22101,"è£ħ饰":22102,"Ġopens":22103,"Ġenabled":22104,"Ġpipe":22105,"UND":22106,"ä¸ĬæĿ¥":22107,"rowing":22108,"ĠNative":22109,"Ġcontest":22110,"æīĶ":22111,"ĠStructure":22112,"Ġmetabolism":22113,"为æŃ¤":22114,"gon":22115,"ĠDutch":22116,"Ġmutual":22117,"aha":22118,"ĠDor":22119,"è¯Ńæĸĩ":22120,"vor":22121,"Ġfon":22122,"æľīåı¯èĥ½":22123,"æĸĩåĮĸçļĦ":22124,"rost":22125,"ä¸į大":22126,"ĠHung":22127,"ìķ¼":22128,"起身":22129,"Ġmeny":22130,"ĠLang":22131,"æĩī該":22132,"Ġdesper":22133,"Ġdelet":22134,"Ġnoch":22135,"auss":22136,"ãģ¹":22137,"coin":22138,"Ġutilized":22139,"欣èµı":22140,"ķĮ":22141,"ĠPear":22142,")]Ċ":22143,"marks":22144,"Details":22145,"Ġméd":22146,"Ġstere":22147,"ű":22148,"麵":22149,"(\"\\":22150,"Ġmanuscript":22151,"(root":22152,"ping":22153,"ç͵éĺ»":22154,"è´¦æĪ·":22155,"Sort":22156,"ĠCategory":22157,"Ġattorney":22158,"ण":22159,"Ġessence":22160,"ÑĩеÑģкого":22161,"游客":22162,"Listener":22163,"pers":22164,"Ġseasons":22165,"å¤ļåħĥ":22166,"ÙĤÙĬ":22167,"foot":22168,"Ġà¦ıবà¦Ĥ":22169,"container":22170,"Ġgovernance":22171,"Ġdag":22172,"Ġincorrect":22173,"Ġaccomplish":22174,"Ġaussi":22175,"Ġnasod":22176,"åѦ家":22177,"Queue":22178,"ĠاÙĦÙĦ":22179,"Ġentertainment":22180,"×§":22181,"亲èĩª":22182,"ĠProduction":22183,"Ω":22184,"Ġcups":22185,"º":22186,"ĠInsp":22187,"Despite":22188,"Ġshooting":22189,"溫":22190,"ĠUSD":22191,"505":22192,".annotation":22193,"ĠÙĨÙħ":22194,"Ġrelate":22195,"ĠRegional":22196,"Ġvessel":22197,"æĹ¥èµ·":22198,"777":22199,"ĠاÙĨت":22200,"å·¥ä¼ļ":22201,"кÑĤи":22202,"Ġdistinction":22203,"adelphia":22204,"liest":22205,"Ġarrival":22206,"èĮĤ":22207,"ĠвÑĭÑģ":22208,"Ġexpend":22209,"çŃĽ":22210,"äll":22211,"630":22212,"Ġconversations":22213,"Ġproportional":22214,"ìĭĿ":22215,".**ĊĊ":22216,"ĠÛĮا":22217,"Ġexecute":22218,"ĠIllinois":22219,"ÑĽÐ¸Ð½":22220,"ĠÑıвлÑıеÑĤÑģÑı":22221,"ä¸Ģä»¶":22222,"ials":22223,"Ġhtml":22224,"ĠPod":22225,"Ġbrothers":22226,"ĠKids":22227,"åĦª":22228,"}ĊĊĊ":22229,"People":22230,"åĽŀå½Ĵ":22231,"andid":22232,",**":22233,"backs":22234,"Ġdramatic":22235,"æ²IJ":22236,"Ġобла":22237,"Ġdess":22238,"åĩºè¡Ģ":22239,"Server":22240,"some":22241,"é«ĺäºİ":22242,"Ġburst":22243,"arched":22244,"ĠMand":22245,"Ġsingular":22246,"以å¤ĸ":22247,"ourt":22248,"Ġcoordinates":22249,"è°ĥç͍":22250,"æĴ°":22251,"Ġexperiencing":22252,"Ġorganisation":22253,"racing":22254,"å»Ĭ":22255,"说ä¸į":22256,"Ġcuts":22257,"ĠColorado":22258,"Ġadmit":22259,"Ġincorporated":22260,"енÑģко":22261,"ĠBras":22262,"ез":22263,"izza":22264,"ĠCollection":22265,"å¿ĺè®°":22266,"Ġschemes":22267,"Ġplates":22268,"helial":22269,"à¤Ĺ":22270,"Ġpode":22271,"Ġmé":22272,"Ġyields":22273,"uti":22274,"\\({}^{\\":22275,"ĠBlock":22276,"ĠmL":22277,"ĶĦ":22278,"ÏĢÏĮ":22279,"ĠObama":22280,"ĠGas":22281,"åħļ建":22282,"Ġassumptions":22283,"\\%":22284,"ĠEditor":22285,"Ġdigest":22286,".ĊĊĊ":22287,"ÙĪÙĬ":22288,"çĩŁ":22289,"Ġmagazine":22290,"ר×IJ":22291,"çļĦ第ä¸Ģ":22292,"ferred":22293,"Ġpossession":22294,"ìĿ´ëĭ¤":22295,"ĠBroad":22296,"389":22297,"Three":22298,"ä¸Ĭæľī":22299,"ĠConnect":22300,"|Ċ":22301,"个æĢ§":22302,"åŁİéķĩ":22303,"оби":22304,"egen":22305,"(*":22306,"423":22307,"Ġestablishing":22308,"417":22309,"Ġstruggling":22310,"ದ":22311,"prot":22312,"ĠMarc":22313,"Ġnavigation":22314,"aura":22315,"ĠPH":22316,"Ġobjet":22317,"çķ«":22318,"ioned":22319,"Ġdurante":22320,"ç¾İæľ¯":22321,"éĥ¨éķ¿":22322,"ĠSolve":22323,"Ġmountains":22324,"ãĥł":22325,"ĠThink":22326,"Ġmistakes":22327,".left":22328,"379":22329,"Ġ모":22330,"BSCRIPT":22331,"æĹłéĻIJ":22332,"kn":22333,"Ġmont":22334,"åĢĭ人":22335,"ान":22336,"info":22337,"ĠÐ´Ð¾Ð¼Ð°ÑĽÐ¸Ð½":22338,"prises":22339,"Ġconfusion":22340,"Ġné":22341,"ĠNicol":22342,"Ġlayout":22343,"ĠConc":22344,"Ġvolunt":22345,"à¸ľà¸¥":22346,"ologically":22347,"meta":22348,"Ġroughly":22349,"ÃŃc":22350,"æ²³åĮĹ":22351,"ischer":22352,"×Ĺ":22353,"443":22354,"vation":22355,"Ġters":22356,"že":22357,"çŀª":22358,"å¹³æĸ¹ç±³":22359,"ĠDue":22360,"Ġpoet":22361,"çļĦçī¹çĤ¹":22362,"080":22363,"亿ç¾İåħĥ":22364,"Ġprotective":22365,"-cl":22366,"Ġmeals":22367,"æ¹ĸåĮĹ":22368,"ĠML":22369,"\\,\\":22370,"die":22371,"çģ£":22372,"ĠMind":22373,"ĠPrimary":22374,"lav":22375,"ictions":22376,"Ġlabels":22377,"iÄħ":22378,"æİ¢è®¨":22379,"ĠEq":22380,"Ġlovely":22381,"Po":22382,"Ġsheets":22383,"Ġprest":22384,"Ġracial":22385,"ÑģÑĥ":22386,"Further":22387,"bie":22388,"Ġeleg":22389,"Resource":22390,"Õ¡ÖĢ":22391,"zenia":22392,"ĠMir":22393,"æĽ´å¥½åľ°":22394,"SW":22395,"åīĸ":22396,"Ġritual":22397,"account":22398,"Ġprevalence":22399,"åıĸæ¶Ī":22400,"วย":22401,"Ġloves":22402,"Ġserum":22403,"ricts":22404,"ĠTok":22405,"avid":22406,"åŃ£åº¦":22407,"å¸ĥç½®":22408,"*i":22409,"elson":22410,"Introduction":22411,"ĠJordan":22412,"ioxide":22413,"Ġoscill":22414,"RC":22415,"ĠAuto":22416,"017":22417,"ĠIslamic":22418,"Ġtunn":22419,"Ġdisaster":22420,"ãģ§ãģĤãĤĭ":22421,"å¿ħè¦ģçļĦ":22422,"åIJijåīį":22423,"Process":22424,"Ġbent":22425,"Ġ문":22426,"Ġempirical":22427,"asan":22428,"rose":22429,"ĠLE":22430,"ĠBry":22431,"ĠOm":22432,"çĿĢçļĦ":22433,"zd":22434,"Ġadjusted":22435,"Options":22436,"ð":22437,"uper":22438,"urches":22439,"ĠHaving":22440,"Ġselecting":22441,"Ġmales":22442,"æ¯Ľæ³½":22443,"illo":22444,"à¹Īาà¸Ļ":22445,"Ġversch":22446,"åĻª":22447,"ĠCer":22448,"ĠBrain":22449,"entry":22450,"475":22451,"zech":22452,"ĠBehavior":22453,"ĠØ£ÙĬ":22454,"Ġutter":22455,"ÑĢÑĥк":22456,"ĠConcept":22457,"ĠITIS":22458,"éģµå®Ī":22459,"Ġchampions":22460,"Ġcake":22461,"ç»´çĶŁç´ł":22462,"490":22463,"ĠпоÑĢ":22464,"è¡¥åģ¿":22465,"Ġsilent":22466,"Ġshorter":22467,"Ġliberal":22468,"udd":22469,"Ġweigh":22470,"Ġgolden":22471,"ĠDemocratic":22472,"abases":22473,"Ġguarante":22474,"æĹ¥çļĦ":22475,"387":22476,"Ġsans":22477,"ÑĺÑĥ":22478,"Ġlegend":22479,"Ġnuest":22480,"Ġcardiac":22481,"pecially":22482,"Ġpractition":22483,"ĠTypes":22484,"emi":22485,"éĺIJ":22486,"ĠOil":22487,"axy":22488,"liers":22489,"Ġling":22490,"Ġactor":22491,"PV":22492,"ĠINT":22493,"æĦıå¿Ĺ":22494,"å¹¿åľº":22495,"ibt":22496,"870":22497,"å¤ļæķ°":22498,"Ġmethodology":22499,"Returns":22500,"eted":22501,"Ġpharmac":22502,"urope":22503,"å¸ĪçĶŁ":22504,"åıijæĶ¾":22505,"段æĹ¶éĹ´":22506,"ĠStation":22507,"Ġzd":22508,"(\"/":22509,"Ġclosest":22510,"ĠPennsylvania":22511,"å¹´çīĪ":22512,"æµ·å¤ĸ":22513,"Ġhospitals":22514,"Ġdifferently":22515,"iale":22516,"Requ":22517,"涯":22518,"DM":22519,"Ġdivine":22520,"BT":22521,"icture":22522,"人åĴĮ":22523,"Ġignore":22524,"Ġbearing":22525,"condition":22526,"Ġkilometers":22527,"ĠÙĪØ£":22528,"*c":22529,"çī©ä½ĵ":22530,"大éĻĨ":22531,"æ·¡æ·¡":22532,"иÑģа":22533,"ÖĦ":22534,".right":22535,"Dev":22536,"ÑģкиÑħ":22537,"ĠBureau":22538,"ĠMultiple":22539,"lab":22540,"è¦ģæľī":22541,"obi":22542,"两ä½į":22543,"æ¯Ľæ³½ä¸ľ":22544,"ĠMeasure":22545,"æŃī":22546,"ĠDark":22547,"ĠاÛĮ":22548,"ç͵åĬ¨":22549,"*y":22550,"source":22551,"\\\\\\\\":22552,"èĨı":22553,"]+":22554,"ailing":22555,"Ġreaches":22556,"èĨĿ":22557,"ĠìŀĪëĭ¤":22558,"Ġgrammar":22559,"verb":22560,".Y":22561,"ĠNurs":22562,"Ġ×ķ×Ķ×":22563,"èijĹåIJį":22564,"оÑģÑģи":22565,"ä¸»å¼ł":22566,"Ġshaped":22567,"ÙĦاÙĦ":22568,".end":22569,"Ġvisits":22570,"arroll":22571,"Ġequality":22572,"ãĤĩãģĨ":22573,"Ġscenes":22574,"analysis":22575,"erge":22576,"iera":22577,"_POSTSUBSCRIPT":22578,"Ġefficacy":22579,"525":22580,"åıįæŃ£":22581,"Ġmeets":22582,"ĠStone":22583,"×ij×":22584,"Ġlebih":22585,"Ġcommands":22586,"ãĤ·ãĥ":22587,"Ġspell":22588,"Ġjack":22589,"ilan":22590,"Ġenf":22591,"毫ä¸į":22592,"änn":22593,"âĦĥ":22594,"çħİ":22595,"çŁŃæľŁ":22596,"ĠBE":22597,"Ġmuseum":22598,"æĹłå¥Ī":22599,"ĠElectric":22600,"Ġedited":22601,"Version":22602,"èħ»":22603,")**":22604,"Ïĥη":22605,"username":22606,"è¶ħ级":22607,"ĠKit":22608,"ĠGuid":22609,"960":22610,"é¡¹çĽ®çļĦ":22611,"Ġattempted":22612,"ynamics":22613,"Ġdesde":22614,"-sm":22615,"Calculate":22616,"çĶŁçIJĨ":22617,"оÑģÑĥ":22618,"Ġtracks":22619,"Menu":22620,"ĠJen":22621,"ĠEconomics":22622,"æī¿è¯º":22623,"æľĽçĿĢ":22624,"æįī":22625,"åįģä¸ĥ":22626,"429":22627,"Ġpubl":22628,"Ġdamaged":22629,"ĠpÅĻÃŃ":22630,"Ġinfected":22631,"Ġcad":22632,"Ġconflicts":22633,"Ġسر":22634,"puter":22635,"ä¼łè¾ĵ":22636,"ĠPeriod":22637,"Ġfluctu":22638,"ÑĪениÑı":22639,"media":22640,"NG":22641,"Ġassuming":22642,"Ġprovince":22643,"Ġanten":22644,"Ú©ÙĨ":22645,"Ġeastern":22646,"Ġdisadvant":22647,"Ġbaseline":22648,"ĠAnderson":22649,"Ġintervals":22650,"ĠDeep":22651,"Ġproces":22652,"Ġdetermines":22653,"空ä¸Ń":22654,"Ġorang":22655,"aying":22656,"ä¼Ĺå¤ļ":22657,"Ġinterrupt":22658,"èħĬ":22659,"Ġ$(":22660,"Ġfiscal":22661,"æĭħå½ĵ":22662,"[[":22663,"à¦¿à§Ł":22664,"Ġlifetime":22665,"ĠInsurance":22666,"ĠPatients":22667,"Ġpursue":22668,"');ĊĊ":22669,"çļĦè¿ĩç¨ĭä¸Ń":22670,"flamm":22671,"Ġpose":22672,"Ġratios":22673,"à§§à§":22674,"health":22675,"Ġfaire":22676,"bas":22677,"Ø¡":22678,"omed":22679,"лÑģÑı":22680,"è¿Ľå±ķ":22681,"Ġcriticism":22682,"stru":22683,"050":22684,"Ġdefines":22685,"Ġà¸ģาร":22686,"omi":22687,"Ġoccurring":22688,"sters":22689,"Ġawarded":22690,"ĠتÙħ":22691,"Ġjury":22692,"æ¸ħçIJĨ":22693,"xxxx":22694,"Ġvu":22695,"ä½ĵåĨħ":22696,"ĠEric":22697,"_at":22698,"acji":22699,"лан":22700,"ostream":22701,"naire":22702,"Ġisolation":22703,"Ġperformances":22704,"Ġró":22705,"滩":22706,"Ġdiscusses":22707,"ÙĤØ·":22708,"æĭĵå±ķ":22709,"åIJĮæŃ¥":22710,"wal":22711,"ĠWars":22712,"ĠÙĬت":22713,"æľīåºı":22714,"asad":22715,"require":22716,"ืà¹Īà¸Ńà¸ĩ":22717,"å··":22718,"448":22719,"........................":22720,"Ġfilling":22721,"ategories":22722,"让ä»ĸ们":22723,"total":22724,"å®Įç¾İ":22725,"iac":22726,"åıijè¨Ģ":22727,"Û¹":22728,"Ġbulk":22729,"è¿Ŀåıį":22730,"éĺŁåijĺ":22731,"bits":22732,"ĠGirl":22733,"éļ¾åº¦":22734,"ĠÑĦÑĥнк":22735,"让åѦçĶŁ":22736,"深深":22737,"Ġsoll":22738,"åĽŀäºĭ":22739,".se":22740,"434":22741,"eno":22742,"çļĦå°±æĺ¯":22743,"èĥĮåIJİ":22744,"ĠSeveral":22745,"Ġrecruit":22746,"etz":22747,"بة":22748,"æĿĤå¿Ĺ":22749,"Ġharmful":22750,"Ġlady":22751,"们çļĦ":22752,"Ġbeer":22753,"è¿Ļä¹Łæĺ¯":22754,"èİİ":22755,"ä¾¿å®ľ":22756,"ĠÑģпоÑģоб":22757,"Ġobs":22758,"rä":22759,"via":22760,"деÑĢ":22761,"sta":22762,"йÑĤе":22763,"Ġamin":22764,"-Z":22765,"Pop":22766,"éľī":22767,"ĠÕ°":22768,"Ñīей":22769,"itance":22770,"ĠSummer":22771,"ishers":22772,"å¤ļæł·":22773,"è²ł":22774,"Ġfunctioning":22775,"ĠDur":22776,"Ġinsulin":22777,"Ġloaded":22778,"åĩ¸":22779,"ĠBor":22780,"ĠMountain":22781,"ÑĥÑĪко":22782,"Ġpolym":22783,"Ġsolved":22784,"(num":22785,"ĠAndroid":22786,"-pl":22787,"æ½ĺ":22788,"ĠSkills":22789,"ĠPu":22790,"ĠLLC":22791,"Ġbases":22792,"aton":22793,"\"),":22794,"Ġëį":22795,"Ġindicators":22796,"528":22797,"çļĦç¡®":22798,"Ġgray":22799,"ĠWales":22800,"ĠBah":22801,"æĸ°åĨł":22802,"第äºĮ天":22803,"Ġlateral":22804,"Ġreasoning":22805,"çij¶":22806,"æļĤæĹ¶":22807,"Ġjuice":22808,"ĠCompet":22809,"éĵ¸":22810,"Close":22811,"iking":22812,"ÏĪ":22813,"ieval":22814,"ĠScript":22815,"äºĶå¹´":22816,"Ġbehavioral":22817,"ĠاÙĦØ«":22818,"ÑģÑĤÑĢо":22819,"ĠFollowing":22820,"ĠFunctions":22821,"åī¥":22822,"éĴĻ":22823,"ä¸įæľĥ":22824,"602":22825,"Ġdecreases":22826,"åĵįåºĶ":22827,"ä½ĵ积":22828,"ĠÐĸенÑģко":22829,"empor":22830,"ä¼ļåijĺ":22831,"Ġsys":22832,"Ġneurons":22833,"ĠVers":22834,"Ġautomatic":22835,"ĠâĬ":22836,"ರ":22837,"Ġcod":22838,"ighth":22839,"两次":22840,"å¿ĥèĦı":22841,"arte":22842,"Ġgrateful":22843,"olves":22844,"Ġscales":22845,"æĬĬå®ĥ":22846,"äºĭå®ŀä¸Ĭ":22847,"æľĢåĪĿ":22848,"ól":22849,"ãĥī":22850,"大家éĥ½":22851,"nut":22852,"ewise":22853,"ĠTele":22854,"Ġtemple":22855,"PG":22856,"ĠMOOC":22857,"约æĿŁ":22858,"ĠRow":22859,"heres":22860,"Ġroutes":22861,"çĵ£":22862,"Given":22863,"Ġounces":22864,"Ġunlikely":22865,"ĠRecord":22866,"×ķ×§":22867,"obic":22868,"Ġmetals":22869,"Ġcamb":22870,"tau":22871,"à´¿à´":22872,"Ġphon":22873,"inton":22874,"ĠCre":22875,"LY":22876,"MF":22877,"ĠDat":22878,"ĠبÙĬ":22879,"ожи":22880,"Äĵ":22881,"è´¼":22882,"press":22883,"Ġsaat":22884,"强大çļĦ":22885,"询éĹ®":22886,"äºĮ次":22887,"ĠÑģоз":22888,"Ġfel":22889,"å¤įåIJĪ":22890,"Ġvalidation":22891,"ĠDeut":22892,"/kg":22893,"Ġsmell":22894,"çϾ年":22895,"Ġassistant":22896,"Ġdescribing":22897,"Only":22898,"éĿ¢åIJij":22899,"ä»¶çļĦ":22900,"ப":22901,"Ġ×Ķ×IJ×":22902,"419":22903,"CON":22904,"侦":22905,"éĢĢä¼ij":22906,"ĠOrrell":22907,"Ġ\"/":22908,"ÑĨиÑİ":22909,"ĠDeg":22910,"Ġextraction":22911,"Ġrounded":22912,"Ġsebagai":22913,"شاÙĨ":22914,"ÑĪение":22915,"ãģĿãĤĮ":22916,"Ġže":22917,"åIJĥäºĨ":22918,"æŁ¯":22919,"Ġphysically":22920,"Ġanat":22921,"iors":22922,"aug":22923,"*d":22924,"Ġworried":22925,"Ġgrasp":22926,"Ġgravity":22927,"gence":22928,"èij±":22929,"ĠпÑĢави":22930,"ació":22931,"Ġmembership":22932,"çªĦ":22933,"UST":22934,"å®ŀä½ĵ":22935,"å¢ĥçķĮ":22936,"æ¶Īæ¯Ĵ":22937,"Ġatomic":22938,"evin":22939,"Ġcohort":22940,"Ġtemporal":22941,"ĠContents":22942,"Ġirrit":22943,"æī¿åĮħ":22944,"Ġcoinc":22945,"æ°´æŀľ":22946,"ç·¨":22947,"Ġtru":22948,"ĠArchitect":22949,"Ġwedding":22950,"ä¸įæĩĤ":22951,"åįķçĭ¬":22952,"è®°è½½":22953,"Ġliterally":22954,"ĠTurkey":22955,"äºĭçī©":22956,"455":22957,"Ġà¸ŀ":22958,"ĠUl":22959,"property":22960,"Ġcited":22961,"$,":22962,"ç»Ħç»ĩçļĦ":22963,"asted":22964,"åĥ§":22965,"Ġpregnant":22966,"è¿ĩäºİ":22967,"ç¼ł":22968,";(":22969,"åIJį为":22970,"常è§Ħ":22971,"aver":22972,"åĪĨæ³Į":22973,"oire":22974,"ï¼ĮãĢĬ":22975,"ĠActivities":22976,"ูà¸ģ":22977,"èĴĻåı¤":22978,"Task":22979,"oline":22980,"馨":22981,"Ġheading":22982,"ледова":22983,"$\\":22984,"киÑħ":22985,"åĸĺ":22986,"Ġanterior":22987,"éĢĿ":22988,"pool":22989,"ĠProfessional":22990,"Ġstocks":22991,"è§ģè¿ĩ":22992,"_date":22993,"760":22994,"Ġmitig":22995,"Ġseam":22996,"大å¹ħ":22997,"olk":22998,"Ġeliminate":22999,"Amount":23000,"ÙħاÙĦ":23001,"oler":23002,"uction":23003,"Ġworkplace":23004,"391":23005,"Ġremembered":23006,"_string":23007,"store":23008,"023":23009,"íı":23010,"å®ŀéªĮ室":23011,"Ġbars":23012,"å¸ĸ":23013,".uk":23014,"Ġexclus":23015,"寫":23016,"Ġought":23017,"ادر":23018,"åľ¨å®¶":23019,"æľĢå°ı":23020,"Head":23021,"èĪŀåı°":23022,"Ġcarcin":23023,"Ġbike":23024,"Ġoste":23025,"¯":23026,"Ġlap":23027,"_value":23028,"累计":23029,"æľīæĹ¶åĢĻ":23030,"ç§įç±»":23031,"Ġnou":23032,"018":23033,"Ġreadily":23034,"æĬĦ":23035,"451":23036,"׾×IJ":23037,"è§Ĩè§ī":23038,"Ġelastic":23039,"Ġelevation":23040,"also":23041,".py":23042,"PI":23043,"ivals":23044,"Ġqualities":23045,"Ġakt":23046,"Ġrejected":23047,"Ġìľł":23048,"oving":23049,"ohyd":23050,"Ġcourage":23051,"Ġartistic":23052,"Ġreceiver":23053,"ĠOwn":23054,"ĠJu":23055,"Ġnella":23056,"Ġà¦ĸ":23057,"culo":23058,"çłĶç©¶çĶŁ":23059,"alling":23060,"Ġbacterial":23061,"ĠнÑĥж":23062,"ticles":23063,"ãģ¾ãģĽ":23064,"æĸ©":23065,"ĠStruct":23066,"çIJĥéĺŁ":23067,"ä¸įå¼Ģ":23068,"Ġgem":23069,"ãĥĭ":23070,"ifiers":23071,"Ġaffairs":23072,"ëªħ":23073,".json":23074,"Bas":23075,"Ġprés":23076,"dec":23077,"认åı¯":23078,"Ġexpanding":23079,"åĨ¥":23080,"èIJĿ":23081,"rizona":23082,"ĠLimited":23083,"vez":23084,"RT":23085,"oped":23086,"è£ħä¿®":23087,"Ġnaar":23088,"人å¿ĥ":23089,"©×¨":23090,"å¾®ç¬ij":23091,"çĻĤ":23092,"Ġcollections":23093,"å½ĵåĪĿ":23094,"Ġ};ĊĊ":23095,"cretion":23096,"Ġcontrary":23097,"ĠPrince":23098,"é«ĵ":23099,"ĠResource":23100,"rors":23101,"NAME":23102,"427":23103,"ĠRequest":23104,"èĹ¥":23105,"forms":23106,"Ġviolent":23107,"/*Ċ":23108,"Ġfeat":23109,"ĠدÛĮ":23110,"çϼçı¾":23111,"avigation":23112,"imetro":23113,"ĠCe":23114,"Ġenhancing":23115,"æĺ¯ä»İ":23116,"idal":23117,"ĠMassachusetts":23118,"åĨĻçļĦ":23119,"Ġsynchron":23120,"445":23121,"Ġtransmit":23122,"397":23123,"\\times":23124,"Ġessere":23125,"fi":23126,"ĠArgent":23127,"ĠVictor":23128,"Ġmuit":23129,"454":23130,"å·®è·Ŀ":23131,"ä¼ļçļĦ":23132,"åıijåĬ¨æľº":23133,"lat":23134,"ĠPosition":23135,"emony":23136,"ï¼ģâĢĿĊ":23137,"ĠLiving":23138,"çļĦåĨħ":23139,"ĠDoc":23140,"ĠобÑĢазова":23141,"Ġunlike":23142,"ĠFern":23143,"iao":23144,"ĠALL":23145,"asser":23146,"forming":23147,"æĥ©":23148,"Ġassociations":23149,"660":23150,"Layout":23151,"453":23152,"æĮĩ令":23153,"Header":23154,"åį¸":23155,"ĠImm":23156,"åĺ¿":23157,"Ġdeck":23158,"ÑĢии":23159,"éĢłåŀĭ":23160,"æĺ¯ä¸ºäºĨ":23161,"Ġ×ŀ×ķ×":23162,"Ġдей":23163,"DNA":23164,"ĠAlt":23165,"Hi":23166,"ĠFox":23167,"ĠDI":23168,"_set":23169,"ĠBody":23170,"ĠRail":23171,"ä¸Ģ樣":23172,"ä½Ĩä»ĸ":23173,"é¢ĸ":23174,"带åĬ¨":23175,"ĠGard":23176,"åıĤè§Ĥ":23177,"ulu":23178,"Å¡t":23179,"Ġcounts":23180,"å·®ä¸įå¤ļ":23181,"comb":23182,"ĠRoll":23183,"ĠMC":23184,"Width":23185,"pus":23186,"Ġsyll":23187,"ĠProperty":23188,"511":23189,"ratic":23190,"ä¸ļ绩":23191,"ĠClassification":23192,"Ġpoison":23193,"IDS":23194,"ĠCole":23195,"à¸Ļà¹ī":23196,"ĠAnth":23197,"Ġlever":23198,"Ġvariant":23199,"Ġangry":23200,"Props":23201,"ĠSab":23202,"Ġcapability":23203,"รà¹Į":23204,"dist":23205,"Ġlying":23206,"437":23207,"ĠHart":23208,"ĠSarah":23209,"Ġpresum":23210,"Ġpept":23211,"ĠÙħد":23212,"çijŁ":23213,"conscious":23214,"Ċ":25097,"ĠDomin":25098,"mus":25099,"æµģéĢļ":25100,"Ġkw":25101,"ĠAfghan":25102,"管éģĵ":25103,"tx":25104,"æĭ¿çĿĢ":25105,"ifi":25106,"yon":25107,"ĠNevertheless":25108,"good":25109,"åħ¹":25110,"lyn":25111,"æĭĺ":25112,"年轻人":25113,"Ġsleeping":25114,"æĪļ":25115,"åĪ©æģ¯":25116,"ì§Ħ":25117,"Ġtends":25118,"Ġgrades":25119,"unnen":25120,"æķĻ室":25121,"491":25122,"第åħ«":25123,"Ġkommer":25124,"477":25125,"Ġcomputed":25126,"è§Ĩ为":25127,"ড":25128,"踢":25129,"Ġlear":25130,"Ġhill":25131,"ĠÃľ":25132,"spect":25133,"Ġmold":25134,"ortic":25135,"Ġstructured":25136,"Ġresident":25137,"Ġwondering":25138,"éĩįéĩı":25139,"innen":25140,"graph":25141,"ä¸įæĪIJ":25142,"Ġprelim":25143,"æĢ»ä¹ĭ":25144,"ursor":25145,"Der":25146,"calcul":25147,"æ³»":25148,"Ġeducators":25149,"éĩįè¦ģçļĦæĺ¯":25150,"omat":25151,"ĠUrban":25152,"Ġcrown":25153,"âĢĿ;":25154,"ciplinary":25155,"代谢":25156,"oscow":25157,"æ¨¡æł·":25158,"enen":25159,"Ġ-ĊĊ":25160,"-St":25161,"çļĦ缮æłĩ":25162,"ĠManufact":25163,"server":25164,"Ġsynthetic":25165,"Sal":25166,"ĠRegular":25167,"730":25168,"çĨ¬":25169,"ydney":25170,"Ġtransm":25171,"æĮ¯åħ´":25172,"éĻķ西":25173,"576":25174,"GET":25175,"æ¯Ķè¼ĥ":25176,"ĠÑĥÑģлови":25177,"atherine":25178,"å¤ļä¹Ī":25179,"cred":25180,"524":25181,"æĽ¿ä»£":25182,"ĠÑģлед":25183,"ãĤĬãģ¾ãģĻ":25184,"apon":25185,"åĩºçı¾":25186,"Ġtempt":25187,"Ġнеп":25188,"ummy":25189,"Ġoccupied":25190,"may":25191,"ĠArg":25192,"make":25193,"Ġabundance":25194,"æĶĢ":25195,"604":25196,"claimed":25197,"ĠHotel":25198,"нова":25199,"ĠContract":25200,"ĠCart":25201,"ĠTony":25202,"á̱":25203,"ĠεÏĢ":25204,"ppe":25205,"Ġhoped":25206,"Ġpreceding":25207,"Ġdifferentiation":25208,"Ġdietary":25209,"ë²ķ":25210,"Ġvoters":25211,"Ġjam":25212,"akespe":25213,"Ġportray":25214,"ĠÐŃÑĤо":25215,"Ġê±":25216,"лоÑģÑĮ":25217,"Ñĩна":25218,"çĨĶ":25219,"ĠDra":25220,"Ġdib":25221,"ĠCustomer":25222,"æĦŁè¦º":25223,"cents":25224,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":25225,"ĠVery":25226,"Ġsustained":25227,"Green":25228,"æŀģ为":25229,"води":25230,"ulse":25231,"urre":25232,"Ġmutations":25233,"031":25234,"强çĥĪ":25235,"çłĶç©¶æīĢ":25236,"çī©ä¸ļ":25237,"èĦĸ":25238,"(node":25239,"Ġmetrics":25240,"å¼Łå¼Ł":25241,"Ġpreference":25242,"Ġrolling":25243,"Ġconsistency":25244,"ilateral":25245,"衬":25246,"Õ¸ÖĤÕ©":25247,"士åħµ":25248,"Ġà¬":25249,"Ġ×Ķ×ķ×IJ":25250,"(key":25251,"ĠPanoramas":25252,"زش":25253,"Ġcommod":25254,"Ġaging":25255,"(list":25256,"ĠOperations":25257,"误差":25258,"vá":25259,"بÛĮ":25260,"Ġalike":25261,"arcel":25262,"Ġdamages":25263,"Ġcasual":25264,"ä¸Ģç³»åĪĹ":25265,"éĴī":25266,"olt":25267,"487":25268,"agine":25269,"aco":25270,"責":25271,"Icon":25272,"606":25273,"ikt":25274,"è°¦":25275,"æĶ¾ä¸ĭ":25276,"Ġìĥģ":25277,"LINE":25278,"argo":25279,"ĠPhase":25280,"æī¶è´«":25281,"575":25282,"ðĿĴ":25283,"ashes":25284,"lov":25285,"Ġdepartments":25286,"even":25287,"å¿ĺäºĨ":25288,"ãĥĥãĥĪ":25289,"çµ²":25290,"ãģªãģı":25291,"檢":25292,"Ġtie":25293,"×ķ×Ĺ":25294,"å·¢":25295,"Ġutilize":25296,"(@":25297,"Ġphenomena":25298,"æĨ¾":25299,"ĠIndians":25300,"nde":25301,"-pr":25302,"Ġletting":25303,"Ġhormone":25304,"å¸ĪèĮĥ":25305,"ĠBarn":25306,"ĠпоÑģле":25307,"Ġdying":25308,"Ġsubset":25309,"Ġfrequencies":25310,"ensed":25311,"Ġcontributes":25312,"åįıåĬ©":25313,"Ġinspection":25314,"yg":25315,"çļĦçĥŃ":25316,"Ġbind":25317,"ĠPeng":25318,"å°ıåŃIJ":25319,"Ġpatch":25320,"overline":25321,"æ°´ä¸Ń":25322,"缺çĤ¹":25323,"Ġalignment":25324,"ĠLater":25325,"ĠAnna":25326,"ĠReviews":25327,"orms":25328,"æĪijåİ»":25329,"Ġmock":25330,"姬":25331,"Ġviolation":25332,"Ġprost":25333,"óÅĤ":25334,"Ġextraordinary":25335,"Ġfue":25336,"å¹³çŃī":25337,"æĸ°é²ľ":25338,"Ġwheat":25339,"611":25340,"Ġip":25341,"Ġhid":25342,"çļĦ管çIJĨ":25343,"å¤Ħç½®":25344,"uese":25345,"гов":25346,"Ġgén":25347,"ĠMembers":25348,"092":25349,".Cont":25350,"Ġorbit":25351,"Ġsphere":25352,"551":25353,"çļĦä¼ģä¸ļ":25354,"562":25355,"æİĪæĿĥ":25356,"åħħåĪĨåıijæĮ¥":25357,"#Ċ":25358,"åΤåĨ³":25359,"Mus":25360,"599":25361,"ĠاÙĦÙħج":25362,"ÑĢован":25363,"延伸":25364,"ÑģÑĤÑĥп":25365,"Ġgathering":25366,"è¿Ļä¹Īå¤ļ":25367,"ĠProc":25368,"å°ıåŃ©":25369,"!=":25370,"Ġcircuits":25371,"ın":25372,"ĠDream":25373,"çĩĥçĥ§":25374,"Ġbrid":25375,"åıijçĹħ":25376,"Ġvalidity":25377,"ĠHours":25378,"æŃ·":25379,"ĠProced":25380,"Ġministry":25381,"910":25382,"ĠChart":25383,"*o":25384,"Ġcock":25385,"дей":25386,"×Ļפ":25387,"Ġsimulations":25388,"imated":25389,"Ġfluores":25390,"лом":25391,"-\\)":25392,"ä¼łæĦŁ":25393,"æıī":25394,"ĠConvers":25395,"ancel":25396,"Ġtermin":25397,"ĠBos":25398,"æĢ»ç»ıçIJĨ":25399,"ÃŃch":25400,"537":25401,"Ġsteam":25402,"592":25403,"Ġtrim":25404,"ĠDonald":25405,"èĬĤå¥ı":25406,"éĢĽ":25407,".min":25408,"æijĶ":25409,"unar":25410,"Ġë¯":25411,"太平":25412,"çļĦ使ç͍":25413,"ä½ĨæĪij":25414,"æĺ¯å¾Ī":25415,"Ġrespondents":25416,"law":25417,"£":25418,"å´Ķ":25419,"Ġjej":25420,"Ġada":25421,"æľŁæľĽ":25422,"526":25423,"adies":25424,"æĺİæĺİ":25425,"\\pi":25426,"Ġcorresponds":25427,"iostream":25428,"United":25429,"Ġmog":25430,"590":25431,"Ġlev":25432,"subscriptðĿij":25433,"875":25434,"nex":25435,"ĠRA":25436,"Ġabroad":25437,"Ġqualitative":25438,"æ¯ķä¸ļçĶŁ":25439,")^{":25440,"esc":25441,"ĠHyd":25442,"ĠTro":25443,"Ġhunting":25444,"uki":25445,"èµ·åΰ":25446,"çĶŁæ°Ķ":25447,"ativo":25448,"ĠÙĬع":25449,"ä¸ĬçıŃ":25450,"Ġ\\-":25451,"åķĨæłĩ":25452,"Ġrestaurants":25453,"ĠCPU":25454,"ĠSound":25455,"ouri":25456,"æ°ijèѦ":25457,"ICAL":25458,"æ¿Ģç´ł":25459,"ìĪ":25460,"Ġcig":25461,"ä¸ĢæĹ¥":25462,"è¯Ńåı¥":25463,"resa":25464,"ç·´":25465,"ĠBR":25466,"Ġsuicide":25467,"æĻ¯åĮº":25468,"Ġmuy":25469,"Ġdrove":25470,"Ġgeneric":25471,"517":25472,"ĠAli":25473,"à®®":25474,"ociated":25475,"åĨłåĨĽ":25476,"ĠSweden":25477,"ulis":25478,"ваеÑĤÑģÑı":25479,"Ġhoney":25480,"frame":25481,"à¯ģà®®à¯į":25482,"Helper":25483,"人ä¹ĭ":25484,"Ġdried":25485,"ìľĦ":25486,"ãģĹãģĦ":25487,"Ġarrested":25488,"Ġglory":25489,"instance":25490,"Ġprescribed":25491,"äºŀ":25492,"inning":25493,"Ġwash":25494,"é»ĺé»ĺ":25495,"Ġdatabases":25496,"Ġmotiv":25497,"543":25498,"å¤ı天":25499,"adtong":25500,"ê¹":25501,"æĸ¯çī¹":25502,"ĠJerusalem":25503,"æĹ¬":25504,"èµ°åĩº":25505,"çĪĨçĤ¸":25506,"ĠRoom":25507,"ÑĩеÑģкий":25508,"Ġinterference":25509,"ĠMAT":25510,"å¸Ĩ":25511,"Ġexplicitly":25512,"Ġdesarroll":25513,"NT":25514,"jango":25515,"ousing":25516,"_number":25517,"Ċ":25762,"apsed":25763,"rah":25764,"èĭ¯":25765,"Ġamongst":25766,"Ġinfinite":25767,"Ġswing":25768,"ĠMeaning":25769,"åĩıè½»":25770,"æĺ¯éĿŀ常":25771,"ĠSchools":25772,"ĉĉĉĉĉĉĉĉ":25773,"ĠDog":25774,"è¿ĻæĿ¡":25775,"é£ŀè¡Į":25776,".put":25777,"hon":25778,"Ġrevel":25779,"ĠTeachers":25780,"Ġratings":25781,"鹤":25782,"Ġcircles":25783,"737":25784,"{al":25785,"489":25786,"<<\"":25787,"ĠWi":25788,"ê·¸":25789,"Pan":25790,"cca":25791,"éħµ":25792,"ĠговоÑĢи":25793,"ximate":25794,"ĠVel":25795,"chell":25796,"Ġobesity":25797,"Ġoutputs":25798,"ĠاÙĦاست":25799,"Cle":25800,"è¿Ļ个æĹ¶åĢĻ":25801,"ieg":25802,"没æľīä»»ä½ķ":25803,"541":25804,"umi":25805,"anyak":25806,"Ġrenal":25807,"Ġbelonging":25808,"Ġfarming":25809,"ä¸īè§Ĵå½¢":25810,"ĠCF":25811,"inned":25812,"ĠAwards":25813,"ãĥij":25814,"Ġproducer":25815,"åıĺæĽ´":25816,"ĠFC":25817,"ĠBh":25818,"494":25819,"ÃŁen":25820,"ç¼ĵç¼ĵ":25821,"æĹłçĸij":25822,"ÙĬرة":25823,".model":25824,"Ġincredibly":25825,"åħ®":25826,"Ġencoding":25827,"{(":25828,"Ġstrains":25829,"Ġ|ĊĊ":25830,"å·«":25831,"Through":25832,".âĢľ":25833,"à§ĩà¦ĸ":25834,"Ġmeanings":25835,"ashi":25836,"write":25837,"ichen":25838,"åģľè½¦":25839,"Ġmortgage":25840,"980":25841,"ĠDA":25842,"åĴĮä»ĸ":25843,"Ġsynthes":25844,"Ġcoupling":25845,"+b":25846,"åı¯æĢķ":25847,"ĠÑįлек":25848,"Make":25849,"atrix":25850,"Ġfatigue":25851,"æ³ķå®ļ":25852,"Ġdivor":25853,"ĠCho":25854,"è¿ľè¿ľ":25855,"pun":25856,"è°Ĭ":25857,"å¼ķåħ¥":25858,"à«į":25859,"pot":25860,"ĠÑĢи":25861,"ä¸į论":25862,"Ġfaz":25863,"èĬ¯çīĩ":25864,"racellular":25865,"Ġmounted":25866,"Ġpaste":25867,"rophy":25868,"Ġoverlook":25869,"Ġconsensus":25870,"Ġplacing":25871,"æĥħçļĦ":25872,"常è§ģçļĦ":25873,"ritt":25874,"Ġinsects":25875,"éĿĴå°ijå¹´":25876,".status":25877,"inian":25878,"atti":25879,"492":25880,"479":25881,"第äºĮ个":25882,"æľĢå¾Į":25883,"Ġengines":25884,"!)":25885,"Java":25886,"Ġearthqu":25887,"ĠLaboratory":25888,"عر":25889,"åij¨å¹´":25890,"ranean":25891,"Ġabsent":25892,"åĴĮåıijå±ķ":25893,"Ġtranscription":25894,"ĠدÙī":25895,"Ġenormous":25896,"omething":25897,"å®īå¾½":25898,"ĠìĤ":25899,"Ġminerals":25900,"Ġecological":25901,"{}":25902,"dan":25903,"æĪijçľĭ":25904,"675":25905,"åĩłç§į":25906,"Space":25907,"arded":25908,"akespeare":25909,"spec":25910,"æİĴåĪĹ":25911,"åıĸå¾ĹäºĨ":25912,"wd":25913,"Ġtranslated":25914,"productive":25915,"окÑĥ":25916,"Ġvolunteers":25917,"éĴ©":25918,"Ġtemps":25919,"Spe":25920,"Ġdol":25921,"çĿ¡çľł":25922,"ĠBridge":25923,"ĠEquation":25924,"ĠSoci":25925,"rapper":25926,"Ġdisabilities":25927,"atro":25928,"Ġpricing":25929,"ĠiPhone":25930,"ĠìĽ":25931,"ĠTar":25932,"å°ıæľĭåıĭ":25933,"Ġcá»":25934,"Ġarom":25935,"-ind":25936,"sto":25937,"ĠÎĵ":25938,"èıĬ":25939,"Ġpanels":25940,"æĥħå½¢":25941,"SI":25942,"بات":25943,"åı¯ä»¥è¯´":25944,"617":25945,"Ġdy":25946,"kol":25947,"Ġapre":25948,"Ġpreferably":25949,"Ġperipheral":25950,"âĪij":25951,"ĠHigher":25952,"akukan":25953,"éĴł":25954,"iances":25955,".âĢĻĊ":25956,"Ġnumerator":25957,"ĠDoctor":25958,"oba":25959,"032":25960,"çļĦæľĭåıĭ":25961,"åħ³èģĶ":25962,"Ġcoin":25963,"Ġupt":25964,"529":25965,"Ġconcert":25966,"Ġsour":25967,"ĠMuch":25968,"çĬ¹è±«":25969,"Ġflesh":25970,"ughed":25971,"Ġandere":25972,"ÙģØŃÙĩ":25973,"ĠHem":25974,"è¾²":25975,"uhan":25976,"æ´»åĬ¨ä¸Ń":25977,"ĠRules":25978,"æ°´æ³¥":25979,"æ´»æĢ§":25980,"ári":25981,"æĪªèĩ³":25982,"ĠForum":25983,"æĶ¿åįı":25984,"Ġuint":25985,"Ġpiano":25986,"osto":25987,"ĠMT":25988,"å®°":25989,"åħĭæľį":25990,"ä¸Ģä¸ĭåŃIJ":25991,"Ġswimming":25992,")|":25993,"618":25994,"Ġgeometric":25995,"ĠíĮ":25996,"Ġseparately":25997,"orf":25998,"夷":25999,"agi":26000,"ä¸İåħ¶":26001,"èģļéĽĨ":26002,"Ġtender":26003,"å¿ĥçģµ":26004,"ĠÑĤом":26005,"æĪIJ为äºĨ":26006,"Program":26007,"zet":26008,"ĠNik":26009,"656":26010,"eous":26011,"Ġstops":26012,"åĪ®":26013,"527":26014,"à¹Ģà¸Ī":26015,"ĠSept":26016,"è¯Ĺ人":26017,"ĠVictoria":26018,"è¶³çIJĥ":26019,"१":26020,"ĠJosh":26021,"Ġdign":26022,"广西":26023,"ĠFacts":26024,"Ġdating":26025,"ÙĥÙħ":26026,"æķ·":26027,"expl":26028,"çŁ¥åIJį":26029,"åľ¨éĤ£":26030,"ãĢĭï¼Ī":26031,"ĠÑĢезÑĥлÑĮÑĤа":26032,"âĸ³":26033,"âĸ¼":26034,"546":26035,"åıįå°Ħ":26036,"ĠNewton":26037,"สม":26038,"alid":26039,"нÑĤи":26040,"ĠOntario":26041,"493":26042,"âĪł":26043,"jamin":26044,"ĠDiet":26045,"åľºçļĦ":26046,"离å©ļ":26047,"April":26048,"ĠRF":26049,"à«įàª":26050,"ĠÙĪØ³":26051,"ĠÑĢеб":26052,"ĠZeit":26053,"ê°ľ":26054,"assium":26055,"Ġeverybody":26056,"olitan":26057,"å¹²æī°":26058,"Ġgenus":26059,"Ġdecreasing":26060,"ĠMorgan":26061,"ĠÃģ":26062,"kind":26063,"रà¥įà¤":26064,"å¹´åºķ":26065,"Ġsensory":26066,"огÑĢам":26067,"ĠâĹı":26068,"Contact":26069,"ei":26070,"Ġprobe":26071,"ëŀĺ":26072,"-style":26073,"é¹°":26074,"bbed":26075,"åīįè¿Ľ":26076,"Ġoffense":26077,"strap":26078,"unting":26079,"Ġadmission":26080,"ĠVector":26081,"ventory":26082,"âĪĤ":26083,"Ġtrusted":26084,"bling":26085,"Ġels":26086,"æĶ»åĿļ":26087,"Ġencountered":26088,"Ġgau":26089,"Ġbasketball":26090,"641":26091,"ɪ":26092,"æ´¾åĩº":26093,"ï¼ı":26094,".split":26095,"Ġextracted":26096,";ĊĊĊ":26097,"æĥ³æĥ³":26098,"Ġnecessity":26099,"æĦŁåĬ¨":26100,"(I":26101,"Ġ{}":26102,"ÅĪ":26103,".Skip":26104,"607":26105,"Ġexplores":26106,"ĠLind":26107,"News":26108,"ï¼ĮãĢĮ":26109,"à¦ķà§įত":26110,"Ġguided":26111,"Ļà§įà¦":26112,"Ġmaint":26113,"åħ¥åı£":26114,"ĠMun":26115,"çĸijæĥij":26116,"ĠJavaScript":26117,"訴":26118,"utz":26119,"çIJĥåijĺ":26120,"Ġhouseholds":26121,"ä¸įæ¸ħ":26122,"åıij表äºİ":26123,"Ġmetall":26124,"ĠпеÑĢи":26125,"ĠÙĦÙĬÙĨ":26126,"åĨħåŃĺ":26127,"Ġconservative":26128,"æ£ļ":26129,"Ġcyber":26130,"Ġexplaining":26131,"ĠMinnes":26132,"ancia":26133,"Ġcancel":26134,"大éĺŁ":26135,"ografia":26136,"Ġnick":26137,"à¦ķà§įষ":26138,"549":26139,"äºĶ个":26140,"MAX":26141,".....":26142,"ĠдÑĢÑĥг":26143,"reatment":26144,"Ġpredictions":26145,"Ġвозмож":26146,"æĺ¯ä½ł":26147,"ĠVo":26148,"Ġcuando":26149,"Ġactivated":26150,"631":26151,"à¸Ķà¹īวย":26152,"ĠMexican":26153,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":26154,"alg":26155,"ĠPain":26156,"Cu":26157,"ï¹":26158,"Ġbass":26159,"561":26160,"ĠSingh":26161,"Ġdiscussing":26162,"ĉwhile":26163,"racks":26164,"568":26165,"Ġwooden":26166,"沸":26167,"ijing":26168,"aram":26169,"Ġengineers":26170,"ĠHoll":26171,"utation":26172,"ÑĩенÑĮ":26173,"ĠAF":26174,"ä¸ĩåIJ¨":26175,"070":26176,"ë¹Ħ":26177,"Ġlaughed":26178,"Ġcombining":26179,"åijµåijµ":26180,"éŃħåĬĽ":26181,"è¦ģæĬĬ":26182,"estamp":26183,"Ġtbsp":26184,"×Ĵ":26185,"çĪŃ":26186,"åĸľæŃ¡":26187,"nek":26188,"ĠTel":26189,"Ġdistances":26190,"Ġki":26191,"498":26192,"viously":26193,"Ġframes":26194,"æĪIJåĬŁçļĦ":26195,"æ´»åĬĽ":26196,"çĪº":26197,"ç´§æĢ¥":26198,"Ġresemb":26199,"åºĶåĬĽ":26200,"ĠÏī":26201,"åĽŃåĮº":26202,"ENCE":26203,"æĽ´æį¢":26204,"Student":26205,"Ġting":26206,"Ġwings":26207,"ĠMarine":26208,"åİ¿å§Ķ":26209,"åįĶ":26210,"èµĦæľ¬ä¸»ä¹ī":26211,"Ġmend":26212,"Ġburied":26213,".write":26214,"appropri":26215,"éļ¶":26216,"Ġenemies":26217,"ĠAlbert":26218,"542":26219,"ĠìŀĪëĬĶ":26220,"ай":26221,"öl":26222,"ĠRather":26223,"åĩºéŨ":26224,"Ġconstitutional":26225,"Ġتج":26226,"Ġद":26227,"èģĸ":26228,"Ġcollaborative":26229,"Ġabundant":26230,"åı¯çα":26231,"ĠFif":26232,"ĠChap":26233,"Ġresidual":26234,"epsilon":26235,"æĪ¶":26236,"çļĦç»ıæµİ":26237,"ĠNich":26238,"Ġbay":26239,"ĠOregon":26240,"Ġпло":26241,"_W":26242,"åĽ¢ä½ĵ":26243,"Init":26244,"Ġchurches":26245,"_str":26246,"ufficient":26247,"views":26248,"Ġinvasion":26249,"Ġaplic":26250,"Ġgases":26251,"å°±æĬĬ":26252,"(q":26253,"åĿ¤":26254,"Ġcollapse":26255,"ĠLuke":26256,"riger":26257,"_)":26258,"å°¬":26259,"åı¯ä»¥ç͍":26260,"Ġvacuum":26261,"Ġtersebut":26262,"ç¼ĸçłģ":26263,"ãĢĭĊ":26264,"à¤ľ":26265,"Ġcement":26266,"erness":26267,"лÑĮно":26268,"construct":26269,"mercial":26270,"Ġsake":26271,"ĠPopular":26272,"Ġbasihan":26273,"Ġworkshop":26274,"unj":26275,"åIJįåįķ":26276,"622":26277,"ĽĦ":26278,"è¾ĪåŃIJ":26279,"Ġtambém":26280,"åĬŁæķĪ":26281,"Ġdelivering":26282,"ĠTaiwan":26283,"Ah":26284,"istor":26285,"çļĦåŃĺåľ¨":26286,"Ġroku":26287,"Ġdecay":26288,"Ġfurniture":26289,"ваеÑĤ":26290,"ãģķãĤĵ":26291,"Ġamid":26292,"Ġâĭħ":26293,"ĠMED":26294,"ÑīеÑģÑĤв":26295,"æĬ±çĿĢ":26296,"endif":26297,"è¿Ļè¾¹":26298,"ĠEssay":26299,"478":26300,"ĠBuy":26301,"aterials":26302,"éĬĢ":26303,"-right":26304,"inth":26305,"Ġmuc":26306,"559":26307,"ä¸įä»ħä»ħ":26308,"Ġpole":26309,"Ġ-=":26310,"äs":26311,"Ġpassive":26312,"curities":26313,"Ġegy":26314,"Ġdefensive":26315,"Ġclouds":26316,"ĠTy":26317,"Ġethics":26318,"Ġstunning":26319,")),":26320,"Ġattending":26321,"åįĴ":26322,"ĠЯ":26323,"anga":26324,"rocal":26325,"Ġslot":26326,"Ġbeans":26327,"æŃ£å¦Ĥ":26328,"à´¾à´":26329,"ĠбÑĭло":26330,"mother":26331,"nals":26332,"ĠKam":26333,"ĠاÙĦز":26334,"ĠìĤ¬ìļ©":26335,"ä¹ĭåľ°":26336,"说åΰ":26337,"Ġmarker":26338,"Ġforgotten":26339,"Ġantibody":26340,"ĉprintf":26341,"iblical":26342,"temp":26343,"éªĦ":26344,"endant":26345,"ãĤĮãģ°":26346,"Ġtrait":26347,"ĠRT":26348,"Ġlid":26349,"åĮł":26350,"éĢī举":26351,"633":26352,"çķĮéĿ¢":26353,"632":26354,"Ġgaps":26355,"Ġcharts":26356,"Ġspecialist":26357,"imos":26358,"Ġbatch":26359,"Ġdeficiency":26360,"åĬ©åĬĽ":26361,"577":26362,"æĭ¿åĩº":26363,"ĠTheorem":26364,"_num":26365,"October":26366,"Ġìĸ´":26367,"Ġsar":26368,"Ġorganism":26369,"르":26370,"ATED":26371,"à°¨":26372,"-order":26373,"_by":26374,"Ñĭм":26375,"Reference":26376,"outheast":26377,"×ķפ":26378,"ockey":26379,"ä¸Ģå¼Ģå§ĭ":26380,"ãģŃ":26381,"iented":26382,"Ġundergo":26383,"æ¼Ķ讲":26384,"arium":26385,"à¹ĥà¸Ī":26386,"åĺ±":26387,":=":26388,"ĠMolecular":26389,"çŁ¥éģĵäºĨ":26390,"Ġeager":26391,"ÃŃfic":26392,"711":26393,"ĠLyn":26394,"illary":26395,"Ġpacked":26396,"ĠCoun":26397,"Ġinterpreted":26398,"åħ·ä½ĵçļĦ":26399,"Ġimperial":26400,"èģªæĺİ":26401,"(H":26402,"624":26403,">=":26404,"_IN":26405,"dz":26406,"644":26407,"以å¾Ģ":26408,"oku":26409,"Ġek":26410,"adj":26411,"585":26412,"å°±è¿Ļæł·":26413,"ĠVin":26414,"àµģà´":26415,"Ġinsect":26416,"æīĵçł´":26417,"Ġhepat":26418,"ç¥ĸåĽ½":26419,"drop":26420,"uccess":26421,"Ġauthorized":26422,"larg":26423,"isy":26424,"Ġdebut":26425,"Ġoverwhelming":26426,"ĠRO":26427,"ĠPE":26428,"ĠStrategy":26429,"æĮ½":26430,"Ġtiem":26431,"/re":26432,"ç¦ħ":26433,"emn":26434,"Ġdun":26435,"Ġ׼׾":26436,"roit":26437,"Ġexped":26438,"绩æķĪ":26439,"ĠCaptain":26440,"Ġinvestigations":26441,"glas":26442,"ĠHarris":26443,"heat":26444,"ĠCovid":26445,"æľŁè´§":26446,"åħ¶å¯¦":26447,"ĠاÙĦÙħر":26448,"zenie":26449,"ĠSamuel":26450,"Ġcelebrated":26451,"Ġ[â̦":26452,"ĠRecords":26453,"ä¸ŃæīĢ":26454,"ĠChat":26455,"ktion":26456,"Ġmethyl":26457,"Ġ-->Ċ":26458,"äºĶåįģ":26459,"ĠAddress":26460,"538":26461,"æĹŃ":26462,"Ġexisted":26463,"ÑĪиÑħ":26464,"inked":26465,"à¹Ģว":26466,"ĠKy":26467,"åѦåijĺ":26468,"ê³µ":26469,"Ġenjoying":26470,"Õ¸ÖĤÕ¶":26471,"Ġسب":26472,"åļ´":26473,"লà§ĩ":26474,"社交":26475,"Ġenzymes":26476,"æħ¨":26477,"Ġsurviv":26478,"Ġattachment":26479,"Ġë¹Ħ":26480,"ĠEM":26481,"ìĽĶ":26482,"abol":26483,"564":26484,"çļĦåŃ©åŃIJ":26485,"Ġtournament":26486,"Ġthreatened":26487,"ĠاÙĦÙħØŃ":26488,"Ġzap":26489,"Ġbyte":26490,"ĠGA":26491,"Õ¡ÖĢÕ":26492,"çα好":26493,"åĬ¿åĬĽ":26494,"Ġ\"\"\"":26495,"ĠCass":26496,"Ġwenn":26497,"Ġfossil":26498,"eron":26499,"636":26500,"æĸ¹éĴĪ":26501,"åĽĽåij¨":26502,"ç®Ģä»ĭ":26503,"马åħĭæĢĿ主ä¹ī":26504,"물":26505,"ĠTak":26506,"interest":26507,"zb":26508,"ĠFibonacci":26509,"Ġpromises":26510,"ĠPow":26511,"æĪijè¿ĺ":26512,"584":26513,"à¸Īำ":26514,"fly":26515,"åħ¬åĬ¡":26516,"ĠJam":26517,"ï¼ĮâĢĿ":26518,"庸":26519,"\"We":26520,"ailure":26521,"ÑĤÑĥÑĢа":26522,"-down":26523,".on":26524,"Ġbaseball":26525,"ocrats":26526,"']Ċ":26527,"Ġintelligent":26528,"çľ¼éĩĮ":26529,"ĠSynt":26530,"仲è£ģ":26531,"ден":26532,"ç«ŀèµĽ":26533,"Ġdispute":26534,"Omega":26535,"application":26536,"940":26537,"Ġoffensive":26538,"ĠMental":26539,"{C":26540,"Ġpneum":26541,"ä¸Ģæī¹":26542,"å¼ĵ":26543,"Ġsail":26544,"Both":26545,"æĹ¥çĽĬ":26546,"{p":26547,"Ġrepository":26548,"æľįåĬ¡çļĦ":26549,"ãģĤãĤĬãģ¾ãģĻ":26550,"èµģ":26551,"553":26552,"860":26553,"åĪĨçļĦ":26554,"ÑĤÑĥÑĢ":26555,"Ġ\\({":26556,"ĠRot":26557,"âĤ":26558,"Ġdrops":26559,"æĦıåĽ¾":26560,"POST":26561,"positive":26562,"館":26563,"etermined":26564,"表çݰ为":26565,"ä¸Ģçķª":26566,"cury":26567,"andy":26568,"alen":26569,"åı¯çŁ¥":26570,"Ġsixth":26571,"jÄĻ":26572,"éĤ£æĹ¶":26573,"ĠNation":26574,"ĠAllah":26575,"Ġvarieties":26576,"Ġcrossed":26577,"Ġdistributions":26578,"ĠÑģем":26579,"西åįĹ":26580,"990":26581,"Ġonset":26582,"ä¸Ģåı£":26583,"**(":26584,"ĠKer":26585,"ł×Ļ×Ŀ":26586,"Ġabandoned":26587,"ĠбÑĭл":26588,"ĠException":26589,"Ġشر":26590,"åħ©åĢĭ":26591,"Ġdok":26592,"Ġexcitement":26593,"天天":26594,"æİĴæĶ¾":26595,"信念":26596,"Ġawards":26597,"å¹³éĿĻ":26598,"998":26599,"incess":26600,"ACH":26601,"690":26602,"ĠاÙĦغ":26603,"ĠMinnesota":26604,"ulsion":26605,"call":26606,"Ġpackages":26607,"-fl":26608,"ĠMarx":26609,"quence":26610,"ást":26611,"671":26612,"ä»İæŃ¤":26613,"Arch":26614,"ichte":26615,"gang":26616,"ierra":26617,"ĠQuick":26618,"NULL":26619,"æĪ²":26620,"æłĭ":26621,"shot":26622,"Ġrég":26623,"plt":26624,"头ä¸Ĭ":26625,"Ġcounterpart":26626,"Ġoldest":26627,"Ġà¸Ń":26628,"Ġnorms":26629,"Ġcompete":26630,"770":26631,"NN":26632,"Ġextens":26633,"+=":26634,"Ġstimulation":26635,"/O":26636,"istration":26637,"ä¸Ģ代":26638,"ĠJSON":26639,"Ġconstitute":26640,"itic":26641,"Ġlesions":26642,"Ø¥ÙĨ":26643,"Ġinflammatory":26644,"Ġholy":26645,"Ġkunnen":26646,"Ġjavax":26647,"à¸ģระ":26648,"çļĦä¸ī":26649,"shared":26650,"ReplyDelete":26651,"è¾IJå°Ħ":26652,"Ġawesome":26653,"Ġbabies":26654,"563":26655,"åıĺåĬ¨":26656,"Ġ\\((":26657,"ÄįnÃŃ":26658,"Ġtimely":26659,"Ġhomework":26660,"639":26661,"Ġ##":26662,"âĢĿãĢģ":26663,"ĠÑģпе":26664,"ģ¬":26665,"ĠRon":26666,"à§įম":26667,"uar":26668,"site":26669,"572":26670,"beg":26671,"Ã¥r":26672,"页éĿ¢":26673,"_user":26674,"cb":26675,"Ġadministered":26676,"×Ļ×Ļ×":26677,"ãĢįï¼Į":26678,"-gener":26679,"\")]Ċ":26680,"653":26681,"çļĦ产åĵģ":26682,"ĠSubject":26683,"Simpl":26684,"ÑīениÑı":26685,"äºīè®®":26686,"Ġanx":26687,"Ġpela":26688,"å¾ĹçŁ¥":26689,"Ġinduction":26690,"eh":26691,"Ġconversions":26692,"ĠÑģозда":26693,"oxic":26694,"ĠWould":26695,"Ġgaze":26696,"ék":26697,"Ġlasting":26698,"ĠÑĤÑĢе":26699,"ĠOthers":26700,"ä½łæĢİä¹Ī":26701,"prop":26702,"Ġlimiting":26703,"ãĥ¥":26704,"mentioned":26705,"åΰæĿ¥":26706,"service":26707,"Ġpeoples":26708,"Ġuk":26709,"èͽ":26710,"Ġcomprehension":26711,"\"\"\"Ċ":26712,"Ġslave":26713,"Ġaltered":26714,"Ġsemiconductor":26715,"Ġprosper":26716,"åĭĺ":26717,"Dto":26718,"Ġchron":26719,"åĿĿ":26720,"á¸":26721,"Ġmph":26722,"ugg":26723,"Ġnúmero":26724,"ancies":26725,"ĠìķĬ":26726,"722":26727,"Ġrolled":26728,"é¤IJåİħ":26729,"Ġboards":26730,"Organ":26731,"ependence":26732,"æĺĬ":26733,"Consider":26734,"åIJĮæĹ¶ä¹Ł":26735,"friend":26736,"Ġdrinks":26737,"Ġodds":26738,"703":26739,"Ġtargeting":26740,"ĠNie":26741,"estone":26742,"ĠSeason":26743,"åIJĪçIJĨçļĦ":26744,"Ġcotton":26745,"Ġremed":26746,"ĠLinked":26747,"Ġgovernor":26748,"对åºĶçļĦ":26749,"phant":26750,"Loc":26751,"Ġsovere":26752,"жениÑı":26753,"768":26754,"ivel":26755,"Ġpromotes":26756,"region":26757,"small":26758,"ĠнеобÑħодимо":26759,"534":26760,"éļıæĦı":26761,"VP":26762,"Ġdiscourse":26763,"ITE":26764,".map":26765,"Я":26766,"æĹ±":26767,"Ġentrepreneurs":26768,"ĠExercise":26769,"ĠÑħаÑĢак":26770,"æĬĢè¡ĵ":26771,"ĠTemple":26772,"Summary":26773,"èĢĮæĿ¥":26774,"Ġprospective":26775,"Ġavoiding":26776,"548":26777,"åĨįåĬłä¸Ĭ":26778,"vised":26779,"ĠFat":26780,"ksi":26781,"iliation":26782,"неÑĢ":26783,"æİ¥åΰ":26784,"çĶŁèĤ²":26785,"crit":26786,"å¤§èµĽ":26787,"ĠÑģооÑĤвеÑĤ":26788,"Ġgamb":26789,"ĠÑĩем":26790,"éĢīç͍":26791,"614":26792,"Mapping":26793,"鸡èĽĭ":26794,"Ġdoctrine":26795,"財":26796,"bound":26797,"ranes":26798,"Ġwondered":26799,"ĠAnda":26800,"ç§Łèµģ":26801,"asets":26802,"æīİå®ŀ":26803,"جÙħ":26804,"Ġchampion":26805,"ermat":26806,"Ġdeemed":26807,"ãģĹãģ¦ãģĦãĤĭ":26808,"owy":26809,"Ïķ":26810,"Ġresidence":26811,"заÑĨии":26812,"ignant":26813,"ĠFisher":26814,"fd":26815,"ĠYu":26816,"Ġcontacts":26817,"ór":26818,"âĦ¢":26819,"ĠPrintable":26820,"Pad":26821,"usr":26822,"ವ":26823,"Score":26824,"Ġappreciation":26825,"Ġbelt":26826,"642":26827,"bsite":26828,"нÑĮ":26829,"交å¾Ģ":26830,"-z":26831,"Ġbibli":26832,"导æ¼Ķ":26833,"_Ċ":26834,"emption":26835,"ĠAsk":26836,"виÑģи":26837,"803":26838,"äºļæ´²":26839,"Ġwages":26840,"äºĨ好":26841,"ÛĮÙĦ":26842,"Ġdividing":26843,"ĠLev":26844,"ам":26845,"诵":26846,"ç´§å¯Ĩ":26847,"ĠEvidence":26848,"xis":26849,"ticas":26850,"Ġsá»ij":26851,"éĢģåΰ":26852,"LR":26853,"ĠHebre":26854,"iversary":26855,"LD":26856,"Ġrevis":26857,"Ġpunishment":26858,"对称":26859,"转åIJij":26860,"à§įল":26861,"Ġvascular":26862,"Ġinquiry":26863,"Ġsubstitute":26864,"æĺ¯çļĦ":26865,"ä»ĸä¸į":26866,"åľ¨å¤ĸ":26867,"Ġvariability":26868,"ĠLCM":26869,"èĢĮåĩº":26870,"仪åύ":26871,"Ġannot":26872,"Ġpresenting":26873,"nas":26874,"-ing":26875,"ateur":26876,"ĠThread":26877,"-md":26878,"Pal":26879,"åı¸æľº":26880,"èĮħ":26881,"akter":26882,"Ш":26883,"ĠKhan":26884,"æĭ¿åΰ":26885,"(false":26886,".md":26887,"Ġmobil":26888,"ĠLower":26889,"Ġrival":26890,"帮å¿Ļ":26891,"Ġutiliz":26892,"Ġovert":26893,"Ġdeposit":26894,"'une":26895,"ĠCE":26896,"Ġvocal":26897,"ĠCommunications":26898,"าà¸Ķ":26899,"[/":26900,"ä¸į容æĺĵ":26901,"Ġdegli":26902,"atto":26903,"uo":26904,"ryption":26905,"Background":26906,"оÑģÑĥдаÑĢ":26907,"Ġwo":26908,",s":26909,"Ġgods":26910,"Ġgy":26911,"Ġvine":26912,"[-":26913,"Ġcholesterol":26914,"ê·":26915,"اÛĮد":26916,"'a":26917,"Ġambient":26918,"Sk":26919,"åIJİèĢħ":26920,"ĠMessage":26921,"éĥ½å¸Ĥ":26922,"ä¸ŃåĽ½åħ±äº§åħļ":26923,"Ġreservoir":26924,"车åŀĭ":26925,"Ġneighbors":26926,"ĠProducts":26927,"Ġsediment":26928,"Ġinhabit":26929,"æļ¨":26930,"Ġincorporating":26931,"Ġaccordingly":26932,"September":26933,"574":26934,"æİ¢ç©¶":26935,"ĠMeeting":26936,"ĠChristianity":26937,"Ġtouched":26938,"Mean":26939,"èĦļæŃ¥":26940,"ç»Ļ她":26941,"Ñıм":26942,"Ġpelo":26943,"ĠBackground":26944,"å±ij":26945,"ä¸Ĭå¸Ŀ":26946,"ĠOriginal":26947,"ìŀ¬":26948,"Ġaudiences":26949,"å¥īçĮ®":26950,"åĬī":26951,"Ġretention":26952,"Ġties":26953,"érie":26954,"à§ľ":26955,"Ġislands":26956,"Ġkun":26957,"041":26958,"Ġveg":26959,"830":26960,"æį§":26961,"Ġpurchasing":26962,"deg":26963,"eor":26964,"è¿Ļåı¥è¯Ŀ":26965,"Ġaument":26966,"ienen":26967,"onomous":26968,"Enc":26969,"Ġsheep":26970,"romagnetic":26971,"ä¼ĺæĥł":26972,"ĠCollabor":26973,"Ġproducers":26974,"ụ":26975,"try":26976,"åį¦":26977,"è´µå·ŀ":26978,"Ġbrack":26979,"Radius":26980,"geq":26981,"Cent":26982,"æĸ¯åĿ¦":26983,"éĤ£å¤©":26984,"Ġpainful":26985,"ributes":26986,"çĹħçļĦ":26987,"媳":26988,"Ġwars":26989,"ynthesis":26990,"æĩĤå¾Ĺ":26991,"书æ³ķ":26992,"Ġcorrelated":26993,"å̼çļĦ":26994,"ĠIng":26995,"缮çļĦæĺ¯":26996,"tterlig":26997,"çłį":26998,"547":26999,"emperature":27000,"Ġpeers":27001,",":27594,"Ġunp":27595,"à¸ŀระ":27596,"ças":27597,"为人":27598,"ĠRemove":27599,"è¿Ļ款":27600,"ĠHoward":27601,"blue":27602,"Ġëĵ±":27603,"amous":27604,"åıĹ伤":27605,"Ġantioxid":27606,"Ġapple":27607,"Ġmai":27608,"-world":27609,"ĠSay":27610,"904":27611,"nte":27612,"دا":27613,"纺":27614,"ĠЦ":27615,"ĠпÑĢоб":27616,"Ġzá":27617,"588":27618,"Ġdik":27619,"è¿Ļæĺ¯ä¸Ģ":27620,"Ġrelevance":27621,"Ġdistinguished":27622,"Pres":27623,"æĸĮ":27624,"ä½łæĥ³":27625,"ĠAy":27626,"ĠDM":27627,"Ġdefence":27628,"ĠStyle":27629,"éªĮæĶ¶":27630,"-ac":27631,"ĠCome":27632,"ĠFish":27633,"Ġtags":27634,"Ġunemployment":27635,"Ġcharacterization":27636,"ĠInstagram":27637,"Range":27638,"Ġfreely":27639,"Ġdamp":27640,"sign":27641,"Ġsunlight":27642,"804":27643,"ĠShel":27644,"è¿Ļä¹Ł":27645,"ités":27646,"Ġpist":27647,"715":27648,"olitical":27649,"Ġfetch":27650,"足以":27651,"ĠìĬ":27652,"ãĤ¤ãĥ³":27653,"icker":27654,"éĢĻæĺ¯":27655,"!!!!":27656,"ĠDuke":27657,"...,":27658,"Ġcooked":27659,"éĻ·åħ¥":27660,"宿èĪį":27661,"ÑĤели":27662,"çļĦ两":27663,"èĤĨ":27664,"651":27665,"/><":27666,"æ¸ħæ´Ĺ":27667,"éĨĭ":27668,"ĠReturns":27669,"è¯ĹæŃĮ":27670,"Ġintroducing":27671,"ä¼łè¯´":27672,"Ġpointing":27673,"ĠBuck":27674,"ç²¾å¿ĥ":27675,"ĠNothing":27676,"ĠαÏĢÏĮ":27677,"ĠTreeNode":27678,"---Ċ":27679,"(in":27680,"Ġpressures":27681,"çĤºäºĨ":27682,"Ġsla":27683,"/x":27684,"itet":27685,"ĠReserve":27686,"ocur":27687,"Ġ)ãĢĤĊĊ":27688,"Ġamplitude":27689,"ĠBron":27690,"ĠUnter":27691,"Ġarchae":27692,"ç»ĨèıĮ":27693,"Ġsword":27694,"æŃ§":27695,"åıĸåĨ³":27696,"ãģ»":27697,"Ġworkforce":27698,"Ġobsc":27699,"elesc":27700,"åĽŀåºĶ":27701,"çĦ¦èĻij":27702,"Ġbreakdown":27703,"Ġgym":27704,"ĠHerm":27705,"erald":27706,"Ġdx":27707,"648":27708,"åĩºæĿ¥äºĨ":27709,"vg":27710,"ĠPitt":27711,"毫米":27712,"ŀ×Ķ":27713,"å·¥ç¨ĭå¸Ī":27714,"/(-":27715,"dk":27716,"Ġdining":27717,"iferation":27718,"rophic":27719,"Ġinadequ":27720,"idge":27721,"Ġintersection":27722,"Ġruled":27723,"iolet":27724,"åľ°è´¨":27725,"aysay":27726,"ÛĮÙĩ":27727,"Ġcalculating":27728,"åĪĿæľŁ":27729,"Ġcitations":27730,"ä»ķ":27731,"iap":27732,"Ġballs":27733,".as":27734,"sson":27735,"Ġapplicant":27736,"Ġlem":27737,"682":27738,"ĠDisney":27739,"ĠWalter":27740,"Ġhosts":27741,"Ġktóry":27742,"Ġcompelling":27743,"Ġlocked":27744,"ĠHyp":27745,"Ġreminded":27746,"clock":27747,"Ġisot":27748,"è§£éϤ":27749,"626":27750,"ĠTestament":27751,"Ġkindergarten":27752,"Ġviel":27753,"Ġdeclare":27754,"身åŃIJ":27755,"æĻ®åıĬ":27756,"Ġbunch":27757,"ij×¢":27758,"Ġà¦Ĩম":27759,"Ġshaping":27760,"å®ĺåijĺ":27761,"æĺ¯æĪij们":27762,"Ġihr":27763,"Ġarbitrary":27764,"August":27765,"728":27766,"人ä¸İ":27767,"Ġshortly":27768,"Educ":27769,"alian":27770,"Ġpremier":27771,"635":27772,"Ġvertex":27773,"Bet":27774,"Ġirregular":27775,"èĢIJå¿ĥ":27776,"arrass":27777,"ĠHold":27778,"æĪijæľī":27779,"æĸ°èĥ½æºIJ":27780,"Ġhypertension":27781,"ĠSnow":27782,"ailand":27783,"Ġreass":27784,"ipation":27785,"ĠAppe":27786,"ĠBL":27787,"Ġappreciated":27788,"ANCE":27789,"UID":27790,"-an":27791,"Ġclubs":27792,"template":27793,"Ġ'../":27794,"缺å°ij":27795,"jes":27796,"sur":27797,"Ñļе":27798,"Ġverm":27799,"åι":27800,"zig":27801,"Ġspokes":27802,"672":27803,"posite":27804,"å¯Ĩçłģ":27805,"Ġespecial":27806,"-ne":27807,"Ġmeer":27808,"Ca":27809,"ĠInvestig":27810,"icion":27811,"-month":27812,"ĠInflu":27813,"ĠSEC":27814,"Ġrevision":27815,"Ġnights":27816,"ĠEmail":27817,"Ġautomated":27818,"Ġ\\)":27819,"å¤ļ人":27820,"(st":27821,"交åıī":27822,"èģļçĦ¦":27823,"system":27824,"ĠعاÙħ":27825,"Ġattempting":27826,"æī©å¼ł":27827,"одÑĥк":27828,"Ġcrystall":27829,"å¼Ĭ":27830,"ĠMaj":27831,"ĠuseState":27832,"对èĩªå·±":27833,"Ġprogn":27834,"569":27835,"721":27836,"tail":27837,"çļĦçݯå¢ĥ":27838,"ÑĦе":27839,"åĮ»èį¯":27840,"duced":27841,"umatic":27842,"è¾¾æĪIJ":27843,"åī¯ä¸»ä»»":27844,"åı¯æĮģç»Ń":27845,"Ġenrich":27846,"Ġcompleting":27847,"Ġah":27848,"严èĤĥ":27849,"[index":27850,"ziaÅĤ":27851,"çļĦåħ·ä½ĵ":27852,"716":27853,"Ap":27854,"Ġaggregate":27855,"otor":27856,"à®±":27857,"Ġkilograms":27858,"Ġartery":27859,"agent":27860,"Ġdepict":27861,"Param":27862,"ĠAnthony":27863,"Ġsufficiently":27864,"Ma":27865,"Ġdesp":27866,"å°±è¡Į":27867,"çļĦèĬ±":27868,"Ġverbs":27869,"é»ĺ认":27870,"hicle":27871,"çĸijéĹ®":27872,"ĠMemory":27873,"اÙĦص":27874,"åįıåIJĮ":27875,"æij©æĵ¦":27876,"Ġassay":27877,"便åĪ©":27878,"Ġimpr":27879,"çijľ":27880,"Ġgrave":27881,"ĠAntonio":27882,"Ġspeeds":27883,"à¹īาà¸Ļ":27884,"mem":27885,"年纪":27886,"åįĹæĸ¹":27887,"ĠBrother":27888,"æĮĸæİĺ":27889,"Ġofficially":27890,"ensation":27891,"éģĹä¼ł":27892,"没æľī人":27893,"åľ¨è¿Ļç§į":27894,"661":27895,"ìĺ¤":27896,"725":27897,"Ġà¦ıà¦ĩ":27898,"Ġsuite":27899,"å¾Īä¹ħ":27900,"æī¾åΰäºĨ":27901,"Ġtam":27902,"ĠRange":27903,"aceae":27904,"Ġdoses":27905,"ĠRGB":27906,"ĠJa":27907,"гоÑĤов":27908,"Xiv":27909,"Ġpemb":27910,"appa":27911,"Ġredirect":27912,"%;Ċ":27913,"åĬĩ":27914,"çīĪæĿĥ":27915,"lay":27916,"ç͍éĢĶ":27917,"Ġshore":27918,"å¹¶æľª":27919,"Ġimped":27920,"kee":27921,"Ġpare":27922,"706":27923,"Ġë°ľ":27924,"ĠLeft":27925,"Ġalien":27926,"æģIJæĥ§":27927,"ição":27928,"Ġgrab":27929,"客人":27930,"Ġwhilst":27931,"inq":27932,"Opt":27933,"uez":27934,"æĪĺ士":27935,"Ġ\\\\ĊĊ":27936,"ä¸Ģä¸Ģ":27937,"Ġë°©":27938,"Ġporque":27939,"ĠLem":27940,"Ġtroubles":27941,"Ġà¹ģ":27942,"ĠFel":27943,"ierung":27944,"Ġlifted":27945,"Ġleak":27946,"æīĢè°ĵçļĦ":27947,"Ġtransc":27948,"åĢºæĿĥ":27949,"æľ¬é¢ĺ":27950,"ĠEmb":27951,".List":27952,"éĺ²çĸ«":27953,"Ġproposals":27954,"unal":27955,"JECT":27956,"varphi":27957,"ĠGO":27958,"模æĿ¿":27959,"ĠëĶ":27960,"Ġmasses":27961,"ĠGB":27962,"?)":27963,"Ġenvelop":27964,"project":27965,"IZE":27966,"Ġsid":27967,"åİ»çľĭ":27968,"Ġexcluded":27969,"æ±ŁåįĹ":27970,"mont":27971,"éĶĻäºĨ":27972,"ariant":27973,".]":27974,"еÑĢез":27975,"IK":27976,"ĠMagazine":27977,"ĠSydney":27978,"ä¸Ģ级":27979,"Ġcorporation":27980,"Ġoutlined":27981,"ìĦł":27982,"Ġconvention":27983,"ĠFormula":27984,"иÑĤÑĮ":27985,"åĬĥ":27986,"ĠPrevention":27987,"upt":27988,"Ġtant":27989,"ç²¹":27990,"è¡«":27991,"èĢģåħ¬":27992,",i":27993,"æĮ«":27994,"çŁ³å¤´":27995,"ĠRevenue":27996,"çĪª":27997,"æĦıè¯Ĩåΰ":27998,"çŃīçĿĢ":27999,"ĠÃħ":28000,"åıĸåĨ³äºİ":28001,"799":28002,"Age":28003,"phones":28004,"ĠChanges":28005,"æĦŁè§īåΰ":28006,"811":28007,"Dan":28008,"éĴĵ":28009,"Ġstreams":28010,"hai":28011,"å¤±æľĽ":28012,"Ġparliament":28013,"Ġlegislative":28014,"ĠYouth":28015,"ÑģÑģа":28016,"à¹ģลà¹īว":28017,"ĠErn":28018,"Ġmonetary":28019,"禮":28020,"ARK":28021,"Ġapproached":28022,"ç¹Ķ":28023,"Ġspinal":28024,"Effect":28025,"days":28026,"aty":28027,"è¿Ļè¯Ŀ":28028,"Ġ'\\":28029,"Ġoxidation":28030,"ikh":28031,"Ġconcentrated":28032,"wt":28033,",k":28034,"Ġpainted":28035,"Ġaudit":28036,"è¯ķåĽ¾":28037,"æĬĬèĩªå·±":28038,"Ġreflecting":28039,"对çħ§":28040,"Ġconsiders":28041,"ĠÑĤÑĭ":28042,"çϾä¸ĩ":28043,"ĠQuarter":28044,"647":28045,"Ġaspir":28046,"bben":28047,"Ġwishes":28048,"Ġcaptain":28049,"iges":28050,"587":28051,"ĠKel":28052,"ĠLock":28053,"æ°®":28054,"Ġdelayed":28055,"Ġfits":28056,"硬件":28057,"ê±°":28058,"é̾":28059,"èĩ´çļĦ":28060,"Ġtoss":28061,"åĢŁåĬ©":28062,"ĠAbd":28063,"Ãģ":28064,"à¯įà®±":28065,"ç»ıæµİçļĦ":28066,"Hash":28067,"allas":28068,"umbled":28069,"Ġventure":28070,"Ġtriple":28071,"ãģ¸":28072,"Ġphysiological":28073,"建设çļĦ":28074,"Ġperme":28075,"Ġfriction":28076,"709":28077,"ÙĦب":28078,"ientos":28079,"Ġdesirable":28080,"Ġmelan":28081,"人æ°ij群ä¼Ĺ":28082,"å°įæĸ¼":28083,"665":28084,"èĦ±è´«":28085,"ĠNormal":28086,"ÙĦÙĬزÙĬØ©":28087,"çľĭå¾Ĺ":28088,"David":28089,"Ġtym":28090,"Ġuncover":28091,"审åΤ":28092,"ĠEmploy":28093,"ĠArticles":28094,"ä¹ĭåĬĽ":28095,"éĢīæīĭ":28096,"Ġì¹":28097,"داÙħ":28098,"ารà¸ĸ":28099,"Ġfounder":28100,"ICS":28101,"Ġfloating":28102,"899":28103,"Ġclicking":28104,"Nas":28105,"Ġriding":28106,"ç§»æ°ij":28107,"ilet":28108,"Students":28109,"ĠTalk":28110,"Ġmodifications":28111,"Ġpremi":28112,"Ġtob":28113,"July":28114,"ìľł":28115,"ĠNach":28116,"oxide":28117,"ensitive":28118,".path":28119,"åĩºç§Ł":28120,"âĶĢâĶĢâĶĢâĶĢ":28121,"ĠHz":28122,"itzerland":28123,"Ġfriendship":28124,"âĢĶâĢĶĊĊ":28125,"åģ¥èº«":28126,"ongo":28127,"Ġjudicial":28128,"å¦Ħ":28129,"ĠÕ´":28130,"Ġrepeatedly":28131,"æĢ¡":28132,"åĮª":28133,"åĽłåŃIJ":28134,"ĠVor":28135,"æĹ¶å°ļ":28136,"Ġï¬ģ":28137,"'S":28138,"æ¶²ä½ĵ":28139,"teenth":28140,"Ġcomplementary":28141,"ĠÑģодеÑĢ":28142,"ognitive":28143,"æĢ»é¢Ŀ":28144,"Ġдоп":28145,"$ĊĊ":28146,"ç¾İ好çļĦ":28147,"Ġunnecessary":28148,"graduate":28149,"Ġsatu":28150,"æīĭä¸ŃçļĦ":28151,"Ġintegrate":28152,"Ġdispers":28153,"ÙĨÚ¯":28154,"ç»ĺåζ":28155,"Û·":28156,"Ġintim":28157,"ิà¸ķ":28158,"035":28159,"ĠChannel":28160,"Login":28161,"Ġlets":28162,"æ³¼":28163,"æī«æıı":28164,"ĠNH":28165,"Ġdominated":28166,"ä¸įåıĹ":28167,"nan":28168,"Ġmutation":28169,"688":28170,"anded":28171,"Ġstrip":28172,"ĠUpdated":28173,"æĿ¡æ¬¾":28174,"996":28175,"åıĺæį¢":28176,"ĠPatient":28177,"itated":28178,"Ġstraightforward":28179,"夢":28180,"ãĢģĊĊ":28181,"Ġviewing":28182,"çį¨":28183,"ĠCorp":28184,"*/Ċ":28185,"éĴ¥":28186,"åŁºç¡Ģ设æĸ½":28187,"ĠMonte":28188,"产çĶŁäºĨ":28189,"Ġrecipes":28190,"acz":28191,"Ġgenerates":28192,"Ġfunded":28193,"urely":28194,"éĤ£åĢĭ":28195,"ĠLäst":28196,"Ġsuspension":28197,"ĠAvenue":28198,"åĩłä½ķ":28199,"personal":28200,"ĠÐĴÑĭ":28201,"ç¡ķ士":28202,"Ġvariants":28203,"ĠMcK":28204,"æĹ¶éĹ´çļĦ":28205,"èİ·å¾ĹäºĨ":28206,"åĩºåĶ®":28207,"ĠVice":28208,"Ġindigenous":28209,"Ùħار":28210,"Ġê³ł":28211,"Û¶":28212,"رÙī":28213,"ĠÙĩÙĬ":28214,"å©·":28215,"าล":28216,"ï½ľ":28217,"Solve":28218,"Ġcomputational":28219,"çļĦçĬ¶æĢģ":28220,"ĠTerms":28221,"æŀģåħ¶":28222,"CB":28223,"992":28224,"ĠContext":28225,"Does":28226,"Öī":28227,"æłĪ":28228,"ohan":28229,"à§Ģর":28230,"ĠExplain":28231,"Ġimproves":28232,"åıijå¸ĥæĹ¶éĹ´":28233,"ãģ§ãģįãĤĭ":28234,"Ġnomin":28235,"åΰä½į":28236,"Ġdeclined":28237,"空è°ĥ":28238,"Term":28239,"awn":28240,"Ġwaited":28241,"å§ĵåIJį":28242,"åħ¨åİ¿":28243,"eri":28244,"Ġsophisticated":28245,"mate":28246,"好å¤Ħ":28247,"à¸ĭึà¹Īà¸ĩ":28248,"ĠTool":28249,"Ġ%>":28250,"Ġconoc":28251,"970":28252,"ĠYOU":28253,"çļĦåĪĨ":28254,"owej":28255,"ĠCrim":28256,"Ġ{}Ċ":28257,"dep":28258,"AME":28259,"ĠConstruction":28260,"Ġnar":28261,"902":28262,"EFA":28263,"ĠLect":28264,"å±ł":28265,"748":28266,"лаÑģÑĮ":28267,"åįıä½ľ":28268,"è¿ĩ度":28269,"FE":28270,"å®ıè§Ĥ":28271,"Ġguides":28272,"598":28273,"åºĵåŃĺ":28274,"ĠÙĪÙĬ":28275,"Ġfired":28276,"ablish":28277,"ĠHS":28278,"Cr":28279,"cite":28280,"ramid":28281,"864":28282,"ĠCold":28283,"ĠPas":28284,"ĠMater":28285,"าà¸Ĺ":28286,"াà¦Ĺ":28287,"Ġanticipated":28288,"Ġloads":28289,"åĴĮå°ı":28290,"èĺŃ":28291,"target":28292,"route":28293,"æĹ¥ä¸ĬåįĪ":28294,"Ġtranslate":28295,"obacter":28296,"623":28297,"Σ":28298,"rological":28299,"æ¥Ĭ":28300,"Ġcavity":28301,"062":28302,"Ġsurvived":28303,"忽çķ¥":28304,"579":28305,"Ġfatal":28306,"997":28307,"kle":28308,"Ġinterven":28309,"å®ĥ们çļĦ":28310,"ç͵åķĨ":28311,"Ġsecurities":28312,"æ©¡":28313,"inent":28314,"åĪĥ":28315,"ĠKelly":28316,"Exper":28317,"Document":28318,"ijIJ":28319,"Ġarth":28320,"ĠDIS":28321,"ÙĢÙĢ":28322,"ä¸ĢéĿ¢":28323,"render":28324,"Ġengineer":28325,"Ġbarrel":28326,"ĠëĤ´":28327,"ĠPolitics":28328,"è¿Ľåº¦":28329,"Ġstrat":28330,"hom":28331,"ä¸ĢåįĬ":28332,"Ġrouter":28333,"Display":28334,"ĠConfig":28335,"åħ¶ä»ĸ人":28336,"ĠEverything":28337,"ç±»åŀĭçļĦ":28338,"027":28339,"æĿı":28340,"ĠRank":28341,"à·Ĭ":28342,"ãĥ¼ãĥĪ":28343,"ĠPok":28344,"åĺ´è§Ĵ":28345,"ĠRecogn":28346,"没æľīä»Ģä¹Ī":28347,"éłĵ":28348,"657":28349,"osten":28350,"ä¹ĭæĹ¶":28351,"ĠFast":28352,"Ġupp":28353,"çļĦéľĢæ±Ĥ":28354,"Ġsettle":28355,"ĠAvoid":28356,"ì¦":28357,"åIJĮæł·çļĦ":28358,"çģ¿":28359,"åİ»åģļ":28360,"Ġsupportive":28361,"ç»ıè´¹":28362,"Ġresearcher":28363,"-des":28364,"incial":28365,"ç³ĸå°¿":28366,"ordinate":28367,"oton":28368,"ç§įç§į":28369,"Ġhect":28370,"Ġdozen":28371,"_.ĊĊ":28372,"à¯ģà®ķ":28373,"zá":28374,"æĵĶ":28375,"è¿ĻéĩĮçļĦ":28376,"ĠKansas":28377,"оне":28378,"פ":28379,"643":28380,"command":28381,"Storage":28382,"ĠBesides":28383,"Ġvic":28384,"稳å®ļçļĦ":28385,"ĠInnovation":28386,"iso":28387,"Ġkes":28388,"593":28389,"大äºĭ":28390,"è¿Ľè¡ĮçļĦ":28391,"çıŃåŃIJ":28392,"Ġlecture":28393,"Ġcorruption":28394,"Explanation":28395,"Ġeu":28396,"Ġimmigration":28397,"Ġneglig":28398,"èµĽåŃ£":28399,"825":28400,"ipes":28401,"Ġprefix":28402,"Ġromantic":28403,"713":28404,"717":28405,"028":28406,"为ä¾ĭ":28407,"ĠClaim":28408,"775":28409,"itten":28410,"gre":28411,"Ġthrew":28412,"Weight":28413,"/.":28414,"727":28415,"ĠTon":28416,"ĠNames":28417,"Ġpersonalized":28418,".web":28419,"Ġgoverning":28420,"çľģå§Ķ":28421,"åı¥åŃIJ":28422,"å¾ħéģĩ":28423,"ĠÑĩелове":28424,"egy":28425,"Ġequival":28426,"Ġتش":28427,"Ġink":28428,"ĠShakespeare":28429,"åľ¨åľ°ä¸Ĭ":28430,"Ġneighbour":28431,"Ġdrought":28432,"Ġreflex":28433,"Ġimplicit":28434,"Ġbell":28435,"ĠMemorial":28436,"(`":28437,"Ġym":28438,"Ġkilometro":28439,"çľģ级":28440,"æĪijéĥ½":28441,"ashboard":28442,"Ġfragment":28443,"ĠLiver":28444,"Ġsensing":28445,"åĮĻ":28446,"åĮºåĪĨ":28447,"第ä¹Ŀ":28448,"EY":28449,"ä¹Łè¢«":28450,"å®ŀåľ°":28451,"Ġconverting":28452,"å¥ł":28453,"ĠSample":28454,"638":28455,"导åIJij":28456,"ophys":28457,"å¸ĪåĤħ":28458,"åı£ç½©":28459,"ĠAppendix":28460,"ר×Ļ×Ŀ":28461,"ä¹°åįĸ":28462,"Ġbleeding":28463,"Ġaffirm":28464,"ç¨įå¾®":28465,"Ġlanding":28466,"Ġrandomly":28467,"Ġphilos":28468,"Ġки":28469,"建éĢł":28470,"),(":28471,"ovat":28472,"antine":28473,")))":28474,"oT":28475,"aq":28476,"ç²¥":28477,"çĶ¨æ°´":28478,"Ġà¦¹à§Ł":28479,"ĠкаÑĢ":28480,"åĢŁæ¬¾":28481,"Ġги":28482,"Ġwalks":28483,"-oriented":28484,"æİ¥ç§į":28485,"æľºåĬ¨":28486,"-value":28487,"ками":28488,"ĠScottish":28489,"éĩĩéĽĨ":28490,"ĠRainfall":28491,"Ġpione":28492,"Ġadop":28493,"578":28494,"enschaft":28495,"ĠKok":28496,"Ġnutrient":28497,"çªĥ":28498,"Ġpipeline":28499,"ĠCOMP":28500,".ch":28501,"ĠBegin":28502,"Ġìĺģ":28503,"Life":28504,"çķĮçļĦ":28505,"686":28506,"Ġcredits":28507,"Ġghost":28508,"744":28509,"Ġguns":28510,"åĭĿ":28511,"Ġimportantly":28512,"ĠWalker":28513,"ä¸Ģ个æľĪ":28514,"Ġbod":28515,"619":28516,"Õ¡Õ¯Õ¡Õ¶":28517,"çķ¢":28518,"Ġconjug":28519,"Configuration":28520,"Ġcattle":28521,"opsis":28522,"æĬ½è±¡":28523,"ikipedia":28524,"飯":28525,"ounding":28526,"åĸľæ¬¢çļĦ":28527,"带æĿ¥äºĨ":28528,"ä¾Ŀçħ§":28529,"ĠSubst":28530,"Ġexamining":28531,"aters":28532,"Ġignor":28533,"Ġ______":28534,"è¼ī":28535,"umat":28536,"ĠVerb":28537,"Ġdiscrete":28538,"Ġ(\\(":28539,"Õ¨":28540,"ĠSide":28541,"Ġnog":28542,"ĠÙħÙĤ":28543,"ĉc":28544,"hd":28545,"Ġshame":28546,"ĠBag":28547,"Ġpriorities":28548,"à¹ĩà¸ģ":28549,"èĤļåŃIJ":28550,"ĠPhilip":28551,"aline":28552,"à®®à¯į":28553,"Ġreplacing":28554,"ãģ¨ãģª":28555,"787":28556,"çļĦéĢŁåº¦":28557,",m":28558,"ĠChronic":28559,"çļĦéĿ¢":28560,"çİĩ为":28561,"ĠÙĦÙĦÙħ":28562,"بÙĨ":28563,"ä¸ŃåįĪ":28564,"{b":28565,"Ġshipping":28566,"ĠÐķÑģли":28567,"Ġब":28568,"Ø®ÙĦ":28569,"èĬ¦":28570,".res":28571,"åĩłåįģ":28572,")/(-":28573,"Ġsoup":28574,"Ġcertification":28575,"Ġdann":28576,"Ġprecipitation":28577,"tha":28578,"ĠØŃد":28579,"ĠThu":28580,"ç»ĻæĪij们":28581,"ĠDrive":28582,"sea":28583,"Ġconsp":28584,"urities":28585,"Ġpackaging":28586,"éĥ¨ä»¶":28587,"ĠPic":28588,"ustral":28589,"Great":28590,"ĠDean":28591,"ĠWu":28592,"åĭ¿":28593,"Ġapr":28594,"Ġretired":28595,"Ġmars":28596,"905":28597,"ĠTABLE":28598,"Tall":28599,"Ġchips":28600,"èįĨ":28601,"597":28602,"Ġsacred":28603,"éĢĢåĩº":28604,"036":28605,"è¾ĥ大çļĦ":28606,"¡×¤×¨":28607,"Mo":28608,"\\\"":28609,"ĠLabour":28610,"ĠStudio":28611,"Ġ{\"":28612,"Ġfranch":28613,"909":28614,"æ¸ĹéĢı":28615,"(user":28616,"atta":28617,"ĠTransfer":28618,"Ġaccessed":28619,"free":28620,"éģİåİ»":28621,"912":28622,"æijĺè¦ģ":28623,"Definition":28624,"ĠPhill":28625,"ĠLawrence":28626,"esa":28627,"issipp":28628,"éģİä¾Ĩ":28629,"otek":28630,"à¹Ģà¸ķ":28631,"esy":28632,"ĠPerspect":28633,"Develop":28634,"Ġ})":28635,"âĢļ":28636,"letic":28637,"åİŁæĸĩ":28638,"requently":28639,"erca":28640,"é«ĺ温":28641,"è¿ŀå¿Ļ":28642,"Ġgraphic":28643,"Ġinserted":28644,"Session":28645,"052":28646,"ĠElectron":28647,"'],Ċ":28648,"ricket":28649,"âij£":28650,"ĠоÑģоб":28651,"-American":28652,"Mac":28653,"æľĢæĹ©":28654,"è°ĪåΤ":28655,"Ġwit":28656,"Ġgraphs":28657,"Ġkinab":28658,"ĠÙģÙĬÙĩا":28659,"çīº":28660,"ĠAlan":28661,"Ġà¦ľà¦¨":28662,"owych":28663,"ÃŃan":28664,"æ¡IJ":28665,"çĹħæĥħ":28666,"723":28667,"ĉcout":28668,"è¿Ļæĺ¯ä¸Ģ个":28669,"วัà¸Ļ":28670,"Super":28671,"ĠâĢĿĊĊ":28672,"寿åij½":28673,"æĹ¥ä¸ĭåįĪ":28674,"ä»ĸ对":28675,"æĪijå¾Ī":28676,"Ġyours":28677,"Ġrhythm":28678,"Ġheeft":28679,"каÑħ":28680,"MW":28681,"ĠDub":28682,"åİ»æī¾":28683,"Ġgrey":28684,"æĺĶ":28685,"673":28686,"âĢķâĢķ":28687,"Ġpersonas":28688,"çľ¯":28689,"agle":28690,"582":28691,"ä¸Ģ个个":28692,"abl":28693,"Ġдиа":28694,"flex":28695,"ãģ¾ãģ§":28696,"Ġ);ĊĊ":28697,"Ġincub":28698,"ĠìĨĮ":28699,"èĥ½åĬĽçļĦ":28700,"687":28701,"åIJ¬è§ģ":28702,"Ġвз":28703,"iffs":28704,"Ġpeny":28705,"nbsp":28706,"ographer":28707,"æĿ¡ä»¶çļĦ":28708,"628":28709,"æĴѿ;":28710,"Ġparticipant":28711,"Auth":28712,"CV":28713,"è¿ĻæĹ¶åĢĻ":28714,"éĥĬ":28715,"Ġtrabal":28716,"Ġsuspended":28717,"etta":28718,"ادÛĮ":28719,"ĠAbb":28720,"Ġcomparative":28721,"æīĵç͵è¯Ŀ":28722,"752":28723,"主è¦ģæľī":28724,"Ġclay":28725,"两èĢħ":28726,"ä»·å̼è§Ĥ":28727,"ãĥĵ":28728,"Ġstreaming":28729,"wind":28730,"Ġlongest":28731,"Ġexclusively":28732,"Ġoccasions":28733,"ĠQU":28734,"她说":28735,"omers":28736,"Ġkam":28737,"ĠAncient":28738,")*(":28739,"åĿĩè¡¡":28740,"zheimer":28741,"ÃŃd":28742,"Ġexpressing":28743,"-second":28744,"!\"ĊĊ":28745,"Ġcentimeters":28746,"control":28747,"Ġbreed":28748,"åıijçݰäºĨ":28749,"ĠEight":28750,"åľ°åĽ¾":28751,"Ġoctal":28752,"806":28753,"åŃ¦æł¡çļĦ":28754,"Ġanomal":28755,"éĢļç͍":28756,"Ġtower":28757,"ĠÙĤد":28758,"ĠعÙĦÙĬÙĩ":28759,"ĠкоÑĤоÑĢÑĭй":28760,"ATIONS":28761,"ĠMaryland":28762,"662":28763,"ôt":28764,"Ġvas":28765,"register":28766,"æĽĿ":28767,"á»ģ":28768,"685":28769,"Ġsink":28770,"udge":28771,"Ġpró":28772,"zin":28773,"Ġalgorit":28774,"Ġencourages":28775,"043":28776,"ä¸įå®ľ":28777,"Ġkasarangang":28778,"Ġtense":28779,"Ġtackle":28780,"inuous":28781,"765":28782,"Ġconsultation":28783,"æĸ°åĮº":28784,"azed":28785,"ä»ĸä»¬åľ¨":28786,"718":28787,"Ġsó":28788,"Ġassim":28789,"ĠÑģоÑģÑĤоÑı":28790,"Ġcoating":28791,"ä¸Ģæł·çļĦ":28792,"Ġdocumented":28793,"Ġlabeled":28794,"çĥŃçα":28795,"樹":28796,"èĥ½å¤ł":28797,"ordered":28798,"Ġvid":28799,"719":28800,"Ġprojection":28801,"æĪĸèĢħæĺ¯":28802,"Global":28803,"Ġmoż":28804,"Ġhorn":28805,"ĠPharmac":28806,"930":28807,"Ġprospect":28808,"Ġbite":28809,"åįĩé«ĺ":28810,"utton":28811,"ĠëıĻ":28812,"ĠmÄĽ":28813,"å¾®åįļ":28814,"èĪħ":28815,"羡":28816,"vid":28817,"seconds":28818,"Ġherb":28819,"æ³ķ人":28820,"ĠبØŃ":28821,"Ġusername":28822,"å¾Ĺå¤ļ":28823,"ASH":28824,"ĠFaculty":28825,"Hey":28826,"精度":28827,"Ġtok":28828,"arde":28829,"Ġangular":28830,"ĠDest":28831,"ÙĨجÙĦÙĬزÙĬØ©":28832,"646":28833,"Ġacadem":28834,"594":28835,"668":28836,"áĥĺáĥ¡":28837,"ĠFootball":28838,"æİĴæŁ¥":28839,"677":28840,"Los":28841,"大æ°Ķ":28842,"Ġrainfall":28843,"unda":28844,"Ġprojected":28845,"驳":28846,"Ġsafegu":28847,"Ġseemingly":28848,"Ġbreeding":28849,"/-":28850,"ĠAbraham":28851,"çŃĸåĪĴ":28852,"ĠBeng":28853,"xtures":28854,"-int":28855,"Ỽ":28856,"ĠÑĥÑĢов":28857,"轨éģĵ":28858,"æĦļ":28859,"idea":28860,",a":28861,"ĠnÄĽ":28862,"ĠEasy":28863,"Ġadaptive":28864,"Ġcrossing":28865,",B":28866,"ylum":28867,"(V":28868,"ĠEpub":28869,"Ġreign":28870,"otto":28871,"Ġ×Ļש":28872,"çĶ«":28873,"ĠHollywood":28874,"æľºåĻ¨äºº":28875,"Ġcareers":28876,"ĠCape":28877,"Ġwinds":28878,"æłĩè®°":28879,"çĶ©":28880,"].Ċ":28881,"ä½łçŁ¥éģĵ":28882,"Ġvow":28883,"éļİ":28884,"Ġâİ":28885,"-o":28886,"ĠKrist":28887,"ä»ĭç»įäºĨ":28888,"Ġsimilarly":28889,"ĠPad":28890,"Ġprejud":28891,"761":28892,"zip":28893,"reas":28894,"Ġoptional":28895,"ç»Ļèĩªå·±":28896,"679":28897,"æĥĬè®¶":28898,"rika":28899,"Ġsubjected":28900,"à¦ķà§ĩ":28901,"Ġsurveillance":28902,"ĠÄĮ":28903,"çĸ¯çĭĤ":28904,"Ġsta":28905,"gment":28906,"Ġconsume":28907,"uden":28908,"ĠWisconsin":28909,"£áĥ":28910,"Ġreasonably":28911,"å½ķåıĸ":28912,"Ġshowc":28913,"æĶ¿åºľçļĦ":28914,"à¯Ĭ":28915,"Ġtiempo":28916,"cen":28917,"-cell":28918,"Ġdrawings":28919,"ĠHil":28920,"å®īæħ°":28921,"658":28922,"wij":28923,"utory":28924,"ĠاÙĦÙħس":28925,"Ġverte":28926,"ðŁĺ":28927,"Ġsuspected":28928,"Ġή":28929,"Ġvotre":28930,"Ġlicensed":28931,"Ġë§Į":28932,"é쏿ĵĩ":28933,"Ġthir":28934,"arious":28935,"Ġpc":28936,"åıijå°Ħ":28937,"ä¸īåĪĨ":28938,"ÑĤие":28939,"736":28940,"ĠAdvent":28941,"åĺ²":28942,"æĩ·":28943,"iker":28944,"income":28945,"Ġshade":28946,"åĮĹæĸ¹":28947,"igkeit":28948,"Ġreset":28949,"è¾ĸåĮº":28950,")))Ċ":28951,"fn":28952,"Fore":28953,"让èĩªå·±":28954,"ĠÑĢабоÑĤа":28955,"ĠSaudi":28956,"ayan":28957,"ваÑİÑĤ":28958,"ìĹŃ":28959,"Ġintensive":28960,"thread":28961,"Ġeds":28962,"æĮģæľī":28963,"âĸº":28964,"ogens":28965,"åī¯ä¹¦è®°":28966,"ĠJose":28967,"663":28968,"ĠGesch":28969,"èIJį":28970,"Overall":28971,"ãĥį":28972,"èĹĿ":28973,"Ġdysfunction":28974,"stock":28975,"NW":28976,"æĬĬæĪij":28977,"995":28978,"Ġjudges":28979,"Ġ_{":28980,"nessee":28981,"à«Ģ":28982,"649":28983,"AX":28984,"è®°ä½ı":28985,"```Ċ":28986,"å¹¶å°Ĩ":28987,"Ġraises":28988,"commun":28989,"Ġdonde":28990,"ëį°":28991,"Ġlapt":28992,"Creat":28993,"oubt":28994,"失åİ»äºĨ":28995,"Ġvapor":28996,"757":28997,"åĩłæ¬¡":28998,"åĨ·åį´":28999,"){":29000,"alia":29001,"ĠNutrition":29002,"nah":29003,"Ġsomebody":29004,"Ġweird":29005,"wend":29006,"Ġbeef":29007,"glich":29008,"Ġkernel":29009,"象å¾ģ":29010,"Ġdesigner":29011,"çij°":29012,"Ġimmers":29013,"æĸĩçī©":29014,"full":29015,"ulg":29016,"676":29017,"ENS":29018,"866":29019,"YY":29020,"Ġহয়":29021,"ண":29022,"ĠIndependent":29023,"981":29024,"ात":29025,"oval":29026,"ĠBreak":29027,"Release":29028,"Ġ**ĊĊ":29029,"Ġباشد":29030,"Ġdonc":29031,"733":29032,"ĠBou":29033,"ĠCould":29034,"Ġ×ĺ":29035,"attice":29036,"681":29037,"Ġenact":29038,"vc":29039,"Ġbeloved":29040,"åĩ¹":29041,"ENC":29042,"ĠOptim":29043,"Ġunlock":29044,"ĠOctal":29045,"Ġknife":29046,"è¯ķçĤ¹":29047,"Ġpositioned":29048,"Ġtear":29049,"ä»Ĭ天çļĦ":29050,"å¼¹æĢ§":29051,"ĠNEW":29052,"ĠнаÑģе":29053,"ajar":29054,"æ¿Ģåħī":29055,"nÄħ":29056,"Ġfailing":29057,"Ġfeaturing":29058,"fü":29059,"åĬłæĭ¿":29060,"çłĶ讨":29061,"ĠGulf":29062,"629":29063,"Ġdepr":29064,"Ġgraphics":29065,"ĠCritical":29066,"åħ¬äº¤":29067,"礼çī©":29068,"æ¼¢":29069,"èĢģå¸ĪçļĦ":29070,"rowse":29071,"¨à¯įத":29072,"peat":29073,"Ġsorts":29074,"æ´»è·ĥ":29075,"edback":29076,"Ġnoble":29077,"oby":29078,"_length":29079,"ABC":29080,"åī©ä¸ĭ":29081,"hh":29082,"ĠUnits":29083,"ĠÑĸ":29084,"ĠAustin":29085,"Ġirrig":29086,"è¶Ĭ大":29087,"è°ĥè§£":29088,"Ġviruses":29089,".Set":29090,"Ġpursuant":29091,"ĠCondition":29092,"-em":29093,"729":29094,"æ°ijçĶŁ":29095,"ĠManchester":29096,"ellation":29097,"ä½ĵéĩį":29098,"ĠWeight":29099,"Ġîn":29100,"uming":29101,"Ġsnap":29102,"Ġresonance":29103,"年第":29104,"809":29105,"ÑĦиÑĨи":29106,"731":29107,"äºĨåĩºæĿ¥":29108,"ĠHttp":29109,"è¿ĩ滤":29110,"ĠдÑĥ":29111,"åŃ¦ä¹łçļĦ":29112,"ÙģØ§Ø¯Ùĩ":29113,"Support":29114,"çĭłçĭł":29115,"Ġreconstruction":29116,"orp":29117,"obin":29118,"座è°Ī":29119,"issue":29120,"annah":29121,"Ġunto":29122,"èĢ³æľµ":29123,"ificant":29124,"Ġpatri":29125,"669":29126,"ĠJoint":29127,"Ġdedication":29128,"Ġcriter":29129,"Ġrevised":29130,"ĠPT":29131,"اÙĥ":29132,"åĩºè¡Į":29133,"ÏĦαι":29134,"Ġcontempl":29135,"ĠConsequently":29136,"816":29137,"747":29138,"Ġvitam":29139,"White":29140,"Ġfavorable":29141,"Ġ×IJ×ķ":29142,"ĠComparison":29143,"Ġkem":29144,"æľīä¸Ģ天":29145,"sted":29146,"ĠFou":29147,"letin":29148,"å°¹":29149,"ĠMig":29150,"Ġinfants":29151,"985":29152,"Iss":29153,"Ġpenc":29154,"olia":29155,"åŃĹæ¯į":29156,"åī©ä½Ļ":29157,"Ġexploit":29158,"åıįæĢĿ":29159,"ä¸ŃåĽ½äººæ°ij":29160,"684":29161,"Ġpunto":29162,"Ġinterf":29163,"Ġmét":29164,"neys":29165,"Ġdrum":29166,"ensch":29167,"ĠMadrid":29168,"Ñĺедина":29169,"çIJª":29170,"Az":29171,"expression":29172,"oden":29173,"ÃŃvel":29174,"èijĹåIJįçļĦ":29175,"avascript":29176,"æľīåĬĽ":29177,"844":29178,"çļĦåIJį":29179,"ç¥Ī":29180,"Ġinspire":29181,"Ġcrash":29182,"kc":29183,",int":29184,"Ġsquad":29185,"Ġunre":29186,"acular":29187,"ushes":29188,"Ġimplied":29189,"pathy":29190,"Ġpressing":29191,"Ġë°ı":29192,"Ġepisodes":29193,"Ġtremend":29194,"ç͍æ³ķ":29195,"Ġjsou":29196,"751":29197,"Ġspectral":29198,"Coll":29199,"unde":29200,"人äºĭ":29201,"Ġelectr":29202,"Chem":29203,"íİ":29204,"arna":29205,"ĠFramework":29206,"Europe":29207,"æĹ¥åīį":29208,"Ġlongitudinal":29209,"åĪĽæĦı":29210,"}}{\\":29211,"åħ¬æŃ£":29212,"051":29213,"ĠÑĤÑĢи":29214,"ús":29215,"634":29216,"ĠRobinson":29217,"typedef":29218,"}`":29219,"ØŃØ©":29220,"pletion":29221,"常ç͍çļĦ":29222,"(file":29223,"cepted":29224,"Ġshower":29225,"Ġsubsid":29226,"Ġlinguistic":29227,"ĠاÙĦاÙĨ":29228,"763":29229,"æĦŁçŁ¥":29230,"éĢĻ裡":29231,"оÑĢов":29232,"Ġthrive":29233,"è§£åĨ³æĸ¹æ¡Ī":29234,"ĠÑĦоÑĢмÑĥ":29235,"ãģĹãģ¾":29236,"羨":29237,"éĺ»æŃ¢":29238,"Ġhasta":29239,"éĺŁéķ¿":29240,"æĻ¯è§Ĥ":29241,"educ":29242,"ä¸įçŃī":29243,"Ġfolk":29244,"819":29245,"åįķä¸Ģ":29246,"uced":29247,"makers":29248,"ĠEmployee":29249,"iran":29250,"ĠJenn":29251,"贯彻èIJ½å®ŀ":29252,"çīºçī²":29253,"815":29254,"ció":29255,"ĠpÅĻed":29256,"Ġinterpre":29257,".component":29258,"ĠBaill":29259,"Ġservers":29260,"Ġauthentic":29261,"Ġ*/":29262,"ç¡Ŀ":29263,"735":29264,"691":29265,"Ġperforms":29266,"));":29267,"èĬĤ约":29268,"ï¼ŁãĢį":29269,"ĠÙĬÙħÙĥÙĨ":29270,"lung":29271,"æ½ľåľ¨":29272,"ität":29273,"izard":29274,".X":29275,"ÑĤомÑĥ":29276,"Ġfranc":29277,"ĠIM":29278,"NH":29279,"Matrix":29280,"Ġoù":29281,"Ġhire":29282,"heter":29283,"\\+":29284,"Ġcomputation":29285,"Ġsecretary":29286,"\"It":29287,"Ġvalidate":29288,"ä¸Ńå±±":29289,"æŁ¥æī¾":29290,"756":29291,"ĠHD":29292,"ĠSri":29293,"Ġshifted":29294,"build":29295,"æĦŁåħ´è¶£":29296,"æĵİ":29297,"Ġ(°":29298,"æĹłåħ³":29299,"Ġemerge":29300,"ники":29301,"ãĢĤ>>":29650,"Ġpúblic":29651,"ĠاÙģ":29652,"èĢĥèĻijåΰ":29653,"اÙĤع":29654,"Ġreproduction":29655,"hex":29656,"èĶĵ":29657,"753":29658,"Ġurine":29659,"ĠBeijing":29660,"寻æ±Ĥ":29661,"ä¸ĭä¸ĢæŃ¥":29662,"Ġcrust":29663,"ä½łæľī":29664,"arcelona":29665,"659":29666,"åĮĸå·¥":29667,"ç¼ĸåı·":29668,"ÙĪØµ":29669,"^^":29670,"ĠSr":29671,"889":29672,"è¶Ł":29673,"åį°åĪ·":29674,"game":29675,"Ġfollowers":29676,"é¹ħ":29677,"èĵī":29678,"Ġphotography":29679,"Ġjou":29680,"Ġinfluential":29681,"Ġhumor":29682,"饶":29683,"çĶ·çĶŁ":29684,"Ġhing":29685,"æŃª":29686,"Ġcoronavirus":29687,"multicolumn":29688,"Blue":29689,"Ġapt":29690,"çľĭäºĨçľĭ":29691,"ánÃŃ":29692,"+ĊĊ":29693,"/core":29694,"916":29695,"Ġbom":29696,"ä¾Ŀ次":29697,"Root":29698,"ĠKin":29699,"ĠScore":29700,"Ill":29701,"Ġphones":29702,"Ġborders":29703,"λογία":29704,"à§ģন":29705,"ש×":29706,"ĠPoss":29707,"Ġhebben":29708,"ĠìĹĨ":29709,"Ġstamp":29710,"满äºĨ":29711,"Ġexhibition":29712,"heel":29713,"å®ŀçļĦ":29714,"éŃĶæ³ķ":29715,"éĺŁçļĦ":29716,"iply":29717,"category":29718,"Ġlacking":29719,"Ġdesires":29720,"å°´å°¬":29721,"813":29722,"781":29723,"íŀ":29724,"ĠStage":29725,"ÑĨÑĥ":29726,"讲座":29727,"Ġattracted":29728,"çİ«":29729,"Ġeines":29730,"ĠTue":29731,"enas":29732,"Ġvest":29733,"081":29734,"系統":29735,"Parent":29736,"æĭ±":29737,"ĠChristopher":29738,"uku":29739,"Ġcreature":29740,"åħ¬æĸ¤":29741,"urchase":29742,"ĠIndiana":29743,"951":29744,"ìļ´":29745,"dem":29746,"stell":29747,"ĠNeuros":29748,"Ġnaz":29749,"ĠAA":29750,"Ġwrapped":29751,"ĠApproach":29752,"Target":29753,"izi":29754,"æľºéģĩ":29755,"Ġrelaxation":29756,"oqu":29757,"éĩĭ":29758,"obil":29759,"uent":29760,"Ġ***":29761,"bank":29762,"emple":29763,"Ġsqrt":29764,"è´ŁåĢº":29765,"Ġëħ":29766,"Ġyd":29767,"åľ°åŁŁ":29768,"negative":29769,"lg":29770,"å¡Į":29771,"-wide":29772,"åĿª":29773,"åĪ©äºļ":29774,"726":29775,"ĠBeck":29776,"Ġpeaceful":29777,"ĠWright":29778,"ĠConservation":29779,"Ġnobody":29780,"ç͍åĬĽ":29781,"operatorname":29782,"ĠاستÙģØ§Ø¯Ùĩ":29783,"_with":29784,"ËIJ":29785,"esian":29786,"814":29787,"828":29788,"olute":29789,"çĶŁåĬ¨":29790,"æ°Ķæģ¯":29791,"bian":29792,"åįłæľī":29793,"(true":29794,"689":29795,"Ġrust":29796,"ç»ĵç®Ĺ":29797,"Ġenv":29798,"менÑĤа":29799,"Ġmaximize":29800,"Ġcatalyst":29801,"æ¯ıæľĪ":29802,"cdots":29803,"-te":29804,"Ê»":29805,"igne":29806,"æĬµæĬ¼":29807,"ĠFont":29808,"Commun":29809,"Ġecosystems":29810,"¡×ĺ":29811,"å¤įæĿĤçļĦ":29812,"February":29813,"economic":29814,"Ġdeposition":29815,"raul":29816,"Ġnat":29817,"988":29818,"ĠCB":29819,"ĠÒ":29820,"Ġmechanics":29821,"æĭIJ":29822,"Ġactu":29823,"Windows":29824,"Ġkap":29825,"žÃŃ":29826,"$$Ċ":29827,"éĻĢ":29828,"æīĭçļĦ":29829,"bourne":29830,"Ġantenna":29831,"Ġgonna":29832,"Ġexert":29833,"903":29834,"زÛĮ":29835,"ĠгодÑĥ":29836,"ÙĨدÙĩ":29837,"November":29838,"ĠOpportun":29839,"দà§įà¦":29840,"(array":29841,"697":29842,"042":29843,"ĠÑĢабоÑĤÑĭ":29844,"Ġnonlinear":29845,"Qs":29846,"Ġbedroom":29847,"ĠÙĩر":29848,"Ġrecommendation":29849,"iliki":29850,"Ġstrictly":29851,"Ġ×Ķר×":29852,"ĠнÑĥжно":29853,"Ġinitiated":29854,"×Ļצ":29855,"ĠJason":29856,"Ġпоказа":29857,"áĢŃ":29858,"walk":29859,"Ġconjunction":29860,"925":29861,"Ġobstacles":29862,"à¸Ļà¹īำ":29863,".remove":29864,"Ġembarrass":29865,"Ġattacked":29866,"Ġvalued":29867,"Ġsimilarity":29868,"Ġduct":29869,")\"":29870,"ĠVeg":29871,"807":29872,"Úº":29873,"ĠKings":29874,"yme":29875,"ĠCertain":29876,"ĠDiscover":29877,"ä½ĽæķĻ":29878,"sti":29879,"Ġpleasant":29880,"çļĦæĹ¥åŃIJ":29881,"Ġshops":29882,"039":29883,"ĠPhoto":29884,"Ġspite":29885,"ÑĤÑĥÑĢÑĭ":29886,"936":29887,"ughters":29888,"akit":29889,"Ġnag":29890,"Ġdistress":29891,"Ġsteep":29892,"åij¨æľ«":29893,"human":29894,"ä½łä¹Ł":29895,"Ġderivatives":29896,"ché":29897,"ÙĤÛĮ":29898,"779":29899,"Ġunclear":29900,"ĠlÃŃ":29901,"æĪĺåľº":29902,"Ann":29903,"åĤ¨å¤ĩ":29904,"Ġnutritional":29905,"ĠÚ©Ùħ":29906,"Ġâĺ":29907,"ç»ĦæĪIJçļĦ":29908,"ahoma":29909,".service":29910,"eenth":29911,"å¥¹åľ¨":29912,"Å¡e":29913,"ĠØ¥ÙĨ":29914,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":29915,"952":29916,".count":29917,"ÙĪØ§Ùĩ":29918,"é¢ĨåŁŁçļĦ":29919,"ĠonClick":29920,"ÑģÑĤвÑĥеÑĤ":29921,"ĠÑģле":29922,"907":29923,"ĠTrack":29924,"ä¸Ģéĥ¨":29925,"-Co":29926,"Ġstro":29927,"à¹Īวà¸Ļ":29928,"icket":29929,"Ġatmospheric":29930,"äºĮ级":29931,"Ġachievements":29932,"Ġexports":29933,"ëĦ":29934,"æķijæı´":29935,"ë§Ī":29936,"Ġbathroom":29937,"ĠWhit":29938,"Ġà®ĩ":29939,".new":29940,"LED":29941,"ĠCrypt":29942,"кÑĢа":29943,"à¸Ħุà¸ĵ":29944,"Ġorb":29945,"ĠVoc":29946,"-co":29947,"åıĤåĬłäºĨ":29948,"éŃĦ":29949,"Ġforum":29950,"pow":29951,"è¡Ŀ":29952,"055":29953,"Ġcontroversial":29954,"/r":29955,"Ġartif":29956,"Ġpublish":29957,"ĠFreder":29958,"ĠиÑģполÑĮзÑĥ":29959,"躯":29960,"Ġeer":29961,"ų":29962,"Ġrandomized":29963,"Ġeconomies":29964,"Ġescol":29965,"×ķס":29966,"Ġtek":29967,"Ġconviction":29968,"ĠDistribution":29969,"Ġembark":29970,"Ġpublishing":29971,"æľĢéĩįè¦ģçļĦ":29972,"åύå®ĺ":29973,"Ġsempre":29974,"wie":29975,"Ġdetector":29976,"æļ´éľ²":29977,"å³»":29978,"776":29979,"åĨ¬åŃ£":29980,"енÑĤов":29981,"ä¸Ģä¹Ŀ":29982,"è¨İ":29983,"045":29984,"ÑĤÑĢо":29985,"æĪijå¸Ĥ":29986,"Ġarrow":29987,"usc":29988,"Ġcompanion":29989,"å°ģéĹŃ":29990,"ĠDad":29991,"Ġtobacco":29992,"人æĸĩ":29993,"Ġamend":29994,"Ġstatute":29995,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":29996,"los":29997,"Ġ_(":29998,"Ġpoorly":29999,"connection":30000,"ον":30001,"ึà¸ģษ":30002,"ä¸Ģä¸Ŀ":30003,"Ġsubjective":30004,"ä¹ĭåĨħ":30005,"832":30006,"Ġsq":30007,"Ġlegitimate":30008,"AH":30009,".title":30010,"à§Ī":30011,"Ġfostering":30012,"Ġsne":30013,"çļĦä¸Ĭ":30014,"Ġattain":30015,"иÑħ":30016,"Success":30017,"маÑĤи":30018,"ף":30019,"Ġpotatoes":30020,"ĠHeritage":30021,"ĠLetters":30022,"Ġbitter":30023,"æŃ¦è£ħ":30024,"ĠRy":30025,"åĨį说":30026,"дÑĮ":30027,"ĠAssert":30028,"ĠBenjamin":30029,"ĠJonathan":30030,"Ġвоп":30031,"Ġsoils":30032,"ĠFlow":30033,"Ġó":30034,"缸çŃī":30035,"Ġwand":30036,"Ġclosure":30037,"ĠJP":30038,"å̤":30039,"ä¼ijéĹ²":30040,"ĠBeyond":30041,"Ġinferior":30042,".Is":30043,"ĠCompl":30044,"ĠExcell":30045,"éĿĴå²Ľ":30046,"vs":30047,"Ġzenith":30048,"å¸Ĥåł´":30049,"ĉĠĠĠ":30050,"è¦ģæĥ³":30051,"Ġburned":30052,"ĠChile":30053,"å²ģæľĪ":30054,"就已ç»ı":30055,".insert":30056,"Ġapproximation":30057,"shore":30058,"宪æ³ķ":30059,"Ġts":30060,"ذÙĦÙĥ":30061,"Ġsupervision":30062,"Ġpuzzle":30063,"bru":30064,"Ġglobe":30065,"ĠChampionship":30066,"Mal":30067,"filter":30068,"Ġgaining":30069,"aments":30070,"è¿Ń":30071,"ucht":30072,"ĠNelson":30073,"âģĦ":30074,"ÃĤ":30075,"lades":30076,"Ġsisters":30077,"تÙģ":30078,"029":30079,"pere":30080,"iatry":30081,"ĠRelease":30082,"éºŁ":30083,"пÑĢе":30084,"以åħį":30085,"éIJµ":30086,"affe":30087,"ä»ĺ款":30088,"961":30089,"etan":30090,"Ġcham":30091,"envol":30092,"ICA":30093,"ORK":30094,"éŀŃ":30095,"ĠدÙĪ":30096,"ĠÎĿ":30097,"igs":30098,"Ġsyntax":30099,"éģĹæĨ¾":30100,"Ġtran":30101,"UV":30102,"æīģ":30103,"Ġhint":30104,"inya":30105,").[":30106,"éłģ":30107,"ooking":30108,"Ġdeterior":30109,"-form":30110,"ãģ£ãģ¦ãģĦãĤĭ":30111,"Ġlemon":30112,"Ġsentiment":30113,"Ñģком":30114,"Ġcabin":30115,"Ġeleven":30116,"Ġapro":30117,"Ġà¦ķরা":30118,"eko":30119,"ĠÑĢÑĥб":30120,"ä»ĸåıĪ":30121,"Ġresistant":30122,"Ġsubmission":30123,"大èĦij":30124,"~~~Ċ":30125,"Ġ;ĊĊ":30126,"æĪIJ人":30127,"ĠFederation":30128,"Ġpotent":30129,"Ñīее":30130,"izione":30131,"æ®ĭçĸ¾":30132,"ĠMuh":30133,"ĠкаÑĩе":30134,"Ġreception":30135,"åŀĭçļĦ":30136,"Ġbore":30137,"绣":30138,"git":30139,"è¾¹çķĮ":30140,"Ġmenos":30141,"æĸ¹å½¢":30142,"738":30143,"åı°çģ£":30144,"937":30145,"Ķ×Ŀ":30146,"åĮ»å¸Ī":30147,"ĠпÑĢимен":30148,"ł×ª":30149,"Ġleverage":30150,"å´ĸ":30151,"çĹħåıĺ":30152,"ä¿®æŃ£":30153,"pleted":30154,"921":30155,"arians":30156,"Ġslavery":30157,"]-":30158,"zon":30159,"762":30160,"Ġseperti":30161,"{a":30162,"amon":30163,"Ġvegetation":30164,".google":30165,"ĠAdapt":30166,"ĠMenschen":30167,"èĤ©èĨĢ":30168,"ã썿ĢĿ":30169,"æľīæķĪåľ°":30170,"ä¹Łéĥ½":30171,"Ġluxury":30172,"Ġprzed":30173,"Ġlimitation":30174,"Ġouts":30175,"ĠинÑĤе":30176,"Ġdieser":30177,"iliary":30178,"Ġtrop":30179,"Ġkur":30180,"-test":30181,"å§IJ妹":30182,"iph":30183,"Ġpresidential":30184,"çµĮ":30185,"تÙĨ":30186,"åĽŀ顾":30187,"-foot":30188,"ĠGC":30189,"Ġ×ŀש×":30190,"лÑĮнÑĭÑħ":30191,"Photo":30192,"856":30193,".index":30194,"imit":30195,"Ġrgb":30196,"æĽ´å¤§çļĦ":30197,"argin":30198,"ниÑİ":30199,"åĨħæ¶µ":30200,"kw":30201,"éŁĵ":30202,"Ġmultic":30203,"ĠиÑģÑģ":30204,"Ġoccupation":30205,"ĠÙĦÙĬÙĨÙĥات":30206,"ĠпаÑĢа":30207,";j":30208,"oming":30209,"San":30210,"Ġore":30211,"å°±è¿ŀ":30212,"ĠAngel":30213,".max":30214,"Russ":30215,"è¦ģçĤ¹":30216,"683":30217,"ختÙĦÙģ":30218,"ĠHaz":30219,"ĠSter":30220,"ĠLik":30221,"Ġkarena":30222,"ĠWeather":30223,"CRE":30224,"rais":30225,"Ġpromoted":30226,"ĠCult":30227,"ĠTokyo":30228,"λα":30229,"three":30230,"Ġglobally":30231,"iele":30232,"她æĺ¯":30233,"ÃŁe":30234,"Ġattach":30235,"848":30236,"Ġgri":30237,"Ġjaw":30238,"Ġpreserved":30239,"ĠOsc":30240,"ä»İä¸Ń":30241,"Ġà¹Ģà¸ŀ":30242,"695":30243,"acu":30244,"åĴª":30245,"Ġrug":30246,"ØŃÙĬ":30247,"æľªèĥ½":30248,"Ġcoaching":30249,"×Ļס":30250,"764":30251,"ä¸ŃåĽ½äºº":30252,"ìľ¡":30253,"ÑĤан":30254,"ĠполÑĥÑĩи":30255,"Ġfuck":30256,"é¢ĨåħĪ":30257,"çľī头":30258,"ĠUsed":30259,"æİı":30260,"éļıåį³":30261,"fare":30262,"اÛĮØ´":30263,"Ġstimulus":30264,"Ġanalytics":30265,"ĠMut":30266,"nings":30267,"}#":30268,"Ġdeployment":30269,"社æľĥ":30270,"-align":30271,"è¤ĩ":30272,"osine":30273,"ì¡":30274,"kap":30275,"ĠMarie":30276,"ç½Ĺ马":30277,"Ġquietly":30278,"861":30279,"ĠVision":30280,"èn":30281,"Ġdraws":30282,"Ġ[...]":30283,"éļ¾é¢ĺ":30284,"852":30285,"ĠBild":30286,"Ġevolutionary":30287,"chter":30288,"Ġpoliticians":30289,"ĠArithmetic":30290,"Ñıв":30291,"å«Ĥ":30292,"mut":30293,"NU":30294,"stre":30295,"Ġlengths":30296,"Ġplanting":30297,"Ġshirt":30298,"Ġneedle":30299,".close":30300,"éħ¯":30301,"èµ°åİ»":30302,"ĠìŬ":30303,"Ġfacial":30304,"个人çļĦ":30305,"ван":30306,"åĴĴ":30307,"Ġstopping":30308,"Õ¶Õ¥ÖĢ":30309,"ÑįÑĤомÑĥ":30310,"à§ĩà¦ķà§ĩ":30311,"ĠDave":30312,"ë£Į":30313,"éĤ£éº¼":30314,"æķ°æį®çļĦ":30315,"}}Ċ":30316,"assets":30317,"主管éĥ¨éŨ":30318,"Ġenhances":30319,"Ġ×ij×IJ":30320,"Ġslip":30321,"Ġpoz":30322,"Ġpermits":30323,"θηκε":30324,"éĻĽ":30325,"åĩºäºİ":30326,"-power":30327,"ä¸įåºĶ":30328,"Ġexplos":30329,"ĠSteps":30330,"ofl":30331,"Ġreceipt":30332,"Ġstead":30333,"ç»ĵæĿŁåIJİ":30334,"ĠArgentina":30335,"éĺ²å¾¡":30336,"ĠListNode":30337,"ĠGarc":30338,"Ġsistem":30339,"à²Ĥ":30340,"ĠChi":30341,"Ġinduce":30342,"离å¼ĢäºĨ":30343,"Ġaluminum":30344,"ocom":30345,"ĠDisplay":30346,"Ġauthentication":30347,"èĢį":30348,"Ġsimplified":30349,"urations":30350,"Ġprima":30351,"åħīæĺİ":30352,"issen":30353,"åľ¨æľ¬":30354,"çıŃ主任":30355,"Ġlipid":30356,"example":30357,"ç͍åĵģ":30358,"Ġfuer":30359,"Ġ×¢×Ŀ":30360,"ĠSqu":30361,"ĠThor":30362,"":30589,"855":30590,"774":30591,"éĹ´æİ¥":30592,"ÑİÑīие":30593,"ä½łå¥½":30594,"åĽ¾ä¸Ń":30595,"Ġlan":30596,"èıĩ":30597,"906":30598,"æīĢæľī人":30599,"ĠPaglinawan":30600,"ĠÑĢо":30601,"ĠJess":30602,"Ġgreenhouse":30603,"Ġsacrific":30604,"мпеÑĢа":30605,"åŁ·":30606,"Ġpodcast":30607,"Percent":30608,"ĠSwitzerland":30609,"Ġqueries":30610,"Ġannoy":30611,"è´Ńçī©":30612,"ĠExperimental":30613,"Ġinterfer":30614,"817":30615,"Department":30616,"944":30617,"ĠFu":30618,"με":30619,"ĠIdentify":30620,"ĠKaz":30621,"ä¼łæŁĵ":30622,"çıĬ":30623,"047":30624,"863":30625,"ĠSort":30626,"epat":30627,"åģ¶å°Ķ":30628,"(function":30629,"èĴľ":30630,"ĠLuther":30631,"Ïĥε":30632,"048":30633,"Donald":30634,"æĬµæĬĹ":30635,"Ġassured":30636,"Ġима":30637,"æıIJéĹ®":30638,"ä¹Łå¥½":30639,"Ġparams":30640,"åĨħå¤ĸ":30641,"sed":30642,"dn":30643,"OCK":30644,"ÑħодиÑĤ":30645,"appropriate":30646,"çIJ¼":30647,"ĠÐĶлÑı":30648,"Ġpaintings":30649,"ĠTheatre":30650,"ressions":30651,"Ġformulation":30652,"ipik":30653,"roscopy":30654,"esi":30655,"Ġappend":30656,"Ġdimens":30657,"hydro":30658,"à¯ĭ":30659,"åŃĺåľ¨çĿĢ":30660,"mediated":30661,"ãģķãģĦ":30662,"Ġwashing":30663,"Ñĩной":30664,"Ġdece":30665,"οÏĤ":30666,"ĠпÑĢодÑĥк":30667,"DOI":30668,"æłĩåĩĨåĮĸ":30669,"Ġcomparisons":30670,"Ġtin":30671,"ĠÚ©Ø´":30672,"ĠíĨµ":30673,"åĩĨåĪĻ":30674,"åıijçĥŃ":30675,"åĽĽå¹´":30676,"书ç±į":30677,"ç¹¼çºĮ":30678,"æ¾ľ":30679,"ĠصÙĪØ±":30680,"èĢģçĪ·":30681,"raining":30682,"Ġdressed":30683,"ç´¹":30684,"Ġprevents":30685,"ĉp":30686,"Ġkeen":30687,"ĠÏģ":30688,"icons":30689,"è¿IJä½ľ":30690,"貨":30691,"876":30692,"ÅĻe":30693,"Ġsuppliers":30694,"ĠScanner":30695,"à¹ĥà¸Ļà¸ģาร":30696,"èĢĮåıĪ":30697,"Ġflags":30698,"Sen":30699,"ĠStream":30700,"ennettuna":30701,"Ġovernight":30702,"åĮĹ京å¸Ĥ":30703,"773":30704,"åľ¨ä¸Ĭ":30705,"è¿İæİ¥":30706,"ĠÏĥÏħ":30707,"اÙĦÙĬ":30708,"914":30709,"upakan":30710,"èĢ»":30711,"Four":30712,"åĴĮä½ł":30713,"IH":30714,"Cy":30715,"Ġshar":30716,"ĠâĻ":30717,"ĠColomb":30718,"Correct":30719,"Tallennettuna":30720,"osit":30721,"061":30722,"Ġдли":30723,"915":30724,"ĠStrategies":30725,"OO":30726,"Ġdeclaration":30727,"Ġdemanded":30728,"两侧":30729,"ové":30730,"åĽºå®ļèµĦ产":30731,"ĠTrip":30732,"851":30733,"Ġdere":30734,"klahoma":30735,"Ġtailored":30736,"erious":30737,"quant":30738,"ĠOak":30739,"å᫿ĺŁ":30740,"Ġreforms":30741,"å¿ĥæĢģ":30742,"Ġforeach":30743,"/pm":30744,"ĠGround":30745,"GM":30746,"usa":30747,"rico":30748,"arkan":30749,"Online":30750,"ç·©":30751,"Ġmerch":30752,"odos":30753,"æĥħèĬĤ":30754,"ĠCul":30755,"esium":30756,"ä¸Ģéģį":30757,"ÙĤر":30758,"èĩ¨":30759,"usamm":30760,"è°Ń":30761,"(System":30762,"åıªä¼ļ":30763,"Ġ»,":30764,"èij£äºĭéķ¿":30765,"Ġfracture":30766,"ĠAlzheimer":30767,"information":30768,"Ġinse":30769,"ĠMissouri":30770,"ÑĦÑĦек":30771,"ĠBrasil":30772,"NOT":30773,"üss":30774,"éϤéĿŀ":30775,"åºĶç͍ç¨ĭåºı":30776,"Ġslaves":30777,"èĢĮåIJİ":30778,"çĿĢåĬĽ":30779,"\\leq":30780,"doc":30781,"Ġobt":30782,"Ġvoy":30783,"Ġindices":30784,"976":30785,"William":30786,"746":30787,"Ġholder":30788,"contr":30789,"Ġtecn":30790,"便äºİ":30791,"ĠVeter":30792,"739":30793,"çļĦçľĭçĿĢ":30794,"Ġvalley":30795,"ĠоÑģÑĤа":30796,"ĠKol":30797,"Ġpassionate":30798,"Ġpublisher":30799,"Ġadolescents":30800,"æµ·åįĹ":30801,"842":30802,"Ġìĺ¤":30803,"Ġclinic":30804,"akat":30805,"ãĤ§":30806,"åľ¨åħ¶":30807,"working":30808,"975":30809,"Ãĵ":30810,"Ġklim":30811,"ìŰ":30812,"äºĨä¸įå°ij":30813,"ÑīаÑı":30814,"ÐĹа":30815,"ĠMOD":30816,"bad":30817,"èݹ":30818,"ç¾İåľĭ":30819,"ĠKinder":30820,"":31104,"amel":31105,"ä¸ŃåĽ½çī¹èī²":31106,"Ġbonus":31107,"South":31108,"ĠгоÑģÑĥдаÑĢ":31109,"ëĬ¥":31110,"æ°ijä¼Ĺ":31111,"ĉcase":31112,"Ġcouples":31113,"å½ĵä¸ĭ":31114,"æĬ¬èµ·":31115,"Ġimmunity":31116,"913":31117,"á½¶":31118,"è§Ĩéĩİ":31119,"নà§įত":31120,"ĠEgyptian":31121,"Ġspecification":31122,"ìĸij":31123,"èŁ¹":31124,"ĠDouglas":31125,"奸":31126,"åĩ°":31127,"odox":31128,"ĠAttorney":31129,"Ġpeuvent":31130,"conduct":31131,"éĤ£ä½į":31132,"æ¶©":31133,"Ġpracticing":31134,"omics":31135,"cknowled":31136,"åŃIJ宫":31137,"무":31138,"å¹´åĪĿ":31139,"Ġgibt":31140,"ç»Ħä»¶":31141,"769":31142,".Ent":31143,"843":31144,"Science":31145,"897":31146,"Ġnurt":31147,"ĠZone":31148,"æĿł":31149,"short":31150,"Design":31151,"ĠEND":31152,"à¸Ńรà¹Į":31153,"ĠпÑĢоÑģÑĤ":31154,"ĠSusan":31155,"Ġestado":31156,"ĠAfghanistan":31157,":[":31158,"Ġscreens":31159,"Ġcnt":31160,".java":31161,"ç͵åύ":31162,"åıĹçĽĬ":31163,"Spring":31164,"Ļà§įà¦Ĺ":31165,"Ġexamines":31166,"atted":31167,"دÙĬ":31168,"Ġдоба":31169,"é®":31170,"Ġcontainers":31171,"JP":31172,"ĠBalt":31173,"设计çļĦ":31174,"ç¯Ħ":31175,"æģĭçα":31176,"064":31177,"opathy":31178,"829":31179,"urally":31180,"Li":31181,"Ġamendment":31182,"è¿Ļä¸ĢåĪĩ":31183,"è¿ij代":31184,"Ġexhibited":31185,"ĉfmt":31186,"Ġorganize":31187,"ä½ľä¸ºä¸Ģ个":31188,"ĠXV":31189,"823":31190,"Ġvulnerability":31191,"审ç¾İ":31192,"Ġcylind":31193,"Rev":31194,"Ġkar":31195,"æ¾Ħ":31196,"ĠKur":31197,"clipse":31198,"-dig":31199,"ĠWa":31200,"éģĹ产":31201,"Ġrecruitment":31202,"ĠгÑĢÑĥп":31203,"])ĊĊ":31204,"è§Ħæł¼":31205,"scan":31206,"ĠÙħÛĮØ´":31207,"871":31208,"Ġtrium":31209,"Ġwrap":31210,"());ĊĊ":31211,"cze":31212,"Ġвла":31213,"è¹²":31214,"ĠLength":31215,"opol":31216,"Har":31217,"éĪ":31218,"hero":31219,"ĠãĢĮ":31220,"ĠÚ©ÙĨد":31221,"program":31222,"borne":31223,"åĽ½æ°ijåħļ":31224,"923":31225,"稳å®ļæĢ§":31226,"918":31227,"éĩĮéĿ¢çļĦ":31228,"enis":31229,"èķ¾":31230,"ĠболÑĮÑĪе":31231,"侨":31232,"ĠجÙħ":31233,"fu":31234,"çĶ¨äºº":31235,"è©©":31236,".query":31237,"Ġleng":31238,"Ġgardens":31239,"encer":31240,"ÑĢоп":31241,"Ġresort":31242,"ĠMunic":31243,"ĠÑĥже":31244,"çļĦè·¯":31245,"æ³ķå®ĺ":31246,"nom":31247,"人æĺ¯":31248,"922":31249,"abi":31250,"ứ":31251,"ਾà¨":31252,"Ġgeographical":31253,"èĮ¨":31254,"Ep":31255,"pendicular":31256,"759":31257,"åıĺ为":31258,"Ġmedieval":31259,"æIJĸ":31260,"ãĤº":31261,"Ġtire":31262,"ãĥ³ãĤ°":31263,"çļĦçŁ¥è¯Ĩ":31264,"ĠTogether":31265,"çŁ¢":31266,"relation":31267,"Ġdeparture":31268,"Ġpassenger":31269,"asket":31270,"Present":31271,"877":31272,"797":31273,"ä¸įå·²":31274,"ä¸Ģ段æĹ¶éĹ´":31275,"......âĢĿĊĊ":31276,"عب":31277,"Remember":31278,"å¸ĺ":31279,"ابة":31280,"impl":31281,"Ġgrief":31282,"Ġzich":31283,"ĠmRNA":31284,"æĢĢåŃķ":31285,"ĠRoutledge":31286,"è½»æĺĵ":31287,"ĠSK":31288,"å½ĵä½ł":31289,"Ġdelve":31290,"908":31291,"ĠFranklin":31292,"åĨħåľ¨":31293,"Ġspectra":31294,"ropolitan":31295,"Ġباز":31296,"ÙĪÙħات":31297,"ä¸į说":31298,"Ġloyalty":31299,"vernment":31300,"彦":31301,"обÑĢаз":31302,".current":31303,"åĪĨå·¥":31304,"Ġamended":31305,"usive":31306,"èģ²éٳ":31307,"endants":31308,"вой":31309,"orum":31310,"ĠнеÑģк":31311,"ìŀij":31312,"è¿ĩ渡":31313,"Ġantigen":31314,"886":31315,"Ġappl":31316,"ĠNotice":31317,"-#":31318,"اک":31319,"ĠImperial":31320,"Ġlease":31321,"ĠHost":31322,"èĤ¡ä»½æľīéĻIJåħ¬åı¸":31323,"Ġincidents":31324,"åĽĽå¤§":31325,"২০":31326,"ÑĩиÑģ":31327,"èīĺ":31328,"ĠMississippi":31329,"Ġsituated":31330,"Ġidentifies":31331,"ĠProvince":31332,"ĠIssues":31333,"Ġeternal":31334,"бÑĢа":31335,"836":31336,"ĠÑĤем":31337,"Ġknees":31338,"ĠModels":31339,"оÑĢа":31340,"edi":31341,"æī¹åΤ":31342,"Ġparks":31343,"主è§Ĵ":31344,"ocl":31345,"à¸Ľà¸£":31346,"Ġstupid":31347,"Ġtrag":31348,"×ķפ×":31349,"arches":31350,"ĠAV":31351,"Ġbeds":31352,"885":31353,"Ġundefined":31354,"çļĦ表çݰ":31355,"Ġii":31356,"ĠDecock":31357,"ĠInit":31358,"åĹ½":31359,"Ġinvested":31360,"ĠìĦ±":31361,"èĻŀ":31362,"Ġtheater":31363,"ĠÑģлÑĥÑĩае":31364,"çļĦåıijçĶŁ":31365,"ä¸įä¸ĭ":31366,"neh":31367,"Ġبعض":31368,"ĠNAT":31369,"éͤ":31370,"Ġsketch":31371,"奢":31372,"éĵħ":31373,"eni":31374,"Ġconting":31375,"Ġpine":31376,"ähr":31377,"ĠCollect":31378,"-alpha":31379,"YK":31380,"083":31381,"Ġdelicate":31382,"Ġhorizon":31383,"본":31384,"åı®":31385,"rr":31386,"Ġcolleges":31387,"ĠدÙĩ":31388,"Ġelite":31389,"ĠExplore":31390,"ĠChallenge":31391,"rawn":31392,"ĠHyper":31393,"×Ļ׳×":31394,"ÑĩеÑģкаÑı":31395,"Ġmao":31396,"à°Ĥà°":31397,"輸":31398,"ienia":31399,"åıĤçħ§":31400,"å¾ģæĶ¶":31401,"ĠCV":31402,"ĠдеÑı":31403,"ìķĪ":31404,"arker":31405,"Ö¹":31406,"ç¬ĶèĢħ":31407,"å¾Īæĺ¯":31408,"Ò£":31409,"ĠBirth":31410,"æİĢ":31411,"主è§Ĥ":31412,"Ġlisting":31413,"Ġrealizing":31414,"Ġвод":31415,"794":31416,"Ġinhibitors":31417,"ĠNas":31418,"urpose":31419,"ç¼ĸç¨ĭ":31420,"çļĦè§Ĵ度":31421,"Ġcontinent":31422,"alin":31423,"Business":31424,"Ġwore":31425,"æııåĨĻ":31426,"ä¸įè¶ħè¿ĩ":31427,"çĥŃéĹ¹":31428,"ophag":31429,"Ġbanyak":31430,"Ġstared":31431,"éĻªä¼´":31432,"бÑĥ":31433,"Ġash":31434,"ĠSpeed":31435,"Ġretreat":31436,"gex":31437,"Ġcortex":31438,"ERROR":31439,"ÏĦÏħμο":31440,"خط":31441,"ĠViews":31442,"顽":31443,"891":31444,"Ġร":31445,"åĪĨåī²":31446,"np":31447,"Por":31448,"ä¸Ĭ课":31449,"hao":31450,"jy":31451,"idata":31452,"Ġsug":31453,"ĠRegulation":31454,"é«ĺè¾¾":31455,"Feature":31456,"698":31457,"ĠHindu":31458,"ä¼ļéķ¿":31459,"945":31460,"åľ¨åīį":31461,"çľ¼æ³ª":31462,"asury":31463,"çľĭä¸Ĭåİ»":31464,"Ġlogged":31465,"äºĭä¸ļåįķä½į":31466,"Ġerg":31467,"Ġsuggestion":31468,"ĠLinear":31469,"åĭĩæķ¢":31470,"tilde":31471,"ä¸ī天":31472,"èĢĮåİ»":31473,"æī¾ä¸įåΰ":31474,"ç®ĬçļĦ":31475,"磮":31476,"å¤ĸ交":31477,"缴纳":31478,"782":31479,"ĠSchul":31480,"ĠYe":31481,"大声":31482,"ãĢijĊ":31483,"Ġendless":31484,"Ġúlt":31485,"James":31486,"èµ·çĤ¹":31487,"åŁİçļĦ":31488,"Ġfailures":31489,"ĠGender":31490,"ĠдвÑĥ":31491,"ĠاÙĦÙĩ":31492,"ĠOptions":31493,"Ġapi":31494,"ĠOlympic":31495,"Ġmodest":31496,"ĠMilitary":31497,"-depth":31498,"ĠElementary":31499,"èĮİ":31500,"ucl":31501,"andal":31502,"visory":31503,"Ġmate":31504,"Ġdeserve":31505,"æİ¥å¾ħ":31506,"ĠSession":31507,"æĭĽåij¼":31508,"Ġdisposal":31509,"ĠPick":31510,"订åįķ":31511,"æĬ¥è¡¨":31512,">&":31513,"计éĩı":31514,"Ġந":31515,"Ġconnectivity":31516,"Ġscholarship":31517,"Ġincur":31518,"Ġindoor":31519,"èĥ½åĬĽåĴĮ":31520,"Ġscientist":31521,"Ġrapport":31522,".val":31523,"ĠÐŀÑĤ":31524,"Ġdump":31525,"tti":31526,"stable":31527,"Ġpuò":31528,"èģĶéĤ¦":31529,"ĩĴ":31530,"æīĵäºĨ":31531,"Ġnetworking":31532,"ĠBaker":31533,"Ġbears":31534,"Ġaccidents":31535,"Ġdefeated":31536,"ÏĦÏħμολογία":31537,"Ġpak":31538,"ciples":31539,"Õ¡Õ´":31540,"983":31541,"ĠPle":31542,"еÑĤа":31543,"çļĦéĥ¨åĪĨ":31544,"ĠSoph":31545,"Ġblessed":31546,"Ġtoxicity":31547,"ä¸ĭéĿ¢çļĦ":31548,"TD":31549,"939":31550,"дан":31551,"Ber":31552,"%%%%":31553,"795":31554,"ĠPubl":31555,"Ġuncomfort":31556,"äºĭçļĦ":31557,"веÑģÑĤ":31558,"ĠìĦł":31559,"ç»ĺçĶ»":31560,"838":31561,"çļĦåīįæıIJ":31562,"ĠرÙĪ":31563,"ÑĢоб":31564,"Ġupward":31565,"ÙĪØ±ÛĮ":31566,"ಲ":31567,"ussen":31568,"åľ¨ä»ĸçļĦ":31569,"Å¥":31570,"ĠCrist":31571,"éĢĤéĩı":31572,"963":31573,"ĠÑįлем":31574,"ä¸Ńèį¯":31575,"俯":31576,"سر":31577,"ĠIndigenous":31578,"Ġprobable":31579,"Ġpt":31580,"Ġranked":31581,"æĺ¯åı¯ä»¥":31582,"854":31583,"ĠEli":31584,"ĠTut":31585,"Ġégal":31586,"·Ċ":31587,"æ·ĭå·´":31588,"Ġadvocate":31589,"Ġcarcinoma":31590,"Ġuniqu":31591,"ç͍å¿ĥ":31592,"ĠSeconds":31593,"788":31594,"è¿Ļ份":31595,"åħħ满äºĨ":31596,"Ġdemanding":31597,"ĠAzure":31598,"اÙĨد":31599,"åħīçļĦ":31600,"793":31601,"867":31602,"ĠIncome":31603,"æī¾åĩº":31604,"Ġassignments":31605,"ä¾µæĿĥ":31606,"ĠDol":31607,"Ġश":31608,"979":31609,"ä¹łè¿ijå¹³æĢ»ä¹¦è®°":31610,"924":31611,"Ġresume":31612,"nm":31613,"Ġguilt":31614,"ìĺĢ":31615,"vt":31616,"ENTS":31617,"éħįå¤ĩ":31618,"Ġtuber":31619,"èµĭäºĪ":31620,"éŨçļĦ":31621,"çĩĥæĸĻ":31622,"ĠElement":31623,"åĭĩæ°Ķ":31624,"Ġ\"@":31625,"Ġreciprocal":31626,"-Based":31627,"Ġhired":31628,"929":31629,"ĠJah":31630,"é¨ĵ":31631,"Ġperceptions":31632,"æ¯Ķéĩį":31633,"Keywords":31634,"success":31635,"Ġprojet":31636,"ĠProgress":31637,"åĽŀäºĨ":31638,"ाल":31639,"Ġpile":31640,"à¹Ĥล":31641,"uba":31642,"被称为":31643,"羣çļĦå¾Ī":31644,"082":31645,"åīįæĻ¯":31646,"à¥ĩà¤Ĥ":31647,"Ġbahwa":31648,"人人":31649,"íĥĢ":31650,"ĠBurn":31651,"Ġcomplexes":31652,"Role":31653,"Ġseasonal":31654,"Ġë°Ķ":31655,"Ġshear":31656,"çļĦéĩįè¦ģæĢ§":31657,"à¹ģà¸ļà¸ļ":31658,"falls":31659,"Ġjoints":31660,"ĠHi":31661,"ĠLoss":31662,"989":31663,"ĠEuropa":31664,"å®Ľ":31665,"è·Łä½ł":31666,"Ġë¶Ħ":31667,"à¸Ľà¸µ":31668,"示ä¾ĭ":31669,"çŃĨ":31670,"ologi":31671,"Ġ\\<":31672,"Ġaccepting":31673,"874":31674,"æĪij们å°Ĩ":31675,"798":31676,"Sever":31677,"sterdam":31678,"Ġwashed":31679,"ĠPlaintiff":31680,"ä»¶äºĭæĥħ":31681,"»ĊĊ":31682,"¶":31683,"ĠREP":31684,"971":31685,"è³ĩæĸĻ":31686,"ĠпоÑĩ":31687,"åIJįè¯į":31688,"ány":31689,"894":31690,"Ġfirmly":31691,"Ġopponent":31692,"Ġë§Ī":31693,"aI":31694,"Ġ미":31695,".Windows":31696,"ç´¢å¼ķ":31697,"Ġexceptions":31698,"Ġcolonies":31699,"лли":31700,"Ġdice":31701,"Ġenterprises":31702,"澡":31703,"Sun":31704,"ศึà¸ģษ":31705,"å®Ŀè´Ŀ":31706,"831":31707,"Ġม":31708,"ãģĦãģŁ":31709,"Widget":31710,"çĶ¨åľ°":31711,"_res":31712,"Ġabsorbed":31713,"Ġexplanations":31714,"äºĤ":31715,"èīĩ":31716,"Elect":31717,"ĠHebrew":31718,"تÙī":31719,"ropic":31720,"ç»ıæµİåѦ":31721,"balance":31722,"ĠPred":31723,"973":31724,"ologÃŃa":31725,"ootstrap":31726,"rollers":31727,"quet":31728,"Ġarising":31729,"åıĺéĿ©":31730,"ä¸Ģå®ļæĺ¯":31731,"iece":31732,"ĠKu":31733,"ĠиÑģк":31734,"nica":31735,"为ä¸Ģ":31736,"ä¸ºåŁºç¡Ģ":31737,"ĠBeat":31738,"å±ķè§Ī":31739,"ĠInstitution":31740,"Ġscanf":31741,"Ġdefect":31742,"Ġprevented":31743,"Ġblocked":31744,"Bre":31745,"Ġhind":31746,"ICT":31747,"ĠProgramming":31748,"Ġdm":31749,"æľīåħ³éĥ¨éŨ":31750,"Ġmaternal":31751,"axies":31752,"Ġcannab":31753,"global":31754,"è´¨çļĦ":31755,"Ġmilliseconds":31756,"bus":31757,"Ú¯ÛĮر":31758,"ributed":31759,"Ġsecrets":31760,"Ġmari":31761,"ización":31762,"产çī©":31763,"Ġacted":31764,"!/":31765,"认åIJĮ":31766,"vic":31767,"ĠCzech":31768,"Ġfantasy":31769,"Ġarte":31770,"827":31771,"oned":31772,"ĠPremier":31773,"796":31774,"865":31775,"Ġalgun":31776,".ap":31777,"人åĿĩ":31778,"868":31779,"931":31780,"Ġдва":31781,"çĶ£":31782,"849":31783,"人们çļĦ":31784,"TM":31785,"åĿİ":31786,"Ġasthma":31787,"ĠInstall":31788,"Ġcompromise":31789,"ιν":31790,"Ġthumb":31791,"ĠXML":31792,"åĬ³åĬ¨åĬĽ":31793,"tree":31794,"Ġspine":31795,"른":31796,"æŃ£å¸¸çļĦ":31797,".Read":31798,"881":31799,"847":31800,"Ġشخص":31801,"lio":31802,"Ġworthy":31803,"isible":31804,"éĢĤå®ľ":31805,"ĠISO":31806,"è°Īè¯Ŀ":31807,"Ġmainstream":31808,"],[":31809,"Ġà¸Ī":31810,"Ġrecom":31811,"Ġlesser":31812,"Ġfragments":31813,"China":31814,"Ġheap":31815,"åįģåĩł":31816,"ĠActions":31817,"ĠRoger":31818,"YP":31819,"Know":31820,"èĬ±åĽŃ":31821,"çĽ£":31822,"095":31823,"×ķ×ŀ":31824,"994":31825,"è¿Ļä¸įæĺ¯":31826,"Children":31827,"çī¹åĪ¥":31828,"éħ¿":31829,"æ²³æµģ":31830,"/e":31831,"æĸ°æĬĢæľ¯":31832,"Ġtras":31833,"èIJĿåįľ":31834,"Ġfocal":31835,"ĠJoin":31836,"Ġwsz":31837,"onometric":31838,"æŃ£éĿ¢":31839,"ãģ¦ãģĦãģŁ":31840,"-bit":31841,"çĶŁäº§çļĦ":31842,"wed":31843,"abetic":31844,"Ġstatistically":31845,"ĠBiden":31846,"hs":31847,"çĦī":31848,"æ¸ħéϤ":31849,"Ġhitting":31850,"tek":31851,"074":31852,"æ°Ķ管":31853,"è¿Ļç§įæĥħåĨµ":31854,"ünd":31855,"Ġplanted":31856,"ĠYellow":31857,"Ġvec":31858,"вание":31859,"ĠAcad":31860,"controller":31861,"Ġmatrices":31862,"ĠVisit":31863,"çķĻåѦ":31864,"Schema":31865,"ียà¸ĩ":31866,"ä¸Ńåįİæ°ijæĹı":31867,"uning":31868,"873":31869,"人åĬĽèµĦæºIJ":31870,"Ġlawyers":31871,"Ġencore":31872,"ĠDecision":31873,"ĠÐłÐ°":31874,"master":31875,"ĠAmer":31876,"ĠUpper":31877,"Ġautomation":31878,"ĠاØŃ":31879,"ç͍æīĭ":31880,"å±±çļĦ":31881,"Ġ%}Ċ":31882,"846":31883,"rv":31884,"è¶ħå¸Ĥ":31885,"Ġrhet":31886,"TI":31887,"举æİª":31888,"ĠMann":31889,"(object":31890,"-Q":31891,"jection":31892,"ĠKB":31893,"Ġrevenues":31894,"ĠPolish":31895,"Ġintroduces":31896,"ä¸ĢåIJĮ":31897,"Ġverification":31898,"882":31899,"ĠGrund":31900,"898":31901,"Ġmening":31902,"`ĊĊ":31903,"åİĨåı²ä¸Ĭ":31904,"Ġvisibility":31905,"955":31906,"ĠVa":31907,"æĮª":31908,"æ±īè¯Ń":31909,"ä¿¡æģ¯çļĦ":31910,"Ġavant":31911,".ac":31912,"Ġspecimens":31913,"Ġfarms":31914,"limited":31915,"Ġsupporters":31916,"æ°Ķæ°Ľ":31917,"Ġmerupakan":31918,"optera":31919,"Ġpond":31920,"Ġдела":31921,"à°®":31922,">{":31923,"Ġcertified":31924,"书éĿ¢":31925,"arga":31926,"åı¯æĢľ":31927,"Ġdetecting":31928,"Ġrewards":31929,"Ġpant":31930,"oggle":31931,"æĩĪ":31932,"ĠSleep":31933,"Ġappet":31934,"Ġett":31935,"Ġfright":31936,"ä¼łè¾¾":31937,"ĠDeutsch":31938,"Ġarrays":31939,"Ġorche":31940,"Ġ'-":31941,"049":31942,"Ġdic":31943,"ĠбÑĭли":31944,"Ġcorporations":31945,"æļ´åĬĽ":31946,"ä¹ĥèĩ³":31947,"norm":31948,"Ġfung":31949,"Ġíĸ":31950,"íŀĪ":31951,"Ġunderground":31952,"ï¼ķ":31953,"932":31954,"Citation":31955,"ĠNetworks":31956,"Ġsymmetry":31957,"068":31958,"ä¸įæĢķ":31959,"ãĤĢ":31960,"Ġainsi":31961,"ĠAlaska":31962,"å½±åĥı":31963,"Ġplots":31964,"\"];Ċ":31965,"å¯Įæľī":31966,"å®Įæ¯ķ":31967,"åĮºéĹ´":31968,"زار":31969,"Ġtitled":31970,"ίαÏĤ":31971,"ĠÐŁÑĢо":31972,"Entry":31973,"ï¼ŁãĢįĊĊ":31974,"λε":31975,"Ġsequencing":31976,"à¸Ńà¸Ķ":31977,"ĠOH":31978,"äch":31979,"ĠCi":31980,"Ġdesigners":31981,"Cost":31982,"ĠMade":31983,"Week":31984,"ogg":31985,"å¼Ģæĭĵ":31986,"962":31987,"phen":31988,"-round":31989,"dfrac":31990,"ĠPand":31991,"ĠCow":31992,"ï¼īï¼ļ":31993,"Those":31994,"çķ¶çĦ¶":31995,"Ġpotassium":31996,"Ġgauge":31997,"Ġempire":31998,"çīĽå¥¶":31999,"ç¼ĸåĨĻ":32000,"agonist":32001,"Ġracing":32002,"Ġnun":32003,"ará":32004,"Ġranking":32005,"ECTION":32006,"_info":32007,"Ġcarbohyd":32008,"åįłæį®":32009,"ĠBudget":32010,"代表大ä¼ļ":32011,"è°¨æħİ":32012,"æĿ¥åΰäºĨ":32013,"åĨĽçļĦ":32014,"Ġfonction":32015,"ĠRace":32016,"ariate":32017,"arser":32018,"ĠPatent":32019,"Ġreluct":32020,"owaÄĩ":32021,"yc":32022,"Ġdairy":32023,"Univers":32024,"Ġclip":32025,"াà¦Ĥ":32026,"禽":32027,"ĠвÑģего":32028,"ĠÐļак":32029,"Ġê°Ļ":32030,"learn":32031,"Ġlamp":32032,"ĠìĦľ":32033,"nowned":32034,"为ä¸Ńå¿ĥ":32035,"ĠGeneration":32036,"ĠÐľÐ¸":32037,"ĠSeattle":32038,"Ġanniversary":32039,"eded":32040,"åĪĨæĪIJ":32041,"Ġinterfaces":32042,",\\,":32043,"Ġcharity":32044,"Ġcompetitors":32045,"ĠTow":32046,"ĠMarshall":32047,"å±±åĮº":32048,"Tim":32049,"atories":32050,"-minute":32051,"Ġarises":32052,"Short":32053,"834":32054,"Õ½":32055,"Ġware":32056,"Ġsymbolic":32057,"并对":32058,"ĠÙĪØ¬ÙĪØ¯":32059,"-X":32060,"/W":32061,"å®¶åħ·":32062,"Ġобе":32063,"Maybe":32064,"Ġ?Ċ":32065,"Answers":32066,"ĠнаÑģ":32067,"ä»Ĩ":32068,"ÑĢави":32069,"unis":32070,"ĠPotential":32071,"讽":32072,"æĶ¾åΰ":32073,"\\]ĊĊ":32074,"Ġlact":32075,"owners":32076,"康å¤į":32077,"osex":32078,"965":32079,"Ġcried":32080,"æįŀ":32081,"gae":32082,"892":32083,"ÏĦά":32084,"Gamma":32085,"å¼Ģå§ĭäºĨ":32086,"åĵĩ":32087,"ĠТа":32088,"hentication":32089,"à§įà¦ļ":32090,"096":32091,"Ġemphasized":32092,"Ġsends":32093,"ĠNar":32094,"Ġflowing":32095,"Ġsoy":32096,"Äģn":32097,"armacy":32098,"union":32099,"ç͵æ°Ķ":32100,"ardi":32101,"ĠGrace":32102,"Ġcri":32103,"Ġprivilege":32104,"Ġsatisfying":32105,"Ġfet":32106,"Ġweaken":32107,"ĠAlgebra":32108,"èĥ°":32109,"ĠDow":32110,"Based":32111,"Ġdeficient":32112,"طة":32113,"iour":32114,"Ġrecycling":32115,"ĠBond":32116,"ä¼ļä¸įä¼ļ":32117,"Ġdrift":32118,"大夫":32119,"Ġapproximate":32120,"ĠArabic":32121,"Ġotros":32122,"969":32123,"ĠBrief":32124,"orse":32125,"Japan":32126,"ricks":32127,"represent":32128,".toString":32129,"Span":32130,"è¿ĺåİŁ":32131,"istem":32132,"initial":32133,"ÙİÙij":32134,"Ġpreservation":32135,"ìłģìĿ¸":32136,"Ġdancing":32137,"Ġworkshops":32138,"ób":32139,"ĠSwedish":32140,"ç»ĵæŀĦçļĦ":32141,"~ĊĊ":32142,"ë°©":32143,"ĠNFL":32144,"ĠкÑĢÑĥ":32145,"使人":32146,"æĪıåī§":32147,"ÑĢеÑģ":32148,"Ġworlds":32149,"ä¸ĵä¸ļçļĦ":32150,"åħļåı²":32151,"Ġconsisted":32152,"ĠBarcelona":32153,"rainian":32154,"Ġbesides":32155,"Ġìļ°":32156,"mn":32157,"iencies":32158,"038":32159,"amiliar":32160,"Ġamen":32161,"ĠRequirements":32162,"ĠEffective":32163,"Ġdz":32164,"ĠwiÄĻ":32165,"ourag":32166,"Ġpunt":32167,"'],":32168,"unden":32169,"OME":32170,"ĠTurkish":32171,"ĠÑĦак":32172,"ĠHier":32173,"abilit":32174,"å¼ıä¸Ń":32175,"åıĤè§ģ":32176,"ĠÑıзÑĭ":32177,"ĠÑģвÑı":32178,"æĬĹæĹ¥":32179,"ĠCarlos":32180,"اÙĪÙĦ":32181,"878":32182,"Ġfisher":32183,"ĠÎŁ":32184,"Ox":32185,"èľĢ":32186,"Ġhosted":32187,"Ġanimation":32188,"Leg":32189,"Ġplanes":32190,"Ġrever":32191,"Average":32192,"åľ¨ç¾İåĽ½":32193,"Ġcamps":32194,"αÏģ":32195,"plet":32196,"Ġتص":32197,"åľ¨æĸ°":32198,"اÙĨÙĬÙĩ":32199,"ç»Ļåĩº":32200,"-party":32201,"ĠKre":32202,"çģ¶":32203,"Ġviable":32204,"æĺ¯å¤§":32205,"phe":32206,"Da":32207,"initions":32208,"ĠChang":32209,"Ġreversed":32210,"à§ĭà¦Ĺ":32211,"Ġigual":32212,"cards":32213,"ĠInv":32214,"Ġdiscomfort":32215,"åĿł":32216,"تÙĩا":32217,"é«ĺ端":32218,".start":32219,"Ġизмен":32220,"ä¸İæŃ¤åIJĮæĹ¶":32221,"ĠBund":32222,"¼":32223,"059":32224,"ĠFinn":32225,"ĠMiami":32226,"Ġtunnel":32227,"phan":32228,"ockets":32229,"Ġepic":32230,"ĠÙħست":32231,"å®ŀæĹ¶":32232,"046":32233,"ĠLen":32234,"ĠMOOCs":32235,"íĹ":32236,"çļĦæłĩåĩĨ":32237,"ä¸Ģåij¨":32238,"Ġpanic":32239,"ä¸Ģå¥Ĺ":32240,"_**":32241,"ĠStress":32242,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":32243,",d":32244,"Ġrestriction":32245,"Ġìĭł":32246,"éĽģ":32247,"zej":32248,"çļĩä¸Ĭ":32249,"Ġgastro":32250,"ÙĨاÙħ":32251,"å¹ħ度":32252,"Template":32253,"ŀר":32254,"Ġstan":32255,"ifiable":32256,"注æĦıåΰ":32257,"Ġexcellence":32258,"Ġhá":32259,"#'":32260,"ĠBuch":32261,"977":32262,"oÅĽci":32263,"OSE":32264,"ĠATP":32265,"REF":32266,"highlight":32267,"vable":32268,"ĠWard":32269,"ĠArn":32270,"Forms":32271,"handle":32272,"æĦ¤æĢĴ":32273,"ĠIce":32274,"Ġurgent":32275,"935":32276,"Ц":32277,"Compar":32278,"Ġslides":32279,"Ġpets":32280,"åIJı":32281,"çļĦ女":32282,"Ġmush":32283,"ĠCommissioner":32284,"Ġholidays":32285,"ãĢħ":32286,"ä¸ŃæľĢ":32287,")?Ċ":32288,"_input":32289,"073":32290,"าà¸Ħ":32291,"лÑĮного":32292,"Cs":32293,"Ġslee":32294,"Ġproposition":32295,"ionale":32296,"æĪijçİ°åľ¨":32297,"ĠKos":32298,"Ġgrip":32299,"ĠSubt":32300,"Ġpharmaceutical":32301,"Ġsurname":32302,"åľĨå½¢":32303,"Hg":32304,"iere":32305,"æľªçŁ¥":32306,"elig":32307,"æĪijåıĪ":32308,"ĠUsers":32309,"Ġanos":32310,"è¿Ļ段":32311,"Ġchopped":32312,"ĠIO":32313,"Conclusion":32314,"haus":32315,"交ç»Ļ":32316,"Ġdisappeared":32317,"ê°ģ":32318,"053":32319,"addle":32320,"ĠпÑĢоблем":32321,"Ġimpairment":32322,"astics":32323,"Ġداش":32324,"USE":32325,"гÑĢÑĥ":32326,"à¸Ħล":32327,"çĭ¬çī¹çļĦ":32328,"Ġfuels":32329,"Land":32330,"ĠCher":32331,"èιèĪ":32332,"ĠEmergency":32333,".<":32334,"éķ¿æ²Ļ":32335,"ï¼Īï¼ī":32336,"å¤ıåŃ£":32337,"Ñļа":32338,"Ġرس":32339,"ãĥ£":32340,"Ġimports":32341,"åĬłæĭ¿å¤§":32342,"лÑĮнÑĭй":32343,"ĠÑĤÑĢеб":32344,"erget":32345,"ĠPul":32346,"Ġbrows":32347,"ĠCris":32348,"人éĹ´":32349,"åıĹçIJĨ":32350,"device":32351,"held":32352,"缸å¤Ħ":32353,"056":32354,"berries":32355,"iken":32356,"aris":32357,"achine":32358,"odi":32359,"ĠasÃŃ":32360,"Ġbenefici":32361,"ylene":32362,"character":32363,"onde":32364,"Come":32365,"ĠCarter":32366,"weise":32367,"Ing":32368,"Ġmemiliki":32369,"=\"{{":32370,"Ġmandatory":32371,"abc":32372,"Ġpartnerships":32373,"Jul":32374,"%).":32375,"itime":32376,"osphere":32377,"Ġadip":32378,"çłĶç©¶çļĦ":32379,"Ġiconic":32380,"Ġbarb":32381,"974":32382,"837":32383,"ä¼ļ被":32384,"Ġmachinery":32385,"JS":32386,"ĠTaking":32387,"Ġproceeds":32388,"Ġslice":32389,"åı³æīĭ":32390,"Ġপার":32391,"Ġkinetic":32392,"879":32393,"ĠClient":32394,"å®ŀéĻħæĥħåĨµ":32395,"far":32396,"æĬ¥çº¸":32397,"Ġprolonged":32398,"Ġpositioning":32399,"Ġshifting":32400,"eca":32401,"Ġbuyers":32402,"åģ´":32403,"Ġupgrade":32404,"çģ¾å®³":32405,"Microsoft":32406,"ĠвÑģеÑħ":32407,"utan":32408,"жен":32409,"ĠLanc":32410,"Ġstoring":32411,"ä¸ĭæĸ¹":32412,"Ġindividually":32413,"ä¸İåħ¶ä»ĸ":32414,"Ġaddiction":32415,"åѤçĭ¬":32416,"Ġ(\\":32417,"Ġallocated":32418,"069":32419,"éľĦ":32420,"Deb":32421,"Ġexterior":32422,"ĠApps":32423,"North":32424,"ÑĢоваÑĤÑĮ":32425,"rene":32426,"ĠMorris":32427,"олов":32428,"াশ":32429,"ãģ¨ãģ¯":32430,"ĠEncyclopedia":32431,"Ġexpecting":32432,"Ġdramatically":32433,"Ġthrowing":32434,"ibus":32435,"صر":32436,"FORM":32437,"NET":32438,"æĪij认为":32439,"Ġconfidential":32440,"Ġर":32441,"çļĦèĦ¸":32442,"Ġà¤Ń":32443,"ĠViewfinder":32444,"etc":32445,"054":32446,"Ġtahun":32447,"earchers":32448,"ĠMiles":32449,"ç§ijåѦ家":32450,"çļĦæµ·":32451,"ĠWol":32452,"Ġdissolved":32453,"psych":32454,"ĠجÙĩ":32455,"Ġcents":32456,"Ġverified":32457,"Ġbesch":32458,"-rich":32459,"-label":32460,"ahimutang":32461,"èµļéĴ±":32462,"ÑĦеÑĢ":32463,"ĠPCR":32464,"诸å¤ļ":32465,"Ġbou":32466,"Ġessays":32467,"ĠÙĪÙĬÙĥ":32468,"ĠÑĢебен":32469,"ยà¹Į":32470,"Ġwaar":32471,"halt":32472,"æīĭæ³ķ":32473,"åĴ³åĹ½":32474,"NR":32475,"ç«ĭè¶³":32476,"Ġpivotal":32477,"Ġsubscription":32478,"ĠAx":32479,"ISS":32480,"Ġziren":32481,"ç¥ģ":32482,"ĠتÙĥ":32483,"æĹłæīĢ":32484,"å¦Ĥæľī":32485,"å®ŀè·µä¸Ń":32486,"æĺ¯ä¸ŃåĽ½":32487,"Ġdischarg":32488,"Ġhighlighting":32489,"949":32490,"æĶ¹ä¸º":32491,"Ġarchitectural":32492,"Îij":32493,"Real":32494,"ĠSources":32495,"ĠVillage":32496,"没人":32497,"._**":32498,"ध":32499,"uzzy":32500,"Ġinhibitor":32501,"çģ«çģ¾":32502,"Ġprescription":32503,"Fact":32504,"å¸ĤæĶ¿åºľ":32505,"è´Łèį·":32506,"åľĭå®¶":32507,"Ġinvite":32508,"ĠPortuguese":32509,"Ġundertaken":32510,"loss":32511,"ĠMg":32512,"ĠTib":32513,"æĥħæ³ģ":32514,"两天":32515,"ç¶ĵæ¿Ł":32516,"ÙĨØ©":32517,"è°ħ":32518,"ĠCampbell":32519,"Ġpurely":32520,"ĠBapt":32521,"Ġdivisions":32522,"Ġথà§ĩà¦ķà§ĩ":32523,"宽度":32524,"ĠEvents":32525,"ĠداÙĨØ´":32526,"termin":32527,"ãĢĤâĢĶâĢĶ":32528,"Ġfinishing":32529,"(map":32530,"Ġétait":32531,"Ġdisclosed":32532,"mans":32533,"ioitu":32534,"Ġdeclar":32535,"ĠTell":32536,"ĠاÙĦØ¢":32537,"Ġseus":32538,"è¿ĻäºĽäºº":32539,".trans":32540,"Ġcargo":32541,"Ġsinger":32542,"[id":32543,"ICAg":32544,"Ġrefuse":32545,"Ġquasi":32546,"ĠQuiz":32547,"Ġbackup":32548,"çłĶç©¶éĻ¢":32549,"åįĬå¾Ħ":32550,"Ġlam":32551,"èĢģåŃIJ":32552,"çķĻè¨Ģ":32553,"।Ċ":32554,".begin":32555,"IJש":32556,"Ġгод":32557,"æIJºå¸¦":32558,"åĴIJ":32559,"sn":32560,"Params":32561,"Ġdepicted":32562,"-der":32563,"orpor":32564,"ä½łäºĨ":32565,"个åĪ«":32566,"زÙĬ":32567,"once":32568,"ĠZn":32569,"Ġvin":32570,"вÑı":32571,"\">ĊĊ":32572,"-side":32573,"standard":32574,"Ġpurchases":32575,"è¿ĩå¤ļ":32576,"iasm":32577,"Ġcombines":32578,"ä¼ŀ":32579,"éĶĢéĩı":32580,"åIJ¬çĿĢ":32581,"ÑĢованиÑı":32582,"ình":32583,"Ġlyrics":32584,"ĠMak":32585,"ĠдеÑĤей":32586,"ĠSF":32587,"åħ¨å¹´":32588,"-est":32589,"Ġyoga":32590,"ĠHend":32591,"ÑĤами":32592,"hund":32593,"}^":32594,"iani":32595,"ĠSad":32596,"奥è¿IJ":32597,"éĢīæĭ©äºĨ":32598,"Ġvaccination":32599,"&\\":32600,"Ġelectroly":32601,"(item":32602,"åĮĸå¦Ĩ":32603,"Ġchloride":32604,"éĹľä¿Ĥ":32605,"ĠÑģвой":32606,"umble":32607,"934":32608,"Ġcaut":32609,"Ġthreads":32610,"Ġana":32611,"ALSE":32612,"Ġinstantly":32613,"éķ¿å¤§":32614,"ÑģÑĤÑĢой":32615,"åϬ":32616,"ĠReports":32617,"åĨ³èµĽ":32618,"{P":32619,"sett":32620,"Ġalc":32621,"åIJijéĩı":32622,"unter":32623,"Ġammon":32624,"ä¾µçĬ¯":32625,"telling":32626,"精确":32627,"çļĦåı£":32628,"958":32629,"å¸ĮèħĬ":32630,"Ġга":32631,"è³£":32632,"Ġlé":32633,"Ġapproaching":32634,"966":32635,"iary":32636,"ä¸Ģ群":32637,"Ġimpressed":32638,"Ġprofes":32639,"Ġfake":32640,"ĠvÃŃ":32641,"obby":32642,"rencies":32643,"çĤ¹äºĨçĤ¹å¤´":32644,"ĸ×Ķ":32645,"ĠRan":32646,"ĠÕ¯":32647,"941":32648,"present":32649,"Ø«ÙĬر":32650,"Ġrectangle":32651,"è¿ľç¨ĭ":32652,"ĠTrends":32653,"ĠServ":32654,"Ġasleep":32655,"ĠAld":32656,"Ġopponents":32657,"Ġmitigate":32658,"former":32659,"ĠOP":32660,"кономи":32661,"æīĢå¾Ĺç¨İ":32662,"_ch":32663,"Ġsb":32664,"Place":32665,"ä¼ĺç§ĢçļĦ":32666,"Ġelekt":32667,"Ġguaranteed":32668,"Ġdebug":32669,"veis":32670,"رس":32671,"огов":32672,"جب":32673,"ĠCatalan":32674,"Ġglasses":32675,"åŁºçĿ£":32676,"rieb":32677,"åı¯ä»¥çľĭåĩº":32678,"ĠCoal":32679,"Ġlav":32680,"ách":32681,"Ġpla":32682,"################################":32683,"_ST":32684,"Ġfluor":32685,"ĠÑĨвеÑĤ":32686,"oard":32687,"ADE":32688,"Detail":32689,"ĠTransl":32690,"ĠCompanies":32691,"ầ":32692,"Ġtodas":32693,"occup":32694,"åłħ":32695,".i":32696,"Ġbother":32697,"è¡Į为çļĦ":32698,"лаг":32699,"ĠEvans":32700,"Ġprize":32701,"/bin":32702,"ĠKnowing":32703,"Ġал":32704,".Name":32705,"ä¸įå¿ĺ":32706,"rir":32707,"Ġconception":32708,"ĠMargaret":32709,"lak":32710,"éĿ¢æĿ¿":32711,"æĺ¯åIJ¦æľī":32712,"roleum":32713,"หว":32714,"Ġleather":32715,"959":32716,"è´ŀ":32717,"883":32718,"æĭ¿èµ·":32719,"initely":32720,"Ġ',":32721,"ĠSympt":32722,"Ġio":32723,"æĶ¾å°Ħ":32724,"ĠPlatform":32725,"Ġfigured":32726,"\"));Ċ":32727,"947":32728,"quin":32729,"tober":32730,"Ġaccountability":32731,"orsch":32732,"Ġanni":32733,"Ġinfectious":32734,"Ġformats":32735,"887":32736,",C":32737,"Ġinstrumental":32738,"Ġvoluntary":32739,"çļĩåIJİ":32740,"Äij":32741,"ĠCash":32742,"ä½ľçī©":32743,"Ġsimplify":32744,"Wed":32745,"å¾Īä¸į":32746,"ĠGraham":32747,"ĠTables":32748,"Ġtablespoon":32749,"دد":32750,"ĠAnat":32751,"Ġspecifications":32752,"ĠGate":32753,"éĢīåıĸ":32754,"æĬķå½±":32755,"âŁ":32756,"å±ĢéĻIJ":32757,"Ġstrikes":32758,"ĠSTAT":32759,"Db":32760,"943":32761,"ĠRand":32762,"ĠLooking":32763,"ĠAuthors":32764,"ĠBelow":32765,"ĠVA":32766,"927":32767,"ĠSolid":32768,"answered":32769,"859":32770,"ãģĮãģĤãĤĭ":32771,"ĠPope":32772,"论述":32773,"à¥ĥ":32774,"066":32775,"Join":32776,"å«Įçĸij":32777,"*)":32778,"ennial":32779,"为ä»Ģä¹Īè¦ģ":32780,"Ġmeditation":32781,"ĠCastle":32782,"091":32783,"âĢĻâĢĻ":32784,"łáĥ":32785,"Hy":32786,"\\).ĊĊ":32787,"åĽ½çİĭ":32788,"ượ":32789,"åѸç¿Ĵ":32790,"aisarv":32791,"ĠMuslims":32792,"mac":32793,"Ġunch":32794,"Ġunpre":32795,"Admin":32796,"ĠDirection":32797,"Ġenroll":32798,"ĠpaÃŃs":32799,"Ġflavors":32800,"ĠExpression":32801,"942":32802,"Company":32803,"Ġপà§įরত":32804,"lings":32805,"926":32806,"respect":32807,"Rober":32808,"nement":32809,"Ġnons":32810,"åİŁæĿ¥çļĦ":32811,"except":32812,"ç»ĦæĪIJéĥ¨åĪĨ":32813,"Ġνα":32814,"making":32815,"åĨįçĶŁ":32816,"\\(-\\)":32817,"æ£Ģå¯ŁéĻ¢":32818,"ĠëıĦ":32819,"Ġrim":32820,"Ñģкие":32821,"ëĭ¹":32822,"ĠProtein":32823,"ĠMRI":32824,"Ġcanal":32825,"åĪ¶çº¦":32826,"åĺ»":32827,"ÙĪÙĬØ©":32828,"æĪijå®¶":32829,"æ²§":32830,"ĠتارÙĬØ®":32831,"Ġnegatively":32832,"Ġwitnessed":32833,"å¦Ĥæŀľæ²¡æľī":32834,"è¿Ļ个人":32835,"ĠíĬ":32836,"ä¸Ĭå¸Ĥåħ¬åı¸":32837,"station":32838,"ĠëĮĢíķľ":32839,"Ġtourist":32840,"products":32841,"hec":32842,"ĠпÑĢав":32843,"exper":32844,"aisarvioitu":32845,"è¡Į使":32846,"è¾ĥé«ĺçļĦ":32847,"ãĥ³ãĥĪ":32848,"æ¸´æľĽ":32849,"*Ċ":32850,"Ġ}}Ċ":32851,"regn":32852,"Ġinevitable":32853,"cano":32854,"Ġprisoners":32855,"esar":32856,"Ġhierarchy":32857,"åĦ¿çļĦ":32858,"ĠÙĩست":32859,"ĠоÑĤвеÑĤ":32860,"æľĪåĪĿ":32861,"ä¸Ńå¹´":32862,"aurus":32863,"jal":32864,"ืà¹īà¸Ń":32865,"Ġcriterion":32866,"Vector":32867,"ĠDiagram":32868,"æ¬²æľĽ":32869,"å®ŀåľ¨æĺ¯":32870,"ĠWebsite":32871,"ĠDelta":32872,"Ġdeput":32873,"Ġgesch":32874,"}\\]ĊĊ":32875,"eking":32876,"Ġté":32877,"æĹ¥èĩ³":32878,"ר׼":32879,"Ġcombustion":32880,"ĠForecast":32881,"Gr":32882,"Ġlogo":32883,"æĬ¥èѦ":32884,")\".":32885,"ĠRena":32886,"ollen":32887,"ન":32888,"_train":32889,"ĠOften":32890,"åľĨ满":32891,"اÙĦب":32892,"ĠRah":32893,"ĠNicolson":32894,"Ġâľ":32895,"ported":32896,"ÂłĊ":32897,"âĸĪ":32898,"ĠVertaisarvioitu":32899,"íķ©ëĭĪëĭ¤":32900,"Ġdelta":32901,"outube":32902,"èĦ±ç¦»":32903,"Ġemphasize":32904,"Obj":32905,"ĠBanglades":32906,"ĠPP":32907,"893":32908,"ĠSalt":32909,"Ġнек":32910,"клÑİÑĩа":32911,"ĠOklahoma":32912,"Ġapopt":32913,"ĠAccessed":32914,"_state":32915,"venile":32916,"Ġtyping":32917,"ä½ıåľ¨":32918,"ĠAnaly":32919,"äºĨä¸Ģåı¥":32920,"Ġseventh":32921,"Ġsusceptible":32922,"书åĨĻ":32923,"াদà§ĩর":32924,"Ñĥг":32925,"iculous":32926,"aused":32927,"åħ¬å®īå±Ģ":32928,"scape":32929,"ĠÑĩаÑģÑĤи":32930,"offs":32931,"ĠStatistical":32932,"Ġinadequate":32933,"967":32934,"ONG":32935,"948":32936,"Ġcalc":32937,"gie":32938,"è¶ķ":32939,"åĽĽåįģ":32940,"éĢļéģİ":32941,"à¹Ģà¸ŀืà¹Īà¸Ń":32942,"Ġtalented":32943,"Ġalternate":32944,"869":32945,"ĠHealthcare":32946,"çĿ̥̿":32947,"ĠKentucky":32948,"OLD":32949,"Ġbackgrounds":32950,"Ġinvestor":32951,"æĭĽæłĩ":32952,"ĠSchedule":32953,"è¿Ļ项":32954,"��������":32955,"Ġoils":32956,"æķĻè®Ń":32957,"ĠFlash":32958,"éĶĪ":32959,"è¡Įæĥħ":32960,"ÑĮÑı":32961,"Ġexpenditure":32962,"[edit":32963,"rowave":32964,"æķ°åŃĹåĮĸ":32965,"umption":32966,"Ġcheer":32967,"Ġpredictive":32968,"Ġnewspapers":32969,"ĠLate":32970,"éϰ":32971,"大èĩ´":32972,"both":32973,"Ġdével":32974,"Save":32975,"ĠDiam":32976,"Ġquestionnaire":32977,"çĶŁåij½çļĦ":32978,"ĠStories":32979,".view":32980,"oux":32981,"icut":32982,"ĠRud":32983,"pathetic":32984,"鼾":32985,"è¨Īç®Ĺ":32986,"psy":32987,"Ġexams":32988,"æľĢåIJİä¸Ģ":32989,"åIJĦéĥ¨éŨ":32990,"æī©æķ£":32991,"inded":32992,"å®ĪæĬ¤":32993,"ĠProtest":32994,"ĠGross":32995,"×Ļ×Ĺ":32996,"å¼Ģå±ķäºĨ":32997,"crease":32998,"羣çIJĨ":32999,"åĿĩ为":33000,"Okay":33001,"953":33002,"ĠVII":33003,"Ġlugar":33004,"лоÑĤ":33005,"Ġplac":33006,"ĠÑĪи":33007,"idis":33008,"åħĭæĸ¯":33009,"Ġenglish":33010,"身çļĦ":33011,"è³¼":33012,"åħ¬åĬ¡åijĺ":33013,"-use":33014,"pf":33015,"ĠAtlanta":33016,"éħ®":33017,"rystal":33018,"Ġattendance":33019,"Ġhungry":33020,"ãĤĭãģ¨":33021,"WR":33022,"ç´į":33023,"otechnology":33024,"大æĪĺ":33025,"vu":33026,"Ġswift":33027,"éĤµ":33028,"Ġolig":33029,"elsius":33030,"Ġcryptocur":33031,"ç͍èį¯":33032,"å»¶ç»Ń":33033,"Ġslower":33034,"ĠBarbara":33035,"åīįæĿ¥":33036,"ä¸įåĥı":33037,"çĹĴ":33038,"ĠPrinciples":33039,"è¶³å¤ŁçļĦ":33040,"ĠStop":33041,"rud":33042,"anium":33043,"ä»Ģä¹Īæł·çļĦ":33044,"è¯Ģ":33045,"еди":33046,"Ġquotes":33047,"NY":33048,"ĠUnknown":33049,"Ġmesh":33050,"Ġczas":33051,"িম":33052,"ĠпÑĢогÑĢам":33053,"ĠBis":33054,"ĠинÑĦоÑĢма":33055,"Ġexhibits":33056,"ĠاÙĨد":33057,"ĠKor":33058,"cery":33059,"æ£į":33060,"ifference":33061,"_dir":33062,"Ġexpectation":33063,"pher":33064,"Video":33065,"æŃ£ä¹ī":33066,"ĠÑĩаÑģÑĤ":33067,"ractice":33068,"vasive":33069,"Ġstairs":33070,"ké":33071,"ylon":33072,"ĠÐĺн":33073,"ÑĨионалÑĮ":33074,"ĠCharlie":33075,"078":33076,"qt":33077,"è°ĥåĬ¨":33078,"Ġneglect":33079,"íĺķ":33080,"Ġglance":33081,"Bal":33082,"íķĺ기":33083,"logo":33084,"纽约":33085,"été":33086,"ĠReyn":33087,"Ġmaintains":33088,"à§įন":33089,"Ġhabag":33090,"Ġunderm":33091,"اÙĦØ¥":33092,"ç«ŀäºīåĬĽ":33093,"份é¢Ŀ":33094,"ï¼Ķ":33095,"Ġflip":33096,"ìĺģ":33097,"Utils":33098,"µľ":33099,"anon":33100,"ĠзавиÑģи":33101,"Ġdismissed":33102,"éĻĽä¸ĭ":33103,"rime":33104,"Ġmens":33105,"Ġstems":33106,"ĠFreedom":33107,"Ġá½":33108,"Settings":33109,"[(":33110,"Ġposting":33111,"Ġcustoms":33112,"Ġtravers":33113,"Ġgebru":33114,"ĠMis":33115,"ĠUniversal":33116,"Modal":33117,"ĠHTTP":33118,"ĠÑĢазлиÑĩ":33119,"è¿ľå¤Ħ":33120,"Ġalgoritmo":33121,"ĠPromise":33122,"isson":33123,"åij³çļĦ":33124,"Ġcute":33125,"Ġrounds":33126,"ĠAdult":33127,"ivial":33128,"æĪijå·²ç»ı":33129,"Ġspirits":33130,"Ġjumped":33131,"Ġبش":33132,"Ġambit":33133,"aggio":33134,"Ġoutlet":33135,"Ġinvestigating":33136,"à¹Ģมืà¹Īà¸Ń":33137,"Ġfires":33138,"Ġmonument":33139,"_map":33140,"Ġpixels":33141,"ÑĤем":33142,"车éĹ´":33143,"èįīåİŁ":33144,"ĠWel":33145,"938":33146,"Ġlocate":33147,".state":33148,"film":33149,"Ġeducated":33150,"å®ŀäºĭ":33151,"å®īç½®":33152,"جة":33153,"æ³ķåζ":33154,"982":33155,"ĠSyria":33156,"ĠLane":33157,"平常":33158,"表çݰåĩº":33159,"Mill":33160,"æĶ¹éĿ©å¼ĢæĶ¾":33161,"jel":33162,"cloud":33163,"Ġpassages":33164,"Ġlogging":33165,"-date":33166,"æ°Ķ象":33167,"Ġcountless":33168,"-me":33169,"åĮĸåIJĪçī©":33170,"Ġbasics":33171,"ç¾İ丽çļĦ":33172,"ĠCreating":33173,"åĪijæ³ķ":33174,"åľ°éĵģ":33175,"Ġoccasional":33176,"RES":33177,"ĠобÑīе":33178,"Ġwished":33179,"099":33180,"Rating":33181,"location":33182,"your":33183,"Ġsins":33184,"Ġvocê":33185,"ĠPrograms":33186,"è´¦åı·":33187,"]{":33188,"954":33189,"wegian":33190,"udad":33191,"ĠSI":33192,"ï¼ļãĢĬ":33193,"Ġdisciplines":33194,"WO":33195,"Ġimg":33196,"Ġmismo":33197,"è§ģäºĨ":33198,"æĥĬåĸľ":33199,"Ġdeciding":33200,"ĠAlliance":33201,"GH":33202,"ĠÙĪØ§ÙĦÙħ":33203,"atalogue":33204,"ç§ijåѦæĬĢæľ¯":33205,"ĠMM":33206,"ä¸į满":33207,"ä¸ī次":33208,"åıĸ代":33209,"contains":33210,"AU":33211,"ĠMAN":33212,"ĠProvide":33213,"Ġversatile":33214,"Ġneat":33215,"Ġmejor":33216,"Ġdiferentes":33217,"Ġabol":33218,"åĨľäº§åĵģ":33219,"æĹ¶ä»£çļĦ":33220,"Ġdeleted":33221,"halten":33222,"级çļĦ":33223,"Ġinnocent":33224,"Ġanchor":33225,"Ġcaracter":33226,"\"))Ċ":33227,"ì¤ij":33228,"apolis":33229,"spot":33230,"Ġincentives":33231,"ĠGauss":33232,"á̽":33233,"Ġrises":33234,"ìĭ¤":33235,"}}ĊĊ":33236,"çŁ¥è¯Ĩ产æĿĥ":33237,"panic":33238,"ĠPresentation":33239,"-inter":33240,"ält":33241,"Ġsuited":33242,"éºĹ":33243,"ĠÑĪе":33244,"èľ¡":33245,"åĩŃè¯ģ":33246,"аÑħ":33247,"ĠHitler":33248,"ä¹ĭéĸĵ":33249,"Ġpractically":33250,".info":33251,"Ġswitched":33252,"ÑĤÑı":33253,"Ġportal":33254,"Ġenjoyable":33255,"ĠRing":33256,"导å¸Ī":33257,"篮çIJĥ":33258,"Ġsemester":33259,"æį¡":33260,"èµ·æĿ¥çļĦ":33261,"ĠFal":33262,"ä½ĵçݰäºĨ":33263,"strom":33264,".first":33265,"Ġrehabilitation":33266,"Ġformulas":33267,"ç´łåħ»":33268,"956":33269,"Ġpesso":33270,"plane":33271,"Ġhue":33272,"Ġunsigned":33273,"åıĻè¿°":33274,"è¨ĵ":33275,"ĠConsumer":33276,"ä¿ŀ":33277,"è§īå¾Ĺèĩªå·±":33278,"ĠGray":33279,"Ġpecul":33280,"Ġinhabitants":33281,"åħ¨éĥ½":33282,"åįĥå¹´":33283,"owania":33284,"ãĤĪãĤĬ":33285,"Ġemphasizes":33286,"Ġlors":33287,"ORS":33288,"Ġfleet":33289,"çĶµæľº":33290,"级åĪ«":33291,"æŃ£æĸĩ":33292,"é¤IJ饮":33293,"athon":33294,"-mediated":33295,"Ġsidebar":33296,"ĠUpon":33297,"åıªéľĢè¦ģ":33298,"污水":33299,"çľĭçļĦ":33300,"×ķס×":33301,"ĠNJ":33302,"Ġmonde":33303,"076":33304,"гÑĢани":33305,"iens":33306,"Ġeq":33307,"Ġtoys":33308,"986":33309,"hello":33310,"zens":33311,"对æĬĹ":33312,"å¿ĥæĢĿ":33313,"åıĮçľ¼":33314,"çļĦç»ĵæŀĦ":33315,"trl":33316,"à¸ŀิ":33317,"èѦåijĬ":33318,"ç©¿è¶Ĭ":33319,"organic":33320,"è¿IJ转":33321,"Ġrestored":33322,"ãĤ±":33323,"ĠFinland":33324,"Ġvaccines":33325,"Ġplt":33326,"åħ¨ä¼ļ":33327,"ستاÙĨ":33328,"Ġnec":33329,"loat":33330,"_add":33331,"еÑĤÑĭ":33332,"æIJħæĭĮ":33333,"Paul":33334,"Ġintentions":33335,"Ġsoldier":33336,"957":33337,"-text":33338,"Ġadjusting":33339,"watch":33340,"ĠGam":33341,"ĠBert":33342,"ĠÙĪØ¹":33343,"åĽŀæĿ¥äºĨ":33344,"åĨĽäºº":33345,"ĠProfile":33346,"éĢĹ":33347,"icus":33348,"ä¹°äºĨ":33349,"ĠExam":33350,"åı¸ä»¤":33351,"Ġscattered":33352,"кое":33353,"强çĥĪçļĦ":33354,"ĠاÙĦÙħت":33355,"ationally":33356,"Ġchairman":33357,"设æľī":33358,"Ġrighteous":33359,"èĮĦ":33360,"ichi":33361,"é¾Ļ头":33362,"Ġstruggled":33363,"}_":33364,"Ġbiomass":33365,"åijķ":33366,"Ġbiodiversity":33367,"ASC":33368,"057":33369,"Ġevolve":33370,"身æĿIJ":33371,"åıĪè¦ģ":33372,"Seg":33373,"Ġपà¥įर":33374,".con":33375,"沿çĿĢ":33376,"GBT":33377,"åįĹåĮĹ":33378,"åħ»æĪIJ":33379,"ின":33380,"ĠSed":33381,"ĠCells":33382,"Family":33383,"جÙĩ":33384,"åįģåŃĹ":33385,"ĠJosé":33386,"ĠGallery":33387,"ibile":33388,"errors":33389,"Ġenerget":33390,"éĴ¦":33391,"西çıŃ":33392,"ĠBloom":33393,"çĥ«":33394,"ĠAustria":33395,"yster":33396,"åħ³éĶ®è¯į":33397,"Copy":33398,"Ú¾":33399,"åĸĥ":33400,"Ġép":33401,"fg":33402,"æĥ³äºĨ":33403,"":33589,"]['":33590,"Ġsits":33591,"Ġsop":33592,"éral":33593,"åı¯åĪĨ为":33594,"-income":33595,"Editor":33596,"Õ¯":33597,"SB":33598,"boards":33599,"ARR":33600,"ĠMix":33601,"Ġmembranes":33602,"ĠElectronic":33603,"ä¸Ģ度":33604,"ç»Ī端":33605,"ViewController":33606,"åıªçľĭ":33607,"wnie":33608,"ĠEthics":33609,"izen":33610,"æľīæĹł":33611,"ĠLabel":33612,"羣缸":33613,"Das":33614,"ุม":33615,"ITION":33616,"ĠCha":33617,"é¢ijç¹ģ":33618,"èIJ½åIJİ":33619,"åįķä½įçļĦ":33620,"Ġpremises":33621,"avan":33622,"Ġfaithful":33623,"à¹Ģà¸Ĺศ":33624,"Ġgenerous":33625,"ä¸ŃåĽ½çī¹èī²ç¤¾ä¼ļ主ä¹ī":33626,"Cond":33627,"ĠEthiop":33628,"osures":33629,"åĮĸåĴĮ":33630,"ĠKiB":33631,"çŃīéĹ®é¢ĺ":33632,"ĠSomething":33633,"สามารà¸ĸ":33634,"ifik":33635,"indo":33636,"FD":33637,"Ġstyl":33638,"Ġcorrespondence":33639,"кан":33640,"ubblic":33641,"Ġafore":33642,"company":33643,"Ġoverlap":33644,"Ġ=>Ċ":33645,"Ġmsg":33646,"indrome":33647,"Still":33648,"ourage":33649,"æĺ¥ç§ĭ":33650,"063":33651,"à·Ĵ":33652,"Ġà¸Ĺีà¹Ī":33653,"Ġdelays":33654,"floor":33655,";\">":33656,"çļĦæł¸å¿ĥ":33657,"Ġcrying":33658,"Ñĩеб":33659,"ahu":33660,"éĺ³å¸Ĥ":33661,"à§ĩত":33662,"ضع":33663,"ä½Ļé¢Ŀ":33664,"è¿ĺ好":33665,"Ġона":33666,"å·²ç»ıæĺ¯":33667,"ä¹ĭåīįçļĦ":33668,"Ġprofic":33669,"Ġperpendicular":33670,"Ġef":33671,"озна":33672,"Ġextern":33673,"漸":33674,"Ġcleared":33675,"ãģ«ãģĬ":33676,"Ġaltogether":33677,"ĠSymbol":33678,"-long":33679,"sing":33680,"KA":33681,"Ġà¹Ģà¸Ľà¹ĩà¸Ļ":33682,"æĿ¥è®²":33683,"è½½ä½ĵ":33684,"Ġìŀij":33685,"Adapter":33686,"ĠÄiji":33687,"Ġchemotherapy":33688,"ä¿ĥ使":33689,"maker":33690,"ĠMI":33691,"ĠбÑĥдÑĥ":33692,"Ġward":33693,"çĶŁæ¶¯":33694,"è§£åĨ³éĹ®é¢ĺ":33695,"ĠPER":33696,".head":33697,"Ġdisciples":33698,"æı®":33699,"åĪĨå¼Ģ":33700,"管çIJĨå±Ģ":33701,"Ġidx":33702,"ĠDeb":33703,"ä¾ĽåºĶåķĨ":33704,"Ġforb":33705,"estock":33706,"ĠColumn":33707,"Ġinvasive":33708,"077":33709,"Ġelectronics":33710,"-run":33711,"Ġdaughters":33712,"詹":33713,"éĢıéľ²":33714,"Ġpseud":33715,"ixel":33716,"Put":33717,"åľºåľ°":33718,"åİ¢":33719,"æīįæľī":33720,"ä¾Ľç͵":33721,"ierte":33722,"ä¸Ĭ次":33723,"еннÑĭÑħ":33724,"».ĊĊ":33725,"QQ":33726,"zza":33727,"Ġterrain":33728,"主ä¹īçļĦ":33729,"Ġtraveled":33730,"ãĢĤâĢĿâĢľ":33731,"Ġexponential":33732,"ÏĦιÏĤ":33733,"Furthermore":33734,"Profile":33735,"Ġrelie":33736,"æİ¨è¡Į":33737,"Ġaffection":33738,"":35407,"ĠÑĢÑĭ":35408,"essages":35409,"éĹ´éļĶ":35410,"æĢİæł·çļĦ":35411,"Ġharass":35412,"LAB":35413,"Ġdocumentary":35414,"owship":35415,"äºĨèĩªå·±çļĦ":35416,"onial":35417,"ĠHalf":35418,"ç¥ŀå¥ĩ":35419,"Quant":35420,"Factor":35421,"Ġwiring":35422,"æ±Łæ¹ĸ":35423,"Ġimagery":35424,"Ġ×ijש×":35425,"-over":35426,"×ķ×ĺ":35427,"Ġfoundations":35428,"Ġultras":35429,"Ġcath":35430,"Ġelectromagnetic":35431,".exports":35432,"нение":35433,"raints":35434,"Ġsuck":35435,"ooks":35436,"Ġinert":35437,"å¤įåħ´":35438,"Ġobserver":35439,"ä½£":35440,"Ġcastle":35441,"屯":35442,"ä½łè¿Ļ":35443,"æĻĤ代":35444,".dart":35445,"RF":35446,"èµĦæºIJçļĦ":35447,"Ġmigrations":35448,"fficial":35449,"were":35450,"Though":35451,"ollo":35452,"ĠKay":35453,"Ġplanets":35454,"Additionally":35455,"jer":35456,"^{*":35457,"-un":35458,"ĠCAR":35459,"Ġë©":35460,"Customer":35461,"Ġdementia":35462,"Ġautonomy":35463,"æ¢Ń":35464,"Profess":35465,"Ġlug":35466,"Ġ-Ċ":35467,"æĹıçļĦ":35468,"è¾īçħĮ":35469,"æµ·åħ³":35470,"ĠClay":35471,"Ġoriented":35472,"ĠValent":35473,"ĠHunter":35474,"ĠLip":35475,"Ñħов":35476,"-gl":35477,"çī¹å®ļçļĦ":35478,"å¹²é¢Ħ":35479,"Ġrectangular":35480,"Ġged":35481,"Ġpizza":35482,"ä¸Ĭçľĭ":35483,"çīĽèĤī":35484,"Ġvein":35485,"urement":35486,"æĢİ麽":35487,"è£ģåΤ":35488,"ÑĤелÑĮнÑĭе":35489,"Ġhealthier":35490,"pal":35491,"ĠMn":35492,"ฤ":35493,"ç¦Ħ":35494,"]),":35495,"åĬ©çIJĨ":35496,"Ġ×ŀת×":35497,"Ġlakes":35498,"Ġhydrox":35499,"ĠTurner":35500,"Ġdecember":35501,"Ġmetros":35502,"USA":35503,"ä½ĵ温":35504,"profit":35505,"ç«ĭ马":35506,"ãĥ»ãĥ»":35507,"ĠConditions":35508,"Ġbankrupt":35509,"ĠØ¢ÙĨÙĩا":35510,"ä¸Ĭæĺ¯":35511,"åŁİå¸ĤçļĦ":35512,"ĠUtil":35513,"ĠStanley":35514,"åĩıå°ı":35515,"ä¸Ŀ毫":35516,"Ġvitamins":35517,"ĠMode":35518,"ĠDJ":35519,"è§ĨåĽ¾":35520,"èĤ¡ä»·":35521,"(null":35522,"é»Ħæ²³":35523,"LT":35524,"ĠÏĥÏĦιÏĤ":35525,"ĠMyst":35526,"ËĨ":35527,"线索":35528,"Ġstaring":35529,"ória":35530,"ĠBir":35531,"çļĦéĩį":35532,"æĬķè¯ī":35533,"Ġdemonstration":35534,"æ²Ļåıij":35535,"unsigned":35536,"表çı¾":35537,"è§Ĥæµĭ":35538,"ĠLinks":35539,"Transaction":35540,"×ķר×":35541,"ĠKalk":35542,"ĠFlore":35543,"าà¸ģาร":35544,"ä¸Ģåįĥ":35545,"ĠTed":35546,"iesz":35547,"Ġpatron":35548,"Ġconstitutes":35549,"ÑĪÑĮ":35550,"mir":35551,"Collect":35552,"èĬ¸":35553,"ĠStanford":35554,"ĠопÑĢеделÑı":35555,"Ġà¸ĭึà¹Īà¸ĩ":35556,"Ġrelativ":35557,"ĠокÑĢÑĥ":35558,"ĠTrail":35559,"Ġtouching":35560,"Ġliberty":35561,"exec":35562,"Ġconstants":35563,"ĠScholars":35564,"Ġanaa":35565,"Ġwhereby":35566,"Ġsubscrib":35567,"Ġconten":35568,"å©Ĩå©Ĩ":35569,"nosti":35570,"ĠΩ":35571,"Ġsimulated":35572,"wig":35573,"好åIJĥ":35574,"Ġspraw":35575,"+y":35576,"Ġsido":35577,"åģ·åģ·":35578,"Util":35579,"æĥ³çļĦ":35580,"çĿĢä»ĸ":35581,"éĩijéĴ±":35582,"éİ®":35583,"ĠSid":35584,"çħ¤çĤŃ":35585,"ĠØ´Ùħا":35586,"รร":35587,"Ġcontinually":35588,"ĠJunior":35589,"å¢ħ":35590,"ĠSecondary":35591,"Ġdinhi":35592,"Ġcareg":35593,"Created":35594,"Ġlicence":35595,"伦çIJĨ":35596,"mia":35597,"ä¸ĩçī©":35598,"áĥĶáĥij":35599,"çļĦåIJįåŃĹ":35600,"æı´åĬ©":35601,"ĠÑĩиÑģло":35602,"(get":35603,"ĠVas":35604,"ÙĦÙĬÙħ":35605,"å¼ĢåѦ":35606,"å½ĵä½ľ":35607,"Ġsimpler":35608,"åĬ¨è¯į":35609,"ĠANY":35610,"ĠTransportation":35611,"Ġmoże":35612,"Ġзд":35613,"ĠDiscuss":35614,"éļ§":35615,"Ġaccompanying":35616,"Issue":35617,"opus":35618,"Ġensemble":35619,"åĮĸè§£":35620,"ĠBiological":35621,"æ±°":35622,"Ġprophe":35623,"Ġrespondent":35624,"ouncing":35625,"Ġdefendants":35626,"ĠÑĩеловек":35627,"èĢĮæĪIJ":35628,"VEL":35629,"׾×Ļ":35630,"ĠEmily":35631,"æĤ£èĢħçļĦ":35632,".bind":35633,"izens":35634,"ĠUntil":35635,"Ġenumer":35636,"ĠLeader":35637,"para":35638,"Ġconductivity":35639,"ÑĩÑĥ":35640,"ujÃŃ":35641,"è¤IJ":35642,"å°ıåŀĭ":35643,"帮æī¶":35644,"è¿Ļäºĭ":35645,"Ġprend":35646,"Ġchromat":35647,",N":35648,"ä¸Ģæľ¬":35649,"à¸Ħà¹Į":35650,"ĠSure":35651,"ĠBA":35652,"æ²¥":35653,"ĠденÑĮ":35654,"Ġcatalog":35655,"ĠогÑĢани":35656,"è§ģçļĦ":35657,"Ġê·":35658,"ĠPrinceton":35659,"adaghan":35660,"éĺ¿å°Ķ":35661,"Ġunh":35662,"ä¸ĢæĶ¯":35663,"Ġucz":35664,"Ġeditors":35665,"Ġtransfers":35666,"Ġantes":35667,"itol":35668,"ÑĤелÑĮной":35669,"Siyent":35670,"åIJĮä¸Ģ个":35671,"çĨŁç»ĥ":35672,"Ġмен":35673,"Ġहà¥Ī":35674,"Ġwrest":35675,"imicro":35676,"áp":35677,"arson":35678,"Ġopera":35679,"Ġunfair":35680,"Ġproximity":35681,"Ġwires":35682,"Ġnouns":35683,"ĠNatur":35684,"ĠÏĥÏĦ":35685,"ductive":35686,"ĠFO":35687,"ĠNuclear":35688,"Sing":35689,"redients":35690,"æĶ¶åĽŀ":35691,"Rate":35692,"å¨ł":35693,"Ġreviewing":35694,"ä¸ĬåѦ":35695,"Ġanalysts":35696,"Ġtalay":35697,"åĨĻåĩº":35698,"èĩªåĪĨ":35699,"Wal":35700,"aras":35701,"ĠHunt":35702,"å°¼äºļ":35703,"æĮijéĢī":35704,"å°¸ä½ĵ":35705,"Ġcran":35706,"Ġjazz":35707,"Ġuncon":35708,"è¯ļä¿¡":35709,"ĠKate":35710,"Ġmodelling":35711,"२":35712,"åİĤåķĨ":35713,"å¿ĥä¸ŃçļĦ":35714,"GI":35715,"Kasarangang":35716,"Ġkainiton":35717,"ordinates":35718,"олни":35719,"Ġcontinuity":35720,"Ġscheduling":35721,"åħĥçļĦ":35722,"俱ä¹IJ":35723,"Ġpest":35724,"å¿ħé¡»è¦ģ":35725,"ulence":35726,"Ġcruise":35727,"澳大åĪ©":35728,"#!/":35729,"æľ¬ç«ł":35730,"Yet":35731,"æĸ¯çļĦ":35732,"KD":35733,"atif":35734,"Ġracism":35735,"Ġê²½ìļ°":35736,"æľŁæľ«":35737,"åĸľçα":35738,"Ġ```":35739,"Master":35740,"äºĽä»Ģä¹Ī":35741,"Ġseverely":35742,"XY":35743,"uet":35744,"Ġà¸ļ":35745,"Ġhalt":35746,"åĵij":35747,"Ġcitation":35748,"ĉĠ":35749,"ĠGit":35750,"èĦĵ":35751,"ĠDallas":35752,"Ġtransistor":35753,"azio":35754,"das":35755,"åĬłå¼ºå¯¹":35756,"è¯ģå®ŀ":35757,"ĠLan":35758,"GeoNames":35759,"Catal":35760,"ĠMAX":35761,"Ġingredient":35762,"éªļ":35763,"èĬ±çļĦ":35764,"based":35765,"ĠTol":35766,"æ³ķåĪĻ":35767,"说说":35768,"সà§įথ":35769,".example":35770,"ĠSupply":35771,"ä¸Ģèµ·æĿ¥":35772,"æ´¾åĩºæīĢ":35773,"åĩ½æķ°çļĦ":35774,"Ġdoen":35775,"Ġobserving":35776,"ĠLiv":35777,"Ġbard":35778,"ĠBitcoin":35779,"Ġsaatavilla":35780,"ĠChallenges":35781,"leans":35782,"ĠÐĴи":35783,"iganos":35784,"Ġaccent":35785,"Ġguiding":35786,"æµij身":35787,"åĺļ":35788,"Contin":35789,"æĪĸåħ¶ä»ĸ":35790,"ArrayList":35791,"stoff":35792,"æĸ°åįİ":35793,"ĠKumar":35794,"ĠعÙĦÙħ":35795,".time":35796,"Ġterritorial":35797,"Ġlightly":35798,"Ġglut":35799,"Ġ??":35800,"è¿ĩæķı":35801,"æŃ¤äºĭ":35802,"expr":35803,"ĠMatter":35804,"ické":35805,"Origin":35806,"Ġdwell":35807,"Ġros":35808,"Ġgraduated":35809,"Ġcytok":35810,"人æ°ijçļĦ":35811,"Ġminist":35812,"Öµ":35813,"Ġfrustration":35814,"Ġventilation":35815,"ĠReligion":35816,"ÑĪие":35817,"<>();Ċ":35818,"ä¸ĬéŨ":35819,"伤åı£":35820,"Ġtimer":35821,"èĩªåĬ¨åĮĸ":35822,"ĠIz":35823,"wort":35824,"רת":35825,"Ġconfigurations":35826,"Ġchick":35827,"oteksti":35828,"inement":35829,"Ġuph":35830,"æľĿé²ľ":35831,"ĠPART":35832,"人对":35833,"Ġmaka":35834,"iona":35835,"Dest":35836,"ĠCrow":35837,"ĠForces":35838,"ä¸Ĭæĸ¹":35839,"ĠCounsel":35840,"Ġlex":35841,"éĤ£æĹ¶åĢĻ":35842,"094":35843,"碰åΰ":35844,"åĹĵ":35845,"Ġmaior":35846,"ĠRespond":35847,"æijĨèĦ±":35848,"Ġendot":35849,"å͝ä¸ĢçļĦ":35850,"åݦéŨ":35851,"νομα":35852,"Ġqueen":35853,"*-":35854,"æĦīå¿«":35855,"éľĩæĥĬ":35856,"ĠEnsure":35857,"çļĦé£İéĻ©":35858,"Ġdissem":35859,"ĠбÑĭла":35860,"ĠOtherwise":35861,"Ġrefugees":35862,"leb":35863,"TF":35864,"-bottom":35865,"Ġissu":35866,"Ġviolations":35867,"especially":35868,"à§İ":35869,"æĬ±æĢ¨":35870,"تع":35871,"writer":35872,"িদ":35873,"ड":35874,"é½Ĭ":35875,"Ġ×Ķר":35876,"ĠSame":35877,"-inch":35878,"VS":35879,"akin":35880,"ä¸įéĶĻçļĦ":35881,"رÙĤ":35882,"åĸĿéħĴ":35883,"ä½ĵæ£Ģ":35884,"ĠSalary":35885,"amide":35886,"ĠKid":35887,"âĢĿ:":35888,"جاÙħ":35889,"QR":35890,"å·²ç»ı被":35891,"笨":35892,"å¤ļ项":35893,"Ġcolours":35894,"æī®æ¼Ķ":35895,"æĬķ票":35896,"ĠVoice":35897,"reading":35898,"Tiganos":35899,"_sub":35900,"ĠWarren":35901,"Ġmidst":35902,"ä¸į管æĺ¯":35903,"?#":35904,"utos":35905,"istle":35906,"Ġconnects":35907,"æĻ¯çĤ¹":35908,"Ġmindset":35909,"Insert":35910,"ĠRC":35911,"Ġestos":35912,"ĠAls":35913,"Ġdall":35914,"inden":35915,"ĠElectrical":35916,"illet":35917,"ĠÙħÙģ":35918,"Ġstresses":35919,"MAN":35920,"大æķ°æį®":35921,"Ġdost":35922,"Ġexempt":35923,"ĠWoman":35924,"ìħ":35925,"testing":35926,"ãĥİ":35927,"Ġsocket":35928,"èĢĥéªĮ":35929,"Ġ\\[\\":35930,"ર":35931,"é«ĺå³°":35932,"è¿Ļ对":35933,"ĠDetroit":35934,"ĠDocuments":35935,"Rob":35936,"Food":35937,"Ġëĭ¨":35938,"illon":35939,"Ġallegations":35940,"çĤ¹è¯Ħ":35941,"ĠPublications":35942,"Ġinspiring":35943,"Changed":35944,"çŀİ":35945,"Ġattraction":35946,"åħĥä»¶":35947,"主è¦ģçļĦ":35948,"çªij":35949,"°,":35950,"ç«ĭæĸ¹":35951,"Ġlavor":35952,"Ġthirteen":35953,"yi":35954,"çľĭäºĨä¸Ģçľ¼":35955,"Ġclimbing":35956,"Ġdowntown":35957,"gate":35958,"线ä¸ĭ":35959,"ĠKeywords":35960,"ìĪł":35961,"Ġangel":35962,"Operation":35963,"Hub":35964,"Ġdemographic":35965,"ĠGuidelines":35966,"Ġbottles":35967,"Ġtragedy":35968,"%ãĢģ":35969,"ĠProte":35970,"à°ķ":35971,"Quaternary":35972,"è¿ĩåİ»çļĦ":35973,".update":35974,"before":35975,"رش":35976,"Ġtokens":35977,"åı£èħĶ":35978,"æĮ¯åĬ¨":35979,"åĴķ":35980,"hir":35981,"stairs":35982,"宵":35983,"Ġdescriptive":35984,"দà§įধ":35985,"çģ¯åħī":35986,"太大":35987,"è¿Ļæł·åģļ":35988,"平稳":35989,"Ġmorphology":35990,"æŀ¶æŀĦ":35991,"Ġgrandes":35992,"Ġlaptop":35993,"ĠStein":35994,"ĠÙħتعÙĦÙĤÙĩ":35995,"Ġendeav":35996,"য":35997,"ãĥ¼ãĤ¸":35998,"ĠInterview":35999,"pent":36000,"ä½łä»¬çļĦ":36001,"äºıæįŁ":36002,"ĠìķĮ":36003,"åıĪä¸į":36004,"ä½łèĩªå·±":36005,"Ġjournalist":36006,"Ġlaughter":36007,"èĦĸåŃIJ":36008,"羣è¯ļ":36009,"ablished":36010,"å¯ĨéĽĨ":36011,"}x":36012,"Ġbucket":36013,"cych":36014,"å§Ķå±Ī":36015,"ĠÑģодеÑĢжа":36016,",T":36017,"ĠPanel":36018,"æĹłåı¯":36019,"Ġsaturated":36020,"ä¾Ĩ說":36021,"诡":36022,"endor":36023,"ettes":36024,"Ġmicrobial":36025,"ĠWikidata":36026,"让åŃ©åŃIJ":36027,"Ġbeste":36028,"Ġcontre":36029,"tainment":36030,"ĠElse":36031,"å¦Ĭå¨ł":36032,"Ġpeculiar":36033,"Ġfuneral":36034,"(size":36035,"offset":36036,"å¢ŀå̼ç¨İ":36037,"éĢļè¿ĩ对":36038,"ÙĦÙĪ":36039,"åºĨç¥Ŀ":36040,"Ñļем":36041,"Ġtwist":36042,"otos":36043,"ĠChel":36044,"Ġgland":36045,"ucker":36046,"={{":36047,"ĠÐIJÑĢ":36048,"æķijåĬ©":36049,"ĠFlu":36050,"âĢ¢ĊĊ":36051,"äºĮåįģ大":36052,"éĩĿ":36053,"好åIJ§":36054,"stop":36055,"/K":36056,"element":36057,"utenant":36058,"Ġcheaper":36059,"accept":36060,"ÅĻed":36061,"Ġtanks":36062,"ighed":36063,"çĭ¬èĩª":36064,"menu":36065,"ĠSTEM":36066,"Ġcompetence":36067,"æĥķ":36068,"çĸ²åĬ³":36069,"Ġév":36070,"ĠtÄĽ":36071,"ÙĬدÙĬا":36072,"Ġ׾×IJ×":36073,"ç¾İåĽ½çļĦ":36074,"Ġundergraduate":36075,"Ġdeer":36076,"æĦŁåĨĴ":36077,"éĺ¿éĩĮ":36078,"تد":36079,"ĠкÑĢи":36080,"ä¸ĭä¸Ģ个":36081,"æ©Łæľĥ":36082,"Ġdisappointed":36083,"æĬ¥èĢĥ":36084,".join":36085,"èªįçĤº":36086,"Ġreplic":36087,"Ġallies":36088,"Ġzwischen":36089,"ĠëͰ":36090,"ĠDEL":36091,"ĠREC":36092,"éĤ±":36093,"AVE":36094,"èĦijæµ·":36095,"Ġfluids":36096,"/annual":36097,"Ġprox":36098,"ución":36099,"اÙĨÙĬØ©":36100,"æİ¨çIJĨ":36101,"à¸ģำ":36102,"à§ĩà¦Ł":36103,"ĠÙħÙĤاÙĦÙĩ":36104,"ĠInn":36105,"_per":36106,"Ġriv":36107,"æĹ¢æľī":36108,"ĠCharlotte":36109,"羣å¿ĥ":36110,"emetery":36111,"alous":36112,"亮çļĦ":36113,"ulose":36114,"-week":36115,"Host":36116,"å°ıç±³":36117,"AST":36118,"å½Ī":36119,"cedented":36120,"storm":36121,"ĠRosen":36122,"Ġtomatoes":36123,"ĠкоÑĺа":36124,"Ros":36125,"Ġwealthy":36126,"Ġintend":36127,"Ġinstability":36128,"Îĵ":36129,"Ġounce":36130,"WD":36131,"prom":36132,"Ġż":36133,"isis":36134,"åŁºåĩĨ":36135,"Ġmonitored":36136,"ĠBangladesh":36137,"Ġprow":36138,"ĠCuba":36139,"常åĬ¡":36140,"常å§Ķä¼ļ":36141,"anych":36142,"åħľ":36143,"欧éĺ³":36144,"åıĹåΰäºĨ":36145,"åĨ·çļĦ":36146,"好å¤ļ":36147,"Ġperc":36148,"ĠGrid":36149,"彬":36150,"MMMMMMMMMMMMMMMM":36151,"Ġearning":36152,"ilingual":36153,"ĠобÑĢазом":36154,"âĸºâĸ¼":36155,"è¶Ĭå¤ļ":36156,"Ġà®İ":36157,"æĭįåįĸ":36158,"Ïģιο":36159,"ä¸įé«ĺ":36160,"Ġanch":36161,"Ġcommerce":36162,"åΰæĹ¶åĢĻ":36163,"ĠDance":36164,"ĠJes":36165,"ĠSpot":36166,"个æķ°":36167,"çļĦåIJĦç§į":36168,"ĠOUT":36169,"amera":36170,"å°ıçϽ":36171,"à¤Ł":36172,"Ïĥα":36173,"æĹ©åľ¨":36174,"é¤ĺ":36175,"ĠкÑĥлÑĮ":36176,"Ġsurprisingly":36177,"ĠعÙħ":36178,"è¿Ļ座":36179,"acerb":36180,"Ġservants":36181,"ĠпÑĢоÑĨеÑģÑģ":36182,"Ġirrad":36183,"agner":36184,"åĬłå¯Ĩ":36185,"#if":36186,"ĠAndy":36187,"bersecurity":36188,"åı¤èĢģ":36189,"Ġsangat":36190,"ä¸įçĿĢ":36191,"Ġcompost":36192,"Ġpeptide":36193,"chte":36194,"æ¶ĪçģŃ":36195,"ammed":36196,"++++":36197,"ĠViewed":36198,"ĠRol":36199,"Ġtreaty":36200,"Ġtemplates":36201,"Ġtá»":36202,"ACC":36203,"Ġmunicip":36204,"Ġbrick":36205,"ĠBI":36206,"禹":36207,"িষ":36208,"è·ijåΰ":36209,"è±ĨèħIJ":36210,"çŀĴ":36211,"äd":36212,"Ġdeposited":36213,"èµ·è¯ī":36214,"ÑģÑĤвова":36215,"ĠDegree":36216,"ä¹ĭæĦı":36217,"Ġsoit":36218,"åİĨç¨ĭ":36219,"Ġsizeof":36220,"çĿĢæīĭ":36221,"ĠEquations":36222,"Ġvisa":36223,"Ġgegen":36224,"ä¸įåĸľæ¬¢":36225,"isplay":36226,"ĠKeith":36227,"Ġnotably":36228,"çĥ·":36229,"ĠAlong":36230,"çİĩçļĦ":36231,"两大":36232,"ĠTechniques":36233,"Ġdownstream":36234,"Ġimpaired":36235,"ĠTHIS":36236,"Ġski":36237,"å¾ĹåĪĨ":36238,"à¸Ĺà¸ĺ":36239,"intage":36240,"веÑģÑĤи":36241,"ĠMatch":36242,"Ġжив":36243,"ĠFourth":36244,"inkle":36245,"ĠAna":36246,"_table":36247,"Ġεί":36248,"ĠìĽIJ":36249,"Ġlud":36250,"éĽ»è©±":36251,"asso":36252,"ĠReform":36253,"adic":36254,"ä¸įåłª":36255,"Ġmodulation":36256,"ĠDateTime":36257,"৪":36258,"=n":36259,"Ġstatutory":36260,".apache":36261,"alph":36262,"ĠاÙĦعرب":36263,"ĠTerrit":36264,"ĠLot":36265,"acchar":36266,"åľ¨ä½ł":36267,"erek":36268,"åı¯ä»¥çľĭåΰ":36269,"å®¶å±ŀ":36270,"Ġdebe":36271,"Ļà¯įà®ķ":36272,"Ġcongress":36273,"Ġreminds":36274,"ãĥķãĤ":36275,"andidate":36276,"Nasod":36277,"oflife":36278,"åķ¸":36279,"Ġenum":36280,"ucc":36281,".show":36282,"Ġrouting":36283,"four":36284,"åIJĦ大":36285,"éij«":36286,"梳çIJĨ":36287,"insula":36288,"ä¸įç®Ĺ":36289,"leading":36290,"etically":36291,"æ¹Ľ":36292,"itably":36293,"ĠOfficial":36294,"flix":36295,"\\to":36296,"{E":36297,"Ġgef":36298,"ĠJS":36299,"è¦ģçŁ¥éģĵ":36300,"compet":36301,"ĠLC":36302,"ringe":36303,"âĢĿ,âĢľ":36304,"Ġterritories":36305,"Ġscroll":36306,"éϤæŃ¤":36307,"å°±ä¸įä¼ļ":36308,"æ¿Ģæĥħ":36309,"Scientific":36310,"ĠAdjust":36311,"ÉĶ":36312,"走访":36313,"Ġmengh":36314,"èļģ":36315,"ĠÑģоп":36316,"è¯ķè¯ķ":36317,"άν":36318,"ĠGun":36319,"ĠĠĊĠĠĊ":36320,"Ġlinking":36321,"hetics":36322,",v":36323,"-white":36324,"Ġils":36325,"pte":36326,"Ġreporter":36327,"ĠXu":36328,"纪å½ķ":36329,"ä¸Ĭæµ·å¸Ĥ":36330,"ĠAgainst":36331,"Ġrotate":36332,"æĺ¯ä¸º":36333,"intestinal":36334,"Ġchromosome":36335,"ĠKnight":36336,".Log":36337,"ĠONE":36338,"Ġlimb":36339,"Ġcontradict":36340,"ĠKEY":36341,"heastern":36342,"subset":36343,"ĠнекоÑĤоÑĢ":36344,"åıijä½ľ":36345,"éħĴç²¾":36346,"Ġning":36347,"Ġdivisor":36348,"Perhaps":36349,"Ġchampionship":36350,"å°ī":36351,"íĺĦ":36352,"Ġà¹ĥà¸Ļ":36353,"Ġimply":36354,"াà¦ķà§ĩ":36355,"urban":36356,"ĠRAM":36357,"äºĨ她":36358,"/tsp":36359,"ç¡«éħ¸":36360,"bast":36361,"Ġ×ķ×IJ×":36362,"ĠBranch":36363,"ĠLis":36364,"Ġdawn":36365,"çļĦæľ¬":36366,"riber":36367,"ĠKap":36368,"çļĦæķĻåѦ":36369,"Ġrespected":36370,"Ġ!ĊĊ":36371,"ampa":36372,"åĪĨæĶ¯":36373,"ĠαÏģ":36374,"Pi":36375,"Ġcv":36376,"屡":36377,"Ġgeneralized":36378,"Ġwounded":36379,"iji":36380,"Ġdigestive":36381,"/he":36382,"çļĦæ¶Īæģ¯":36383,"åľ¨æĦı":36384,"pler":36385,"饥":36386,".catalogue":36387,"à¸Ĵà¸Ļ":36388,"ĠSul":36389,"Ġneon":36390,"×ķ×ļ":36391,"ĠÎĻ":36392,"-associated":36393,"Ġtijd":36394,"çļĦåĽ½å®¶":36395,"Ġmuss":36396,"Ġhighway":36397,"Ġspecialists":36398,"ä¸įæĺİ":36399,"вÑĢа":36400,"Ġrotating":36401,"Ïĥει":36402,"elong":36403,"Ġencompass":36404,"Ġstark":36405,"Ġautumn":36406,"è¿ĺæľīä¸Ģ个":36407,"GRAP":36408,"é»ŀéłŃ":36409,"Ġelaborate":36410,"æ²»å®ī":36411,"ãĤ½":36412,"èµĦ产éĺ¶çº§":36413,"--;Ċ":36414,"Ġinstructor":36415,"çĥĽ":36416,"æĸĭ":36417,"æ¸ħæ°´":36418,"åģ¶çĦ¶":36419,"Ġefect":36420,"ÙĬÙĪ":36421,"好äºĭ":36422,"ĠMaine":36423,"Ġsurvivors":36424,"eba":36425,"交äºĴ":36426,"Ġbuyer":36427,"ä¸Ģ身":36428,"নà§ĩর":36429,"ĠClose":36430,"gree":36431,"Ġenlarg":36432,"]).":36433,"Ġà¦Ł":36434,"Ġ×ŀ×Ķ×":36435,"设å¤ĩçļĦ":36436,"(['":36437,"unted":36438,"èħIJè´¥":36439,"Tab":36440,"è·µè¡Į":36441,"Ġdispatch":36442,"illation":36443,"RODU":36444,"åĢ©":36445,"èħIJèļĢ":36446,"ĠNash":36447,"Ġsealed":36448,"Ġnevertheless":36449,"ëłĪ":36450,"åıijæĢ§":36451,"scale":36452,"'A":36453,"Ġrobots":36454,"Ġclarify":36455,"ĠChan":36456,"Ġتأ":36457,"098":36458,"Ġreconc":36459,"Ġ×§×":36460,".catalogueoflife":36461,"079":36462,"Ġconditioning":36463,"Fran":36464,"éĬ·":36465,"alawigan":36466,"#endif":36467,"Ġ[-":36468,"паÑĢа":36469,"ĠApply":36470,"dale":36471,"è´©":36472,"åºĶä»ĺ":36473,"Ġboats":36474,"-checklist":36475,"Ïĥι":36476,"åĬĽåѦ":36477,"à¹Ħร":36478,"Ġcaptivating":36479,"schen":36480,"åħ¸åŀĭçļĦ":36481,"ĠëĺIJ":36482,"Ġmultif":36483,"ë¡ł":36484,"ưá»Ŀ":36485,"ĠEntre":36486,"jug":36487,"ducing":36488,"blank":36489,"python":36490,"Ġfiring":36491,"ĠMoz":36492,"ĠÙħÙħÙĥÙĨ":36493,"×Ļ׳×ķ":36494,"[a":36495,"æµ·åĨĽ":36496,"Ġlearner":36497,"åľ¨è¿Ļä¸Ģ":36498,"éĢīé¢ĺ":36499,"Ġdés":36500,"Ġcharm":36501,"Ġsoap":36502,"iba":36503,"arius":36504,"Ġblast":36505,"Ġpreserving":36506,"çĸ®":36507,"italic":36508,"ĠÙħÙĪØ±Ø¯":36509,"ĠJefferson":36510,"Ġtrapped":36511,"grid":36512,"tera":36513,"æĦŁåĴĮ":36514,"ç«ĭä½ĵ":36515,"bird":36516,"ĠRobin":36517,"Learning":36518,"Ġlobby":36519,"Ġinability":36520,".o":36521,"Ġtraces":36522,"ĠZar":36523,"ĠJung":36524,"cit":36525,"è¯ķåį·":36526,"ĠGuy":36527,"ĠarXiv":36528,"è¿Ľåζ":36529,"Ġdorm":36530,"ĠPray":36531,"Ġsocially":36532,"juana":36533,"ĠFractions":36534,"éĿ¢åĮħ":36535,"ä¸ŃæľŁ":36536,"ĠCycl":36537,"Ġমà§ģ":36538,"course":36539,"Ġconqu":36540,"boolean":36541,"åĪĨè£Ĥ":36542,"Ġgrandmother":36543,"_G":36544,"isine":36545,"ाम":36546,"西èĹı":36547,"Ġlaughing":36548,"ĠÔ±":36549,"Ġnome":36550,"Turn":36551,"proof":36552,"Cart":36553,"quier":36554,"Ġundergoing":36555,"æĪĺèĥľ":36556,"+-":36557,"ĠRating":36558,"ĠPowers":36559,"ĠâĤ":36560,"已被":36561,"æľ¯åIJİ":36562,".Drawing":36563,"Ġproblematic":36564,"Ġurge":36565,"ĠExperiment":36566,"ĠHawaii":36567,"ÑģÑĤÑĢÑĥк":36568,"Ġradial":36569,"强èĢħ":36570,"Ġsensation":36571,"origin":36572,"ĠBew":36573,"Õ¡Õ½":36574,"ĠCele":36575,"ĠUSB":36576,"Ġë³Ģ":36577,"åıĪç§°":36578,"åľĵ":36579,"è¶ĬæĿ¥è¶Ĭå¤ļ":36580,"Ġultra":36581,"çļĦåIJİ":36582,"ä¸įæĦ¿æĦı":36583,"Try":36584,"Ġimpose":36585,"é»ijé¾Ļ":36586,"Ġbicy":36587,"ä¼½":36588,"Ġenergies":36589,"ä½Ĩæĺ¯åľ¨":36590,"ISA":36591,"Ġbeet":36592,"-inf":36593,"Ġhoe":36594,"׾×ķ":36595,"èĩªè¡Į车":36596,"atio":36597,"ĠвопÑĢоÑģ":36598,"_class":36599,"Ġweiter":36600,"æ²īæ·Ģ":36601,"ĠMaths":36602,";->":36603,"身å¿ĥ":36604,"太è¿ĩ":36605,"-app":36606,"зм":36607,"Whether":36608,"assador":36609,"dal":36610,"çļĦå¸Ĥåľº":36611,"Chinese":36612,"ĠRomans":36613,"对éĿ¢":36614,"@\"":36615,"ariance":36616,"ĠMovie":36617,"Ġattenu":36618,"பà¯įப":36619,"äºĨèĩªå·±":36620,"Ø·ÙĨ":36621,"Ġà¤Ĺ":36622,"Ġ×ŀ×¢":36623,"Physical":36624,"atori":36625,"Ġstolen":36626,"ĠHein":36627,"çļĦæ³ķå¾ĭ":36628,"ĠBachelor":36629,"大åѸ":36630,"ĠDenmark":36631,"ĠÐijе":36632,"温馨":36633,"æĪªæŃ¢":36634,"çľ¼åħī":36635,"ĠRemote":36636,"ë°ľ":36637,"Ġgates":36638,"Ġnowhere":36639,".be":36640,"{B":36641,"ĠMY":36642,"ĠGET":36643,"æľīäºĽäºº":36644,"Ġadopting":36645,"Ġreactor":36646,"nos":36647,"ÑģÑĤавлÑıеÑĤ":36648,"-head":36649,"ĠDiss":36650,"Dir":36651,"æĪijä¸Ģ":36652,"ĠTin":36653,"è¡Ģç³ĸ":36654,"èīºæľ¯å®¶":36655,"Ġreinforce":36656,"â̦âĢĿ":36657,"ĠDrawing":36658,"å»īæĶ¿":36659,"Ts":36660,"ĠØ¢ÙħÙĪ":36661,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":36662,"声éģĵ":36663,"ास":36664,"ĠSurgery":36665,"_model":36666,"æĬķæłĩ":36667,"uilt":36668,"ãģ®ãģ§ãģĻ":36669,"éĽĮ":36670,"è¾ľ":36671,"attr":36672,"ĠاÙĦتع":36673,"Ġrecognised":36674,"Ġacoustic":36675,"æĢ§åĪ«":36676,"è¯ŀçĶŁ":36677,"澳大åĪ©äºļ":36678,"Ġreversal":36679,"ĠCraft":36680,"Ġtennis":36681,"ĠëĦ":36682,"éĺ²å®Ī":36683,"Ġnerves":36684,"Ġperturb":36685,"Sometimes":36686,"æĻ®éĢļçļĦ":36687,"etto":36688,"Ġromance":36689,"人äºĨ":36690,"ĠJin":36691,"åľºåIJĪ":36692,"ĠSoutheast":36693,"Ġtego":36694,".Tasks":36695,"æĽ¦":36696,"æĺ¯ä¸ĢåĢĭ":36697,"ĠHour":36698,"Ùĥز":36699,"æ¯Ķäºļ":36700,"ĠController":36701,"Ġнад":36702,"åĮħ裹":36703,"Ġsubm":36704,"âĪ«":36705,"ĠParad":36706,"Ġsoccer":36707,"Study":36708,"ĠPf":36709,"ĠFREE":36710,"andle":36711,"åıĬåħ¶ä»ĸ":36712,"âĢ¢Ċ":36713,"choice":36714,"ä¸Ĭä¼ł":36715,"ärvi":36716,"Ġbiz":36717,"åŃĹæ®µ":36718,"çļĦ说":36719,"ĠSARS":36720,"ĠAw":36721,"à¸Ĺำà¹ĥหà¹ī":36722,"еми":36723,"Ġsupplier":36724,"å¤Ħå¤Ħ":36725,"å¿ĹæĦ¿æľįåĬ¡":36726,"æĴ¤éĶĢ":36727,"âŀ":36728,"å¤ĸç§ij":36729,"ĠLiverpool":36730,"æĢİä¹Īä¼ļ":36731,"ĠÑĥÑĩаÑģÑĤ":36732,"åºķéĥ¨":36733,"ÅĦst":36734,"常å§Ķ":36735,"Ġassisted":36736,"Ġrepublic":36737,"Ġ\"-":36738,"ĠзнаÑĩи":36739,"表达å¼ı":36740,"Ġlawn":36741,"-solving":36742,"Ġsouls":36743,"Ġexcuse":36744,"ĠCompare":36745,".char":36746,"Ġdare":36747,"ĠMine":36748,"اعة":36749,"(type":36750,"নà§įদ":36751,"èĬĻ":36752,"Ġtart":36753,"ĠArtificial":36754,"Ġtorque":36755,"Ġcompiled":36756,"Ġ....":36757,"াà¦ĵ":36758,"ãĢĢãĢĢ":36759,"omp":36760,"ç¥ŀçļĦ":36761,"Ġsupplements":36762,"Education":36763,"ĠEpid":36764,"论è¯ģ":36765,"Ġà¦Ĩর":36766,".U":36767,"اغ":36768,"ä½İ头":36769,"ĠCommercial":36770,"ĠIh":36771,"attery":36772,"èĢĥãģĪ":36773,"doors":36774,"Ġquadratic":36775,"å°ıæĹ¶åĢĻ":36776,"utral":36777,"à¦¿à¦ľ":36778,"ä¹ĭéģĵ":36779,"Ġweaknesses":36780,"'en":36781,"-work":36782,"ĠпÑĢоÑĤив":36783,"Low":36784,"Ġelong":36785,"ç͵ç¼Ĩ":36786,"Ġtwin":36787,"ĠTower":36788,"-sub":36789,"å°ıå°ıçļĦ":36790,"dest":36791,"fact":36792,"表çļĦ":36793,"å¼ĢéŨ":36794,"Ġinaug":36795,"éĿŀ常çļĦ":36796,"å¤ľæĻļ":36797,"isode":36798,"çij¾":36799,"Ġhydroc":36800,"媳å¦ĩ":36801,"æĺ¯æ²¡æľī":36802,"Ġcommentary":36803,"ала":36804,"Primary":36805,"å¹´éĹ´":36806,"Cache":36807,"нÑĨи":36808,"åIJĮæľŁ":36809,"à¦ķার":36810,"Ġsafer":36811,"ários":36812,"åľ¨åĵªéĩĮ":36813,"occus":36814,"izophren":36815,"/V":36816,"æµ·ä¸Ĭ":36817,"Ġpayload":36818,"对æĸ¹çļĦ":36819,"Ġdile":36820,"ogan":36821,"ĠSurv":36822,"Ġtomato":36823,"Ġnaming":36824,"ĠFresh":36825,"åIJİåĨį":36826,"à«ĭ":36827,"bia":36828,"æĦıä¹īçļĦ":36829,"Never":36830,"Ġprac":36831,"ĠExplanation":36832,"ä¸ĢæĿ¥":36833,"约为":36834,"Ġtelesc":36835,"ĠSwitch":36836,"ĠCAN":36837,"ellington":36838,".cn":36839,"STEM":36840,"Ġbonding":36841,"天使":36842,"å¹¾åĢĭ":36843,"许åı¯è¯ģ":36844,"ĠIntegration":36845,"ONT":36846,"ĠAlexand":36847,"ĠGeography":36848,"াবà§ĩ":36849,"ãĥı":36850,"åĴĮå°ļ":36851,"ุà¸Ļ":36852,"umlah":36853,"éĩijèŀįæľºæŀĦ":36854,"大éŨ":36855,"è·¯çļĦ":36856,"ிà®ķ":36857,"Ġhunger":36858,"æŁIJæŁIJ":36859,"Ġdrying":36860,"Ġseptember":36861,"Selected":36862,"Ġtemporarily":36863,"Ċ":39278,"çļĦä¸Ģ次":39279,"Ġadjective":39280,"Ġincentive":39281,"ĠÙĪÙĦ":39282,"their":39283,"Ġmt":39284,"ĠDefine":39285,"Ġactivate":39286,"ä¸Ģ个æĺ¯":39287,"à¦¿à¦Ł":39288,"èı±":39289,"Ġliabilities":39290,"Ġtragic":39291,"oprotein":39292,"Ġeighth":39293,"thy":39294,"æĹ©æĻ¨":39295,"{T":39296,"ĠLORD":39297,"Ġ'.":39298,"ç¯ī":39299,"Ġcredibility":39300,"Ġberk":39301,"Ñģо":39302,"ogly":39303,"-page":39304,":/":39305,"ниÑĤÑĮ":39306,"Ġpollut":39307,"åįĥç±³":39308,"ÛĮا":39309,"Ġpuis":39310,"à¹Ħà¸Ĺย":39311,",r":39312,"Ġfibr":39313,"Peter":39314,"Ġlane":39315,"éĢĻæ¬¡":39316,"Ġperimeter":39317,"Ġadren":39318,"Ġobed":39319,"Ġmedio":39320,"Integ":39321,"Ġdependency":39322,"Ġgrocery":39323,"०":39324,"Lower":39325,"каÑı":39326,"-Al":39327,"ersistence":39328,"ĠHob":39329,"èĢģ師":39330,"Without":39331,"å®ļä»·":39332,"æıļ":39333,"ECD":39334,"ĠíĮĮ":39335,"unge":39336,"Bean":39337,"èĤ¯å®ļæĺ¯":39338,"Ġyeah":39339,"Ġrealization":39340,"essment":39341,"Ġtreats":39342,"ır":39343,"aru":39344,"Ġmins":39345,"ĠLav":39346,"ðŁĶ":39347,"æĹ¢æĺ¯":39348,"åζåĬ¨":39349,"Ġnd":39350,"rocy":39351,"upa":39352,"è¶Ĭé«ĺ":39353,"Ġevenly":39354,"Ġgeen":39355,"çĿ£å¯¼":39356,"save":39357,"çļĦåħ¶ä»ĸ":39358,"è̽":39359,"ountain":39360,"å§¥":39361,"ÑĤин":39362,"ĠFeature":39363,"åı¯ä¸įæĺ¯":39364,"é¢Ĩåıĸ":39365,"ĠInitiative":39366,"Ġpartie":39367,"ocrine":39368,"Ġalgo":39369,"Ġjul":39370,"mathfrak":39371,"Ġscripts":39372,"Nav":39373,"ĠWorkers":39374,"Ġfaint":39375,"Ġunstable":39376,"ounters":39377,"èĩªå·±åľ¨":39378,"éĢīä¸Ń":39379,"Ġveterans":39380,"-fold":39381,"posure":39382,"à¸Ħำ":39383,"month":39384,"Desc":39385,"Ġcurved":39386,"Ġpersec":39387,"æĥħæĻ¯":39388,"çļĦ身影":39389,"rale":39390,"æł·å¼ı":39391,"ĠREL":39392,"åıĺåİĭ":39393,"Ġideology":39394,"管çIJĨå·¥ä½ľ":39395,"书çļĦ":39396,"Ġgallery":39397,"Ġembracing":39398,"اصÙĦ":39399,"丫头":39400,"ĠпÑĢоиÑģ":39401,"Ġsout":39402,"سبب":39403,"åĩłçĤ¹":39404,"Ġà¦ĺ":39405,"Ġprinter":39406,"æĢĿæĥ³æĶ¿æ²»":39407,"è§ĴçļĦ":39408,"à§ĩà¦ĵ":39409,"Month":39410,"olan":39411,"é£ŁåłĤ":39412,"ISO":39413,"Ġnour":39414,"Ġadventures":39415,"ç°¡åĸ®":39416,"Ġmemorial":39417,"ISE":39418,"ìĭ¬":39419,"丨":39420,"弥补":39421,"Ġhipp":39422,"-inflammatory":39423,"Grade":39424,".random":39425,"Ġinvitation":39426,"ä½İä¸ĭ":39427,"Ġvisitor":39428,"ĠÑģледÑĥÑİÑīи":39429,"èĩ´åĬĽäºİ":39430,"aho":39431,"é̲åħ¥":39432,"æķĻä¼ļ":39433,"æħķ容":39434,"кий":39435,"ĠYOUR":39436,"ä¸į好æĦıæĢĿ":39437,"ï¼ŀ":39438,"ĠJennifer":39439,"纺ç»ĩ":39440,"iendo":39441,"ĠStorm":39442,"çļĦåıį":39443,"Ġcounterparts":39444,"Ġinjust":39445,"Ġbladder":39446,"ĠBT":39447,"Ġtilt":39448,"æĢĢéĩĮ":39449,"ĠLucas":39450,"Ġconferences":39451,"ÑĢоÑģÑĤ":39452,"éĹ»è¨Ģ":39453,"Ġlacks":39454,"æ¯Ķèµ·":39455,"交èѦ":39456,"Austral":39457,"Ġkings":39458,"ä»ĸæĬĬ":39459,"çĻĮçĹĩ":39460,"ĠGrammar":39461,"Ġrang":39462,"Ġanalysed":39463,"çģ«ç®Ń":39464,"ÃŃnh":39465,"Ġbee":39466,"Ġ×ŀצ":39467,"ÑĴа":39468,"UF":39469,"ĠKaren":39470,"è¡ĮæĿİ":39471,"åħ¥åѦ":39472,"å̾æĸľ":39473,"flation":39474,"chos":39475,"-grade":39476,"Ġperce":39477,"Ġanatomy":39478,"Ġenfer":39479,"ĠÙħختÙĦÙģ":39480,"Ïģε":39481,"олож":39482,"Thomas":39483,"çľ¼éķľ":39484,"صد":39485,"Ãĩ":39486,"ĉĉĉĊ":39487,"ĠADHD":39488,"ĠبÛĮÙĨ":39489,"Ġdetr":39490,"_title":39491,"éļ§éģĵ":39492,"Ġ({":39493,"ÑĢев":39494,"Äĥm":39495,"[p":39496,"ĠOscar":39497,"èİ·å¥ĸ":39498,"oft":39499,"Ġencaps":39500,"antics":39501,"ienie":39502,"Ġtutti":39503,"ä¸ĩç¾İåħĥ":39504,"Ġamplifier":39505,"ä¸ĢæĿ¯":39506,"ï¼Ľï¼Ī":39507,"Ġmounting":39508,"ĠBent":39509,"ä½ľç͍çļĦ":39510,"éĤ£äºº":39511,"ĠAssignment":39512,"andel":39513,"ÑĢен":39514,"æĽ´æĶ¹":39515,"ĠÑĢазвиÑĤиÑı":39516,"ĠButter":39517,"åĽ½å®¶åĴĮ":39518,"çī¡":39519,"олез":39520,"ĠColombia":39521,"å¿ľ":39522,"è¹Ħ":39523,"Ġproton":39524,"Ġmedicines":39525,"çĶŁæ´»ä¸ŃçļĦ":39526,"ä¿Ŀåħ»":39527,"rowned":39528,"Ġarriving":39529,"Ġregulating":39530,"à¸Ĺีà¹Īมี":39531,"Ġsurrounds":39532,"Ġsis":39533,"Ġdesktop":39534,"aaaa":39535,"éĹ®æĪij":39536,"ĠHarrison":39537,"adays":39538,"ÑĢабоÑĤ":39539,".http":39540,"racht":39541,"Ġprimera":39542,"æĥ©ç½ļ":39543,"ĠvÄĽ":39544,"ĠThroughout":39545,"ĠSELECT":39546,"+c":39547,"Ġstip":39548,"Ġinterpretations":39549,",S":39550,"å·¥å§Ķ":39551,"ĠScre":39552,"ĠMeasurement":39553,"Ġ(ĊĊ":39554,"!(":39555,"Ġbang":39556,"ĠØ£Ùħ":39557,"Ġvalidated":39558,"Robert":39559,".sw":39560,"-speed":39561,"åijĬ訴":39562,"French":39563,"Ġblogs":39564,"Law":39565,"è´µæĹı":39566,"çļĦåij³éģĵ":39567,"Ġsorting":39568,"ĠÙĨظر":39569,"çĪ±ä½ł":39570,"Ġfaut":39571,"Ġfounding":39572,"Ġinqu":39573,".key":39574,"ĠTM":39575,"æ·ĺæ±°":39576,"æııç»ĺ":39577,"Ġrenewed":39578,"ĠEstate":39579,"æ¯ıä¸Ģ次":39580,"å¿«æį·":39581,"Ġbasal":39582,"è¿Ŀ约":39583,"Ġionic":39584,"ĠArkansas":39585,"ĠSons":39586,"Õ¡Õ¼":39587,"åĮ¿":39588,"éģĵæŃī":39589,"Ġcourtesy":39590,"Ġantibiotic":39591,"gia":39592,"Ġwinners":39593,"**ãĢIJ":39594,"ä½Ĩä¸į":39595,"ennium":39596,"itives":39597,"æįIJèµł":39598,"Mag":39599,"é¦Ļåij³":39600,"henera":39601,"Ġindirectly":39602,"ĠнеÑģколÑĮко":39603,"çļĦæ¯Ķä¾ĭ":39604,"ä¸ĢèάæĿ¥è¯´":39605,"Ġcoven":39606,"Ġaprend":39607,"Additional":39608,"ĠMale":39609,"åĨ³å®ļäºĨ":39610,"oment":39611,"Ġgenetics":39612,"nem":39613,"anden":39614,"Ġ×Ķת":39615,"Ġceremon":39616,"abanay":39617,"ÃŃculo":39618,"/st":39619,"Ġپر":39620,"çŃ¾çº¦":39621,"óm":39622,"温åĴĮ":39623,"人æĢ§":39624,"×Ļ×ķף":39625,"venir":39626,"Ġretrieve":39627,"Ġseamless":39628,"æľĪ亮":39629,"ÑĪий":39630,"Ask":39631,"èĥ³èĨĬ":39632,"ĠиÑģ":39633,"prob":39634,"Ġaffair":39635,"Ġlover":39636,"ebab":39637,"楽":39638,"èĦīåĨ²":39639,"{v":39640,"ĠÑĢÑĥÑģ":39641,"ĠPatri":39642,"å©ļ礼":39643,"chod":39644,"ĠMasters":39645,"Ġformerly":39646,"[int":39647,"integer":39648,"å·¦æīĭ":39649,"Ġgeomet":39650,"Ġdesarrollo":39651,"ĠRecovery":39652,"Ġgenius":39653,"?\"Ċ":39654,"ĠNicholas":39655,"кова":39656,"ĠConnection":39657,"ä¹ĭæĥħ":39658,"å¿ĥå¾Ĺ":39659,"Ġанали":39660,".config":39661,"æľī害":39662,"毫åįĩ":39663,"Exploring":39664,"Ġdull":39665,"Ġcyan":39666,"Ġexecutives":39667,"èĬĤçľģ":39668,"ĠÙħÛĮØ´ÙĪØ¯":39669,"à¸ķัà¹īà¸ĩ":39670,"Ġsuccessive":39671,"Ġlac":39672,"limit":39673,"Ġtravés":39674,"ĠNP":39675,"iquit":39676,"pués":39677,"Ġdevastating":39678,"AMA":39679,"ĠÐĿи":39680,"æ±Łèĭıçľģ":39681,",M":39682,"__________":39683,"Ġempl":39684,"بÙĪ":39685,"ĠGCF":39686,"éĤ®ç®±":39687,"ĠSecurities":39688,"=new":39689,"coll":39690,"ç¡®è¯Ĭ":39691,"Ġexclude":39692,"usan":39693,"è¥Ħ":39694,"éĩĩç͍äºĨ":39695,"Ġì§Ħ":39696,"Ġcompatibility":39697,"è§ģè¯ģ":39698,"æ´½":39699,"Ġlum":39700,"Ġelic":39701,"না":39702,"åı³ä¾§":39703,"íĻĺ":39704,"Ġagua":39705,"ieron":39706,"é¢Ħ约":39707,"åįĺ":39708,"modal":39709,"Ġdiesel":39710,"_from":39711,"ĠMIN":39712,"ĠChrom":39713,"ĠÄIJ":39714,"ĠAlternatively":39715,"Ġfluorescence":39716,"iciones":39717,"åį´åıĪ":39718,"çŁ¥èŃĺ":39719,"ĠAj":39720,"-def":39721,"ç«ĭçļĦ":39722,"主æĮģ人":39723,"lf":39724,"umerable":39725,"Ġarguing":39726,"产å̼":39727,"ĠSalv":39728,"à²ķ":39729,"ighting":39730,"Ġanjara":39731,"å°ijéĩı":39732,"ÑĢиан":39733,"ĠAudio":39734,"ourd":39735,"åı¯ä»¥è¯´æĺ¯":39736,"æ¼Ķ示":39737,"çļĦçľ¼ç¥ŀ":39738,"ppo":39739,".User":39740,"ĠOst":39741,"ä¿Ŀ管":39742,"éĢīæĭ©é¢ĺ":39743,"Ġfetal":39744,"åºĹéĵº":39745,"DateTime":39746,"ä¿Ŀå®Ī":39747,"Ġeurope":39748,"Ġpolymorph":39749,"Ġclearance":39750,"ĠMeet":39751,"çĶ³è¯·äºº":39752,"éĸī":39753,"ĠVincent":39754,"}:":39755,"سÙĦ":39756,"ਰ":39757,"/fl":39758,"ç»ŀ":39759,".pl":39760,"ctica":39761,"ĠÑĩаÑģÑĤÑĮ":39762,"Ġtoute":39763,"ê±´":39764,"iveau":39765,"ĠполÑĥÑĩ":39766,"ר×Ĵ":39767,"ĠChain":39768,"ĠIsaac":39769,"Ġvé":39770,"å¥¹ä¹Ł":39771,"Ġscanner":39772,"Ġgrupo":39773,"ĠGand":39774,"éĿĻæĢģ":39775,"æĺ¨æĹ¥":39776,"Ġprofitable":39777,"×ķתר":39778,"ĠOw":39779,"Ġsunshine":39780,"æķ£æĸĩ":39781,"}\\\\":39782,"++.":39783,"Ġadministrator":39784,"dates":39785,"çĶŁäº§åĬĽ":39786,"èį£èİ·":39787,"æµİåįĹ":39788,"iothe":39789,"à´¤":39790,"ĠBrew":39791,"ĠÑģÑĤÑĢо":39792,"Ġtakże":39793,"ëĵł":39794,"ĠBM":39795,"Ġspouse":39796,"æķ°åĪĹ":39797,"Ġcampo":39798,"uego":39799,"çĭ¬ç«ĭçļĦ":39800,"ĠInside":39801,"å¾Īéĩįè¦ģ":39802,"Posts":39803,"Ġevangel":39804,"ðŁĮŁ":39805,"IPS":39806,"Ġlithium":39807,"ĠDiscovery":39808,"ĠاÙĦبÙĬ":39809,"orneys":39810,"ĠÙĬÙĥÙĪÙĨ":39811,"ĠLocations":39812,"ä¸ľäº¬":39813,"åĪļ好":39814,"ĠiOS":39815,"æijĦåĥı":39816,"ĠоÑĤкÑĢÑĭ":39817,"Ġbend":39818,"Ġunconscious":39819,"isha":39820,"-all":39821,"åħ¨æĺ¯":39822,"ĠBowl":39823,"Ġhumble":39824,"ĠÑĢади":39825,"qi":39826,"Ġemotionally":39827,"_error":39828,"Ġchuck":39829,"lez":39830,"Ġcorrelations":39831,"ç³Ļ":39832,"çĹħçIJĨ":39833,"ĠWildlife":39834,"交代":39835,"erner":39836,"ĠDynamics":39837,"被ä»ĸ":39838,"ĠCAP":39839,"Ġcease":39840,"ìŀĦ":39841,"Ġconvinc":39842,"ĠDescribe":39843,"меÑĢи":39844,"ĠобÑĬек":39845,"Ġsint":39846,"Ġpathogens":39847,"ÅĻi":39848,"Ġdeut":39849,"ĠHoff":39850,"LM":39851,"onics":39852,"jectives":39853,"uario":39854,"ĠMull":39855,"enes":39856,"ĠSV":39857,"ĠIoT":39858,"åij½åIJį":39859,"ĠFrederick":39860,"arcin":39861,"ĠTRUE":39862,"'=>":39863,"Ġeliminating":39864,"ĠPredict":39865,"é£Łåĵģå®īåħ¨":39866,"å²©çŁ³":39867,"phys":39868,"itters":39869,"ä¿®è¡Į":39870,"Ġcondem":39871,"Ġ×ķ×ij×":39872,"âĭ":39873,"×ķ×Ķ":39874,"åĩ¤åĩ°":39875,"ĠLux":39876,"Ġaccelerated":39877,"çľĭä¸įåΰ":39878,"éĻIJ度":39879,"Ġmagnesium":39880,"ĠEntertainment":39881,"Ġrigorous":39882,"Ġcultura":39883,"Ġturnover":39884,"gunakan":39885,"Ġsuppression":39886,"æŃ¤åIJİ":39887,"Ġss":39888,"Ġbump":39889,"å¢ŀéĢŁ":39890,"ĠSudan":39891,"Ġpork":39892,"åºļ":39893,"ĠJO":39894,"Ġstretching":39895,"Ġeligibility":39896,"creasing":39897,"ĠLebens":39898,"å̡坼":39899,"ãģªãĤī":39900,"ধà§įয":39901,"Ġstationary":39902,"Ġrewarding":39903,"ĠAcid":39904,"Ġzip":39905,"åĮºå§Ķ":39906,"ĠÑģлова":39907,"åĨ³è®®":39908,"Ġcommodity":39909,"ĠLanka":39910,"头çĹĽ":39911,"DH":39912,"primary":39913,"tri":39914,"åĨħèĴĻåı¤":39915,"ç»ıæµİå¢ŀéķ¿":39916,"ências":39917,"Basic":39918,"Ta":39919,"å¸ĪèĮĥ大åѦ":39920,"Ġdados":39921,"Ġthé":39922,"Ġlord":39923,"ĠMorning":39924,"Ġinfluenza":39925,"Ġcoping":39926,"Ġavoir":39927,"%ãĢĤĊ":39928,"à¦Ĩ":39929,"Ġprimes":39930,"Ġzab":39931,"æĪij们éľĢè¦ģ":39932,"ï¼½":39933,"èģ½åΰ":39934,">\\":39935,"national":39936,"ĠGirls":39937,"-ey":39938,"产èĥ½":39939,"equal":39940,"ÑģÑĤоÑıн":39941,".htm":39942,"ĠSCH":39943,"Ġ(%)":39944,"اÙĪÙī":39945,"ä½Ĩçͱäºİ":39946,"以çĤº":39947,"äººä¹Ł":39948,"帶èijĹ":39949,")\\),":39950,"åľ¨åħ¨åĽ½":39951,"èĢĥçĤ¹":39952,"eeper":39953,"ĠRou":39954,"ĠZhou":39955,"æ³ķåºŃ":39956,"缮æ¨Ļ":39957,"ç²¾ç»Ĩ":39958,"ï¼»":39959,"ĠAlternative":39960,"Ġprosperity":39961,"Ġoutward":39962,"夫å¦ĩ":39963,"èŀºæĹĭ":39964,"çļĦå®¶":39965,"ĠLac":39966,"hingga":39967,"celand":39968,"Ġpont":39969,"Ġaria":39970,"ä»ij":39971,"rove":39972,"uria":39973,"ä¸ŃåİŁ":39974,"Ġschon":39975,"Ġвол":39976,"Ġcultiv":39977,"å®ŀè¯Ŀ":39978,"ĠWelcome":39979,"ĠÑĥпÑĢав":39980,"Hot":39981,"Ġpall":39982,"Ġsinus":39983,".use":39984,"uwe":39985,"Ġwiel":39986,"ÑģÑĤеÑĢ":39987,"ναι":39988,"Ġmast":39989,"ваÑĤ":39990,"ĠاÙĦÙħÙĪ":39991,"izio":39992,"ä½ķæĹ¶":39993,"çݯå¢ĥä¿ĿæĬ¤":39994,"ÙĬج":39995,"ĠParameters":39996,"ocate":39997,"ç¼ķ":39998,"Ġtours":39999,"تÙĪØ§ÙĨ":40000,"éĢıè¿ĩ":40001,"ainen":40002,"åİĨåı²çļĦ":40003,"otics":40004,"_pos":40005,"ĠREAD":40006,"_col":40007,"assy":40008,"ĠDublin":40009,"ŀת":40010,"å®ĹæĹ¨":40011,"Ġotro":40012,"Ġinteracting":40013,"ään":40014,"åķª":40015,"åĵĪå°Ķ":40016,"åŃķå¦ĩ":40017,".charAt":40018,"ози":40019,"(.":40020,"Ġruler":40021,"çĮľæµĭ":40022,"ishi":40023,"Methods":40024,"Ġfungi":40025,"([Ċ":40026,"ÃŃo":40027,"μÏĢ":40028,"Ġtransported":40029,"ĠOperating":40030,"ĠJobs":40031,"ĠLatest":40032,"éı¡":40033,"ĠRural":40034,"èħ¥":40035,"ç´Ľ":40036,"зма":40037,"ĠReserved":40038,"ĠArchae":40039,"Ġunb":40040,"æĮ£æīİ":40041,"-direct":40042,"ĠÏĦον":40043,"/web":40044,"ÑĶ":40045,"ENSE":40046,"Ġconna":40047,"Ġrabbit":40048,"Ġwrist":40049,"ĠدÙĪÙĦ":40050,"Ġcallback":40051,"管çIJĨéĥ¨éŨ":40052,"Ġrefined":40053,"ĠNeural":40054,"ìĹIJê²Į":40055,"jm":40056,"Ġglimpse":40057,"ullivan":40058,"ĠDipl":40059,"ĠJulia":40060,"ĠgÅĤ":40061,"ĠاÙħا":40062,"(test":40063,"ĠNovel":40064,"ĠImportant":40065,"Ġdifférent":40066,"Very":40067,"Ġmosquito":40068,"éľĸ":40069,"å̼çıŃ":40070,"à¸ļริ":40071,"Ġencont":40072,"oplast":40073,"Ġadvise":40074,"Ġcommence":40075,"attered":40076,"Ġtimeline":40077,"decl":40078,"_token":40079,"Ġshocked":40080,"owane":40081,"-spe":40082,"ĠParents":40083,"Crit":40084,"å¦Ĥæŀľè¯´":40085,"Ġexhausted":40086,"ayers":40087,"ĠÑĥда":40088,"ĠLar":40089,"管çIJĨ人åijĺ":40090,"ĠÙĬÙĤ":40091,"åıªæľīä¸Ģ个":40092,"ç¹ģæ®ĸ":40093,"Tri":40094,"coma":40095,"Ġpriests":40096,"çľ¼çļĦ":40097,"缸è¿ŀ":40098,"ĠÏĢε":40099,"×ķ×£":40100,"æīĭå·¥":40101,"idet":40102,"åĨħ容çļĦ":40103,"è¿IJåĬ¨çļĦ":40104,"ĠMAC":40105,"åıªæĺ¯ä¸Ģ":40106,"Ġlasted":40107,"олÑĮзова":40108,"Ġunderneath":40109,"ีà¸ģ":40110,"é¢Ĩ导å°ıç»Ħ":40111,"udent":40112,"åı¶åŃIJ":40113,"grav":40114,"Ġ॥":40115,"ĠMHz":40116,"æĪijä¸įçŁ¥éģĵ":40117,"èļķ":40118,"ĠÏĢο":40119,"ĠRecently":40120,"XXXXXXXX":40121,"éĢļè¡Į":40122,"åĬłæ·±":40123,"ÐĶлÑı":40124,"Ġгла":40125,"æĬijéĥģ":40126,"Ġমান":40127,"-ed":40128,"oleh":40129,"ĠHughes":40130,"ĠGroups":40131,"Ġdéf":40132,"Ġantioxidant":40133,"ĠMove":40134,"åĨĴéĻ©":40135,"\\Models":40136,"ĠBorder":40137,"æĮĩçļĦæĺ¯":40138,"ÙĬÙĨØ©":40139,"ĠDH":40140,"åħĪè¡Į":40141,"åĪĨæĺİ":40142,"Ġcharacterize":40143,"Ġsore":40144,"ÑĤелÑĮноÑģÑĤÑĮ":40145,"çļĦç¬ij":40146,"æĺ¯éĿŀ":40147,"inv":40148,"LV":40149,"Ġaccessing":40150,"ĠSIM":40151,"ĠLost":40152,"VENT":40153,"mation":40154,"ç»ıæµİæķĪçĽĬ":40155,"ä»İæĿ¥æ²¡æľī":40156,"(text":40157,"åı²ä¸Ĭ":40158,"åΤå®ļ":40159,"æŀģ端":40160,"ĠJustin":40161,"éħ°":40162,"æĿ¥å¾Ĺ":40163,"Ġfreeze":40164,"Computer":40165,"ĠLP":40166,"ÑĢез":40167,"нами":40168,"Ġprobabilities":40169,"ĠìļĶ":40170,"Ġbeberapa":40171,"ĠGov":40172,"Ġbagi":40173,"Ġdecimals":40174,"ĠSoon":40175,"æŃ¤ç±»":40176,"Ġrelying":40177,"Ġencoded":40178,"Ġsurplus":40179,"æķ°ä¸º":40180,"åݿ级":40181,"ç»ĵæĿŁäºĨ":40182,"ĠMEDLINE":40183,"/ha":40184,"åıĺåĮĸçļĦ":40185,"Ġmasks":40186,"Ġviscosity":40187,"between":40188,"urst":40189,"Å¡tÄĽ":40190,"Ġsummarized":40191,"å°įæĸ¹":40192,"ĠCommunist":40193,"social":40194,"ĠART":40195,"icky":40196,"Ġadministrators":40197,"ĠBil":40198,"ä¼ģä¸ļåľ¨":40199,"ä¸įéĢĤ":40200,"/mL":40201,"illery":40202,"ä»ĸæĥ³":40203,"æķĻç§ij":40204,"-ben":40205,"ä¸įæĸŃæıIJé«ĺ":40206,"ĠASC":40207,"žit":40208,"ãģıãģłãģķãģĦ":40209,"ĠPoor":40210,"åĪĨåħ¬åı¸":40211,"Ġpedest":40212,"ĠSerial":40213,"ielle":40214,"Ġhumanitarian":40215,"%=":40216,"Ġtema":40217,"Ġtriangles":40218,"lb":40219,"åѦ年":40220,"ulsive":40221,"è·¨è¶Ĭ":40222,"èµ¢å¾Ĺ":40223,"âŃIJ":40224,"Ġgit":40225,"agh":40226,"Arrays":40227,"ĠÑħоÑĢо":40228,"à¶±":40229,"Ġslices":40230,"Ġש×Ķ":40231,"ĠDarwin":40232,"èĨ³":40233,"tionary":40234,"isco":40235,"Ġlomb":40236,"ĠFunctional":40237,"åİ¥":40238,"Ġbelieving":40239,"conc":40240,"åIJĦæľī":40241,"ĠPun":40242,"uebl":40243,"LAN":40244,"Ġexpans":40245,"acja":40246,"Ġinterrog":40247,"Ġcausa":40248,"帮æĪij":40249,"ĠFerr":40250,"زر":40251,"protected":40252,"Ġuter":40253,"ĠíĻķ":40254,"ương":40255,"åİŁåĽłæĺ¯":40256,"å°±åı¯ä»¥äºĨ":40257,"ĠUkrainian":40258,"ãģ²":40259,"à¸ĺรรม":40260,"ĠOlympics":40261,"çĽ¸å¯¹äºİ":40262,"ç¼ļ":40263,"åİĭè¿«":40264,"Ġpracticed":40265,"ÑĹ":40266,"ðŁĵ":40267,"css":40268,"çĥŃæ°´":40269,"׾ק":40270,"Ġzelf":40271,"訪":40272,"rano":40273,"Ġaccelerate":40274,"rots":40275,"为æĪij们":40276,"ĠGermans":40277,"osomal":40278,"çľĭå¾ħ":40279,",and":40280,"Ġ'',":40281,"неÑĤ":40282,"æĹłå£°":40283,"Ġproactive":40284,"Ġrelacion":40285,"elsh":40286,"ĠValid":40287,"Ġquestioning":40288,"æłijæľ¨":40289,"Ġসà§įব":40290,"ĠPrincipal":40291,"ĠFourier":40292,"-is":40293,"Var":40294,"Ġmicrowave":40295,"Sche":40296,"rability":40297,"ĠIdeas":40298,"Ġsperm":40299,"weights":40300,"Ġsalaries":40301,"ĠOracle":40302,"ĠWays":40303,"ĠCoach":40304,"×Ļ×Ĵ":40305,"Ġìłģ":40306,"ĉString":40307,"Ġburg":40308,"ĠRR":40309,"å°±è¿Ļä¹Ī":40310,"Ġaquatic":40311,"Ġhike":40312,"å¿ĥèĤĮ":40313,"æľºæŀĦçļĦ":40314,"Ġlou":40315,"æ¤ľ":40316,"Ġartifacts":40317,"使ä¹ĭ":40318,"ilians":40319,"åĽºä½ĵ":40320,"Ġabrupt":40321,"ĠVenez":40322,"èģĶåĬ¨":40323,"Ġinformación":40324,"âĢĿ),":40325,"æ¤Ń":40326,"Ġdav":40327,"ublik":40328,".]ĊĊ":40329,"Ġlod":40330,"Ġ'#":40331,"ilters":40332,"ĠUnless":40333,"æ¯Ķçī¹":40334,"èİ«åIJį":40335,"Ġ$_":40336,"Ġanimated":40337,"å½±åŃIJ":40338,"åĩĦ":40339,"DER":40340,"å®ĥåľ¨":40341,"Ġkeine":40342,"ĠBath":40343,"Ġprotects":40344,"çŃĽéĢī":40345,"olecules":40346,"àµĨ":40347,"æīĢèĩ´":40348,"ĠNM":40349,"ĠØŃتÙī":40350,">>&":40351,".âĢĻâĢĻĊĊ":40352,"ä½ĵè´¨":40353,"åĸĩ":40354,"Ġeighteen":40355,"ĠEste":40356,"ĠPrincess":40357,"ï¼īï¼Ī":40358,"xxx":40359,"Ġclassrooms":40360,"�Ċ":40361,"Constructor":40362,"Ġhä":40363,"æĺ§":40364,"Ġdecir":40365,"æľīæĦı":40366,"Ġbugs":40367,"åѦçĶŁåľ¨":40368,"åIJ¯åıij":40369,"DN":40370,"Dear":40371,"large":40372,"ĠÑįкономи":40373,"(void":40374,"çļĦä¸Ģèά":40375,"Ġupdating":40376,"GDP":40377,"ĠاÙĦدر":40378,"Ñĩки":40379,"ëĵ¤ìĿ´":40380,"åħļä¸Ń央":40381,"åĪĽå§ĭ":40382,"ĠдвÑĥÑħ":40383,"prising":40384,"Ġsuccession":40385,"å®ļåζ":40386,"åĩºèµĦ":40387,"éĦī":40388,"詳":40389,"Ġад":40390,"(is":40391,"çļĦ身ä½ĵ":40392,"ĠShaw":40393,"scanf":40394,"æĻı":40395,"Ġsynthesized":40396,"observ":40397,"Ġsuas":40398,"леÑĢ":40399,"Ġhanya":40400,"ä¼ļ导èĩ´":40401,"integ":40402,"Ø®ÛĮ":40403,"Ġdiesem":40404,"Ġgloss":40405,"ä¸Ģåħ±":40406,"ĠSale":40407,"åįģ大":40408,"@gmail":40409,"respons":40410,"Connect":40411,"gow":40412,"нем":40413,"å·²æĺ¯":40414,"kim":40415,"Ġpsychiatric":40416,"ĠNeither":40417,"ÑİÑīий":40418,"ĠMt":40419,"ĠWhatever":40420,"Ġcommitments":40421,"Occ":40422,"åŃĺæĶ¾":40423,"-effective":40424,"IMA":40425,"å®ĺç½ij":40426,"ä¼ļ对":40427,"åĨįæĿ¥":40428,"bing":40429,"Ġpremature":40430,"ÑĤелÑĮного":40431,"à¸Ķู":40432,"Sample":40433,"غر":40434,"Îķ":40435,"Ġtended":40436,"ĠSold":40437,"æĽ´å®¹æĺĵ":40438,"\",&":40439,"Ñīие":40440,"ÃŃses":40441,"ĠÙĪÙĦا":40442,"ĠAntar":40443,"Ga":40444,"Ġclearing":40445,"åªĴä»ĭ":40446,"Ġglyc":40447,"icles":40448,"ĠSettings":40449,"Ġmolto":40450,"ãģ¨ãģį":40451,"Ġساز":40452,"åĨ°ç®±":40453,"ĠìłĢ":40454,"Ġinstructional":40455,"ÙĬØ·":40456,"Ġköz":40457,"Ġ,ĊĊ":40458,"ãģĭãģ£ãģŁ":40459,"Ġpalace":40460,"Ġcf":40461,"ĠApache":40462,"ĠHaus":40463,"çĦı":40464,"Ġneighboring":40465,"ĠÑĪколÑĥ":40466,"ĠAid":40467,"åĵ¨":40468,"ĠÑģвÑıза":40469,"Ġ{%":40470,"ĠThan":40471,"ç½®äºİ":40472,"á»ĩn":40473,"天èµĭ":40474,"Ġfork":40475,"ĠMeng":40476,"Ġpenny":40477,"é¦ĸåħĪè¦ģ":40478,"தà¯ģ":40479,"ĠProcedure":40480,"æľīçĽĬ":40481,"Ġ}})":40792,"hte":40793,"Ġbeautifully":40794,"urious":40795,"ä¸Ģè¾Ĩ":40796,"disciplinary":40797,"Ġlibert":40798,"Ġ-----":40799,"Ġenvironmentally":40800,"ĠÑĤÑĢан":40801,"Pool":40802,"Ġmond":40803,"ĠتØŃت":40804,"ÐķÑģли":40805,"Ġramp":40806,"job":40807,"alert":40808,"roz":40809,"ĠVik":40810,"×ķ׳×":40811,"-set":40812,"Ġcient":40813,"Ġrhe":40814,"åħŃ个":40815,".un":40816,"ĉprint":40817,"Ġprobl":40818,"à®Ļà¯įà®ķ":40819,"å¤ļ彩":40820,"Ġpolymers":40821,"ĠGenetic":40822,"Ġposture":40823,"ophyll":40824,"ÑĪин":40825,"nda":40826,"éĺ³æĢ§":40827,"Ġsql":40828,"Ġfilt":40829,"å¹£":40830,"Ġabsor":40831,"çļĦä½ľåĵģ":40832,".mod":40833,"é϶çĵ·":40834,"ä½įç§»":40835,"åIJĦ種":40836,"سÙĩ":40837,"ĠMedium":40838,"Ġcivilian":40839,"Ġdors":40840,"ãĥĶ":40841,"线æĿ¡":40842,"Ġdonations":40843,"Ġoptimum":40844,"æĥħåł±":40845,"ĠGraduate":40846,"ĠâĬ¢":40847,"Ġbilateral":40848,"gio":40849,"entre":40850,"Ġterj":40851,"åIJĪä½ľç¤¾":40852,"ĠAve":40853,"åij½çļĦ":40854,"তা":40855,"aders":40856,"ĠJug":40857,"Ġ×Ķ×Ļ×Ķ":40858,"åIJĥå®Į":40859,"Ġmediated":40860,"ppy":40861,"Ġsteering":40862,"äºĶè¡Į":40863,"ĠÙĨاÙħ":40864,"Ġplugin":40865,"Ġhydraulic":40866,"iksi":40867,"Ni":40868,"_out":40869,"Ġмом":40870,"éĻIJäºİ":40871,"FROM":40872,"åĪĹåħ¥":40873,"Ġ×ij×Ķ×":40874,"Ġkole":40875,">{{":40876,"è¯·ä½ł":40877,"ÑĤок":40878,"æİĴåĩº":40879,"ç®ĢåĮĸ":40880,"Ġاع":40881,"åĥıç´ł":40882,"Um":40883,"ĠBilly":40884,"å·´å·´":40885,"åĵĪåĵĪåĵĪ":40886,"HO":40887,"Ġents":40888,"Ġcurv":40889,"çͲçĬ¶":40890,"Ġмеди":40891,"-equ":40892,"å¹»æĥ³":40893,"æľĢé«ĺçļĦ":40894,"两åıª":40895,"Ġdeserves":40896,"foo":40897,"Ġclue":40898,"Arab":40899,"Ġdoit":40900,"ÑĩиÑĤÑĮ":40901,"Ġproposes":40902,"Ġsond":40903,"ãĢĤ[":40904,"ĠTranslation":40905,"绸":40906,"ĠJar":40907,"éĩįç»Ħ":40908,"åĩłä½į":40909,"OVE":40910,"ĠBulletin":40911,"Ġappetite":40912,"_:":40913,"Ġstor":40914,"Within":40915,"Ġsecretion":40916,"Ġboring":40917,"ullah":40918,"^(":40919,"ĠTemplate":40920,"产éĺ¶çº§":40921,"sequence":40922,"Ġenfants":40923,"Partic":40924,"ĠCBD":40925,"èĢ¿":40926,"ä¿¡ç͍åį¡":40927,"ĠDial":40928,"bold":40929,"çļĦåĪĨæŀIJ":40930,"ĠاÙĦاط":40931,"å¤ĦåĪĨ":40932,"ĠHolland":40933,"occo":40934,"ĠìĤ¬ëŀĮ":40935,"éĹº":40936,"ضÙĪØ¹":40937,"rational":40938,"大ãģį":40939,"çľĭè¿ĩ":40940,"ä¸ĵç§ij":40941,"inol":40942,"èĤĭ":40943,"Ġfacilitating":40944,"Round":40945,"Ġcontraction":40946,"ensk":40947,"Readers":40948,"à¦¿à§Łà§ĩ":40949,"Ġshy":40950,"Ġvalores":40951,"婦":40952,"aniu":40953,"Ġff":40954,"ĠMechanical":40955,"Ġconvex":40956,"ĠWaste":40957,"Ġthromb":40958,"éĻ¡":40959,"åħļç»Ħ":40960,"Ġautomotive":40961,"èIJ½åΰ":40962,"division":40963,"jours":40964,"ðŁı":40965,"Enum":40966,"olah":40967,"Ġbeams":40968,"ä¸į顾":40969,"acker":40970,"à¸Ķัà¸ļ":40971,"å¾Ģä¸ĭ":40972,"Ġmagazines":40973,"Psych":40974,"ĠëįĶ":40975,"±ħ":40976,"ĠAthens":40977,"Ġforgiveness":40978,"×ķ׳×Ļ×Ŀ":40979,"Ġoft":40980,"Ġvil":40981,"ajÃŃ":40982,"ĠJimmy":40983,"æīĢåľ¨åľ°":40984,"Phi":40985,"voc":40986,"archar":40987,"Ġathletic":40988,"dim":40989,"è¶ħ声":40990,"è¾£æ¤Ĵ":40991,"rg":40992,"èµ°åľ¨":40993,"emplates":40994,"WM":40995,"chy":40996,"Ġpermanently":40997,"Siyentipik":40998,"itung":40999,"æĪijä»¬ä¹Ł":41000,"none":41001,"æħ£":41002,"以åĨħ":41003,"çŁŃä¿¡":41004,"ĠGaussian":41005,"íĭ":41006,"-minded":41007,"åĴĮæĬĢæľ¯":41008,"anan":41009,"çĭ¸":41010,"¨àµįà´":41011,"Ġ'__":41012,"สาร":41013,"Ġresidue":41014,"ŀצ":41015,"baum":41016,"Ġleap":41017,"Ġjego":41018,"Ptr":41019,"ĠCoron":41020,"Ġsuspicious":41021,"Ġkat":41022,"ĠVienna":41023,"/her":41024,"они":41025,"$$\\":41026,"ä¸ĢåĪĨ":41027,"ìĿ´ëĿ¼":41028,"å¼ĢåıijåĮº":41029,"×Ļ׳×Ķ":41030,"æķ°é¢Ŀ":41031,"relations":41032,"ĠVR":41033,"Ġcharming":41034,"ĠGreater":41035,"Ġdisadvantages":41036,"Ġphosphory":41037,"ÙĬÙij":41038,"Ġsmartphone":41039,"Ġsoluble":41040,"(url":41041,"verting":41042,"ĠParkinson":41043,"Distance":41044,"rb":41045,"Ġcomplexities":41046,"Ġbored":41047,"aptic":41048,"èĦĪ":41049,"两çĤ¹":41050,"ĠClassical":41051,"ä¸ĸéĹ´":41052,"ĠزÙħاÙĨ":41053,"[u":41054,"ĠPlayers":41055,"åζæĪIJ":41056,"ALE":41057,"Ġcelebrating":41058,"辦æ³ķ":41059,"Friday":41060,"Ġsynonyms":41061,"');":41062,"Women":41063,"Ġextensions":41064,"(response":41065,"Ġbiblical":41066,"lette":41067,"ç¾İ人":41068,"history":41069,"ĠWritten":41070,"нал":41071,"async":41072,"æ¢ĵ":41073,"æĺ¯å°Ĩ":41074,"ĠREF":41075,".send":41076,"åħ±è®¡":41077,"Ġcrafts":41078,"ĠWayback":41079,"Siyentipikinhong":41080,"Hard":41081,"çļĦæķ°éĩı":41082,"çľĭå®Į":41083,"ĠDenver":41084,"æ·¹":41085,"ĠTodd":41086,"以åħ¶":41087,"Ġcela":41088,"seen":41089,"è´±":41090,"Monday":41091,"Ġcricket":41092,"å°±ä¸įèĥ½":41093,"èĦ«":41094,"оза":41095,"çļĦæĹł":41096,"Ġopenly":41097,"ĠÙħÙħ":41098,"ĠDip":41099,"çļĦ对":41100,"Ġtyl":41101,"çĩĴ":41102,"å¸®ä½ł":41103,"ĠCelsius":41104,"ĠÚ©ÙĨÙĨد":41105,"ĠHopkins":41106,"æĢªçī©":41107,"ĠUniverse":41108,"æ¼Ķç»ĥ":41109,"Ġrecurrent":41110,"Ġnano":41111,"LES":41112,"\\theta":41113,"åı·åı¬":41114,"Ġëį°":41115,"ì¤Ģ":41116,"ÃŃp":41117,"åĮºåĿĹ":41118,"ounces":41119,"è¦ģæ¯Ķ":41120,"ĠÙĤاÙĦ":41121,"dar":41122,")){Ċ":41123,"ĠTA":41124,"Ġduck":41125,"ĠNova":41126,"Ġvaluation":41127,"Ġexpresses":41128,"ĠInfluence":41129,"agar":41130,"ĠMini":41131,"ĠDefinitions":41132,"çļĦæīĭ段":41133,"ĠнаÑĢÑĥ":41134,"à±įà°°":41135,"ï¼ľ":41136,"ĠاÙĦاجتÙħاع":41137,"HH":41138,"Ġzwei":41139,"Ġsiblings":41140,"×ķ׾×Ŀ":41141,"äºĨè¿ĩåİ»":41142,"lando":41143,"Ġslipped":41144,"åıijæī¬":41145,"icator":41146,"ĠHindi":41147,"ĠFrequency":41148,"æ£Ħ":41149,"-defined":41150,"Ġshells":41151,"Ġר×ij":41152,"matics":41153,"Ġdatetime":41154,"ĠRadi":41155,"Ġ×IJ×ij":41156,"çļĦæľįåĬ¡":41157,"রা":41158,"Ġours":41159,"Ġç":41160,"Ġcrowded":41161,"ĠWayne":41162,"catch":41163,"èµĦè´¨":41164,"Ġmanages":41165,"aben":41166,"å½ĵæĪij们":41167,"ĠPhoenix":41168,"à¦¾à¦Ľ":41169,"Trace":41170,"istes":41171,"æīĢ说":41172,"åı£æĦŁ":41173,"Complete":41174,"é¦ĸéĥ½":41175,"ĠPublishers":41176,"emor":41177,"mith":41178,"Ġmillimeter":41179,"ÏīÏĤ":41180,"igers":41181,"Ġaggression":41182,"iras":41183,"ç¼ħ":41184,"Ġfinanc":41185,"หรัà¸ļ":41186,"erten":41187,"ç»ĵçĤ¹":41188,"çİĭçļĦ":41189,"Ġalgebraic":41190,"Ġlined":41191,"vering":41192,"ĠIntrodu":41193,"æľīä¸Ģ次":41194,"æĥħå¢ĥ":41195,"Single":41196,"ĠFormation":41197,"ĠIk":41198,"Ġdrunk":41199,"Ġnau":41200,"æĶ¿æ³ķ":41201,"Ġtensions":41202,"åıįæĬĹ":41203,"çªŁ":41204,"ä»ĭè´¨":41205,"/,":41206,"ricted":41207,"Õ¡Õ¾":41208,"éĶĻ误çļĦ":41209,"Ġamendments":41210,"åĽ³":41211,"æŃ£è§Ħ":41212,"å´Ľ":41213,"оÑĤе":41214,"Ġotras":41215,"ĠDepending":41216,"Ġfertilizer":41217,"ĠImplementation":41218,"ÑıвлÑı":41219,"ä¸į妨":41220,"éĿĴå±±":41221,"ĠOptional":41222,"ĠFill":41223,"导弹":41224,"Ġfis":41225,"Ġons":41226,"ĠConverter":41227,"ĠпомоÑīÑĮÑİ":41228,"ĠAnders":41229,"éĦĻ":41230,"×ķ×ŀר":41231,"[c":41232,"åĩºèº«":41233,"Ġgeme":41234,"Ñĩного":41235,"æľīéĴ±":41236,"Being":41237,"éī´åĪ«":41238,"-owned":41239,"ĠShar":41240,"clamation":41241,"èİŀ":41242,"Ġips":41243,"å¤ļå¤ļ":41244,"াথ":41245,"θε":41246,"TON":41247,"Ġthunder":41248,"anche":41249,"ĠвелиÑĩи":41250,"ĠJahren":41251,"çļĦä¸ŃåĽ½":41252,"éĺ´å½±":41253,".delete":41254,"Ġwholly":41255,"Ġà¹Ĩ":41256,"éĨĴæĿ¥":41257,"Ġsoda":41258,"Ġculmin":41259,"ç¤İ":41260,"Ġfebru":41261,"ĠTet":41262,"Ġproves":41263,"Ġibabaw":41264,"missible":41265,"{X":41266,"Ġhover":41267,"æĹ¶éĹ´åĴĮ":41268,"Ġdigestion":41269,"Ġinitiate":41270,"çļĦæĿĥåĪ©":41271,"ãĢĤ\"":41272,"Ġarchive":41273,"ĠEdinburgh":41274,"Phys":41275,"cken":41276,"Õ¡Õ®":41277,"主æĦı":41278,"ĠCox":41279,"Ġsorrow":41280,"Ġfats":41281,"åıĺçļĦ":41282,"Ġppm":41283,"ALS":41284,"ĠSor":41285,"åĨħå¿ĥçļĦ":41286,"åħ©äºº":41287,"{aligned":41288,"again":41289,"Ġwholes":41290,"opa":41291,"Ġadherence":41292,"Ġabnormalities":41293,"Ġbending":41294,"æĪijéĻ¢":41295,"ĠMerc":41296,"Ġhazardous":41297,"uber":41298,"ä¼łåªĴ":41299,"åķıéģĵ":41300,"Ġery":41301,"ération":41302,"EMA":41303,"Ġaston":41304,"Ġtheor":41305,"Ġyn":41306,"eso":41307,"è¿Ľåħ¥äºĨ":41308,"Ġprince":41309,"â̦âĢĿĊĊ":41310,"é»ijé¾Ļæ±Ł":41311,"çĶ¢åĵģ":41312,"éĩĩæł·":41313,"Numbers":41314,"åĸĶ":41315,"çıŃçļĦ":41316,"æĭ¢":41317,"Ġexpose":41318,"Ġrecipients":41319,"Ġmint":41320,"Ġsimultaneous":41321,"ĠFrame":41322,"effective":41323,"åŃĹèĬĤ":41324,"Ġthereafter":41325,"à·Ģ":41326,"æłħ":41327,"Ġmarijuana":41328,"ĠCarr":41329,"Ġdepuis":41330,"Ġencouragement":41331,"ä¸Ģæµģ":41332,"Ġpolarization":41333,"\\-":41334,"Ġscholar":41335,"çķ¢ç«Ł":41336,"Early":41337,"Ġclues":41338,"Ġrocket":41339,"{y":41340,"伺":41341,"obia":41342,"Ġinfrared":41343,"ãĥ¼ãĤ¿":41344,"ê°ķ":41345,"ç»İ":41346,"ĠPub":41347,"onen":41348,"æİ¥åľ°":41349,"ç¿ĺ":41350,"ieri":41351,"Ġxml":41352,"æķ£åıij":41353,"CPU":41354,"ĠповеÑĢÑħ":41355,"DST":41356,"Ġsurgeon":41357,"Ġfamilia":41358,"Ġлег":41359,"/second":41360,"׳×":41361,"ĠShanghai":41362,"zten":41363,"æĿ¿çļĦ":41364,"quiries":41365,"Ġseas":41366,"æľ¬ç«Ļ":41367,"Music":41368,"Ġfs":41369,"ĠBacter":41370,"è¦ģä¸į":41371,"ĠÙĪÙĩÙĪ":41372,"çªĹæĪ·":41373,"Mass":41374,"ĠProof":41375,"ÑĤони":41376,"ĠJiang":41377,"ĠмеÑģÑı":41378,"Ġunve":41379,"Ġcatching":41380,"nsic":41381,"NI":41382,"\\{":41383,"alent":41384,"Ġ\"\";Ċ":41385,"Ġcleaned":41386,"ĠCitations":41387,"ĠÐŁÐ¾Ñģ":41388,"佬":41389,"ĠInterface":41390,"ĠPittsburgh":41391,"Ġà¤ī":41392,"äºĨè¿ĩæĿ¥":41393,"enie":41394,"å¦Ĥæŀľä¸įæĺ¯":41395,"Ġentropy":41396,"Ġthankful":41397,"ĠGust":41398,"æ±Ĥè§£":41399,"arently":41400,"optional":41401,"ικÏĮÏĤ":41402,"ÑĮе":41403,"è¿ĺ羣":41404,"vl":41405,"Ġhemat":41406,"Ġillnesses":41407,"ĠÑĩиÑģла":41408,"iasis":41409,"Ġsegmentation":41410,"Ġmaker":41411,"Ġnewborn":41412,"ê¸Ī":41413,"ç¼´è´¹":41414,"Ġballoon":41415,"横åIJij":41416,"ilage":41417,"ĠGren":41418,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":41419,"Ġcomunic":41420,"ĠNum":41421,"çļĦèĮĥåĽ´":41422,"æĿijéĩĮ":41423,"ä¸Ńåħ±ä¸Ń央":41424,"วม":41425,"helm":41426,"Ġawful":41427,"ï¼ĸ":41428,"Ġarena":41429,"Ġembargo":41430,"Ġcasos":41431,"wic":41432,"ĠIranian":41433,"ä¹°çļĦ":41434,"Ġgalaxy":41435,"Engineering":41436,"-mark":41437,"殿ä¸ĭ":41438,"ĠобÑĢазоваÑļем":41439,"Ġdecentral":41440,"ман":41441,"Phil":41442,"ĠDivide":41443,"å¼Ģå¹ķ":41444,"ĠUC":41445,"Ġpaired":41446,"Ġpoured":41447,"patic":41448,"-service":41449,"ĠbyÄĩ":41450,"éģĩåΰäºĨ":41451,"hole":41452,"èµ·ä¹ī":41453,"Ġextr":41454,"Ġoneself":41455,"Ġmpandray":41456,"é¢Ĭ":41457,"å¾Īå°ı":41458,"arming":41459,"ĠплоÑīа":41460,"Expression":41461,"çĤ«":41462,"Ġepile":41463,"Ġkab":41464,"ĠPeel":41465,"ÙĪØ§Ø¹":41466,"Gre":41467,"ä¹ĭæĹ¥èµ·":41468,"Ġà¤ĩ":41469,"/I":41470,"Ġowns":41471,"Ġministers":41472,"опи":41473,"åĪĽæĸ°çļĦ":41474,"rops":41475,"ĠZhao":41476,"Ġpercept":41477,"ä¸īåįĥ":41478,"åķŁ":41479,"gos":41480,".archive":41481,"人类çļĦ":41482,"ç͵ç«Ļ":41483,"Ter":41484,"Ġbail":41485,"sheet":41486,"Ġbacking":41487,"ĠSau":41488,"ĠMeter":41489,"éļĭ":41490,"åŃ¦ä¹łåĴĮ":41491,"å®ģæ³¢":41492,"jÅ¡ÃŃ":41493,"ĉconst":41494,"ç»Ĩèĩ´":41495,"Ġreleasing":41496,"åı¯å¾Ĺ":41497,"Ġxx":41498,"ĠShi":41499,"Ġreligions":41500,"ĠPedro":41501,"ĠIncreased":41502,"ĉvoid":41503,"Ġcapitalism":41504,"ĠìĪ":41505,"Ġcuisine":41506,"Ġpubblic":41507,"coh":41508,"Ġwt":41509,"زÙħ":41510,"Ġsummit":41511,"Ġviene":41512,"ĠAlpha":41513,"BY":41514,"Ġpear":41515,"à¹Ħม":41516,"Ġbert":41517,"Ġextracts":41518,"rÃŃa":41519,"irmingham":41520,"hadap":41521,"-high":41522,"parameter":41523,"ÛĮدÙĩ":41524,"é£İåIJ¹":41525,"Ġinstalling":41526,"Ġbanned":41527,".getElement":41528,"éĩİçĶŁ":41529,"Ġfathers":41530,"ĠDEC":41531,"ĠRidge":41532,"ĠTeach":41533,"Ġott":41534,"è´Łè½½":41535,"িয়া":41536,"Ġwool":41537,"ĠArbe":41538,"æīĵåį¡":41539,"åı¯ä»¥å°Ĩ":41540,"yu":41541,"åĩºèĩª":41542,"orig":41543,"æĤį":41544,"/de":41545,"Ġinvaluable":41546,"弹簧":41547,"ĠStarting":41548,"Rule":41549,"Ġorbital":41550,"final":41551,"Ġconced":41552,"ä»ĸè¦ģ":41553,"Ġcaptures":41554,"ĠMunicipal":41555,"_ARG":41556,"ĠизвеÑģÑĤ":41557,"ĠÑĩаÑģÑĤо":41558,"è¿ĩæĿ¥çļĦ":41559,"éģĩè§ģ":41560,"Ġmandate":41561,"ä¸įèĤ¯":41562,"=\"./":41563,"ä¼ĺéĽħ":41564,"ãĥľ":41565,"uchen":41566,"border":41567,"angen":41568,"leted":41569,"Ġcertainty":41570,"ĠÑħаÑĢакÑĤеÑĢи":41571,"ĠWeekly":41572,"ĠRaf":41573,"大åİħ":41574,"ä¼ģä¸ļå®¶":41575,"Ġalterations":41576,"æĪijä¸įæĺ¯":41577,"ĠInj":41578,"就好äºĨ":41579,"менно":41580,"Mary":41581,"äºĨä¸ĢåĢĭ":41582,"âĶĤ":41583,"task":41584,"ulle":41585,"æģ³":41586,".This":41587,"èĩªæĦ¿":41588,"Ġspacing":41589,"æĸ¯åŁº":41590,"Subjects":41591,"Policy":41592,"Het":41593,"}package":41594,"mma":41595,"ĠSett":41596,"Ġsolic":41597,"Ġecc":41598,"æ´ĹåĩĢ":41599,"Ġpredecess":41600,"idikan":41601,"ĠReich":41602,"çļĦéĢļçŁ¥":41603,"оÑĤи":41604,".Length":41605,"Ġsurviving":41606,"iev":41607,"æĢ»çļĦ":41608,"çĺĢ":41609,"Their":41610,"è¿Ļ个ä¸ĸçķĮ":41611,"Ġnast":41612,"Altern":41613,"ابÙĦ":41614,"auth":41615,"#import":41616,"Ġbras":41617,"Ġdeform":41618,"ESCO":41619,"Ġ×Ķס":41620,"æĽ¹æĵį":41621,"åIJĦæĸ¹":41622,"ارة":41623,"ĠDetection":41624,"edo":41625,"Ġunified":41626,"æľĪç»ı":41627,"èĮĥæĸĩ":41628,"_image":41629,"etter":41630,"Ġprat":41631,"ç£ħ":41632,"lysis":41633,"Ġcreator":41634,"Ġdeceased":41635,"Dictionary":41636,"Ġcredentials":41637,"Ġlazy":41638,"Ġciudad":41639,"ĠGrande":41640,"æĹ¶ç©º":41641,"Ġainda":41642,"ĠdB":41643,"çļĦåı¯èĥ½æĢ§":41644,"ï¼įï¼į":41645,"åĴ§":41646,"ĠVlad":41647,"ök":41648,".en":41649,"Ġrape":41650,"Gal":41651,"ç®ĹäºĨ":41652,"}}}":41653,"ãģ©ãģĨ":41654,"plementation":41655,"Ġê°ķ":41656,"над":41657,"Ġlightning":41658,"ĠرÙĪØ²":41659,"Normal":41660,"(std":41661,"ĠTrig":41662,"Ġintuitive":41663,"_item":41664,"æĪĺæľ¯":41665,"Ġowing":41666,"ãĤģãĤĭ":41667,"Ġunchanged":41668,"ĠبÛĮشتر":41669,"æĭĤ":41670,"Ġpresumably":41671,"Ġtoda":41672,"quisition":41673,"ĠFemale":41674,"Ġtraff":41675,"åζå¤ĩ":41676,"uzione":41677,"-read":41678,"Ġesse":41679,"-do":41680,"{u":41681,"èĶļ":41682,"zioni":41683,"ĠDit":41684,"éªij士":41685,"Ġtherein":41686,"ILITY":41687,"ĠGabriel":41688,"ĠTCP":41689,"äºĮçϾ":41690,"ĠSak":41691,"éģĹåĿĢ":41692,"ĠÑĤÑĥ":41693,"Ġsino":41694,"ĠÑįÑĤоÑĤ":41695,"Ġmentor":41696,"ĠPret":41697,"ĠCharter":41698,"ĠCountries":41699,"Ġlleg":41700,"æīĭåĨĮ":41701,"Ġverses":41702,"pta":41703,"åīįè¾Ī":41704,"ophy":41705,"éĹ´éļĻ":41706,"ĠÙħجÙħÙĪ":41707,"ĠEF":41708,"ecil":41709,"god":41710,"à¸ģิà¸Ī":41711,"Åijl":41712,"/sh":41713,"UTION":41714,"ĠMarcus":41715,"Rank":41716,"Ġarose":41717,"flies":41718,"éĥ½å·²ç»ı":41719,"rames":41720,"ĠRogers":41721,"åľ°åľ¨":41722,"ĠModeling":41723,"ensi":41724,"ĠPerry":41725,"ä¸Ģç»Ħ":41726,"ä¸»æľº":41727,"管çIJĨèĢħ":41728,"rowning":41729,"ificar":41730,"ç͵ç½ij":41731,"Ġaxes":41732,"åŁºç¤İ":41733,"ä¸ĩä¸Ģ":41734,"ĠíķĦ":41735,"Ġadvertis":41736,"ĠGew":41737,"osome":41738,"æ²Ļæ¼ł":41739,"_score":41740,"çļĵ":41741,"çļĦåĴĮ":41742,"åIJĪåĬĽ":41743,"Ġparl":41744,"}\\).ĊĊ":41745,"ĠáĥĽ":41746,"Ġcollectively":41747,"Services":41748,"Jesus":41749,"èĤłéģĵ":41750,"Ġlitigation":41751,"[][]":41752,"Ġllam":41753,"estial":41754,"Ġoccupational":41755,"æį®äºĨè§£":41756,"ĠWiktionary":41757,"Ol":41758,"沿海":41759,"ä¸ĢåĬ¨":41760,"Ġcompressed":41761,"ÑĨенÑĤ":41762,"ĠReed":41763,"çľ¼åīįçļĦ":41764,"ĠRas":41765,"chin":41766,"Ġdenial":41767,"æĪªéĿ¢":41768,"åĬłè½½":41769,"Ġ×Ķש":41770,"éł»":41771,"ä¸į为":41772,"ĠYale":41773,"注æĺİ":41774,"Ġdelivers":41775,"ÃŃr":41776,"Progress":41777,"Ñĺи":41778,"-items":41779,"Ġpoco":41780,"å¯ĨçļĦ":41781,"à¸ŀัà¸Ļ":41782,"Ġtranslations":41783,"ĠCot":41784,"ä¸ŃéĢīæĭ©":41785,"Ľ×ª":41786,"ä¿®åħ»":41787,"tk":41788,"Thursday":41789,"خداÙħ":41790,"ئÙĬس":41791,"Books":41792,".date":41793,"Ġfinances":41794,"Ġculinary":41795,".dat":41796,"ĠDyn":41797,"ĠاÙĦبر":41798,"ãĥĦ":41799,"æŃ¤å¤Ħ":41800,"Ñīен":41801,"ĠTopic":41802,"--ĊĊ":41803,"Ġtravelers":41804,"æĻ¯è±¡":41805,"Ġhammer":41806,"jas":41807,"Ġmang":41808,"EU":41809,"foreach":41810,"æ¬§çĽŁ":41811,"æĪijåĽ½çļĦ":41812,"......âĢĿ":41813,"ifica":41814,"Ġpencil":41815,"کاÙĨ":41816,"Ġexagger":41817,"æľ½":41818,"ituation":41819,"ĠMik":41820,"нев":41821,"çͲæĸ¹":41822,"Ġepithelial":41823,"Contract":41824,".Id":41825,"Ġminimizing":41826,"Ġstarter":41827,"resse":41828,"Ġrag":41829,"izophrenia":41830,"Ċ":42797,"Bill":42798,"èĥĮæĻ¯ä¸ĭ":42799,"Lic":42800,"Ġpacking":42801,"ubic":42802,"-ins":42803,"ĠBass":42804,"æĪijäºĨ":42805,"æ²Ľ":42806,"enson":42807,"ĠMilan":42808,"Ġanda":42809,"ĠØ£ÙĨÙĩ":42810,"Ġlistener":42811,"footer":42812,"åĽĽåŃ£":42813,"åĪĽåĬŀ":42814,"è´ŁéĿ¢":42815,"è¿ģç§»":42816,"ulators":42817,"çľŁç©º":42818,"çĦ¶çļĦ":42819,"çł´ç¢İ":42820,"erton":42821,"ÑĤемаÑĤи":42822,"ĠFilip":42823,"Ġmakeup":42824,"ĠÙĦÙĪ":42825,"ACS":42826,"Ġ×IJ׾":42827,"stad":42828,"åįłç͍":42829,"Ġparagraphs":42830,"ĠÑī":42831,"ĠHudson":42832,"ĠBaptist":42833,"Ġvide":42834,"Ĺ×§":42835,"æīĢæľī人éĥ½":42836,"material":42837,"Ġbenz":42838,"Ġست":42839,"%~":42840,"ÑĪаÑı":42841,".empty":42842,"ä¸įæĸ·":42843,"æľīæľĽ":42844,"童年":42845,"Ġ,Ċ":42846,"Ġcitt":42847,"æīĵéĩı":42848,"Ġcrist":42849,"©ëĭĪëĭ¤":42850,"лий":42851,"visited":42852,"ĠÙĩÙĨا":42853,"-learning":42854,"å¤ļå®¶":42855,"atorial":42856,"நà¯įத":42857,"çļĦäºĨ":42858,"ĠобÑıза":42859,"usage":42860,"диÑĤе":42861,"rimp":42862,"ĠWikimedia":42863,"åīįåĪĹ":42864,"etrics":42865,"Ġë§İ":42866,"ĠMaz":42867,"Ġrespects":42868,"è¯ı":42869,"æĪijåıª":42870,"åºĶ注æĦı":42871,"modern":42872,"ĠCrisis":42873,"ä¸īæľĪ":42874,"ĠAns":42875,"以ä¸ĭç®Ģç§°":42876,"åĵ®":42877,"สุà¸Ķ":42878,"hart":42879,"arus":42880,"èħ¾è®¯":42881,"à¸Ńาà¸Ī":42882,"arie":42883,"ographers":42884,"letics":42885,"owi":42886,"COM":42887,"Ġmasa":42888,"åĭī强":42889,"篷":42890,"objects":42891,"aksi":42892,"ĠContinue":42893,"(line":42894,"Unfortunately":42895,"éľĩèį¡":42896,"ĠPOL":42897,"emd":42898,"æĭīæĸ¯":42899,"çĿĢä¸Ģ个":42900,"æľīæľºä¼ļ":42901,"oblast":42902,"ĠдеÑıÑĤелÑĮноÑģÑĤи":42903,"upper":42904,"æĪij们ä¼ļ":42905,"Ġjel":42906,"ĠTot":42907,"ordin":42908,"ä½łåı¯":42909,"Ġara":42910,"ĠклаÑģÑģ":42911,"okin":42912,"çī¢åĽº":42913,"æĪĸèĢħ说":42914,"Ġmistaken":42915,"Ġrecession":42916,"Ġadhere":42917,"ĠLiteracy":42918,"ĠApost":42919,"ä¸ºæł¸å¿ĥ":42920,"ĠCOR":42921,"ाद":42922,"Visual":42923,"Ġimpedance":42924,"çķ¶æĻĤ":42925,"Ġcows":42926,"Ġseized":42927,"ĠاÙĦÙħؤ":42928,"otechn":42929,"ĠTamb":42930,"éįµ":42931,"ptide":42932,"çħĻ":42933,"Ġmercury":42934,"gens":42935,"Ġ(`":42936,"æĹłå½¢":42937,"loop":42938,"Wrapper":42939,"éĶĤ":42940,"段çļĦ":42941,"Ġugly":42942,"}}(":42943,"å¼ĤæĢ§":42944,"èļĬ":42945,"è¿ĻåĦ¿":42946,"æĽ¿æį¢":42947,"ración":42948,"æĿ¥è¿Ľè¡Į":42949,"ugu":42950,"Pot":42951,"Ġparser":42952,"Ġdisasters":42953,"links":42954,"Ġconventions":42955,"èĹ©":42956,"Sunday":42957,"Ġ\"\"Ċ":42958,"ĠبÛĮÙħ":42959,"лÑĮнаÑı":42960,"没æľīäºĨ":42961,"Ġtribute":42962,"Ġbelly":42963,"اعات":42964,"Counter":42965,"Ġinh":42966,"åĨįä¹Ł":42967,"itching":42968,"æĿĨèıĮ":42969,"اÙĨÙĪÙĨ":42970,"Ġlisteners":42971,"Ġindividu":42972,"Ġíı¬":42973,"ä½ı宿":42974,"à¾":42975,"lining":42976,"æĹ¥åľ¨":42977,"Ġcharter":42978,"à¸ļุ":42979,"Ġpenetration":42980,"éĥĿ":42981,"éĽį":42982,"Ġstadium":42983,"plik":42984,"æ±Ľ":42985,"åħŃå¹´":42986,"Ġfaç":42987,"ë¦Ħ":42988,"Ġlar":42989,"ĠBehind":42990,"ĠRav":42991,"ética":42992,"immune":42993,"ãĤīãĤĮãĤĭ":42994,"ải":42995,"Ġvier":42996,"antu":42997,"Ġseule":42998,"Ġplusieurs":42999,"zan":43000,"å®¶åĽŃ":43001,"å°Ĩäºİ":43002,"======":43003,"Ġtoujours":43004,"磷éħ¸":43005,"ÙĪØ©":43006,"alar":43007,"ĠFul":43008,"ваÑİÑĤÑģÑı":43009,"æį®æĤī":43010,"Ġreportedly":43011,"ê²°":43012,"ĠSine":43013,"اÙĬرÙĩ":43014,"Ġâ̦Ċ":43015,"aphyl":43016,"abile":43017,"ιÏĥ":43018,"manager":43019,"Middle":43020,"æīĭä¸ĭ":43021,"Ġکتاب":43022,"Ġdiplomatic":43023,"åijķåIJIJ":43024,"Ġmatang":43025,"æĢİä¹ĪäºĨ":43026,"ाव":43027,"ĠاÙĦعÙħÙĦ":43028,"çģ¾éļ¾":43029,"Ġmaximal":43030,"à§Į":43031,"Ġmotions":43032,"chanical":43033,"Ġdiscovering":43034,"ĠPrepare":43035,"ât":43036,"Ġinnings":43037,"èį·åħ°":43038,"Register":43039,"ĠBasin":43040,"ĠHalloween":43041,"Hence":43042,"ĠHolmes":43043,"åĨįåİ»":43044,"Ġhorrible":43045,"Ġrecurrence":43046,"vä":43047,"osse":43048,"é«ĺéĢŁåħ¬è·¯":43049,"Ġpendant":43050,"keiten":43051,"icides":43052,"æ²ī积":43053,"íĤ¤":43054,"æĩµ":43055,"ĠÑĤÑĭÑģÑı":43056,"åĨĻäºĨ":43057,"økt":43058,"колÑĮ":43059,"Ġprognosis":43060,"Cong":43061,"ãģĭãĤĬ":43062,"czas":43063,"ĠÙħÙĪØ¬":43064,"Ġhydroph":43065,"برÙĩا":43066,"ĠGott":43067,"ĠDirectors":43068,"Ġchez":43069,"Ġangl":43070,"å»īæ´ģ":43071,"ĠСо":43072,":int":43073,"ÙĩÙĦ":43074,")),Ċ":43075,"Τ":43076,"è¶ħåĩº":43077,"ĠIPv":43078,"京åŁİ":43079,"羣å®ŀçļĦ":43080,"被æīĵ":43081,"ãģ¨ãĤĤ":43082,"Ġdiplom":43083,"})ĊĊ":43084,"Ġ(=":43085,"ç¥ŀè¯Ŀ":43086,"Ġkons":43087,"ĠTransition":43088,"ĠLogic":43089,"åľ¨æł¡":43090,"ลัà¸ĩ":43091,"Ġheterogeneous":43092,"mare":43093,"ĠVS":43094,"ysk":43095,"\"`Ċ":43096,"]ãĢĤ":43097,"Ġвнима":43098,"Ġcuenta":43099,"çļĦåģļæ³ķ":43100,"argest":43101,"åĢļ":43102,"olph":43103,"åĪĹ车":43104,"services":43105,"Ġevaluations":43106,"ĠBD":43107,"rst":43108,"à¥Į":43109,"Ġgenres":43110,"ĠÐŁÐ¾Ð´":43111,"Ġhiking":43112,"åΰ大":43113,"Ġlesion":43114,"ĠMaintenance":43115,"ĠMaximum":43116,"«çĹ":43117,"лок":43118,"Camb":43119,"Ġتت":43120,")}\\":43121,"igens":43122,"ĠPeru":43123,"çĽı":43124,"çħ©":43125,"Ġتض":43126,"蹦":43127,"ĠEUR":43128,"éĥ½è¯´":43129,"Ġdevil":43130,"iterator":43131,"diff":43132,"ĠEssential":43133,"Ġsensible":43134,"ĠZhu":43135,"OUND":43136,"Tuesday":43137,"Ġcompares":43138,"åĽ¾è¡¨":43139,"edic":43140,"Ġï¼īĊĊ":43141,"Ġmigrants":43142,"ĠCarn":43143,"parents":43144,"ä¸Ńå¤ĸ":43145,"conv":43146,"åĬ³åĬ¨åIJĪåIJĮ":43147,"'))Ċ":43148,"onces":43149,"Ġbronze":43150,"ĠDP":43151,"ãĢĭãĢģ":43152,"人æľī":43153,"Ġmenggunakan":43154,"ç»Ī身":43155,"榮":43156,"peg":43157,"Ġelectoral":43158,"Ġalbums":43159,"ĠDF":43160,"說話":43161,"éĶ®çĽĺ":43162,"çĮĽåľ°":43163,"Ġpeel":43164,"halb":43165,"Ġsteadily":43166,"pex":43167,"Ġreflective":43168,"ÙIJÙĨ":43169,"radesh":43170,"ä¹¾éļĨ":43171,"lakan":43172,"raska":43173,"xico":43174,"ĠÃĦ":43175,"HY":43176,"{L":43177,"Ġyearly":43178,"ĠGarcia":43179,"Ġcompartment":43180,"лÑĮное":43181,"Ġallergic":43182,"ĠRect":43183,"度åģĩ":43184,"Available":43185,"Files":43186,"iros":43187,"Ġfrank":43188,"Ġdamaging":43189,"ULTS":43190,"å¦Ĵ":43191,"Ġformulated":43192,"è¡Ģæ¸ħ":43193,"Ġbrightness":43194,"Ġpseudo":43195,"ocaust":43196,"Fire":43197,"åľĨæŁ±":43198,"ĠÙħعÙĦ":43199,"Ġvague":43200,"为ä»ĸ":43201,"ä¼ļ使":43202,"umbai":43203,"ÙĦÙĬÙĩ":43204,"客æĪ·ç«¯":43205,"ĠHorse":43206,"cke":43207,"ï¼Łï¼Ī":43208,"Ġhandles":43209,"evity":43210,"è¯Ħ为":43211,"Ġinterrupted":43212,"ROP":43213,"ĠAlcohol":43214,"Ġexemplary":43215,"廳":43216,"iper":43217,"Ġkond":43218,"Ġdoubled":43219,"çĨŁæĤīçļĦ":43220,"活泼":43221,"å®łçī©":43222,"è¦ıå®ļ":43223,"noun":43224,"!\\":43225,"üler":43226,"Watch":43227,"Ġclamp":43228,"ç·£":43229,"à¹Ģหล":43230,"river":43231,"Arm":43232,"_cl":43233,"éłħ缮":43234,"à¸ķุ":43235,"æľīä½Ļ":43236,"Ġappre":43237,"ĠAreas":43238,"á¹£":43239,"æĪij没":43240,"ä¸į好çļĦ":43241,"ĠíĽĦ":43242,"ĠÑģÑĤи":43243,"é«ĺä¸ī":43244,"è£Ķ":43245,"æ¯ıä¸Ģä½į":43246,"ÙĦÙĬÙĦ":43247,"{-":43248,"è¿Ļèά":43249,"ĠSilva":43250,"ĠGH":43251,"å½ĵæĪIJ":43252,"Ġpastor":43253,"åı£ä¸Ń":43254,"Ġretire":43255,"æ±īåŃĹ":43256,"Ġrecommends":43257,"表çİ°åľ¨":43258,"ĠJoshua":43259,".url":43260,"Ġtraumatic":43261,"Ġtransformative":43262,"Ġtrash":43263,"াà¦ļ":43264,"巴西":43265,"ĠHolid":43266,"æ·Ģç²ī":43267,"Ġпода":43268,"ĠTHAT":43269,"éļ¾å¾Ĺ":43270,"Ġclimbed":43271,"ाय":43272,"Ġetern":43273,".":43386,"×ķ×ĸ":43387,"Ġsuccessor":43388,"Ġdelighted":43389,"âĢĻï¼Į":43390,"æ¯ĶèµĽä¸Ń":43391,"Ġcomposer":43392,"ĠDriver":43393,"castle":43394,"عات":43395,"ĠVeget":43396,"STRACT":43397,"Ġhelper":43398,"jos":43399,"umas":43400,"Ġfinest":43401,"輩":43402,"ĠEcology":43403,"Ġrównież":43404,"èµ°è¿ĩ":43405,"Ġgravitational":43406,"edes":43407,"Ġtenth":43408,"ĠWave":43409,"ĠExcept":43410,"çļĦå¼Ģ":43411,"Ø´ÙĨ":43412,"ĠSek":43413,"çķĻä¸ĭäºĨ":43414,"ì§ģ":43415,"Ġtastes":43416,"enser":43417,"Ġcorpus":43418,"ĠExport":43419,"Ġstorms":43420,"Ġendothelial":43421,"湯":43422,"å¤ĦçIJĨåύ":43423,"ällor":43424,"Ġrevelation":43425,"اÛĮت":43426,".height":43427,"æĶĿ":43428,"одов":43429,"sek":43430,"Ġмоде":43431,"Ġtwisted":43432,"ä¸į让":43433,"Ġsew":43434,"Ġcommittees":43435,"ĠButler":43436,"Ġmarvel":43437,"Ġì¶ľ":43438,"Ġalleviate":43439,"ĠÙħÙĩÙħ":43440,"ÑĨиали":43441,"Ġbreeze":43442,"åĨįä¸Ģ次":43443,"åī§æľ¬":43444,"ystems":43445,"ĠMicrobiol":43446,"ĠRein":43447,"Ġеди":43448,"-controlled":43449,"portional":43450,"ĠKash":43451,"онов":43452,"Ġdzieci":43453,"à¸ı":43454,"\\({}^{-":43455,"ĠGalaxy":43456,"æ¾³éŨ":43457,"ĠÙĪÙĬÙĥÙĬÙ¾ÙĬدÙĬا":43458,"Ġnelle":43459,"Ġpredators":43460,"Ġbek":43461,"ær":43462,"çļĦãģª":43463,"äºĨä¸Ģ次":43464,"åIJįä¹ī":43465,"ĠResolution":43466,"ä¹ĥæĺ¯":43467,"çϽèī²çļĦ":43468,"iliters":43469,"Ġrefund":43470,"Ġhostile":43471,"Ġnouve":43472,"Ġdefending":43473,"èĭ±èªŀ":43474,"ĠFailure":43475,"æĺ¥åŃ£":43476,"Ġmiser":43477,"Ġterminals":43478,"Remove":43479,"åı°åĮĹ":43480,"Õ¡Öģ":43481,"Ġproceso":43482,"ĠDD":43483,"Ġвек":43484,"å°ıæķ°":43485,"å¾Ĺä¸Ĭ":43486,"Digital":43487,"åĩºç¤º":43488,"ä¼ģä¸ļåĴĮ":43489,"ĠEinstein":43490,"Ġrunner":43491,"ãĥĥãĥĹ":43492,"Dat":43493,"ĠVehicle":43494,"Ġautre":43495,"ÑĪим":43496,"prints":43497,"ĠEen":43498,"æĪij们æĺ¯":43499,"æ¿ĥ":43500,"å°ı麦":43501,"åıªæľīåľ¨":43502,"éĹªçĥģ":43503,"Ġchef":43504,"structure":43505,".has":43506,"ĠHistor":43507,"colo":43508,"ĠCompared":43509,"rase":43510,"upun":43511,"Ġundertake":43512,"δÏģ":43513,"Ġinterviewed":43514,"æĺĨæĺİ":43515,"Ġ(.":43516,"anos":43517,"åIJijå¾Ģ":43518,"åĪ©äºİ":43519,"Ġsurpass":43520,".Component":43521,"Ġleer":43522,"éĽ¯":43523,"Ġslots":43524,"å®ŀçī©":43525,"ĠVictorian":43526,"ymi":43527,"ĠResistance":43528,"opia":43529,"å¢ĥå¤ĸ":43530,"ĠInteractive":43531,"Ġapprove":43532,"Ġwarfare":43533,"Ġpam":43534,"éĢĢå½¹":43535,"irtschaft":43536,"-cost":43537,"visors":43538,"umed":43539,"åħ¨æĸ°":43540,"ĠпÑĢедпÑĢи":43541,"Ġscent":43542,"亲æĪļ":43543,"åı¯è¾¾":43544,"ĠAllow":43545,"ĠNetflix":43546,"çļĦéĩij":43547,"åĴı":43548,"pecial":43549,"geben":43550,"анд":43551,"ĠSkin":43552,"(ãĢĬ":43553,"ĠпоÑħа":43554,"对èĩªå·±çļĦ":43555,".ãĢĬ":43556,"Ġorden":43557,"çĿĢæĪij":43558,"à§įযান":43559,"Ġtimber":43560,"সà§įত":43561,"診":43562,"å͝æľī":43563,"contract":43564,"åĴĮæĸ¹æ³ķ":43565,"ç²¾çģµ":43566,"Ġikke":43567,"ĠHels":43568,"身é«ĺ":43569,"éĢĻ麽":43570,"ĠDisorders":43571,"":44611,"ĠGamb":44612,"à¸Ķà¹īาà¸Ļ":44613,"ĠTD":44614,"ĠProtestant":44615,"çĶº":44616,"Ġmethodologies":44617,"Interval":44618,"Ġmasters":44619,"রà§įত":44620,"ä¸Ģèµ·åİ»":44621,"ÐĴÑĭ":44622,"åŃ¦ä¹łæķĻèĤ²":44623,"rott":44624,"Ġlys":44625,"Scale":44626,"Ġtribal":44627,"Ġintercept":44628,"done":44629,"èĩ³åħ³":44630,"Changes":44631,"cula":44632,"ĠвоздÑĥ":44633,"è¿ijä¼¼":44634,"Ġgovernmental":44635,"Ġinspir":44636,"Ġnovember":44637,"Õ¸Õ¾":44638,"Ġreductions":44639,"奴éļ¶":44640,"Ġequitable":44641,"Ġdesenvolv":44642,"æľĪä¸Ń":44643,"اسة":44644,"Ġlifelong":44645,"Ġtomb":44646,"åºŁçī©":44647,"Ġmagnificent":44648,"Ġjealous":44649,"AAA":44650,"encias":44651,"ĠJD":44652,"ç´Ĭ":44653,"তà§įত":44654,"à¸ĺี":44655,"Ġ×ijת×":44656,"Apr":44657,"ieux":44658,"DOCT":44659,"rache":44660,"avorite":44661,"ë°±":44662,"érés":44663,"ĠClinic":44664,"ĠDiamond":44665,"ĠMETHOD":44666,"ë¶ĢíĦ°":44667,"æŃ¹":44668,"åĵŁ":44669,"usammen":44670,"ä¿Ŀå¯Ĩ":44671,"flows":44672,".the":44673,"chard":44674,".xml":44675,".Z":44676,"ĠСи":44677,"ĠاÙĦÙħستÙĤ":44678,"Òĵ":44679,"çĿĢçľ¼":44680,"Ġdilakukan":44681,"çĶŁæ´»åľ¨":44682,"çݯå¢ĥä¸Ń":44683,"Ġdefinitive":44684,"çļĦ两个":44685,"izado":44686,"éĤ£æł·çļĦ":44687,"uded":44688,"åĵº":44689,"ÑĤÑĢен":44690,"Ġacidic":44691,"ĠBirmingham":44692,"ĠÑĥни":44693,"afood":44694,"ĠHass":44695,"azo":44696,"à§Ģয়":44697,"é²ľè¡Ģ":44698,"æĹ¥åĨĽ":44699,"书åºĹ":44700,"NotFound":44701,"bind":44702,"ĠIcon":44703,"Ġenrollment":44704,":h":44705,"è¶Ĭæĺ¯":44706,"ĠAccept":44707,"Ġmolar":44708,"apest":44709,"ĠGoth":44710,"ãĤ¶":44711,"丽çļĦ":44712,"gonal":44713,"å®ŀ践活åĬ¨":44714,"Ġcontextual":44715,"Ġaltri":44716,"éĽŀ":44717,"Ġautomobile":44718,"Ġprzyp":44719,"Ġй":44720,"åIJİæĸ¹":44721,".lang":44722,"ĠØ´ÙĨ":44723,"ä¾µçķ¥":44724,"ãĢij**":44725,"ä¿Ń":44726,"Ġestabl":44727,"Ġcirculating":44728,"ragen":44729,"Ġkay":44730,"à¸Ļิ":44731,"æĮĩçĿĢ":44732,"第äºĮèĬĤ":44733,"ĠCities":44734,"ĠMeyer":44735,"ä¸ĭå±ŀ":44736,"-data":44737,"æĮīä¸ĭ":44738,"ابع":44739,"RODUCTION":44740,"ä¸įæĺ¯å¾Ī":44741,"Ġfir":44742,"Ġcompile":44743,"åIJĮå¹´":44744,"æ¦Ĥè¿°":44745,"Ġfibre":44746,"çļĦç¨ĭ度":44747,"Ġsurrender":44748,"Ġlimbs":44749,"åľ°éģĵ":44750,"Ġfeminist":44751,"ại":44752,"\"{":44753,"sworth":44754,"ĠоÑģнова":44755,"æĹ¶å¸¸":44756,"çͱæĸ¼":44757,"ĠÑĥÑĩеб":44758,"æ¶²åİĭ":44759,"ĠFlorence":44760,"Ы":44761,"æĢ»å±Ģ":44762,"å¸ĪåħĦ":44763,"ç²¾èĩ´":44764,"ĠDong":44765,"Ġ\"${":44766,"ĠKw":44767,"亲åŃIJ":44768,"Ġtenant":44769,"stellung":44770,"Ġoktober":44771,"ä»ĭçŁ³":44772,"idian":44773,"Ġpoles":44774,"ĠWalt":44775,"Pattern":44776,"Ġwilderness":44777,"ä¸Ĭç½ij":44778,"ĠCRC":44779,"-edge":44780,"å¾Ĺä¸įåΰ":44781,"æĻ®éĢļ人":44782,"Spanish":44783,"ä¼łåĬ¨":44784,"ĠSimpl":44785,"ĠLloyd":44786,"Far":44787,"inae":44788,"video":44789,"è¯Ŀè¯Ń":44790,"èİ·å¾ĹçļĦ":44791,"ĠاÙĨجاÙħ":44792,"erl":44793,"Ġà¦ķà§ĭন":44794,"Ġíİ":44795,"çѾåIJį":44796,"ĠTyler":44797,"ĠDaw":44798,"Ġcompart":44799,"Ġsimulate":44800,"Ġmitigation":44801,"Ġhierarchical":44802,"è°ĭåĪĴ":44803,",P":44804,"ĠLambda":44805,"STM":44806,"å§Ĩæĸ¯":44807,"usually":44808,"Ġlonely":44809,"Ġnarrator":44810,"åĤ¾":44811,"uesto":44812,"odia":44813,"佩æĪ´":44814,"ÑĪки":44815,"Ġangels":44816,"人æīįåŁ¹åħ»":44817,"ĠEuler":44818,"Ġqualification":44819,"Princ":44820,"rile":44821,"δα":44822,"ĠKlein":44823,".Value":44824,"æģ°å½ĵ":44825,"Ġmah":44826,"ĠContainer":44827,"Ġtray":44828,"-link":44829,"æijĩäºĨæijĩ头":44830,"icha":44831,"lice":44832,"riages":44833,"Ġcustomary":44834,"æĶ¹éĿ©çļĦ":44835,"Ġfortunate":44836,"çIJ¢":44837,"ãģªãģĮ":44838,"ãĥĨãĤ£":44839,"اÙĤØ©":44840,"æ¬ºè´Ł":44841,"åĸ§":44842,"ĠClip":44843,"Ġmidnight":44844,"è¿ĩ大":44845,"prefix":44846,"ุà¹Īม":44847,"Ġresultado":44848,"Ġbush":44849,"ÑĨией":44850,"Ġnacional":44851,"Ġantagon":44852,"Side":44853,"ÙĤدÙħ":44854,"ĠпоÑĤÑĢеб":44855,"Ġfinancially":44856,"ĠPreparation":44857,"zem":44858,"ĠDé":44859,"ĠWere":44860,"åĽ½åľŁ":44861,"ĠØ¢ÙħÙĪØ²Ø´":44862,"被迫":44863,"身ä½ĵåģ¥åº·":44864,"åĮĹéĥ¨":44865,"æĬµè¾¾":44866,"Ġvista":44867,"Ġzach":44868,"Resources":44869,"Ġseparating":44870,"ì¸":44871,"isot":44872,"çĶĦ":44873,"ĠGonz":44874,"ĠпÑĢоÑģÑĤо":44875,"ĠصÙĪØ±Øª":44876,"ortal":44877,"è½®å»ĵ":44878,"æµıè§Īåύ":44879,"uvre":44880,"Ġদà§ĩà¦ĸ":44881,"Ġaugust":44882,"Ġbracket":44883,"à¯Ĥ":44884,"Management":44885,"itone":44886,"å¾µ":44887,"columns":44888,"ĠKällor":44889,"两边":44890,"รี":44891,"大åѦåĩºçīĪ社":44892,"åĽ½æ°ijç»ıæµİ":44893,"ĠConfiguration":44894,"æ°´èĤ¿":44895,"çŁ¥è¯ĨåĴĮ":44896,"Ġrisen":44897,".Model":44898,"Äħd":44899,"à±Ĩ":44900,"ŀ×ķת":44901,"ä¸Ńå¿ĥçļĦ":44902,"ĠExplorer":44903,"ÑģÑĤвиÑı":44904,"说起":44905,"ĠMeas":44906,"ĠبشÙĥÙĦ":44907,"à¸łà¸²à¸©":44908,"ĠÙĦØ£ÙĨ":44909,"Ġâμ":44910,"Energy":44911,"Ġunfortunate":44912,"Ġgoed":44913,"Ġseniors":44914,"èµ°ä¸Ĭ":44915,"iameter":44916,"Ġnn":44917,"Something":44918,"ĠFY":44919,"ĠStir":44920,"Ġà¦Ľà¦¿à¦²":44921,"ucay":44922,"ĠÙĨسب":44923,"ત":44924,"ĠListen":44925,"House":44926,"bread":44927,"×Ļת×":44928,"Ġskew":44929,"ĠпиÑĤа":44930,"Ġconverts":44931,"Ġlicensing":44932,"roscopic":44933,"èµ·ä¼ı":44934,"questions":44935,"ä¸įä»ħä»ħæĺ¯":44936,"ĠRB":44937,"ĠEg":44938,"Ġobjection":44939,"æĸĩ件夹":44940,"Ġwelcoming":44941,"ì¦Ŀ":44942,"Chart":44943,"ĠSuz":44944,"éĴ¢çIJ´":44945,"ĠWelsh":44946,"Ġجز":44947,"UX":44948,"æ¸Ŀ":44949,"ERC":44950,"ÑĤелÑĮнÑĭй":44951,"European":44952,"Ra":44953,"ä¸ĢåĪ»":44954,"å¯Įè£ķ":44955,"Ġlengthy":44956,"å°ıéķĩ":44957,"ä¸Ģå¤ľ":44958,"Ġresponds":44959,"ĠChin":44960,"Ġshrink":44961,"å¹¶ä¸įèĥ½":44962,"ĠBrothers":44963,"à°¦":44964,"pshire":44965,"âĿ¤":44966,"派人":44967,"Visit":44968,"Ġ리":44969,"Ġdiabetic":44970,"âĸĪâĸĪ":44971,"-par":44972,"ãģĻãĤĭãģ¨":44973,"ĠInfant":44974,"åıĪåľ¨":44975,"Ġbeast":44976,"Ġdois":44977,"Ġprompts":44978,"มูล":44979,"iffe":44980,"åĽºå®ļçļĦ":44981,"à¹Ģวล":44982,"riv":44983,"tem":44984,"Ġà¦ıর":44985,"haw":44986,"çļĦé¡¹çĽ®":44987,"èĩ³æŃ¤":44988,"ĠMultiply":44989,"ĠIndustries":44990,"esters":44991,"æ´Ĺ澡":44992,"鼷éĶĭ":44993,"Ġdiscoveries":44994,"Ġencompasses":44995,"Ġdebts":44996,"ului":44997,"scriber":44998,"ä¿®çIJĨ":44999,"Ġpromoter":45000,"aient":45001,"ENCES":45002,"ĠAur":45003,"èĤĺ":45004,"æķĻèĤ²å±Ģ":45005,"Ġcigarette":45006,"Ġbeats":45007,"endum":45008,"-water":45009,"antiago":45010,"æĦıä¹īä¸Ĭ":45011,"Ġtaké":45012,"Ġrituals":45013,"æ³ķåĴĮ":45014,"ĠExtension":45015,"é¢ģå¸ĥ":45016,"æĥ³äºĨæĥ³":45017,"Sa":45018,"å°ıäºĭ":45019,"Smith":45020,"Ġpockets":45021,"uncan":45022,"éĽĨä¸Ńåľ¨":45023,"enze":45024,"ousse":45025,"Ġfactories":45026,"Ġvanilla":45027,"æİ§åζåύ":45028,"âĢĵâĢĵ":45029,"Ġridiculous":45030,"Daniel":45031,"çļĦ计ç®Ĺ":45032,"Mrs":45033,"positions":45034,"ä¸ĢçĤ¹çĤ¹":45035,"counter":45036,"(message":45037,"Ho":45038,"çģ½":45039,"Ġsap":45040,"èµ°å»Ĭ":45041,"ĠPrevent":45042,"Ġforecasts":45043,"-source":45044,"å»¶è¿Ł":45045,"격":45046,"ĠFifth":45047,"ĠпопÑĥ":45048,"uteur":45049,"Ġbuses":45050,"Ġdisclose":45051,"è¶£åij³":45052,"review":45053,"หลาย":45054,"Ġcheeks":45055,"{bmatrix":45056,"-hydro":45057,"å¾Ĺä¸į":45058,"áĢŃá̝áĢ":45059,"tip":45060,"åĽ½åºĨ":45061,"çļĦ人éĥ½":45062,"Ġphysic":45063,"严åİī":45064,"Ġresistor":45065,"Ġmilligrams":45066,"XXX":45067,"Ġginger":45068,"à¦ł":45069,"ä»¿çľŁ":45070,"èµ°åIJ§":45071,"ĠдоÑģÑĤа":45072,"Ġmuest":45073,"å¹¶æĹł":45074,"...\"":45075,"Ġgravel":45076,"Ġpretend":45077,"æģ¶å¿ĥ":45078,"Ġscaff":45079,"Ġ_____":45080,"목":45081,"æ·±åĪ»çļĦ":45082,"Ġmapped":45083,"ĠREG":45084,"Ġerst":45085,"à·ı":45086,"nate":45087,"Ġlun":45088,"Ġspecially":45089,"Ġvä":45090,"itto":45091,"Ġdedic":45092,"éķĢ":45093,"æĸ¯ç§ij":45094,"Ġantara":45095,"ĠлÑİдей":45096,"ĠиÑģÑģледова":45097,"ĠAnthrop":45098,"Ġhistories":45099,"ĠAnglo":45100,"Ġsehingga":45101,"ĠопÑĢеделен":45102,"ĠBac":45103,"åĽłä¸ºå®ĥ":45104,"åħ¥ä¾µ":45105,"åĶIJ代":45106,"ĠÙ¾ÛĮ":45107,"ĠAkt":45108,"acet":45109,"å¥ĩ迹":45110,"å¨ģåĬĽ":45111,"ĠTribunal":45112,"à§ģম":45113,"brand":45114,"Ġeviden":45115,"Symbol":45116,"Ġfixture":45117,"umab":45118,"nak":45119,"Ġtransverse":45120,"çļĦéŨ":45121,"è¦ģ以":45122,"Ġjuvenile":45123,"ĠDidži":45124,"å±ģèĤ¡":45125,"ĠAlban":45126,"ĠConstruct":45127,"dit":45128,"Prop":45129,"ÃŃsticas":45130,"è±ģ":45131,"HB":45132,"Ġ×ŀס":45133,"owo":45134,"ĠFixed":45135,"åĸĢ":45136,"第ä¸ĢèĬĤ":45137,"åĵĹ":45138,"Richard":45139,"Ġsystematically":45140,"áĥķ":45141,"library":45142,"ç§ij室":45143,"Ġgluten":45144,"elah":45145,"æĬĬèĩªå·±çļĦ":45146,"ĠEngl":45147,"οÏħν":45148,"Ġwarnings":45149,"Ġpulses":45150,"ĠÙħÙĦ":45151,"-first":45152,"满æĦıçļĦ":45153,".$":45154,"åĽ¾å±Ĥ":45155,"Êĥ":45156,"ONS":45157,"èĢģå®ŀ":45158,"çĽ²çĽ®":45159,"antal":45160,"-la":45161,"被åijĬ人":45162,"vre":45163,"onders":45164,"Ġversa":45165,"å¼ĢèĬ±":45166,"æĬ¥ä»·":45167,"çŀ»":45168,"å®ļé¢Ŀ":45169,"ç£ķ":45170,"缮çļĦåľ°":45171,"ĠAbucay":45172,"ãģĹãģ¦ãģĦãģ¾ãģĻ":45173,"Ġmicroorganisms":45174,"交è°Ī":45175,"غة":45176,"æ³°åĽ½":45177,"ĠDidžiulis":45178,"åħĪåīį":45179,"ä¹ĭä¸ĢçļĦ":45180,"âīĪ":45181,"Bell":45182,"Ġoceans":45183,"Ġcrushed":45184,"ĠÑģни":45185,"lists":45186,"Ġdisadvantage":45187,"zas":45188,"Ġtrivial":45189,"Lib":45190,"Ġcylindrical":45191,"ĠTong":45192,"Ġutilities":45193,"Ġcoup":45194,"Ġuncertainties":45195,"Ġprecedes":45196,"帽åŃIJ":45197,"idan":45198,"ĠAdm":45199,"æĺ¯æĪijåĽ½":45200,"ĠرÙĪØ´":45201,"è¡Įåĭķ":45202,"澤":45203,"æīĢ示çļĦ":45204,"velle":45205,"ĠShell":45206,"Ġconsid":45207,"вали":45208,"днако":45209,"车ç«Ļ":45210,"Ġrealities":45211,"ÏĢει":45212,"代表çļĦ":45213,"NL":45214,"igung":45215,"Ġpurity":45216,"Flow":45217,"Ġace":45218,"Ġshru":45219,"Due":45220,"Ġthigh":45221,"çIJĨæĻº":45222,"è±IJ":45223,"GV":45224,"ç»Īç©¶":45225,"Ġoptimistic":45226,"åįī":45227,"Ġcorpo":45228,"ĠSyndrome":45229,"-winning":45230,"ĠHampshire":45231,"Ġhockey":45232,"Ġzaj":45233,"Ġخاص":45234,"Ġscary":45235,"ĠHybrid":45236,"æ¶µçĽĸ":45237,"ĠLibr":45238,"BOOK":45239,"Ġjika":45240,"æĿ¥å®ŀçݰ":45241,"å¾ĹæĽ´":45242,"åĿĩæľī":45243,"康çĨĻ":45244,"ানà§ĩ":45245,"ĠعÙĨÙĪØ§ÙĨ":45246,"ĠSOC":45247,"ĠGRO":45248,"Ġtegen":45249,"Bes":45250,"æľī人说":45251,"Vertex":45252,"culation":45253,"ardship":45254,"ĠPartners":45255,"Ġpodemos":45256,"Ġliquidity":45257,"à§įà¦Ľ":45258,"åĢĻéĢī":45259,"çİĭçĪ·":45260,"eches":45261,"odor":45262,"ĠRico":45263,"ĠگرÙģ":45264,"Õ¡Õ¬":45265,"ëĶĶ":45266,"å¤ļå¹´æĿ¥":45267,"thalm":45268,"তি":45269,"cido":45270,"ĠVilla":45271,"photo":45272,"Ġসাল":45273,"(my":45274,"mage":45275,"åįķ个":45276,"¤×¢":45277,"itches":45278,"©×IJ":45279,"Ġathlete":45280,"esteem":45281,"Identifier":45282,"idelity":45283,"гÑĢÑĥз":45284,"ÑĤки":45285,"æļĩ":45286,"Ġshortest":45287,"ĠавÑĤом":45288,"Ġshades":45289,"Ġwines":45290,"ĠRise":45291,"ĠOrig":45292,"æĭĩ":45293,"æµģ失":45294,"docs":45295,"ocia":45296,"çŃīåĽłç´ł":45297,"Ġdisplaced":45298,"åķĬåķĬ":45299,"åĽĽå·Ŀçľģ":45300,"ĠBlake":45301,"ðĿĽ":45302,")âĪĴ":45303,"ĠAdvance":45304,"å´©æºĥ":45305,"æľīä½ķ":45306,"æī¹åıij":45307,"å¿įåıĹ":45308,"Ġdeliberate":45309,".label":45310,"Ġselector":45311,"æĪIJ为ä¸Ģ个":45312,"Ġobstacle":45313,"Ġnuestro":45314,"Ġquo":45315,"ÙıÙħ":45316,"Ġstair":45317,"çĶµè§£":45318,"è¯¥ä½ľèĢħ":45319,"lek":45320,"çľĮ":45321,"Ġ'<":45322,")));Ċ":45323,"Expert":45324,"éĿĴæµ·":45325,"æĮ«æĬĺ":45326,"维度":45327,"Ġretrospective":45328,"ÑĩÑĮ":45329,"Ġcardinal":45330,"QA":45331,"表ä¸Ń":45332,"folk":45333,"Py":45334,"Ġdamit":45335,"Ġodpow":45336,"zar":45337,"ĠAristotle":45338,"angg":45339,"èt":45340,"à¦ī":45341,"ĠاÙĦتÙĪ":45342,"è·ĿéĽ¢":45343,".it":45344,"ÌĢ":45345,"ŀ×Ļ":45346,"esser":45347,"Ġupright":45348,"æ°ijæĶ¿":45349,"å¿«è¦ģ":45350,"详æĥħ":45351,"åľ¨å¤©":45352,"Ġbelievers":45353,"Ġmembuat":45354,"isma":45355,"Ġcrian":45356,"çļĦæĢ§è´¨":45357,"ä¿ı":45358,"æĢ§ä¸İ":45359,"Ġneighborhoods":45360,"-ci":45361,"Ġrotor":45362,"ĠPend":45363,"ç¾İåij³":45364,"osin":45365,"Ġnoon":45366,"Ġprecursor":45367,"νακÏĦήθηκε":45368,"amment":45369,"Ġseiner":45370,"åľ¨ä½¿ç͍":45371,"Ġ\"'":45372,"Ġ[]ĊĊ":45373,"ĠPink":45374,"avin":45375,"uca":45376,"åĽŀä¾Ĩ":45377,";âĢľ":45378,"对æĪij们":45379,".Re":45380,"Ġdeaf":45381,"ĠHern":45382,"Ġzou":45383,"ESSION":45384,"Ġì¢":45385,"Ġsignatures":45386,"æĺİæľĪ":45387,"ارج":45388,"ç´§æİ¥çĿĢ":45389,"\"],":45390,"iau":45391,"éĩĮæľī":45392,"Ġguerra":45393,"ĠSuff":45394,"Ġróż":45395,"clean":45396,"ĠDeutschland":45397,"uyen":45398,"Ġpremière":45399,"æķĻèĤ²æķĻåѦ":45400,"ĠÐŀн":45401,"ä¼ļ计å¸Ī":45402,"(min":45403,"oret":45404,"Ġkinderen":45405,"Ġtenure":45406,"Gra":45407,"über":45408,"íĸĪ":45409,"Ġdealer":45410,"Prob":45411,"çļĦé»ij":45412,"ĠCorinth":45413,"Send":45414,"Ġcite":45415,"rington":45416,"ermal":45417,"Ġproficiency":45418,"带ä¸Ĭ":45419,"Ġthoughtful":45420,"matical":45421,"ĠØ«Ùħ":45422,"ĠCes":45423,"ĠFAQs":45424,"Ġhabe":45425,"oscopy":45426,"Ġrodz":45427,"æ»ijåĬ¨":45428,"å°Ĩä»ĸ":45429,"Ġcultivated":45430,"çĶŁäºİ":45431,"ä¸ĢåIJij":45432,"ulag":45433,"Ġsep":45434,"å¤ļåIJį":45435,"æ°ĶåĬ¿":45436,"ziel":45437,"Ġgloves":45438,"ĠÙĬتÙħ":45439,"Ans":45440,".ca":45441,"Ġnumeric":45442,"åĴ¬çīĻ":45443,"Ġideals":45444,"è¦ģä¸įè¦ģ":45445,"divid":45446,"ĠForward":45447,"ĠØŃÙĪÙĦ":45448,"Ġsuffix":45449,"ĠGiov":45450,"Ġsulph":45451,"å´İ":45452,"Parameters":45453,"å¼ķ导åѦçĶŁ":45454,"çĶŁçĹħ":45455,"Ġtrades":45456,"Ġcd":45457,"Ġwithstand":45458,"Ġtopology":45459,"ĠDew":45460,"ĠOrt":45461,"-length":45462,"طع":45463,"ĠSlow":45464,"ாà®ķ":45465,"Ġinduces":45466,"竣工":45467,"à§ĭর":45468,">ĊĊĊ":45469,"ĠSach":45470,"াষ":45471,"Ġcybersecurity":45472,"Ġsync":45473,"ä¸ĥå¹´":45474,"Theme":45475,"embro":45476,"Ġnegotiation":45477,"åĨ·ç¬ij":45478,"å®ĮæĪIJçļĦ":45479,"Ġdominate":45480,"**_":45481,"ç²¾åįİ":45482,"Ġintracellular":45483,"Ġconson":45484,"legraph":45485,"shape":45486,"Ġrenewal":45487,"eston":45488,"Ġjog":45489,"ĠIngl":45490,"าห":45491,"Ġhedge":45492,"çѾåŃĹ":45493,"ĠÛģ":45494,"Ġadhesive":45495,"paid":45496,".IO":45497,"Ġoutlets":45498,"íĴĪ":45499,"Ġbeim":45500,"Ġrelieve":45501,"UA":45502,"Ġsolids":45503,"stvÃŃ":45504,"립":45505,"ĠOpportunities":45506,"ĠHarvey":45507,"rizz":45508,"ä¸Ģæĥ³":45509,"çĿĢ她":45510,"å¨ĺå¨ĺ":45511,"ührt":45512,"ĠâĢĵĊĊ":45513,"çħ¤çŁ¿":45514,"[t":45515,"ĠÚ©ÙĪØ¯":45516,"å°±åĥıæĺ¯":45517,"naments":45518,"Ġhari":45519,"Ġ×ķ×¢":45520,"ĠÑĦоÑĢми":45521,"æİĪ课":45522,"交éĢļè¿IJè¾ĵ":45523,"æĤ²åī§":45524,"follow":45525,"Ġslopes":45526,"Speed":45527,"Ãĺ":45528,"nung":45529,"мÑĸ":45530,"cios":45531,"Ġprisoner":45532,"Tu":45533,"Ġ\")Ċ":45534,"Ġsolub":45535,"Ġzuen":45536,"ajaran":45537,"颤æĬĸ":45538,"Dom":45539,"æ¨Ļæºĸ":45540,"édia":45541,"Eds":45542,"Ġdiving":45543,"çļĦåŁİå¸Ĥ":45544,"iates":45545,"å¾¹":45546,".junit":45547,"Ġspill":45548,"æĭīåħĭ":45549,"Ġdehyd":45550,"æĿ̿ѻ":45551,"çĸ¾çĹħçļĦ":45552,"\".[":45553,"ÐĻ":45554,"chel":45555,"chief":45556,"Ġdug":45557,"å°±åİ»":45558,"Ġpuzzles":45559,"ĠÙĪÙĦÙĥ":45560,"ĠÙĪØ¥":45561,"ç»Ŀ大å¤ļæķ°":45562,"}\\)-":45563,"æĪijå°±æĺ¯":45564,"ĠVern":45565,"ĠTimothy":45566,"è¿Ŀæ³ķè¡Į为":45567,"åħ¶æīĢ":45568,"arnings":45569,"ĠCosine":45570,"ienn":45571,"Ġcompliment":45572,"ØŁĊĊ":45573,"Ġmyocardial":45574,"ÑĩнÑĭй":45575,"å¼ĢéĢļ":45576,"manuel":45577,"Ġoccupy":45578,"éļ¾åıĹ":45579,"Ġconvincing":45580,"åıijå¸ĥçļĦ":45581,"Ġscholarly":45582,"Õ¤":45583,"åİĦ":45584,"Give":45585,"Ġnah":45586,"idase":45587,"Determ":45588,"Ġrhetoric":45589,"Vo":45590,"Wil":45591,"Ġville":45592,"ĠDNS":45593,"mis":45594,"éªĨ":45595,"ĠExpect":45596,"Ġcrowds":45597,"Bytes":45598,"åħīæ»ij":45599,"Ġcrises":45600,"Ġunint":45601,"æĸ½è¡Į":45602,"ĠëijIJ":45603,"ĠFahrenheit":45604,"Ġtrailer":45605,"èĩªå·±æĺ¯":45606,"績":45607,"ĠLiberty":45608,"ĠAlberta":45609,"ĠNGO":45610,"åķĨä¼ļ":45611,"æIJĵ":45612,"æ®ĭçĸ¾äºº":45613,"è¨Ģä¹ĭ":45614,"-gu":45615,"à¹Īวม":45616,"Ġspecialty":45617,"Ġfighter":45618,"å°±éľĢè¦ģ":45619,"Ġworkout":45620,"é£İåħī":45621,"Ġà°µ":45622,"_DIR":45623,"Ġfazer":45624,"ê¹Įì§Ģ":45625,"Ġprocessors":45626,".items":45627,"ipl":45628,"Ġranged":45629,"Ġexceeded":45630,"ĠRailway":45631,"bare":45632,"Ġfacilitates":45633,"ĠPound":45634,"Events":45635,"attribute":45636,"æIJģ":45637,"à¥įद":45638,"Ġinterconnected":45639,"梧":45640,"Ġcapacities":45641,"Begin":45642,"ĠPsychological":45643,"Front":45644,"Ġsentido":45645,"åĪļå¼Ģå§ĭ":45646,"ترÛĮÙĨ":45647,"ưá»Ŀng":45648,"'(":45649,"åĽĽç§į":45650,"Ġtouchdown":45651,"ìĬ¤íĬ¸":45652,"heap":45653,"Ġpetitioner":45654,"è¿Ļ两ç§į":45655,"ĉstd":45656,"ĠBiochem":45657,"Ġdarker":45658,"Ġadvisor":45659,"Ġmożna":45660,"ubre":45661,"ĠFunding":45662,"unta":45663,"å¼Ģè¾Ł":45664,"ä»ĸ们æĺ¯":45665,"çİĭåĽ½":45666,"ĠDeal":45667,"ĠDelivery":45668,"ató":45669,"Ġnonetheless":45670,"è¿Ľåĩºåı£":45671,"anyon":45672,".ToString":45673,"ĠUb":45674,"Ġdiper":45675,";//":45676,"Ġestable":45677,"Ġgrouped":45678,"ĠBryan":45679,"Ġvolcano":45680,"åħ«åįģ":45681,"常æķ°":45682,"Ġpraying":45683,"collect":45684,"íķĺë©´":45685,"боÑĢа":45686,"acic":45687,"å®ļæĹ¶":45688,"Ġeventual":45689,"aal":45690,"ĠLep":45691,"æ´Ľéĺ³":45692,">":45693,"isu":45694,"à°µ":45695,"ährend":45696,"Ġ$$ĊĊ":45697,"Ġtug":45698,"æľĶ":45699,"ĠTier":45700,"أس":45701,"Ġsuspicion":45702,"ĠTernary":45703,"ĠбеÑĢе":45704,"Ġtuned":45705,"Ġsummarize":45706,"主義":45707,"Series":45708,"ĠMeth":45709,"ctal":45710,"ĠBound":45711,"Products":45712,"Ġìŀ¬":45713,"posing":45714,"æĶ¹æŃ£":45715,"çķĮéĻIJ":45716,"Ġassays":45717,"ĠÑĩеÑĢ":45718,"imburs":45719,"ĠFBI":45720,"Ġmeantime":45721,"Ġjanuari":45722,"åģıåģı":45723,"Ġmalignant":45724,"ÃŃl":45725,"åĽĽå¤Ħ":45726,"Ġmodeled":45727,"ĠMonths":45728,"ům":45729,"ĠNeeds":45730,"æIJºæīĭ":45731,"à¸ŀร":45732,"æ©ĭ":45733,"ĠBMI":45734,"ĠLingu":45735,"-pe":45736,"ĠFIN":45737,"æľŁéĸĵ":45738,"Ġmatt":45739,"Ġunwanted":45740,"ĠUnderg":45741,"ĠDennis":45742,"Ġaromatic":45743,"Tech":45744,"Ġwyst":45745,"ä»ĸå·²ç»ı":45746,"Ġdigunakan":45747,"Ġconstructive":45748,"square":45749,"Ġruntime":45750,"GROUND":45751,"竳ç¨ĭ":45752,"ĠBiom":45753,"èij¡èIJĦéħĴ":45754,"ĠÑįÑĦÑĦекÑĤив":45755,"åħĪæĺ¯":45756,"ussels":45757,"ä¸įæľį":45758,"Ġcollector":45759,"ĠNegro":45760,"æºĥçĸ¡":45761,"åľ°è¯´éģĵ":45762,"Ġresultados":45763,"âĪ¥":45764,"SCs":45765,"kappa":45766,"ucked":45767,"礼è²Į":45768,"è´¿":45769,"Ġglow":45770,"rels":45771,"Ġ\\,":45772,"ĠAssume":45773,"à¹Ģà¸Ħร":45774,"(error":45775,"åħļæĶ¿":45776,"åĦ¿å¥³":45777,"æĺĵäºİ":45778,"ĠNavig":45779,"Ġsubscribe":45780,"Ġmurm":45781,"Ġdecorated":45782,"å¾Ĺ太":45783,"inic":45784,"Ġmondo":45785,")).Ċ":45786,"åıªçľĭè¯¥ä½ľèĢħ":45787,"Ġdegener":45788,"ĠSomeone":45789,"ĠÎķÏĦÏħμολογία":45790,"Pack":45791,"Ġtf":45792,"迦":45793,"çĹħåĽł":45794,"Ġbrake":45795,"ĠконÑĤÑĢ":45796,"éĶļ":45797,".persistence":45798,"ĠFeedback":45799,"Ġcomprend":45800,"示æĦıåĽ¾":45801,"ĠRichmond":45802,"çļĦåĽłç´ł":45803,"ĠJi":45804,"æĭ¯":45805,"Ġmethane":45806,"ĠپاÛĮ":45807,"Ġexpenditures":45808,"ожа":45809,"第ä¸ī竳":45810,"说æľį":45811,"ĠпоÑįÑĤомÑĥ":45812,"Ġعدد":45813,"à¥įà¤Ł":45814,"ãģ«ãĤĪãĤĭ":45815,"Ġexpedition":45816,"Ġphilosopher":45817,"ä¸ī个æľĪ":45818,"Ġcarbohydrates":45819,"Ġlatent":45820,"Hon":45821,"ש×Ķ":45822,"éĻªåIJĮ":45823,"ĠElection":45824,"ipeline":45825,"Ġinterpreting":45826,"Ġrefreshing":45827,"头顶":45828,"Ġemerges":45829,"/hour":45830,"BUG":45831,"群ä¼ĹçļĦ":45832,"á¿¶":45833,"_len":45834,"æłijèĦĤ":45835,"ĠExtra":45836,"ä¸ĢçļĦ":45837,"Ġattractions":45838,"Private":45839,"DOCTYPE":45840,"lagen":45841,"ĠاÙĦعاÙħ":45842,"Ġworkflow":45843,"Ġpersistence":45844,"ä½İ温":45845,"红èī²çļĦ":45846,".).ĊĊ":45847,"èģĶæĥ³":45848,"Ġfuncion":45849,"units":45850,"angk":45851,"âĢĿï¼Ľ":45852,"ä¿Ŀ温":45853,"Ġangi":45854,"-generation":45855,"çµķå°į":45856,"ĠبÙħ":45857,"Ġharassment":45858,"Ġsympathetic":45859,"Ġpolitician":45860,"ĠFen":45861,"çħ§å°Ħ":45862,"Ġболез":45863,"ÑģÑĤиÑĤÑĥ":45864,"Ġprecautions":45865,"igram":45866,"Ġdownloaded":45867,"Ġsadness":45868,"currency":45869,"-centered":45870,"ç»ĵæĻ¶":45871,"Ġescrit":45872,"ĠReporting":45873,"丶":45874,"ĠIncludes":45875,"åħ¬çº¦":45876,"Ġодна":45877,"etÃł":45878,"管çIJĨåĬŀæ³ķ":45879,"Ġserm":45880,"isol":45881,"æĬĢæľ¯åĴĮ":45882,"å¨ĥå¨ĥ":45883,"å¸ĮæľĽèĥ½":45884,"ĠпÑĢом":45885,"гии":45886,"(number":45887,"ược":45888,"domain":45889,"Ġspinning":45890,"çŀĦ":45891,"éĶĢåĶ®é¢Ŀ":45892,"ĠBennett":45893,"æĿ¥ä¸´":45894,"èijµ":45895,"Ġfic":45896,"ellate":45897,"çłģ头":45898,"Ġambition":45899,"anca":45900,"beck":45901,"ĠObst":45902,"ĠClarke":45903,"ortium":45904,"ä¸İ人":45905,"缮æłĩçļĦ":45906,"Ġwelding":45907,"ĠCHAPTER":45908,"HTTP":45909,"æ£īèĬ±":45910,"continue":45911,"Ġempres":45912,":@\"":45913,"etes":45914,"ĠQueensland":45915,"æĽ¾åľ¨":45916,"ĠEntity":45917,"é«ĺçŃīæķĻèĤ²":45918,"íĮĮ":45919,"Ġwyn":45920,"è¿Ļæĺ¯åĽłä¸º":45921,"çļĦå¾Ī":45922,"çŁ©å½¢":45923,"าะ":45924,"åĩłä¸ªæľĪ":45925,"ĠзнаÑĩениÑı":45926,"絡":45927,"Ġاص":45928,"{t":45929,"Ġbrit":45930,"çĤºä»Ģ麼":45931,"çļĦåīįæıIJä¸ĭ":45932,"éħĮ":45933,"roads":45934,"ĠÑįлекÑĤÑĢо":45935,"ĠHB":45936,"æľįåĬ¡ä¸ļ":45937,"ôle":45938,"å¿ĥ头":45939,"ä»·æ¯Ķ":45940,"绣ä¸ĢçļĦ":45941,"ä¸įå°ıå¿ĥ":45942,"éĻ¢çļĦ":45943,"ĠKont":45944,"ĠCauses":45945,"æł¸éħ¸æ£Ģæµĭ":45946,"åģĩå®ļ":45947,"ĠHello":45948,"ãĥªãĥ¼":45949,"æįķæįī":45950,"Und":45951,"Ġslept":45952,"WC":45953,"ĠÙĪÙĨ":45954,"Ġcosa":45955,"èĴĭä»ĭçŁ³":45956,"ĠKill":45957,"Ġplea":45958,"asto":45959,"åĪĹå®ģ":45960,"ĠSimulation":45961,"硬çĽĺ":45962,"Scroll":45963,"WORD":45964,"Ġworkload":45965,"èmes":45966,"ë³Ħ":45967,"aso":45968,"гда":45969,"aille":45970,"å±±å¸Ĥ":45971,"ĠÑĥÑĢ":45972,".ru":45973,"ĠBee":45974,"à¹ĥหà¸į":45975,"Ġcatalytic":45976,"ĠObjectives":45977,"æ«":45978,"ément":45979,"ými":45980,"inez":45981,"ר×Ļ":45982,"Ġsensit":45983,"åıijåħī":45984,"Ġinfring":45985,"ç»Ĩèħ»":45986,"Ġfragile":45987,"Ġà¤ı":45988,"[y":45989,"strate":45990,"×Ļ׾×Ķ":45991,"ë¸":45992,"Ġsupreme":45993,"fund":45994,"agination":45995,"é«ĺéĵģ":45996,"лин":45997,"ucket":45998,"iotherapy":45999,"-esteem":46000,"诸èijĽ":46001,"Ġviele":46002,"é¢Ĩè¢ĸ":46003,"榴":46004,"纳ç¨İ人":46005,"Ġmighty":46006,"gomery":46007,"åīįéĿ¢çļĦ":46008,"arial":46009,"Ġleveraging":46010,"-[":46011,"Ġfox":46012,"å᳿ĺ¯":46013,"ços":46014,"尾巴":46015,"ĠSustainability":46016,"bero":46017,"å½ĵåľº":46018,"ä¸ŃåĴĮ":46019,"::::":46020,"ï½ŀĊ":46021,"伤å¿ĥ":46022,"ĠÙĪÛĮ":46023,"oslav":46024,"åı¯ä»¥éĢīæĭ©":46025,"àªĤ":46026,"Ġneuronal":46027,"-commerce":46028,"ĠImproved":46029,"Ġprose":46030,"validate":46031,"ĠThreat":46032,"ĠSUB":46033,"ĠFergus":46034,"梵":46035,"Ġappliances":46036,"-frequency":46037,"ел":46038,"ä¸ĢåıĮ":46039,"ĠThirty":46040,"ĠReally":46041,"Almost":46042,"Ġfootage":46043,"ĠÑģÑĤан":46044,"eee":46045,"ĠManhattan":46046,"Ġpé":46047,"ktor":46048,"atics":46049,"积æŀģåıĤä¸İ":46050,"jam":46051,"å®ļåIJij":46052,"vr":46053,"Ġries":46054,"à¹Ħว":46055,"ĠChelsea":46056,"selling":46057,"ĠKoh":46058,"-methyl":46059,"ëijIJ":46060,"Ġpréc":46061,"ĠMeasures":46062,"èĢĮä¸Ķè¿ĺ":46063,"§×ĺ":46064,"arah":46065,"/nm":46066,"åĪĨåĮº":46067,"å¥ij约":46068,"ĠIst":46069,"Ġarrows":46070,"Ġأع":46071,"ĠTerra":46072,"ĠдолжнÑĭ":46073,"itating":46074,"å¤ļãģı":46075,"Ġrage":46076,"detailed":46077,"çĭ®åŃIJ":46078,"åĪĩåī²":46079,"ήÏĤ":46080,"Ġlime":46081,"Indust":46082,"æķĻ导":46083,"Ġawe":46084,"+)":46085,"Ġgarbage":46086,"arat":46087,"âĢį":46088,"ĠNorthwest":46089,"ĠRepresentatives":46090,"äºĶ大":46091,"Ġgasoline":46092,"Ġisolates":46093,"holding":46094,"ần":46095,"Ġscrutiny":46096,"ä¸Ģé¦ĸ":46097,"аÑĤи":46098,"Apple":46099,"Ġindispensable":46100,"缴è§Ĵ":46101,"гÑĢе":46102,"Ġadhesion":46103,"Ġpian":46104,"ĠWagner":46105,"ĠAdmin":46106,"ĠMai":46107,"æ¨Ĭ":46108,"глÑı":46109,"ĠAnimalia":46110,"Ġcreep":46111,"Ġfutures":46112,"verages":46113,"åĿIJçĿĢ":46114,"ĠHenri":46115,"æ¦ľæł·":46116,"Working":46117,"Dao":46118,"emporal":46119,"usement":46120,"ĠDental":46121,"寻常":46122,"ï¼Įï¼Į":46123,"Ġemitted":46124,"ĠRunning":46125,"ĠBuddha":46126,"Library":46127,"ŀ×Ļ×ĵ":46128,"ĠHaven":46129,"iks":46130,"Ġignoring":46131,"Ġglands":46132,"æĭĨéϤ":46133,"วà¸Ķ":46134,"ĠND":46135,"ãģ§ãģĹãĤĩãģĨ":46136,"ä¸Ģå¿ĥ":46137,".contains":46138,"Ġanalogy":46139,"æĢİä¹Ī说":46140,"Ġgenuinely":46141,"ĠScheme":46142,"xia":46143,"Ġcompiler":46144,"ĠTheme":46145,"Ġemperor":46146,"èĤĿèĦı":46147,"_result":46148,"ìĦ¤":46149,"nisse":46150,"Ġdonne":46151,"ĠYES":46152,"ót":46153,"wright":46154,"ĠSchne":46155,"âķIJâķIJ":46156,"bery":46157,"彪":46158,"interpret":46159,"Ġassertion":46160,"媽媽":46161,"ï¼ĺ":46162,"journal":46163,"ĠÙħاÙĨ":46164,"涤":46165,"-binding":46166,"é»ijèī²çļĦ":46167,"erns":46168,"Asked":46169,"ä¼´æľī":46170,"ĠImagine":46171,"parameters":46172,"èķī":46173,"沦":46174,"åįĬå¤ľ":46175,"Ġ×¢×ĵ":46176,"ĠClearly":46177,"ritic":46178,"Ġconstrained":46179,"{Z":46180,"奧":46181,"ĠBrooks":46182,"Ġignorance":46183,"åı¯ä»¥å¸®åĬ©":46184,"aż":46185,"Ġpov":46186,"ĠMetro":46187,"Ġjewelry":46188,"ĠÐļон":46189,"ĠصÙģØŃ":46190,"Ġpermett":46191,"Ġauthenticity":46192,"äºĭå®ľ":46193,".fr":46194,"ĠCDC":46195,"ç£ģåľº":46196,"ĠCriteria":46197,"Ġdre":46198,"ucid":46199,"Ġdiscourag":46200,"Ġbiochemical":46201,"ä¹ĺæ³ķ":46202,"ä¹ŁéľĢè¦ģ":46203,"æ¼ĵ":46204,"以ä¸ĭçļĦ":46205,"ä¸ĬæĬ¥":46206,"brate":46207,"Ġtand":46208,"Ġglue":46209,"Inv":46210,"ĠNatl":46211,"Languages":46212,"润æ»ij":46213,"éĻĦè¿ijçļĦ":46214,"ĠpolÃŃtica":46215,"ËĮ":46216,"ĠSuperior":46217,"ĠEventually":46218,"xa":46219,"Ġlend":46220,"æ¶Īè´¹èĢħçļĦ":46221,"ÑģÑĤвии":46222,"Generator":46223,"lear":46224,"æĥ«":46225,"ĠاÙĦØ£ØŃ":46226,"Ġpancreatic":46227,"ä¸Ĭ线":46228,"ĠInterestingly":46229,"Ġmushrooms":46230,"ĠperÃŃ":46231,"æĺ¯æĪijçļĦ":46232,"å°±å¾Ī":46233,"Ġsnack":46234,"Ġexh":46235,"ä¹ĭèī²":46236,"Ġascending":46237,"_content":46238,"rone":46239,"classes":46240,"åľ¨ä¸į":46241,"価":46242,"å´ĩæĭľ":46243,"ĠÑĪа":46244,"Ġпоб":46245,"комен":46246,"ĠÑĢебенка":46247,"cod":46248,"ÙĨÙī":46249,"(model":46250,"æªĶ":46251,"Duration":46252,"Ġoverly":46253,"ĠмаÑģÑģ":46254,"Ġfins":46255,"ĠSanskrit":46256,"æĪijä¸Ģ缴":46257,"æİ¥ä¸ĭæĿ¥çļĦ":46258,"á»ĵ":46259,"ĠPalm":46260,"ĠGenesis":46261,"াà¦ĸ":46262,"车åŃIJ":46263,"denly":46264,"Ġcooler":46265,"Ġlining":46266,"ĠMats":46267,"ĠColumbus":46268,"ĠVerg":46269,"å¤ļ为":46270,"輯":46271,"ä»ĸçŁ¥éģĵ":46272,"ancellor":46273,"ä¸Ģè¾ĪåŃIJ":46274,"__':Ċ":46275,"ĠEdu":46276,"Ġascertain":46277,"è¿Ľåıĸ":46278,"Ø·ØŃ":46279,"ĠExercises":46280,"Ġprocurement":46281,"ĠتÙģ":46282,"ĠTourism":46283,"åIJĵå¾Ĺ":46284,"algorithm":46285,"ĠMario":46286,"Ġvá":46287,"æħ¢æħ¢çļĦ":46288,"à¹Ģลืà¸Ń":46289,"Ġirrelevant":46290,"Ġcancellation":46291,"aye":46292,"Ġcuc":46293,"posal":46294,"ĠاÙ쨲":46295,"]))Ċ":46296,"Ġà¸Ľ":46297,"ç»ĻåŃ©åŃIJ":46298,"æİĮæİ§":46299,"Ġunderest":46300,"çĶ·æľĭåıĭ":46301,"Ġpsycho":46302,"riad":46303,"ÑħÑĥ":46304,"ĠInsect":46305,"Central":46306,"Ġretaining":46307,"æĢĿæĥ³çļĦ":46308,"æĭĨè¿ģ":46309,"chu":46310,"пÑĢо":46311,"ĠGrey":46312,"Ġawaken":46313,"ÙIJÙij":46314,"å¹¾ä¹İ":46315,"Ġabb":46316,"-game":46317,"Ġballot":46318,"capt":46319,"nc":46320,"Ġcob":46321,"议论":46322,"æĺ¯å¥¹":46323,"Ġ>>>":46324,"orns":46325,"å»ºè®¾é¡¹çĽ®":46326,"è·¡":46327,"ĠNUM":46328,"æīĢ以说":46329,"Contents":46330,"Ġadvisory":46331,"åĮħ容":46332,"Ġciting":46333,"Ġcolleague":46334,"trim":46335,"Ġhemorrh":46336,"ĠAware":46337,"èĦļä¸ĭ":46338,"ĠÑĦÑĥнкÑĨии":46339,"ä¸įäºĪ":46340,"人æ°ijæ£Ģå¯ŁéĻ¢":46341,"Ġenergetic":46342,"åIJĪ计":46343,"Opp":46344,"ĠNarr":46345,"åħ¬ç§¯éĩij":46346,"æĬĺ磨":46347,"éľīç´ł":46348,"Ù¹":46349,"ĠonChange":46350,"-product":46351,"æľīä¸ī":46352,"spNet":46353,"attering":46354,"Ġsentenced":46355,"åįĹæµ·":46356,"çŀ¥":46357,"üssen":46358,"Ġpore":46359,"ĠLR":46360,"åı¯ä½¿":46361,"insky":46362,"}}_{":46363,"oluble":46364,"ablo":46365,"åĪĽéĢłäºĨ":46366,"(ii":46367,"-so":46368,"-sensitive":46369,"-tra":46370,"à³ĩ":46371,"ÑĢоме":46372,"_CH":46373,"ĠWOR":46374,"ĠNigerian":46375,"TB":46376,"éĩįå¿ĥ":46377,"ä¸ĢåĪĩéĥ½":46378,"ĠLiberal":46379,"Gi":46380,"Sine":46381,"ghi":46382,"Ġpoetic":46383,"è½®èĥİ":46384,"å·¥ä½ľåĴĮ":46385,"ĠTamil":46386,"Ġpathogen":46387,"çģ°èī²":46388,"Ġbats":46389,"Ġ×ij×Ļף":46390,"-container":46391,"umann":46392,"plies":46393,"ricia":46394,"_trans":46395,"ĠSaw":46396,"Ġnullptr":46397,"Ðļа":46398,"Ġligand":46399,"Ġrecruited":46400,"åĬłçıŃ":46401,"åύæĿIJ":46402,"oplasm":46403,"éŨè¯Ĭ":46404,"dh":46405,"èIJ±":46406,"å°±è¡ĮäºĨ":46407,"æ´Ĺæīĭ":46408,"Ġenjoys":46409,"oors":46410,"æĢĿ念":46411,"볨":46412,"ÙĬØ´":46413,"Ñĭми":46414,"Ġвме":46415,"Ġпоме":46416,"zm":46417,"Ġاگر":46418,"ATIONAL":46419,"éĩįåĬĽ":46420,"ä¸ĸçļĦ":46421,"ĉa":46422,"ĠHir":46423,"οÏģ":46424,"Chicago":46425,"èŃ¦ç¤º":46426,"ĠаÑĥÑĤо":46427,"ĠTru":46428,"มิ":46429,"å½ķéŁ³":46430,"åĨ³å®ļçļĦ":46431,"à³įರ":46432,"Ġmixtures":46433,"viation":46434,"课æĹ¶":46435,"ĠMayo":46436,"Ġannées":46437,"kiem":46438,"æµĵéĥģ":46439,"oretical":46440,"Ġinnate":46441,"Ġnicely":46442,"çļĦä¸Ģå¹´":46443,"kou":46444,"Ġremembering":46445,"çļĦè¯Ńè¨Ģ":46446,"Ġinterim":46447,"language":46448,"window":46449,"æİ¥è¿ĩ":46450,"Ġvowel":46451,"alli":46452,"æ¸ħæĻ¨":46453,"ĠFuj":46454,"人æīĢ":46455,"pleasant":46456,"Ġ?>Ċ":46457,"ÑĢениÑı":46458,"ĠмеÑĢ":46459,"æĺ¯æĹł":46460,"oelectric":46461,"ĠполиÑĤи":46462,"Ġked":46463,"oxyl":46464,"ä»įåľ¨":46465,"нÑİ":46466,"merge":46467,"æĿijçļĦ":46468,"aude":46469,"Validation":46470,"Ġcuer":46471,"åįĵè¶Ĭ":46472,"pb":46473,"ниÑĨи":46474,"é¢Ħå¤ĩ":46475,"ĠSony":46476,"-consum":46477,"rifug":46478,"Ġallem":46479,"ITED":46480,"ä»»ä½ķä¸Ģ个":46481,"æĤ²ä¼¤":46482,"å¥Īä½ķ":46483,"ë²Ħ":46484,"ARCHAR":46485,"Ġmedicinal":46486,"ennen":46487,"ĠмеÑģÑĤ":46488,"ĠRew":46489,"ĠÙħÙĪÙĤع":46490,"ĠLor":46491,"unders":46492,"cribing":46493,"Ġpoets":46494,"Ġsiempre":46495,"Ġbyl":46496,"obo":46497,"ningen":46498,"å°ı红":46499,"ĠJulie":46500,"æĥħæĢĢ":46501,"нен":46502,"ĠÕ¿":46503,"Ġsulfate":46504,"ĠInto":46505,"æłijçļĦ":46506,"ĠÙĥتاب":46507,"-economic":46508,"Ġcompetit":46509,"jk":46510,"Tel":46511,"Ġwives":46512,"èµ·çłģ":46513,"Ġfabrication":46514,"ĠÚ©Ø´ÙĪØ±":46515,"Ġapril":46516,"亥":46517,"æĮĩ导ä¸ĭ":46518,"å°±æĥ³":46519,"ìłĢ":46520,"Ġqualifying":46521,"åı¯æĮģç»Ńåıijå±ķ":46522,"Ġseismic":46523,"Ġrecreational":46524,"tbody":46525,"ĠGor":46526,"ĠXIX":46527,"Ġszcz":46528,"Ġcriticized":46529,"hit":46530,"å«ī":46531,"éĨĴäºĨ":46532,"ÏģÏĩ":46533,"Ġcleaner":46534,"ĠOpera":46535,"_),":46536,"Ġಮ":46537,"ĠQuarterly":46538,"ĠStru":46539,"عار":46540,"Ġmodular":46541,"æĿ¡çº¦":46542,"[Ċ":46543,"ĠSig":46544,"าà¸ķิ":46545,"ĠLINE":46546,"å¤¸å¼ł":46547,"çļĦå®ŀéĻħ":46548,"contrib":46549,"Õ¢":46550,"Ġregimes":46551,"Ġparenting":46552,"åįłåľ°":46553,"pragma":46554,"Ġcollapsed":46555,"ĠPerspective":46556,"Ġprograma":46557,"Ġruin":46558,"Ġenacted":46559,"jed":46560,"åģľçķĻ":46561,"Ġaveraged":46562,"èij«":46563,"ĠCitizens":46564,"ĠDubai":46565,"rze":46566,"_base":46567,"Ġundes":46568,"Ġindicative":46569,"ĠпÑĢовед":46570,"Ñıви":46571,"èĢģèĻİ":46572,"ĠSchema":46573,"odont":46574,"人éĻħ":46575,"ĠGastro":46576,"æĪijè¿Ļ":46577,"èĥ±":46578,"Ġindustri":46579,"(obj":46580,"çļĦåİŁ":46581,"Ġether":46582,"æĢĿç´¢":46583,"=f":46584,"Ġbic":46585,"管åζ":46586,"Ġ/**":46587,"Ġduplicate":46588,"(req":46589,"pering":46590,"Ġdias":46591,"ĠSummit":46592,"å®īåħ¨æĢ§":46593,"ĠJohannes":46594,"cyl":46595,"éĴ§":46596,"Ġcyclic":46597,"左边":46598,"ĠmiR":46599,"Dam":46600,"ä½Ĩå®ĥ":46601,"Win":46602,"สà¸Ķ":46603,"ĠÑģобой":46604,"(left":46605,"ître":46606,"Ġworries":46607,"å¥ijæľº":46608,"éric":46609,"Ġмилли":46610,"icidal":46611,"ĠDivine":46612,"Ġoptimizing":46613,"Ġpossesses":46614,"Ġsuperficial":46615,"ounder":46616,"inin":46617,"Ġbaked":46618,"ĠPOST":46619,"Ġseated":46620,"à´Ĥ":46621,"ĠоÑģÑĥ":46622,"çĿĢäºĨ":46623,"ÑĤвеÑĢ":46624,"éĩıåĮĸ":46625,"模åħ·":46626,"Sem":46627,"èĢģé¼ł":46628,"Ġstiffness":46629,"PAR":46630,"ĠLif":46631,"Ġsuatu":46632,"блÑİ":46633,"Ġmiracle":46634,"ĠSatan":46635,"chair":46636,"ĠConfeder":46637,"ivism":46638,"دÙģ":46639,"Mg":46640,"Ġà¦ķরতà§ĩ":46641,"Ġfairness":46642,"hatan":46643,"ILD":46644,"ît":46645,"asper":46646,"olla":46647,"Ġsólo":46648,"Ġlively":46649,"ĠWasser":46650,"ä¸Ĭåij¨":46651,"Ġaviation":46652,"æĺ¥é£İ":46653,"Ġdisreg":46654,"ç¿ħèĨĢ":46655,"AMS":46656,"Ġcertificates":46657,"ĠFreud":46658,"alter":46659,"对çŃĸ":46660,"Ġcounc":46661,"Ġrecruiting":46662,"udy":46663,"æľĢä¼ĺ":46664,"Ġstellar":46665,"ĠRonald":46666,"หมาย":46667,"ÑĭÑĤа":46668,"ä¹³èħº":46669,"æĪĬ":46670,"Ġideological":46671,"à¸ŀัà¸Ĵà¸Ļ":46672,"ĠSara":46673,"ĠPale":46674,"actual":46675,"ç»ı纪":46676,"\"],Ċ":46677,"ĠMob":46678,"Ġsympathy":46679,"Ġexpres":46680,"Ġexceeding":46681,"Ġperché":46682,"Ġinsult":46683,"пÑĢимеÑĢ":46684,"æŁ¿":46685,"线åľĪ":46686,"Ġotra":46687,"Ġsouvent":46688,"å¹´å¼Ģå§ĭ":46689,"èĩªæĿĢ":46690,"ĠShadow":46691,"ĠGeoNames":46692,"Ġfigur":46693,"Ġmanifestations":46694,"(word":46695,"ĠTangent":46696,"æĪIJä¸Ģ":46697,"raisal":46698,"Ġincorporates":46699,"did":46700,"æ¦Ĩ":46701,"Mont":46702,"jour":46703,"樣åŃIJ":46704,"Ġrailroad":46705,"ĠCubic":46706,"ĠRepresentative":46707,"ä½łä¸įæĺ¯":46708,"Ġscandal":46709,"Ġpunct":46710,"عداد":46711,"ĠPictures":46712,"Tele":46713,"ĠAnswered":46714,"åͱæŃĮ":46715,"Ġzoom":46716,"Ġpeque":46717,"èĥ½çļĦ":46718,"raits":46719,"ÙĩاÛĮÛĮ":46720,"åı¯ä»¥çĽ´æİ¥":46721,"æł¼åħ°":46722,"Ġmisc":46723,"ĠEisen":46724,"Ġpremise":46725,"ç¬Ķè®°æľ¬":46726,"ĠLakes":46727,"Ġgrim":46728,"è¯Ħå®ļ":46729,"pn":46730,"ographs":46731,"-negative":46732,"Ġwarmer":46733,"åĺī宾":46734,"Enabled":46735,"ĠLeaf":46736,"éĽ»å½±":46737,"Happy":46738,"uron":46739,"ĠMing":46740,"âĢĶâĢĶâĢĶ":46741,"çݰæľīçļĦ":46742,"éĹ®ä»ĸ":46743,"Ġsuppressed":46744,"ĠScar":46745,"ł×¡":46746,"ä¸įåħģ许":46747,"bestos":46748,"à¦Ľà§ĩ":46749,"Ġreflections":46750,"ÑĥÑĩа":46751,"заÑĨиÑı":46752,"Ġsized":46753,"åΰè¿ĻéĩĮ":46754,"èµ·æŃ¥":46755,"ç²Ĺç³Ļ":46756,"PB":46757,"ĠGuinea":46758,"Ġpunctu":46759,"Ġfestivals":46760,"èĬ·":46761,"Ġmisleading":46762,"னà¯į":46763,"ĠТе":46764,"ĠDefendant":46765,"åľ¨éĢĻ":46766,"Ñĸв":46767,"Ġliquids":46768,"entric":46769,"æĢ»æķ°":46770,"缺失":46771,"Genre":46772,"Ġteenagers":46773,"Ġenclosed":46774,"ĠNZ":46775,"glass":46776,"Ġporous":46777,"ĠMcDonald":46778,"IQ":46779,"ĠLayer":46780,"ä¹ĭæīĢ":46781,"Ġlista":46782,"Ġillusion":46783,"à¸ķà¹Į":46784,"Ġforecasting":46785,"ĠÐĽÐ¸":46786,"Ġlarvae":46787,"主治":46788,"είο":46789,"å½ĵä»ĸ":46790,"Ġдом":46791,"åĩłä¸ªäºº":46792,"Ġrealise":46793,"Appro":46794,"åı¯ä¸į":46795,"çªľ":46796,"Ġlaunching":46797,"èĥĸåŃIJ":46798,"æµ·æĭĶ":46799,"æ¡ĵ":46800,"è·ijæŃ¥":46801,"æĹ¥æĻļ":46802,"çIJĥè¿·":46803,"çĢij":46804,"LCM":46805,"ĠvỼi":46806,"Ġembraced":46807,"Ġilleg":46808,"Ġohne":46809,"Ġsiècle":46810,"Ġrearr":46811,"ĠuseEffect":46812,"ĠменÑĮ":46813,"ĠÑıвлÑıÑİÑĤÑģÑı":46814,"å´Ľèµ·":46815,"FB":46816,"ÙĨاÙĨ":46817,"æįĨ":46818,"ĠNebraska":46819,"Hor":46820,"åѵ":46821,"à¸Ļวà¸Ļ":46822,".message":46823,"Ġcommercially":46824,"ĠJulian":46825,"Ġheterogeneity":46826,"!)ĊĊ":46827,"ĠWinds":46828,"Ġà®Ĩ":46829,"Ġতিনি":46830,"Ġargu":46831,"Ġexcitation":46832,"Ġpropaganda":46833,"Ġornament":46834,"))/(":46835,"ĠÄijá»Ļ":46836,"Writing":46837,"ය":46838,"Tangent":46839,"Ġemploys":46840,"Ġessa":46841,"åŃĺåľ¨çļĦéĹ®é¢ĺ":46842,"ĠHungary":46843,"odus":46844,"Ġtorture":46845,"ãģĹãģı":46846,"Divisors":46847,"ĠÅĤ":46848,"Ġgou":46849,"Ġinsist":46850,"åľĨçļĦ":46851,"Ġspos":46852,"两岸":46853,"à¥įम":46854,"Ġtutto":46855,"oultry":46856,"个å°ıæĹ¶":46857,"ĠManuel":46858,"ç¨İçİĩ":46859,"ultura":46860,"ĠCroat":46861,"embrane":46862,"Ġéqu":46863,"Ġlightweight":46864,"Ùĥتب":46865,"Ġrepetitive":46866,"ĠDebt":46867,"Ġviewer":46868,"實åĬĽ":46869,"wiÄħz":46870,"Ġvalves":46871,"agna":46872,"ãģ¾ãĤĬ":46873,"Young":46874,"Ġpollutants":46875,"Ġrecycled":46876,".conf":46877,"Ġclo":46878,"à¸IJาà¸Ļ":46879,"emer":46880,"Ġactress":46881,"ÏĥηÏĤ":46882,"ĠHydrology":46883,"èĬ±çĶŁ":46884,"Ġsalts":46885,"organization":46886,"ĠFriedrich":46887,"(This":46888,"å°½åĬĽ":46889,"æİ§åζçļĦ":46890,"åĨįç͍":46891,"å±ħå®¶":46892,"Ġwarehouse":46893,"Ġmun":46894,"ificance":46895,"æĪij们éĥ½":46896,"Ġceramic":46897,"ĠReligious":46898,"Ġtö":46899,"inline":46900,"ç±½":46901,"æ£Ģä¿®":46902,"পন":46903,"ë°Ķ":46904,"Ġmerged":46905,"便æį·":46906,"ĠInstrument":46907,"æĦıè¯ĨçļĦ":46908,"ç¨ħ":46909,"ιÏĤ":46910,"plicates":46911,"Ġchrist":46912,"å¼ĢåĪĽ":46913,"Ġexotic":46914,"裳":46915,"yson":46916,"ĠOutcomes":46917,"ĠDevices":46918,"Msg":46919,"对该":46920,"Ġpersever":46921,"ائÙĬ":46922,"ä¾ĥ":46923,"genic":46924,"æĤłæĤł":46925,"äºĨä½ł":46926,"inee":46927,"çĹħæĪ¿":46928,"à¹Ĥย":46929,"Ġ(%":46930,"ĠXI":46931,"-load":46932,"Ġremotely":46933,"Ġweit":46934,"å¨ħ":46935,"atuak":46936,"ĠPriority":46937,"lip":46938,"׾×ķת":46939,"Ġcivilians":46940,"switch":46941,"Ġ×ij×ĵ":46942,"ĠCRE":46943,"Ġactivist":46944,"å·²ç»ıæĪIJ为":46945,"ĠNatal":46946,"太å¤ļçļĦ":46947,"Ġbooking":46948,"严峻":46949,"Ġanticipation":46950,"ĠRuby":46951,"æīĵæī®":46952,"ĠпÑĢинима":46953,".isEmpty":46954,"igos":46955,"Ġdele":46956,"ãģ«éĸ¢":46957,"Ðŀб":46958,"Ġpraised":46959,"ĠNaval":46960,"ÙģØ±Ø§Ø¯":46961,"ĠTall":46962,"å¸ĤçļĦ":46963,"ĉcin":46964,"ĠSax":46965,"骨头":46966,"æĺİç¡®çļĦ":46967,"åĭĩäºİ":46968,"ÑĤÑĥа":46969,"è¾¼":46970,"åIJĮç±»":46971,"Ġextracellular":46972,"ç§ijæĬĢåĪĽæĸ°":46973,"'])Ċ":46974,"ÑĤеÑĤÑĥ":46975,"ĠSynthesis":46976,"NEW":46977,"æĺ¯çͱäºİ":46978,"ÃŃlia":46979,"Ġauxiliary":46980,"Ġtires":46981,"ĠLoren":46982,"grave":46983,"ä¸įæĺİçϽ":46984,"iquity":46985,"િ":46986,"Solved":46987,"town":46988,".Date":46989,"é³³":46990,"{f":46991,"ä½łçİ°åľ¨":46992,"Ġobstruct":46993,"ĠWeeks":46994,"Ġsociale":46995,"éĩı为":46996,"èĬ±éĴ±":46997,"Transform":46998,"Ġcongru":46999,"Qual":47000,"豪åįİ":47001,"enum":47002,"åħħåĪĨçļĦ":47003,"滤波":47004,"imat":47005,"Ġhaul":47006,"ĠANS":47007,"Ġspider":47008,"åħĶåŃIJ":47009,"äºĨä¸ĢæĿ¡":47010,".equals":47011,"-direction":47012,"èģĮåľº":47013,"Ġbiopsy":47014,"ĠÏĦη":47015,"Ġcautious":47016,"Ġplag":47017,"à¸Ĺั":47018,"æ°¨éħ¸":47019,"arbeit":47020,"Ġestablishes":47021,"Ġairline":47022,"ĠÑģпеÑĨиалÑĮ":47023,":\",":47024,"Ġ(\\(\\":47025,"Community":47026,"çļĦè¡Ģ":47027,"пов":47028,"ÙIJÙĬ":47029,"ĠÏĥÏĦην":47030,"'-":47031,"earth":47032,"é©´":47033,"ÑĢÑĥд":47034,"fica":47035,"纳米":47036,"Ġnails":47037,"Ġgek":47038,"åı¯éĿłæĢ§":47039,"OOL":47040,"Ġarteries":47041,"Ġattorneys":47042,"çļĩåŃIJ":47043,"getto":47044,"३":47045,"Ġcontributors":47046,"æ¯Ķçİĩ":47047,"ä¸ĭçıŃ":47048,"³³³³³":47049,"Ġubiquit":47050,"çĽijæĬ¤":47051,"calculation":47052,"/{":47053,"ĠpÅĻi":47054,"cro":47055,"èĩ´å¯Į":47056,"ög":47057,"added":47058,"翼翼":47059,"Ġtransmitting":47060,"ĠÙĪÙĤد":47061,"ĠÙĦØ£":47062,"Ġphosphorus":47063,"ĠUniv":47064,"ĠبÙĩا":47065,"Tests":47066,"çļĦé£Łçī©":47067,"åĿĩåı¯":47068,"Ġmessaging":47069,"ĠPlato":47070,"Nature":47071,"-count":47072,"Ġtweet":47073,"ĠبÙĪ":47074,"æĮģä¹ħ":47075,"æĭ¥æĬ±":47076,"Ġconstituents":47077,"ĠSang":47078,"-energy":47079,"涨å¹ħ":47080,"ĠDrag":47081,"лой":47082,"åįģæľĪ":47083,"çļĦæľĢä½³":47084,"çļĦæĥħå½¢":47085,"æ»ĭåij³":47086,"æ§Ľ":47087,"Ġviability":47088,"ĠاÛĮراÙĨ":47089,"Ġtatto":47090,"å¸ľ":47091,"Camp":47092,"åĴĮ个人":47093,"rients":47094,"åĽĽèĤ¢":47095,"ARI":47096,"sam":47097,"Ġrefusal":47098,"aucoup":47099,"'))":47100,"åĽļ":47101,"Ġcores":47102,"ĠWeber":47103,"Ġmonarch":47104,"çĶ¨ä½ľ":47105,"ç³Łç³ķ":47106,"ĠBod":47107,"eil":47108,"ĠAndrea":47109,"æľ¨æĿIJ":47110,"Ġprivileges":47111,"祷":47112,"×ķ׼":47113,"Ġankle":47114,"results":47115,"ĠMedicaid":47116,"}}=":47117,"çĶŁæľº":47118,"å·ħå³°":47119,"ĠCrystal":47120,"ĠLov":47121,"Ġযà¦":47122,"ĠAdobe":47123,"è¡ĮæĶ¿æľºåħ³":47124,"President":47125,"éĢ¾æľŁ":47126,"/z":47127,"âĢĿâĢĶâĢĶ":47128,"ĠHod":47129,"Ġellos":47130,"Ġaggregation":47131,"æĸ¹æīį":47132,"Ġtexte":47133,"Until":47134,"ĠDirectory":47135,"æĹĭå¾ĭ":47136,"+n":47137,"Ġ:-":47138,"ĠAboriginal":47139,"'));Ċ":47140,"产åĵģè´¨éĩı":47141,"Ġì¢ħ":47142,"ÙĬاة":47143,"formatics":47144,"çļĦåĬŀæ³ķ":47145,"马车":47146,"Ġmd":47147,"ÙĦÙĩا":47148,"à¸Ńม":47149,"Liter":47150,")\\,":47151,"ĠÙħÙĨØ·":47152,"Ġnood":47153,"δο":47154,"Ġnickel":47155,"Ġpins":47156,"Ġexcluding":47157,"åĪĽå§ĭ人":47158,"ologous":47159,"Ġsuccesses":47160,"ĠSuite":47161,"à§ĩà¦Ľà¦¿à¦²":47162,"Ġconhec":47163,"被害":47164,"ĠJazz":47165,"apia":47166,"atient":47167,"Imp":47168,"æĭŃ":47169,"Ġë¹":47170,"Ġbutt":47171,"Ñĩник":47172,"ĠObviously":47173,"Ġtuberculosis":47174,"ä¸Ĭè¯ī":47175,"Ġeffet":47176,"çļĦéĴ±":47177,"à±Ģ":47178,"published":47179,"åıĺå¼Ĥ":47180,"èĥĮåIJİçļĦ":47181,"baar":47182,"ãĢijï¼ļ":47183,"creased":47184,"Ġsweep":47185,"å¼Ī":47186,"ophage":47187,"Ġbýt":47188,"-id":47189,"ĠãĢĤâĢĿ":47190,"elastic":47191,"ç§ĭåŃ£":47192,"ĠIndividuals":47193,"ĠPorter":47194,"Å£":47195,"fon":47196,".hpp":47197,"Applic":47198,"ĠGRE":47199,"ĠItems":47200,"Õ¡Õ£":47201,"ĠOrganisation":47202,"俺":47203,"åºķ线":47204,"اعدة":47205,"+a":47206,"تÙİ":47207,"ĠоÑĨен":47208,"ĠTesla":47209,"ĠGilbert":47210,"Ġdagat":47211,"Ġyr":47212,"ç͍以":47213,"æ¸ħæĸ°":47214,"ĠSolving":47215,"esthetic":47216,"å¹¶éĢļè¿ĩ":47217,"Ġrespiration":47218,"Ġdiffuse":47219,"è¦ģ说":47220,"вÑĭе":47221,"bau":47222,"åľ¨åĽ½åĨħ":47223,"ogra":47224,"Ġrisky":47225,"Ġfoolish":47226,"äºĨä¸Ģä¼ļåĦ¿":47227,"Ġjudgement":47228,"Ġtul":47229,"ungkin":47230,"xf":47231,"å½ĵåį³":47232,"atorio":47233,"Ġdisappointment":47234,"%d":47235,"ĠCalendar":47236,"ICH":47237,"ĠResearchers":47238,".View":47239,"年以æĿ¥":47240,"æĬķ稿":47241,"ĠسÙĦ":47242,"ĠVietnamese":47243,"refer":47244,"ĠWriter":47245,"èĢģ太太":47246,"ิม":47247,"continu":47248,"borough":47249,"è¿Ļä»¶äºĭæĥħ":47250,"è°Īåΰ":47251,"Html":47252,"wat":47253,"{g":47254,"çļĦèīºæľ¯":47255,"å·į":47256,"ĠComposite":47257,".Size":47258,"éĤ®æĶ¿":47259,"оÑģÑĤи":47260,"rl":47261,"Ġstochastic":47262,"ĠEpidem":47263,"Ġsells":47264,"ĠTah":47265,"ĠFix":47266,"åľŁè±Ĩ":47267,"ĠTitan":47268,"Ġantimicrobial":47269,"Ġtransformer":47270,"ä¸įçͱå¾Ĺ":47271,"rometry":47272,"æĽī":47273,"Ġmultitude":47274,"('.":47275,"irie":47276,"ä¸Ńéĥ¨":47277,"Ġ'+":47278,"à¸ģาย":47279,"Ġinternally":47280,"-General":47281,"ĠпеÑĢеда":47282,"ĠHosp":47283,"гоÑĢ":47284,"å¤įèĭı":47285,"Ġlump":47286,"Ġmultimedia":47287,"Ġshrugged":47288,"Ġdemo":47289,"人们对":47290,"ä¸ĭ车":47291,"ategor":47292,"ĠDefence":47293,"Ġbun":47294,"aways":47295,"åĦĢ":47296,"çł´è£Ĥ":47297,"cale":47298,"оÑĤо":47299,"åľ¨åĨħçļĦ":47300,"çŀŃ":47301,"ĠQuantity":47302,"åIJijä»ĸ":47303,"ĠSTUD":47304,"严谨":47305,"ĠÎļÏį":47306,"isz":47307,"æ²Į":47308,"Ġнаиб":47309,"ĠObjective":47310,"\"\"\"ĊĊ":47311,"Major":47312,"สิà¹Īà¸ĩ":47313,"ĠJessica":47314,"Ġ×ŀש":47315,"Ġmicroseconds":47316,"stitutional":47317,"Ġmerits":47318,"Ġcustomized":47319,"ĠDiff":47320,"ç®Ģæ´ģ":47321,"ĠMaintain":47322,"ĠMarkets":47323,"Ġneuron":47324,"orro":47325,"åĩºåĽ½":47326,"幫åĬ©":47327,"ĠÙĬÙĨ":47328,"ĠHav":47329,"akk":47330,"å¾ĢæĿ¥":47331,"ĠزÛĮر":47332,"×ķ׾×Ķ":47333,"Ġbour":47334,"idable":47335,"æ°ijæ³ķ":47336,"Ġhappily":47337,"审议":47338,"ĠпÑĢаво":47339,"даг":47340,"Ġgj":47341,"ç»ĪçĤ¹":47342,"Ġgoddess":47343,"ĠPros":47344,"åͮ价":47345,"ä»ĸ没æľī":47346,"åѦçĶŁä»¬":47347,"çϾç§ij":47348,"Ġfmt":47349,"èĥ½å¤Łåľ¨":47350,"RM":47351,"ĠTheater":47352,"ç͍æĪ·çļĦ":47353,"вÑĢоп":47354,"iliations":47355,"Ġundertaking":47356,"<>":47357,"kph":47358,"Ġflowering":47359,"ĠTrou":47360,"å°ıä¼Ļä¼´":47361,"Ġsplitting":47362,"ĠEngineers":47363,"ĠPG":47364,"ĠÑĦа":47365,"Ġtej":47366,"好人":47367,"èªł":47368,"биÑĢа":47369,"Ġwitch":47370,"ĠFortunately":47371,"Ġвам":47372,"çī©ä»·":47373,"Eth":47374,"Ġfungal":47375,"è·¯éĿ¢":47376,"åĺİ":47377,"Ïħνα":47378,"宣åijĬ":47379,"inology":47380,"ä¿ĺ":47381,"Ġvomiting":47382,"ĠâĶĤ":47383,"Ġstare":47384,"åı«æĪij":47385,"acqu":47386,"èģĨ":47387,"ĠHarbor":47388,"Ġdespués":47389,"å½ĵæĹ¥":47390,"ĠÐľÐ¾Ñģк":47391,"ĠWend":47392,"åĩºèī²":47393,"ãģĵãĤį":47394,"çī©è´¨çļĦ":47395,"ĠвÑĭÑĢа":47396,"éĺ¶æ®µçļĦ":47397,"Bio":47398,"ĠAcadem":47399,"ĠScientists":47400,"æĭ§":47401,"aporation":47402,"åĽŀè·¯":47403,"()).":47404,"eke":47405,"ÏģÏĮ":47406,"ĠChicken":47407,"'ex":47408,"ãģĪãģ¦":47409,"渲":47410,"Ġendangered":47411,"æ²ī浸":47412,"Assert":47413,"Ġmane":47414,"ä¹Łä¸º":47415,"ĠÑģка":47416,"æĸ°èģŀ":47417,"æĸ¹å¼ıçļĦ":47418,"нÑıÑı":47419,"èĥĮå½±":47420,"Ġমধà§įয":47421,"ĠcientÃŃfic":47422,"_-":47423,"ãĥĽ":47424,"ĠDir":47425,"èĩªè±ª":47426,"ĠÅĽw":47427,"ãģİ":47428,"强迫":47429,"çĽijè§Ĩ":47430,"ĠYank":47431,"×Ļר×Ķ":47432,"Ġprotagonist":47433,"Ġdotted":47434,"åĺ¶":47435,"ĠÙħÙĪØ¬ÙĪØ¯":47436,"Between":47437,"æĶ¾è¿ĩ":47438,"ÑĤоÑĩно":47439,"Ġproceeded":47440,"ä»İ严":47441,"æ·¨":47442,"ிற":47443,"اÙģØª":47444,"Ġalbeit":47445,"éĵ²":47446,"ä¼°ç®Ĺ":47447,"Ġnich":47448,"ä¸Ĭå®ĺ":47449,"è¿ĩå¹´":47450,"ظر":47451,"Ġunfamiliar":47452,"Aw":47453,"ĠпÑĢоÑĦе":47454,"çĸ²æĥ«":47455,"ĠMentions":47456,"ĠTN":47457,"Ġberg":47458,"Ġداد":47459,"Ġinitialize":47460,"Ġsaber":47461,"*sqrt":47462,"ĠHerbert":47463,"Ġkills":47464,"Ġarche":47465,"nick":47466,"enario":47467,"Ġconcise":47468,"åĽ°æĥij":47469,"ĠFormer":47470,"desc":47471,"æĹĹä¸ĭ":47472,"ĠиÑģполÑĮзоваÑĤÑĮ":47473,"cedure":47474,"Ġcomplained":47475,"ķàµįà´":47476,"æĪijå¸ĮæľĽ":47477,"æĭ¼éٳ":47478,"/ch":47479,"ozo":47480,"åĸĬéģĵ":47481,"ĠChapters":47482,"åIJijå¤ĸ":47483,"these":47484,"askan":47485,"Ġutf":47486,"å¾ĴåĪij":47487,"KT":47488,"Ġ×Ķ×ŀ×ķ×":47489,"Ġgad":47490,"ellite":47491,"æĺ¯ä¸Ģä»¶":47492,"hey":47493,"stage":47494,"ĠContinuous":47495,"å°ıæĻĤ":47496,"ä¸īæĺŁ":47497,"Bay":47498,"Ġsei":47499,"окÑĢа":47500,"/?":47501,"Ġds":47502,"çļĦè§ĤçĤ¹":47503,"Ġtelev":47504,"ggle":47505,"諾":47506,"æĪijå°Ĩ":47507,"æĪijä¹Łä¸į":47508,"_on":47509,"าà¸Ń":47510,"æĭīå¼Ģ":47511,"cciones":47512,"æł¼æĸ¯":47513,"Ġcontinuation":47514,"湿度":47515,"ĠÙĨØ´":47516,"Ġlabeling":47517,"çļĦç»ıéªĮ":47518,"åĨħåľ°":47519,"Ġbiases":47520,"ä¸ĸä¸Ĭ":47521,"ĠاÙĦÙħص":47522,"'''":47523,"nehmen":47524,"دÛĮ":47525,"íķŃ":47526,"æĦıè¯Ĩå½¢æĢģ":47527,"Ġpulls":47528,"wicklung":47529,"å¿ħé¡»åľ¨":47530,"Loader":47531,"Ġpractitioner":47532,"'an":47533,"ZZ":47534,"(df":47535,"à¸ĸูà¸ģ":47536,"Ġcongen":47537,"dzie":47538,"ibi":47539,"othermal":47540,"лиÑĩа":47541,"ĠکردÙĩ":47542,"Surely":47543,"arettes":47544,"([]":47545,"éĩıåŃIJ":47546,"å®¶å±ħ":47547,"ëĵ¤ìĿĢ":47548,"ÑĤоÑĢÑĭ":47549,"Õ¬":47550,"ĠTanz":47551,"Ġzona":47552,"è¿ĻåĿĹ":47553,"ĠVac":47554,"iencia":47555,"-title":47556,"Ġoversight":47557,"åĤ¨èĵĦ":47558,"Ġninth":47559,"rijk":47560,"第ä¸Ģ竳":47561,"åı¤åŁİ":47562,"तà¥ĩ":47563,"ĠColonel":47564,"é³¥":47565,"ĠOpinion":47566,"racting":47567,"ابر":47568,"checked":47569,"åŁĥåıĬ":47570,"Ġconspiracy":47571,"太æŀģ":47572,"'''Ċ":47573,"ĠKS":47574,"yarakat":47575,"ĠPhen":47576,"念头":47577,"development":47578,"Ġlearnt":47579,"ĠÙħÙĨÙĩا":47580,"ĠFields":47581,"Ġarchaeological":47582,"ĠYa":47583,"-tion":47584,"ĠÙĦد":47585,"ellectual":47586,"onio":47587,"-primary":47588,"âμ":47589,"ĠвÑĭÑĪе":47590,"Ġarbitration":47591,"æīĢæľīæĿĥ":47592,"Combine":47593,"ç¿¡ç¿ł":47594,"Ġregisters":47595,"Ġbog":47596,"ĠÑģÑĭ":47597,"æ·¤":47598,"venous":47599,"Ġinfar":47600,"ACKGROUND":47601,"Verse":47602,"å·®ä¸į":47603,"ĠHerz":47604,"opal":47605,"ĠBren":47606,"à¸Ĺุà¸ģ":47607,"ɾ":47608,"Ġendure":47609,"Ġsyllables":47610,"Ġtheat":47611,"ĠØŃر":47612,"Ġê°IJ":47613,"ĠAppeal":47614,"çݰè¡Į":47615,"ãĥ©ãĤ¤":47616,"ĠاÙĦشع":47617,"[l":47618,"arag":47619,"ĠOle":47620,"Ġquien":47621,"Ġsexuality":47622,"ĠFear":47623,"Ġpollen":47624,"Could":47625,"continued":47626,"Ġείναι":47627,"Ġeb":47628,"assign":47629,"çļĦåħ¬åı¸":47630,"Ġmucho":47631,"move":47632,"ä»»åij½":47633,"èį£èĢĢ":47634,"Ġdischarged":47635,"ĠCAS":47636,"ãĤĦãģĻ":47637,"ãģŁãĤģãģ«":47638,"ĠUrs":47639,"ĠInterior":47640,"Ġ\\\\Ċ":47641,"sets":47642,"ĠOwen":47643,"ansen":47644,"Rh":47645,"ĠÑĢиÑģ":47646,"Five":47647,"cot":47648,"ĠGeneva":47649,"ĠÙħعÙĦÙĪÙħات":47650,"ĠWorth":47651,"æģ°å¥½":47652,"urate":47653,"Ġhatte":47654,"Ġsele":47655,"Ġsaya":47656,":.":47657,"ï¼Ĺ":47658,"æĻ¯èī²":47659,"éĺ´éģĵ":47660,"åIJĮä¼´":47661,"æŃ·åı²":47662,"Jer":47663,"hematic":47664,"çļĦéĤ£ä¸ª":47665,"гÓĢ":47666,"ĠتÙĪÙĦ":47667,"Ġbalances":47668,"nice":47669,"ãģIJ":47670,"Ġdistortion":47671,"Ġveterin":47672,"Ġfö":47673,"ĠMile":47674,"Ġtranslates":47675,"ĠTommy":47676,"èĢħåľ¨":47677,"jj":47678,"à¸Ĭà¹Īวย":47679,"/hess":47680,"Ġstrand":47681,"ĠDesert":47682,"éĶĻè¿ĩ":47683,"Ġchromatography":47684,"вид":47685,"æĺŁè¾°":47686,"Charles":47687,"Ġmuseums":47688,"letters":47689,"Ġpled":47690,"learning":47691,"ivitÃł":47692,"éĶħçĤī":47693,"Ñģкое":47694,"Ġeinf":47695,"Éij":47696,"Ġcaregivers":47697,"转头":47698,"Ġpopula":47699,"ĠмеÑģÑĤо":47700,"MY":47701,"bah":47702,"å¢Ĺ":47703,"èĦ¸ä¸ĬçļĦ":47704,"marine":47705,"underline":47706,"dens":47707,"ĠFranco":47708,"Ġmelted":47709,"igesimal":47710,"Ġacon":47711,"èĭĵ":47712,"Ġfundamentally":47713,"-sided":47714,"&#":47715,"/km":47716,"Ġά":47717,"ĠMontana":47718,"quez":47719,"ç§Łéĩij":47720,"Ġhospitality":47721,"Ġteż":47722,"ĠìĹ°êµ¬":47723,"Pin":47724,"éĹ®é¢ĺæĺ¯":47725,"Ġdevotion":47726,"Ġenjoyment":47727,"è§ĦèĮĥåĮĸ":47728,"Oxford":47729,"ðĿij¡":47730,"ĠGenetics":47731,"åĽ¾çļĦ":47732,"Ġней":47733,"Ġlearns":47734,"Ġunfold":47735,"ĠCollections":47736,"Ġsleeve":47737,"à¸Ħิà¸Ķ":47738,"ä¸ŃåĮ»èį¯":47739,"ĠBeautiful":47740,"éģı":47741,"该å¦Ĥä½ķ":47742,"Ġselfish":47743,"Ġbiography":47744,"WF":47745,"yz":47746,"robl":47747,"ĠLay":47748,"以èī²":47749,"aths":47750,"eker":47751,"对ä¸ŃåĽ½":47752,"ÙĩÙĪØ±":47753,"èİĬ":47754,"ĠSector":47755,"Ġbef":47756,"jÃł":47757,"æĪij羣çļĦ":47758,"Recomm":47759,"}^{(":47760,"Cosine":47761,"Ġtaxi":47762,"Ġmassively":47763,"ĠÅŁ":47764,"Bur":47765,"Ġexaminations":47766,"Ġpossono":47767,"ĠBle":47768,"ä½ĵç³»çļĦ":47769,"æ¯ĽçĹħ":47770,"Sol":47771,"就说":47772,"Ġpredictable":47773,"ĠSlov":47774,"volution":47775,"Ġpotentials":47776,"Tags":47777,"UAL":47778,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":47779,"ĠRaymond":47780,"ä¸įåIJĪ":47781,"à¹ģห":47782,"ä¸İåıijå±ķ":47783,"Ġpainter":47784,"Ġdividends":47785,"ĠEu":47786,"ĠSpencer":47787,"ĠMyth":47788,"ĠEther":47789,"ä¹ħäºĨ":47790,"åζå®ļäºĨ":47791,"管家":47792,"(double":47793,"Ġвид":47794,"Ġremoves":47795,"Ġcrore":47796,"ãģ¸ãģ®":47797,"gef":47798,"ĠDemand":47799,"Ġrelaxing":47800,"Ġutterly":47801,"ãģ§ãģĻãģĮ":47802,"gende":47803,"èĩªæľī":47804,"Received":47805,"ète":47806,"ç²¾ç¥ŀçļĦ":47807,"etur":47808,"ä¸į该":47809,"çķĻä¸ĭçļĦ":47810,"é¼ĵèĪŀ":47811,"Ġdiffraction":47812,"bullet":47813,"_idx":47814,"ĠBuddhism":47815,"еÑĨ":47816,"Ġalkyl":47817,"Ġoverload":47818,"ĠнаÑĢод":47819,"ĠíķĺëĤĺ":47820,"ĠAirl":47821,"Ġartikel":47822,"Ġдов":47823,"ĠGrass":47824,"/wiki":47825,"Ġdiarrhea":47826,";Ċ":48535,"sy":48536,"à¹Ģหà¹ĩà¸Ļ":48537,"⣩":48538,"brates":48539,"Ġexplosive":48540,"Ú¯ÛĮرÛĮ":48541,"Ġchooses":48542,"ĠInstructions":48543,"nee":48544,"æĶ¶æĶ¯":48545,"Ġgraphical":48546,"Ġeig":48547,"ĠAsc":48548,"ĠMBA":48549,"ÏĦÏĮ":48550,"Ġcountryside":48551,"è¦ģä¸įæĺ¯":48552,"érica":48553,"æĭ¼åij½":48554,"ÑĤаÑĢ":48555,"ĠNurse":48556,"-digit":48557,"ĠÑĤова":48558,"Ġgateway":48559,"ĠFew":48560,"Ġcortical":48561,".at":48562,"borg":48563,"ANN":48564,"isate":48565,"åĬ¨éĿĻ":48566,"廢":48567,"Ġsliced":48568,"ushi":48569,"Ġполож":48570,"è¯ł":48571,"æµ·æ°´":48572,"ĠGenet":48573,"äºĶæľĪ":48574,"listed":48575,"Ġstesso":48576,"ĠHF":48577,"äºī夺":48578,"JD":48579,"_page":48580,"è§£éĩĭ":48581,"_SIZE":48582,"Ġproprio":48583,"ĠHeight":48584,"rupted":48585,"Ġcurvature":48586,"cv":48587,"ĠDoor":48588,"以ä¸ĭåĩłä¸ª":48589,"örd":48590,"ĠиÑģÑĤоÑĢи":48591,"(head":48592,"è¿Ļåıª":48593,"Ġestudio":48594,"饲åħ»":48595,"éĽĨ群":48596,"éĽ·è¾¾":48597,"夯":48598,"渺":48599,"нÑĥÑĤ":48600,"ulagway":48601,"iï¬ģ":48602,"inki":48603,"ndef":48604,"ĠAdoles":48605,"ziÄĻ":48606,"ĠRebecca":48607,"Ġchase":48608,"мÑı":48609,"天åłĤ":48610,"Ġrevenge":48611,"Ġfade":48612,"ĠSCI":48613,"身éĤĬ":48614,"Ġculturally":48615,")\\).":48616,"ç«ĻçĤ¹":48617,"kinson":48618,"(vector":48619,"è´ŃæĪ¿":48620,"ĠÑģÑĤаÑĢоÑģ":48621,"PLC":48622,"xl":48623,"å¿ĥåĬ¨":48624,"çľŁäºº":48625,"Ġportrayed":48626,"imas":48627,"Ġ%>%Ċ":48628,"feeding":48629,"à¦Ĺà§ģল":48630,"Ġdeficits":48631,"Mah":48632,"æ½®æµģ":48633,"ispatch":48634,".NET":48635,"修饰":48636,"/news":48637,"ĠEllis":48638,"Ġihrer":48639,"cg":48640,"ĠMumbai":48641,"ä¹ĭç±»":48642,"radio":48643,"ãģ¤ãģ®":48644,"ĠBios":48645,"expensive":48646,"åIJĦçķĮ":48647,"ANGE":48648,"ĠÑĤÑı":48649,"Ġobservers":48650,"γγ":48651,"åįķåħĥæł¼":48652,"ç¿©":48653,"ultat":48654,"çIJħ":48655,"èĭ·":48656,"å¤įåį°":48657,")]ĊĊ":48658,"ç͍èĩªå·±çļĦ":48659,"书çĶ»":48660,"çĮ¶":48661,"ĠIntervention":48662,"]}":48663,"ĠSequ":48664,"Ġic":48665,"ĠBiomed":48666,"åħ»æĬ¤":48667,"漫çĶ»":48668,"Ġmillones":48669,"Ġtubuh":48670,"çķħéĢļ":48671,"ĠKunst":48672,"inan":48673,"ĠEG":48674,"Ġleaning":48675,"jang":48676,"ĠTik":48677,"Ġmurdered":48678,"Ġlifespan":48679,"certain":48680,"minent":48681,"éϤå¤ĸ":48682,"Ñĸд":48683,"Japanese":48684,"uur":48685,"Ġconjunto":48686,"uang":48687,"wikk":48688,"Ġforemost":48689,"Ġsexually":48690,"Ġdisturbed":48691,"-ter":48692,"çļĦæĹ¶ä»£":48693,"à¥įन":48694,"檬":48695,"াà¦Ĥল":48696,"ecd":48697,"失误":48698,"Ġfluorescent":48699,"Ġdaar":48700,"flux":48701,"代çIJĨ人":48702,"欺éªĹ":48703,".oz":48704,"ས":48705,"CRIPTION":48706,"upan":48707,"ìķ½":48708,"achen":48709,"Ġkina":48710,"extension":48711,"Ġmerger":48712,"çľ¶":48713,"Ġdiagnose":48714,"为主è¦ģ":48715,"iq":48716,"çļĦéģĵ":48717,"Ġmindful":48718,"Ġdiminished":48719,"uds":48720,"Ġmorphological":48721,"ä»ĸæľī":48722,"Ġвза":48723,")>":48724,"ä¸Ģ款":48725,"Ġcapita":48726,"NBA":48727,"åľ°å°Ĩ":48728,"oths":48729,"Ġconsegu":48730,"é¡¶éĥ¨":48731,"åĴĮæĸ°":48732,"ĠAthen":48733,"Ġcelle":48734,"Ġaktiv":48735,"Ġszer":48736,"Ġregulates":48737,"------------------------------------------------":48738,"uffs":48739,"Ġfighters":48740,"ĠдÑĢÑĥгие":48741,"ç»Ļå®ļ":48742,"ãģĹãģ¾ãģĹãģŁ":48743,"Ġdirecting":48744,"Ġantis":48745,"ä¹Łä¸įè¦ģ":48746,"Ġyielded":48747,"Ġyo":48748,"é¡Ĩ":48749,"æĪ·åı£":48750,"ĠاÙĦÙħتØŃ":48751,"Ġclasse":48752,"æīĭæ©Ł":48753,"éĩijå±±":48754,"ç¾İåѦ":48755,"opez":48756,"ĠобÑĢазованиÑı":48757,"Ġdash":48758,"æ»Ķ":48759,"à¹Ģà¸Ľà¸¥":48760,"Ġcush":48761,"equiv":48762,"Ġpenyakit":48763,"åħ«å¹´":48764,"ĠGeme":48765,"Ġcontour":48766,"Ġlign":48767,"èĢģèĢħ":48768,"ĠShift":48769,"Russian":48770,"Ġìļ°ë¦¬":48771,"Ġdeed":48772,"ĠLed":48773,"Ġmoi":48774,"оÑĢонÑĭ":48775,"something":48776,"çļĦæ¨¡æł·":48777,"Ġ\"__":48778,"вол":48779,"ä½įæķ°":48780,"versal":48781,"æĬĽå¼ĥ":48782,"URN":48783,"ĠPalmer":48784,"ä¸ĢæıIJ":48785,"Ñīем":48786,"racies":48787,"Ġcommanded":48788,"ĠÑģÑĤаÑĢоÑģног":48789,"Ġowe":48790,"ÑĤелÑĮÑģÑĤва":48791,"Ġtransferring":48792,"ĠStructures":48793,"Ġhepatic":48794,"åĽ°æī°":48795,"ĠTechnol":48796,"BU":48797,"Ġ×IJ×Ŀ":48798,"à³įತ":48799,"ĠMG":48800,"åħ¨æĸ¹ä½į":48801,"otions":48802,"åºĶæĶ¶":48803,"Ġconfirms":48804,"ĠChildhood":48805,"ä¸ºè¿Ľä¸ĢæŃ¥":48806,"Ġquand":48807,"ĠÑģÑĤÑĢÑĥк":48808,"à¹ĥหà¸įà¹Ī":48809,"Ġstern":48810,"éĹ´è·Ŀ":48811,"லà¯į":48812,"ĠÑĤок":48813,"Ġ목":48814,"æĭĩæĮĩ":48815,"ä»»èģĮ":48816,"çĶ»åĩº":48817,"åį³ä½¿æĺ¯":48818,"ynthetic":48819,"Ġprestigious":48820,"uristic":48821,"ĠLeonard":48822,"ĠBun":48823,"é¢Ħæ¡Ī":48824,"/Tropical":48825,"Ġharmonic":48826,"typen":48827,"Ġlicenses":48828,"Ġà¹Ģà¸ŀืà¹Īà¸Ń":48829,"çľĭ好":48830,"-sid":48831,"Ġfacilitated":48832,"ĠSullivan":48833,"venues":48834,"ãģ¨ãģ®":48835,"Ġaccumulate":48836,"ĠActually":48837,"Ġrotational":48838,"jin":48839,"ursion":48840,"Ġsvilupp":48841,"аÑĤ":48842,"ìľ¼ë©°":48843,"Ġspeculation":48844,"bring":48845,"Station":48846,"è©¢":48847,"ленÑĭ":48848,"Ġblades":48849,"å¤ļ大":48850,"Ġнего":48851,"åħĥç´łçļĦ":48852,"填空":48853,"erschied":48854,"olithic":48855,"allo":48856,"Ġcoarse":48857,"WHERE":48858,"æ¸ħç®Ĺ":48859,"rang":48860,"Ġzeros":48861,"Ġindications":48862,"ĠراÙĩ":48863,"Ġlaundry":48864,"Au":48865,"Ġcalor":48866,"Ġaloud":48867,"æŀĦä»¶":48868,"Ġsignifica":48869,"ç°ĩ":48870,"Ġblended":48871,"aktor":48872,"Btn":48873,"è§ģäºİ":48874,"Period":48875,"太平æ´ĭ":48876,"Ġbullying":48877,"Ġtrafficking":48878,"天çĶŁ":48879,"飾":48880,"åıªåī©ä¸ĭ":48881,"Ġcleans":48882,"纪æ£Ģ":48883,"Ġfavorites":48884,"éĢĥéģ¿":48885,"Ġsystème":48886,"alog":48887,"ĠBoot":48888,"زاÙĨ":48889,"æĹģè¾¹çļĦ":48890,"Ġattracting":48891,"çĮªèĤī":48892,"ĠCardinal":48893,"ksen":48894,"Ġwasted":48895,"ãĤ´":48896,"ç¬¬åĽĽç«ł":48897,"太éĺ³èĥ½":48898,"=\"_":48899,"xygen":48900,"Ġharvesting":48901,">\"":48902,"Ġhatred":48903,"SET":48904,"ĠPes":48905,"æľī大":48906,"Ġosm":48907,"èĢģå®¶":48908,"Ġnaw":48909,"åΰä¸Ģ个":48910,"Ġupside":48911,"ç»ĵå°¾":48912,"渲æŁĵ":48913,"Ġspaced":48914,"does":48915,"Ġsic":48916,"Objects":48917,"äºĨåĩºåİ»":48918,"ĠArtist":48919,"Ġsculpture":48920,"à§Ģব":48921,"ĠRV":48922,"åĮ»æĬ¤":48923,"ĠâĪĨ":48924,"ä¸ĥåįģ":48925,"Ġfres":48926,"ĠDrugs":48927,"å·®çļĦ":48928,"arthy":48929,"-resolution":48930,"æ¶Īè²»":48931,"़":48932,"ativos":48933,".sp":48934,"ĠHispanic":48935,"尺度":48936,"æľī好":48937,"]))":48938,"........................................................":48939,"Ġhath":48940,"Ġnth":48941,"ĠWhereas":48942,"amorph":48943,"urned":48944,"举åĬ¨":48945,"ĠSamsung":48946,"åıĺäºĨ":48947,"Ġtexto":48948,"Ġà¤ķर":48949,"éļ¨èijĹ":48950,"Ġcompromised":48951,"ĠBMC":48952,"åľ¨é«ĺ":48953,"Ġspike":48954,"Ġfunción":48955,"ä»ĭç´¹":48956,"Ġविà¤":48957,"ĠTox":48958,"vos":48959,"åĽŀéģ¿":48960,"çĺ¾":48961,"æĹ¥æľ¬äºº":48962,"Ġล":48963,"å¹´éĻIJ":48964,"楼主":48965,"aul":48966,"Ġ(<":48967,"ĠDiese":48968,"åıªå¾Ĺ":48969,"åħĥå¹´":48970,"åIJ¬åΰäºĨ":48971,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶĊĊ":48972,"Ġtriangular":48973,"Ġwallet":48974,"èĬ¹":48975,"Ġ×ij×Ĺ":48976,"COL":48977,"ĠDetailed":48978,"payers":48979,"å¹²çļĦ":48980,"å½±åĵįçļĦ":48981,"Ġunsure":48982,"è¿ŀéĶģ":48983,"åı¦æľī":48984,"Ġsprings":48985,"åľ¨åŃ¦æł¡":48986,"Ġovarian":48987,"atas":48988,"离åİ»":48989,"ĠÑĢазме":48990,"Ġogs":48991,"Ġvertically":48992,"Ġfavored":48993,"Ġfights":48994,"æĪIJè¯Ń":48995,"rera":48996,"æļ´éĽ¨":48997,"Ġmoh":48998,"åİĭåζ":48999,"Notification":49000,"èĤ¥èĥĸ":49001,"Îļ":49002,"лением":49003,"ä¸įåģļ":49004,"Ġconsolidation":49005,"ä¸į便":49006,"æĪijåĸľæ¬¢":49007,"çļĦç¬ij容":49008,"urry":49009,"_The":49010,"Ġrz":49011,"avo":49012,"Ġvolcanic":49013,"rosso":49014,"Ġuniformly":49015,"=s":49016,"HER":49017,"Mit":49018,"å¾ģæ±Ĥ":49019,"zyme":49020,"æ°¨åŁºéħ¸":49021,"Ġdiffering":49022,"åĽ¢å§Ķ":49023,"incinn":49024,"æĥ³å¿ħ":49025,"Blog":49026,"ówn":49027,"ä¼ļ产çĶŁ":49028,"ĠBuen":49029,"éĸĭåı£":49030,"Ġcovenant":49031,"ĠRouter":49032,"cci":49033,"线段":49034,"Ġspokesman":49035,"Ġcrafted":49036,"çĿ£æŁ¥":49037,"ê²ĥ":49038,"ĠFung":49039,"Ġì¤":49040,"ierre":49041,"Ġsilica":49042,"è³ĩæºIJ":49043,"component":49044,"以èī²åĪĹ":49045,"brella":49046,"ĠDoll":49047,"Lord":49048,"Ġrelieved":49049,"cock":49050,"çļĦä¸ĢçĤ¹":49051,"ĠBristol":49052,"essel":49053,"ĠGut":49054,"ÑijÑĤ":49055,"ĠStockholm":49056,"Ġacademics":49057,"ĠZa":49058,"Ġfrontier":49059,"åĢŁåı£":49060,"ERSION":49061,"Ġtrabaj":49062,"Comm":49063,"åĬłçĽŁ":49064,"ç´¯äºĨ":49065,"isle":49066,"å±ĭéĩĮ":49067,"ĠHC":49068,"ĠOptimization":49069,"çļĦä¸Ģ天":49070,"Ġà°¨":49071,"ÅĤu":49072,"ĠMartha":49073,"Ġpolicym":49074,"大è¡Ĺ":49075,"ä¹ĭåĴĮ":49076,"Ġallocate":49077,"Ġاخ":49078,".prot":49079,"ĠFlag":49080,"Ġaxi":49081,"매":49082,"ceptive":49083,"ĠLevels":49084,"Ġadulthood":49085,"Ġ×Ļ׼":49086,"ç¥Ńç¥Ģ":49087,"ĠSupplement":49088,"ĠTrinity":49089,"Ġperiodically":49090,"ĠBehavioral":49091,"Ëľ":49092,"ĠMcL":49093,"пла":49094,"åijĪçݰåĩº":49095,"ĠFriend":49096,"OLOGY":49097,"belie":49098,"/images":49099,"KO":49100,"emann":49101,"ĠEA":49102,"Ġreservation":49103,"Ġverw":49104,"æ°Ķè¡Ģ":49105,"ĠReduction":49106,"ëĵ±":49107,"墩":49108,"æıIJåıĬ":49109,"ĠSyrian":49110,"ĠHomework":49111,"ĠUses":49112,"design":49113,"ajÄħc":49114,"ĠAntib":49115,"Ġunderwater":49116,"å°Īæ¥Ń":49117,"éĤ¢":49118,"Ġconflicting":49119,"Ġnested":49120,"åĮ»åĬ¡":49121,"Ġmarble":49122,"¤":49123,"ĠAri":49124,"ä¸įåĥħ":49125,"ĠNest":49126,"ÑĩеÑĤа":49127,"Ġdai":49128,"osol":49129,"çŃīä½ł":49130,"以åīįçļĦ":49131,"bier":49132,"ĠDob":49133,"Ġallowance":49134,"Ġjeho":49135,"BLE":49136,"åİĭæĬij":49137,"Ġstressful":49138,"ä¸Ĭå¹´":49139,";Ċ":49962,"ayo":49963,"ç쵿ķı":49964,"é¢Ĩ导çıŃåŃIJ":49965,"\\alpha":49966,"mass":49967,"ðĿijĽ":49968,"Ġsesu":49969,"YC":49970,"lijk":49971,"ĠLegacy":49972,"Handle":49973,"Ġpoisoning":49974,"ĠOrthodox":49975,"Ġturtle":49976,"iore":49977,"缸äºĴä½ľç͍":49978,"ĠDeW":49979,"Ġ//Ċ":49980,"ĠHeavy":49981,"Ġmortal":49982,"æijĨæĶ¾":49983,"ĠBasket":49984,"åħīèį£":49985,"-play":49986,"Ġgol":49987,"nant":49988,"ĠAde":49989,"Ġparamount":49990,"ĠPROC":49991,"èĴ¸åıij":49992,"ĠWide":49993,"Ġadditions":49994,"ĠÑĢеак":49995,"imentos":49996,"Ġë°Ľ":49997,"çļĦéĩįçĤ¹":49998,"Ġíĸī":49999,"stick":50000,"ĠNeurosci":50001,"Ġbubbles":50002,"Pair":50003,"olate":50004,"accur":50005,"æľīä¸įå°ij":50006,"åĹ£":50007,"ĠCosts":50008,"è¿Ļ段æĹ¶éĹ´":50009,"Ġmicrom":50010,"Ġenfermed":50011,"Think":50012,"thead":50013,"Ġ\"\",Ċ":50014,"Ġflush":50015,"æĻļæľŁ":50016,"æĪijæīĢ":50017,"Ġfixation":50018,"Dou":50019,"VAL":50020,"ĠSaved":50021,"æ¼Ķç»İ":50022,"+i":50023,"hidden":50024,"Ġinorganic":50025,"Ġkans":50026,"Ġdeviations":50027,"ä¼ĺè¶Ĭ":50028,"મ":50029,"Ġmultiples":50030,"æģ¶åĬ£":50031,"Ġresent":50032,"à·ĥ":50033,"ä¼ļå½±åĵį":50034,"ä»»åĭĻ":50035,"uckland":50036,"phthalm":50037,"Ġsos":50038,"Äįi":50039,"Ġunrelated":50040,"lieÃŁ":50041,"Fort":50042,"ีà¸Ļ":50043,"acceptable":50044,"রণ":50045,"å®ģéĿĻ":50046,"incinnati":50047,"ĠGang":50048,"Ġsolidarity":50049,"/y":50050,"Ġrept":50051,"ĠDisorder":50052,"ĠVenezuela":50053,"é¦ħ":50054,"Ġseamlessly":50055,"æ·ĺå®Ŀ":50056,"ëĸ":50057,"Ġ];Ċ":50058,"ĠNoah":50059,"tl":50060,"Ġкм":50061,"æĵį纵":50062,"Ġfactorisate":50063,"Ġbeaucoup":50064,"åıªæĺ¯ä¸Ģ个":50065,"æ¶īåıĬåΰ":50066,"çī¹çļĦ":50067,"éĺŁåıĭ":50068,"Ñijм":50069,"hall":50070,"Ġinstructors":50071,"Ġresurrection":50072,"лад":50073,"ÐłÐ°":50074,"çĶŁäºĨ":50075,"ĠAtomic":50076,"ĠAdministrator":50077,"деÑģÑıÑĤ":50078,"åIJĦèĩªçļĦ":50079,"æľīéĹľ":50080,"ĠMant":50081,"ĠElder":50082,"Ġgenera":50083,"注å®ļ":50084,"Ġpolitique":50085,"Market":50086,"Ġgon":50087,"æķĻçļĦ":50088,"åIJĦçľģ":50089,"Ġpassport":50090,"ĠØ¥ÙĦا":50091,"_X":50092,"ubbed":50093,"Ġkeg":50094,"describe":50095,"zhou":50096,"Ġcondemned":50097,"Ġcomma":50098,"à¹ĩà¸ļ":50099,"ĠProphet":50100,"-vis":50101,"ĠBench":50102,"ä»ĸåĢijçļĦ":50103,"è§£æĶ¾åĨĽ":50104,"åĮºåĨħ":50105,"ĠmPa":50106,"æĴŃç§į":50107,"ĠпÑĢоп":50108,"Ġconstructs":50109,"Ġglycol":50110,"Ġmüssen":50111,"Ġdaylight":50112,"-comp":50113,"Ġpatrol":50114,"Ġcondemn":50115,"ĠÑĢазде":50116,"ÙĬÙĪÙĨ":50117,"èĢĮæĿ¥çļĦ":50118,"åIJĪèµĦ":50119,"afka":50120,"innings":50121,"ĠElsevier":50122,"æ¯ģçģŃ":50123,"æµģåŁŁ":50124,"Ġquantify":50125,"ĠCamera":50126,"Express":50127,"å¹¶è´Ń":50128,"Ġsyrup":50129,"Ġfictional":50130,"ाह":50131,"Ġimplementations":50132,"Ġsnapped":50133,"ubes":50134,"ç¢ĺ":50135,"xff":50136,"ĠSG":50137,"teil":50138,"Ġprere":50139,"Jim":50140,"ÙĬار":50141,"ĠOncol":50142,"Ġdeterminants":50143,"ĠGreeks":50144,"sem":50145,"arya":50146,"Ġmůže":50147,"Ġmmol":50148,"২০১":50149,"ĠÐłÐ°Ð·":50150,"cipline":50151,"Drop":50152,"åľ¨ä»ĸ们":50153,"plate":50154,"è²ĵ":50155,"Ġsurveyed":50156,"Ġflock":50157,"ĠClare":50158,"Ġradicals":50159,"credit":50160,"park":50161,"Ġeerste":50162,"ĠRum":50163,"Ġposible":50164,"žd":50165,"Ġsettlers":50166,"Ġcrisp":50167,"Ġselectively":50168,"Ġcriminals":50169,"éĺ¿æĭī伯":50170,"å͝çī©":50171,"Ġprésent":50172,"ĠداشتÙĩ":50173,">::":50174,"ĠEb":50175,"лки":50176,"Visible":50177,"Ġkaj":50178,"Ġnausea":50179,"ĠGalile":50180,"åıįæĩī":50181,"Driver":50182,"èĸªéħ¬":50183,"ĠHg":50184,"ĠAssign":50185,"æłijæŀĹ":50186,"Grand":50187,"Ġthirst":50188,"athing":50189,"ficiency":50190,"olymer":50191,"raviolet":50192,"breaks":50193,"èĤĽ":50194,"ĠSeller":50195,"unts":50196,"é͝":50197,"Ġbehaviours":50198,"観":50199,"æ®ĺ":50200,"Ġtuig":50201,"ĠRig":50202,"ymal":50203,"Ġtrong":50204,"าà¸Ĭ":50205,"ä¸Ńä¹Ł":50206,"ostasis":50207,"çĿĢèĩªå·±çļĦ":50208,"awat":50209,"ç¥ŀä»Ļ":50210,"æIJľéĽĨ":50211,"ĠHind":50212,"éré":50213,"Ġবà§įযব":50214,"yled":50215,"ĠRee":50216,"Ġà¦Ĺà§įর":50217,"åľ¨ä¹İ":50218,"weather":50219,"Ġairway":50220,".domain":50221,"ĠGovernance":50222,"ĠDanny":50223,"ستÙĩ":50224,"åĥ¹æł¼":50225,".image":50226,"åľ°è¿Ľè¡Į":50227,"åī§çĥĪ":50228,"Ġconserved":50229,"åĽłä¸ºä»ĸ们":50230,"ĠпÑĢинÑı":50231,"Ġহবà§ĩ":50232,"ÑĪем":50233,"驼":50234,"overty":50235,"çļĦäºĭçī©":50236,"rede":50237,"intendent":50238,"çŃīæĥħåĨµ":50239,"çĶŁçĶŁ":50240,"Ġaddedge":50241,"ãģĵãģ¨ãģ¯":50242,"Ġworrying":50243,"ĠPunj":50244,"ayed":50245,"rare":50246,"OCs":50247,"Ġmassage":50248,"Need":50249,"åij¨åĽ´çļĦ":50250,"ĠPure":50251,"Ġcler":50252,"çĸĻ":50253,"åIJĦåįķä½į":50254,"Ġnr":50255,"Ġ%ĊĊ":50256,"Ġpalab":50257,"ĠArmstrong":50258,"ä¸Ģ種":50259,"ĠBU":50260,"ĠDuncan":50261,"anson":50262,"warz":50263,"ĠLil":50264,"ankton":50265,"Ġcultured":50266,"Ġfeathers":50267,"رÙĬÙģ":50268,"\\displaystyle":50269,"æ±IJ":50270,"æīĵçļĦ":50271,"Ġcalibr":50272,"Ġelectroph":50273,"æĢİä¹Īåģļ":50274,"Ġcultivate":50275,"åŃľ":50276,"лÑĭÑħ":50277,"ymn":50278,"zych":50279,"itza":50280,"'es":50281,"Ġclearer":50282,"cdn":50283,"Ġintuition":50284,"åĴļ":50285,"Ġplaque":50286,"åIJĪèĤ¥":50287,"สà¸ĩ":50288,"Ġbeverage":50289,"Illuminate":50290,"grades":50291,"æĭ¬åı·":50292,"Ġsher":50293,"ĠParallel":50294,"wear":50295,"ĉr":50296,"odore":50297,"çłĶç©¶ä¸Ńå¿ĥ":50298,"Ġanalytic":50299,"phase":50300,"æĬķéĻį":50301,"tm":50302,"èĢĮæľī":50303,"СС":50304,"*.":50305,"Ġcasi":50306,".env":50307,"æµĭè¯Ħ":50308,"轿车":50309,"jun":50310,"æ°´åĴĮ":50311,"ä»ĸæīĢ":50312,"Ġ¦":50313,"ĠImprovement":50314,"สูà¸ĩ":50315,"ĠHect":50316,"Ġaboard":50317,"åľ°æĸ¹æĶ¿åºľ":50318,"uker":50319,"Particip":50320,"Ġattained":50321,"ĠAmericas":50322,"Ġobservational":50323,"åŃĻåŃIJ":50324,"Ġctx":50325,"çĤĻ":50326,"ĠInstituto":50327,"Ġscarce":50328,"ÑĭÑĪ":50329,"Ġersten":50330,"Ġsafeguard":50331,"ä¹ĭ大":50332,".entity":50333,"çļĦ说æ³ķ":50334,"à¸Ħรัà¹īà¸ĩ":50335,"Ġcelestial":50336,"SV":50337,"Ġtransforms":50338,"icals":50339,"ĠBeispiel":50340,"ĠCAT":50341,"åľ¨æķ´ä¸ª":50342,"িà¦ķà§įষ":50343,"Ġjeden":50344,"ĠMeters":50345,"Ġsingles":50346,"ĠChoosing":50347,"æĽ´èĥ½":50348,"Ġ׾ש×":50349,"४":50350,"åĵªç§į":50351,"volving":50352,"ĠÃĢ":50353,"ĠAnalyst":50354,"ä¸Ģè¶Ł":50355,"åѦä¸ļ":50356,"æĬ¢æķij":50357,"åıİ":50358,"complete":50359,"Ġburns":50360,"åĵ©":50361,"urus":50362,"Ġmemo":50363,"ião":50364,"åħ¬ç¤¾":50365,".!":50366,"ª":50367,"ÑģÑĤÑĢи":50368,"产åIJİ":50369,"Ġblowing":50370,"ä½Ĩå¦Ĥæŀľ":50371,"ĠÑĤеÑħнологи":50372,"ustin":50373,"match":50374,"æī¿åĬŀ":50375,"onden":50376,"âĪĨ":50377,"номеÑĢ":50378,"ä¸ºæľ¬":50379,"信访":50380,"ĠAndre":50381,"Physics":50382,"vir":50383,"ниÑĨÑĭ":50384,"ä¹Ĵ":50385,"ĠPeg":50386,"åĩºå¤Ħ":50387,"ĠMontgomery":50388,"çŃīæĪij":50389,"Ġregistry":50390,"Rightarrow":50391,"ä¸įéļ¾":50392,"ĠWen":50393,"Ġcredited":50394,"ĠRut":50395,"izo":50396,"æ¸ħ代":50397,"æķĪèĥ½":50398,"Ġrealizar":50399,"rophe":50400,"ĠCanal":50401,"Daily":50402,"Ġsubsidiary":50403,"ÙĬÙĦØ©":50404,"ĠCurtis":50405,"æ¯Ķè¾ĥ好":50406,"åģĩçļĦ":50407,"ogel":50408,"æİĴæĸ¥":50409,"пад":50410,"æĭ¯æķij":50411,"ĠWARR":50412,"åĽŀæĿ¥çļĦ":50413,"没èĥ½":50414,"appen":50415,"ĠUNESCO":50416,"\"This":50417,"è¡ĮäºĨ":50418,"Ġfacil":50419,"çĨĦ":50420,"ĠbyÅĤ":50421,"åļ·":50422,"Ġkek":50423,"åĬĽäºī":50424,"Scope":50425,"FM":50426,"iÃŁ":50427,"лений":50428,"çľĭä¸įè§ģ":50429,"Ġlineage":50430,"Ġorthogonal":50431,"Ġrecuper":50432,"Tell":50433,"Ġİ":50434,"mino":50435,"rador":50436,"ophysical":50437,"æĹ¶åºĶ":50438,"dea":50439,"è®¤çľŁçļĦ":50440,"Ġuneven":50441,"ĠFactory":50442,"Ġhän":50443,"ĠAlger":50444,"Ġquella":50445,"Website":50446,"åĦĴå®¶":50447,"åĵĪå°Ķ滨":50448,":%":50449,"-effect":50450,"Material":50451,"alcul":50452,"ä¸įæŃ»":50453,"Ġeighteenth":50454,"ermann":50455,"ĠÃĺ":50456,"asci":50457,"cid":50458,"named":50459,"Ġprogen":50460,"æĬĢæľ¯äººåijĺ":50461,"åŃ¦è¯´":50462,"âĢĶâĢĶâĢľ":50463,"five":50464,"é«ĺä¸Ģ":50465,"OVA":50466,"æłı缮":50467,"Ġá±":50468,"ĠOlive":50469,"æĦŁåıĹåΰäºĨ":50470,"Ġpilots":50471,"Ġmerchand":50472,"ãģįãģ¾ãģĻ":50473,"flat":50474,"ĠPerspectives":50475,"/common":50476,"ĠSantiago":50477,"ĠKang":50478,"ä¹Įåħĭåħ°":50479,"ruce":50480,"Ġaerial":50481,"ä¸įæĩĪ":50482,"çĦ¶åIJİåľ¨":50483,"Identity":50484,"Speaking":50485,"Ġ׼×IJ×":50486,"éĿ¢éĥ¨":50487,"jected":50488,"^-":50489,"hibition":50490,"Ġintimid":50491,"ä½łè§īå¾Ĺ":50492,"åIJį人":50493,"Ġrecursive":50494,"Upper":50495,"å¼¹åĩº":50496,"çļ±çľī":50497,"Heb":50498,"ä¼ģåĽ¾":50499,"åĨĻéģĵ":50500,"ilus":50501,"Ġminorities":50502,"ĠGeschichte":50503,"าà¸ļ":50504,"åıĹä½ĵ":50505,"à·Ķ":50506,"èĸĦå¼±":50507,"Ġlandmark":50508,"iciary":50509,"ĠвнÑĥÑĤÑĢен":50510,"ĠGrowing":50511,"Ġptr":50512,"Ġbride":50513,"许å¤ļ人":50514,"ানà§įত":50515,"Lou":50516,"æĬĺæĹ§":50517,"ãĤ°ãĥ©":50518,"Ġsanté":50519,"ĠWing":50520,"ovirus":50521,"Ġmonkey":50522,"[row":50523,"ина":50524,"马æĿ¥":50525,"ĠVec":50526,"Ġmatag":50527,"åįĥä¸ĩä¸įè¦ģ":50528,"ÏģίοÏħ":50529,"Ġwatches":50530,"èħ¹æ³»":50531,"could":50532,"Ġvort":50533,"Ġobstruction":50534,"ĠHij":50535,"åij¨è½¬":50536,"ĠWordPress":50537,"Ġpied":50538,"èµ·å§ĭ":50539,"æ¸ħçĥŃ":50540,"Ġalles":50541,"scribed":50542,"ä¼ĬæľĹ":50543,"Ġdummy":50544,"åıĺéĢŁ":50545,"ĠSeed":50546,"åĩıå¼±":50547,"رÙĬاض":50548,"Ġmelakukan":50549,"ĠاÙĦØŃد":50550,"icrosoft":50551,"ĠEsta":50552,"nf":50553,"ÏģαÏĨ":50554,"Ġdisturbing":50555,"Ġindexes":50556,"è¶ħè¿ĩäºĨ":50557,"Focus":50558,"หลัà¸ģ":50559,"FOR":50560,"brain":50561,"Ġdub":50562,"inational":50563,"ĠSyst":50564,"ummer":50565,"Ġquindi":50566,"Ġиде":50567,"ĠJC":50568,"Cho":50569,"Setting":50570,"ä¸į被":50571,"Ġদà§ģ":50572,"Ġruins":50573,"Ġfried":50574,"Patent":50575,"åĪĨæŀIJåĴĮ":50576,"Ġknowledgeable":50577,"typename":50578,"ÙĪÙĬÙĩ":50579,"Proc":50580,"ĠJamie":50581,"ĠоÑĢи":50582,"ĠPLA":50583,"å¼Ģ头":50584,"è½§":50585,"åĨ·åĨ·":50586,"useum":50587,"ÎĽ":50588,"å¿ħå¤ĩ":50589,"Adam":50590,"æĢ¥æķij":50591,"Ġ%>%":50592,"é¢Ĩ导çļĦ":50593,"Ġsparse":50594,"hiyon":50595,"à¹Ģà¸Ķà¹ĩà¸ģ":50596,"#ĊĊ":50597,"èĢĮèµ·":50598,"ĠColin":50599,"ĠDios":50600,"å¢ŀé«ĺ":50601,"å¤ĦçIJĨçļĦ":50602,"à¸Ľà¸£à¸°à¸ģ":50603,"àŃįà¬":50604,"ĠAwareness":50605,"inel":50606,"erna":50607,"TG":50608,"ĠSki":50609,"åѦåŃIJ":50610,"Enumerable":50611,"çļĦå½¢æĪIJ":50612,"ducible":50613,"Ġdestiny":50614,"iage":50615,"-Man":50616,"åĪĽç«ĭ":50617,"å¤ŁäºĨ":50618,"ĠLemma":50619,"Sql":50620,"ĠKN":50621,"Ġzat":50622,"ĠRomania":50623,"/data":50624,"aÄĩ":50625,"Í¡":50626,"ĠLebanon":50627,"ĠíĻľ":50628,"Ġdelegate":50629,",Y":50630,"-exp":50631,"Ġattra":50632,"æ¯ĴæĢ§":50633,"onc":50634,"Ġpredicts":50635,"Ġstimulated":50636,"çĬĢ":50637,"ĠRé":50638,"æµģåĩº":50639,"???":50640,"Ġskeleton":50641,")âĢĿ":50642,".--":50643,"æĺıè¿·":50644,"Ġnumbered":50645,"Ġmanifestation":50646,"Ġpoi":50647,"öglich":50648,"Ġtimeless":50649,"ému":50650,"Ġanalogous":50651,"Ġodpowied":50652,"Ren":50653,"ulp":50654,"à³įಲ":50655,"ä¹Łå¾Ĺ":50656,"æºī":50657,"èle":50658,"à¹Īวà¸ĩ":50659,"ÑģÑĤвÑĥÑİÑĤ":50660,"å¼Ģéĩĩ":50661,"马æĭī":50662,"диви":50663,"æ»ļåĬ¨":50664,"Fragment":50665,"Ġintermitt":50666,"åıĪ被":50667,"åºķå±Ĥ":50668,"Ġcalorie":50669,"éĹŃåIJĪ":50670,"ĠApollo":50671,"ĠArrays":50672,"æĬ±æŃī":50673,"Ġaggress":50674,"ircle":50675,"JC":50676,"ĠGI":50677,"å°±ä¸įæĺ¯":50678,"Ġcompetitions":50679,"Ġsnacks":50680,"Ġzoals":50681,"å¤ĸåľ°":50682,"Ġconfronted":50683,"Õ£":50684,"forall":50685,"Sat":50686,"åľ¨å®ŀéĻħ":50687,"ĠKyle":50688,"ר×ķת":50689,"network":50690,"Ġrichness":50691,"ĠConsidering":50692,"Ġultr":50693,"æľĢç¾İ":50694,"justed":50695,"ĠOutlook":50696,"-conf":50697,"Ġcaffeine":50698,"楷":50699,"даеÑĤ":50700,"Ġthriving":50701,"Site":50702,"Ġsubstract":50703,"ĠÙĪÙĦÙĥÙĨ":50704,"èݽ":50705,"иÑģÑĮ":50706,"Ġreadiness":50707,"Ġlottery":50708,"ĠWine":50709,"Ġbipolar":50710,"Ġnale":50711,"ĠNCAA":50712,"æĺ¾çݰ":50713,"éĺ¶å±Ĥ":50714,"Ġblacks":50715,"ãģĤãģ£ãģŁ":50716,"é²ľèĬ±":50717,"ãģijãģ¦":50718,"Ġinjustice":50719,"-Geiger":50720,"å¹´è¼ķ":50721,"Ġpolys":50722,"å°±æĺ¯è¿Ļæł·":50723,"asic":50724,"avian":50725,"สำหรัà¸ļ":50726,"ĠDiagnosis":50727,".Com":50728,"Ġslid":50729,"Ġkissed":50730,"Ġbachelor":50731,"åĨįè§ģ":50732,"à¸Ħัà¸į":50733,"Ġmemoir":50734,"\"A":50735,"Ġgdy":50736,"жÑĥ":50737,"னà¯įà®±":50738,"è¿Ļæĺ¯ä¸Ģç§į":50739,"åīįéĢĶ":50740,"è¿ĿèĥĮ":50741,"Ġperk":50742,"Ġkomun":50743,"ĠNNE":50744,"inees":50745,"æķ°æĺ¯":50746,"æĶ¯æ°Ķ管":50747,"colm":50748,"給æĪij":50749,"Ġhumid":50750,"ĠTransformation":50751,"çļĦ建设":50752,"庶":50753,"Ġoutreach":50754,"ĠAirlines":50755,"Ġrushing":50756,"Ġdigging":50757,"Ġpersonalities":50758,"Ġhandsome":50759,"족":50760,"Ġress":50761,"üd":50762,"åIJĦç§įåIJĦæł·çļĦ":50763,"à¹Ģà¸ģีà¹Īยว":50764,"ç쵿ĦŁ":50765,"Canada":50766,"欢åĸľ":50767,".au":50768,"åĢ«":50769,"ĠFindings":50770,"ĠMéxico":50771,"Ġrud":50772,"Ġdetention":50773,"å¾Ĺèµ·":50774,"åįĹæĺĮ":50775,"ĠAH":50776,"ĠMORE":50777,"Ġenvision":50778,"åѦéĩij":50779,"ĠзанÑı":50780,"ĠÙĨشاÙĨ":50781,"ĠGeometric":50782,"Ġscreaming":50783,"åIJ¯ç¤º":50784,"洪水":50785,"Ġceux":50786,"Ġfairy":50787,"æ¯Ĵç´ł":50788,"ĠOslo":50789,"Ġsegregation":50790,"Ġethnicity":50791,"Ùĥس":50792,"æĶ¯éĺŁ":50793,"ĠðŁĵ":50794,"ÙĦÛĮÙĦ":50795,"atri":50796,"Ġsponsor":50797,"Ġexercised":50798,"Ġhopeful":50799,"å¿ĥ裡":50800,"ã썿ĢĿãģĦãģ¾ãģĻ":50801,"Ġplausible":50802,"йн":50803,"ä»·çļĦ":50804,"Ġnutrit":50805,"èĿī":50806,"ĠFAQ":50807,"Ġfiltration":50808,"vine":50809,"кам":50810,"磨æįŁ":50811,"Axis":50812,"Fa":50813,"Ġcompress":50814,"æ¯ıä½į":50815,"ĠMarvel":50816,"第ä¸īèĬĤ":50817,"Ġاش":50818,"è¿ĺæľīä¸ĢäºĽ":50819,"Nom":50820,"боÑĤа":50821,"åĹħ":50822,"Ġunpredictable":50823,"åħ¥éŨ":50824,"Ġsenza":50825,"Ġ×ijר":50826,"Ġì¦Ŀ":50827,"неÑģ":50828,"ারা":50829,"âĹİ":50830,"æ¢ħèĬ±":50831,"овой":50832,"Ġmonitors":50833,"Ġprocedural":50834,"Letter":50835,"take":50836,"éĿϿѢ":50837,"ĠÑĤеÑĩение":50838,"Ġspanning":50839,"oplasmic":50840,"å°±åı¯":50841,"ĠëķĮ문":50842,"Ġassisting":50843,"Validator":50844,"Ġkinaug":50845,"å®ŀæĪĺ":50846,"让大家":50847,"ziej":50848,"æķĻèĤ²åĴĮ":50849,"å½±åĵįäºĨ":50850,"analy":50851,"ä»ĸæīį":50852,"åıĹæ¬¢è¿İ":50853,"èħij":50854,"Ġdenying":50855,"Ġeuros":50856,"Ġkinabasaan":50857,"Ġkinaugahan":50858,"vod":50859,"Ь":50860,"使ä»ĸ":50861,"Ã¶ÃŁ":50862,"ĠÑĤоÑĢ":50863,".my":50864,"åĩ¿":50865,"å·¡è§Ĩ":50866,"ê¸ī":50867,"pod":50868,"ĠBayesian":50869,"åľ¨è¿Ļç§įæĥħåĨµä¸ĭ":50870,"Animal":50871,"(target":50872,"Ġrebellion":50873,"æĢ»çĽij":50874,"è¶ĬæĿ¥è¶Ĭå¤ļçļĦ":50875,"KR":50876,"rets":50877,"Ġvamp":50878,"Ġrelent":50879,"ä¸ĩ亩":50880,"æĿłæĿĨ":50881,"à§ĩস":50882,"Ġsaint":50883,"ĠPete":50884,"ĠNucle":50885,"ĠKurt":50886,"æľŁåĨħ":50887,".doi":50888,"à¹Ĥลà¸ģ":50889,"Ġnephe":50890,"åŁ¹è®ŃçıŃ":50891,"ÅĤaw":50892,"Wood":50893,"zne":50894,"积æŀģçļĦ":50895,"Ġê°ģ":50896,"Ġelectrolyte":50897,"IELD":50898,"strip":50899,"Office":50900,"Jon":50901,"Ġsuperv":50902,"ĠMaxim":50903,"Ġizan":50904,".Request":50905,"Ġsailing":50906,"ĠëͰëĿ¼":50907,"-air":50908,"çļĦä¸ĵä¸ļ":50909,"Ġgaan":50910,"åĩºåľŁ":50911,"è¡Į人":50912,"æīĢæľª":50913,"Ġdepicts":50914,"Ġstaat":50915,"ahkan":50916,"å±ķ示äºĨ":50917,"sq":50918,"bly":50919,"ç¶ĵé©Ĺ":50920,"ĠConversely":50921,"ISC":50922,"Statement":50923,"ĠPars":50924,"åľ¨è¿Ľè¡Į":50925,"åħ¨å®¶":50926,"ĠÙĪÙģÙĬ":50927,".wikipedia":50928,"Ġroster":50929,"ymmetric":50930,"utations":50931,"geois":50932,"éķ¿è¾¾":50933,"izzazione":50934,"(U":50935,"ĠWon":50936,"ivent":50937,"adu":50938,"Ġpigment":50939,"Ġparc":50940,"наÑĩа":50941,"çΏçΏå¦Īå¦Ī":50942,"ĠLore":50943,"Ġlocks":50944,"ozzá":50945,"ĠAstronomical":50946,"ĠMond":50947,"-dom":50948,"Else":50949,"-ulan":50950,"è§ģè¯Ĩ":50951,"è¾¹å¢ĥ":50952,"Ġhardest":50953,"ĠRepresentation":50954,"ä¸įå¹³":50955,"èĤ¯å®ļä¼ļ":50956,"[List":50957,"è¿ĺæĥ³":50958,"ĠPlaces":50959,"afety":50960,"Ġmalign":50961,"زÙĬاØŃ":50962,"ÑĤно":50963,")x":50964,"ãģ¥":50965,"Ġrides":50966,"ä¸įåı¯æĢĿ":50967,"åIJŀåϬ":50968,"å¾ģæľį":50969,"plotlib":50970,"Ġcreators":50971,"ãĥ¼ãĤ¯":50972,"Ġshelves":50973,"tel":50974,"ç͵èį·":50975,"Ġmodelo":50976,"æĺ¯ä¸ĢäºĽ":50977,"ннÑĭÑħ":50978,"åģľä¸ĭ":50979,"URR":50980,"Ġalternating":50981,"uko":50982,"Ġterrestrial":50983,"澳洲":50984,"VERTIS":50985,"èĩªå®ļä¹ī":50986,"بÙĬØ©":50987,"ĠбÑĢа":50988,"Consulta":50989,"´ij":50990,"Ġseasoned":50991,"{I":50992,"æ¯Ĵåĵģ":50993,"Ġphilosophers":50994,"Ġaudi":50995,"à¸Ĺีà¹Īà¸Īะ":50996,"Ġcongestion":50997,"natal":50998,"ĠÖĩ":50999,"Ġvind":51000,"æŃ¤äºº":51001,"Ġcollateral":51002,"ĠÙĬÙı":51003,"ĠIndonesian":51004,"åĬŁèĥ½ä»ĭç»į":51005,"ëł¹":51006,"ĠKitchen":51007,"Ġθη":51008,"Ġwreck":51009,"ಯ":51010,"两个æľĪ":51011,"ç§ijæĬĢæľīéĻIJåħ¬åı¸":51012,"å¯ĦåŃĺ":51013,"Sheet":51014,"entence":51015,"åıĹæįŁ":51016,"åŃĹåħ¸":51017,"ĠвÑģегда":51018,"ĠTiger":51019,"çŃīå¤ļ":51020,"Ġimplication":51021,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":51022,"theme":51023,"ĠEstim":51024,"ĠSv":51025,"ĠdziaÅĤ":51026,"Ġcontinuum":51027,"âĢĿï¼ĮâĢľ":51028,"æīĴ":51029,"iai":51030,"æ±¹":51031,"atha":51032,"äre":51033,"è¨Ģ论":51034,"ĠابÙĨ":51035,"ructuring":51036,"æķ´æ´ģ":51037,"Ġaugment":51038,"骨骼":51039,"ä¹łä¿Ĺ":51040,"brace":51041,"ìĥī":51042,"Ġalloys":51043,"çĿĢèĩªå·±":51044,"apses":51045,")=(":51046,"æ®ĭéħ·":51047,"ĠSets":51048,"lys":51049,"ĠBless":51050,"ĠZur":51051,"ĠMARK":51052,"ä½ľä¸ºä¸ĢåIJį":51053,"лоп":51054,"ÑģÑĤÑĢÑĥ":51055,"æĬĹåİŁ":51056,"幸ç¦ıçļĦ":51057,"lop":51058,"Ġvrij":51059,"ãģĸ":51060,"Ġstrengthened":51061,"æ´»çļĦ":51062,"ĠÙĩÙĬا":51063,"å°¼æĸ¯":51064,"ĠPatt":51065,"åģļ人":51066,"åıijçĶŁåıĺåĮĸ":51067,"ç¾½æ¯Ľ":51068,"é£İæ°´":51069,"Ġfasting":51070,"两åIJį":51071,"ä¸ŃæĢ§":51072,"issent":51073,"Ġبأ":51074,"osaic":51075,"amba":51076,"æŃ¦å¸Ŀ":51077,"Ġvoir":51078,"Ġempresa":51079,"åıĥåĬł":51080,"ných":51081,"Ġmorb":51082,"ĠRegulatory":51083,"æ²īéĩį":51084,"çļĦæĥħæĦŁ":51085,"/search":51086,"åĪĩçļĦ":51087,"åģ¿è¿ĺ":51088,"/)":51089,"Ġbild":51090,"çī¹ç§į":51091,"ä¸įåĪĨ":51092,"Ġunaware":51093,"éķ¿æĺ¥":51094,"ä»ĸ人çļĦ":51095,"Ġâĺħ":51096,"-ref":51097,"Ġkata":51098,"Ġridge":51099,"Eq":51100,"school":51101,"ĠÑģогла":51102,"Ġبخش":51103,"ĠWals":51104,"engl":51105,"Ġempowerment":51106,"isations":51107,"ĠвÑĭде":51108,"æĬĵç´§":51109,"/json":51110,"Ġhonesty":51111,"Ġobscure":51112,"Ġbishop":51113,"usters":51114,"ĠÙģÙĤØ·":51115,".frame":51116,"Ġlocus":51117,"Ġordinal":51118,"Paris":51119,"Ġcatalysts":51120,"ĠLily":51121,"æµģ转":51122,"imbabwe":51123,"IEEE":51124,"è¿Ļéģĵ":51125,"æł¹æºIJ":51126,"/qt":51127,"ÙĮ":51128,"éĩijåĪļ":51129,"Throughout":51130,"ÑģÑģе":51131,"Ñģким":51132,"Ġfucking":51133,"Ġcoordinator":51134,"çζæ¯įçļĦ":51135,"Ġweighing":51136,"mant":51137,"ÏĥειÏĤ":51138,"-%":51139,"Zero":51140,"ivist":51141,"ĠObl":51142,"å¿ĥçIJĨåģ¥åº·":51143,"Ġfoil":51144,"ä¸Ģä¸ĩ":51145,"ĠEnerg":51146,"histoire":51147,"Ġdonnées":51148,"ĠобÑĥÑĩ":51149,"Ġbargaining":51150,"Ġdefender":51151,"ÑģколÑĮ":51152,"ĠSue":51153,"વ":51154,"ĠLeben":51155,"Ġstainless":51156,"Ġmodifying":51157,"èij«èĬ¦":51158,"æľºç»Ħ":51159,"ographie":51160,"ĠTrading":51161,"è¯łéĩĬ":51162,"Ġvoll":51163,"å®¶åºĦ":51164,"Ġblown":51165,"Adding":51166,"人æ°ijåĮ»éĻ¢":51167,"èµ£":51168,"Ġнал":51169,"ä¸İä¼ļ":51170,"ðĿIJ´":51171,"ĠWaters":51172,"acry":51173,"åı¯ä»¥æĬĬ":51174,")âĢĶ":51175,"ffent":51176,"ĠMotors":51177,"ĠPassword":51178,"Ġdor":51179,"ÏħÏĥ":51180,"ĠJJ":51181,"Ġhippoc":51182,"{{":54703,"mitter":54704,"stellen":54705,"Ġalteration":54706,"ã±":54707,"å°Ħ线":54708,"ĠBroadcast":54709,"_[":54710,"estly":54711,"tat":54712,"ĠFo":54713,"Ġcans":54714,"åı¯ç͍äºİ":54715,"åįĹ宫":54716,"嬷":54717,"cmp":54718,"ĠPseud":54719,"çīĨ":54720,"Ġincremental":54721,"ĠVladimir":54722,"izioni":54723,"æľ¬ä¾Ĩ":54724,"没æ³ķ":54725,"Ġfinishes":54726,"Ġnilai":54727,"Payment":54728,"Links":54729,"à¸Ħวร":54730,"éĨ«éĻ¢":54731,".root":54732,"Ġtx":54733,"ĠFellow":54734,"çľ¼åºķ":54735,"Ġmelody":54736,"Ġforte":54737,"antis":54738,"æīĢè¬Ĥ":54739,"ноÑģи":54740,"Ġhos":54741,"computer":54742,"æ¤ħåŃIJ":54743,"é»ıèĨľ":54744,"Ġlavoro":54745,"pekt":54746,"ĠнаÑģÑĤоÑı":54747,"æ¸ħæľĿ":54748,"å±ıéļľ":54749,"ä½İ声":54750,"ÑĨеп":54751,"à¦ģ":54752,"à§ģà¦ķà§įত":54753,"æĪijåĢijçļĦ":54754,"åī¿":54755,"é»ijè¡£":54756,"ĠJugend":54757,"leaf":54758,"ĠAuch":54759,"好好çļĦ":54760,"Ġrelación":54761,"Ġcarrots":54762,"ĠAutomation":54763,"Mas":54764,"æĬ¥å¤į":54765,"åºŀ大çļĦ":54766,"éĶĻçļĦ":54767,"空éĹ´çļĦ":54768,"Ġzurück":54769,"×ķ×ij×":54770,"佩æľį":54771,"ĠSubstant":54772,"ÑĢоÑģÑĤÑĢан":54773,"Ġнам":54774,"ĠORDER":54775,"Ġdarah":54776,"bab":54777,"ĠThr":54778,"è¯ģä»¶":54779,"×ŀף":54780,"Ġprojekt":54781,"åľ¨ç¤¾ä¼ļ":54782,"壯":54783,"åĪĽä¼¤":54784,");//":54785,"çѹå¤ĩ":54786,"Ġnurturing":54787,"_check":54788,"ĠLond":54789,"ĠоÑĩе":54790,"åıªæĥ³":54791,"-)":54792,"Ġindoors":54793,"Ġillustrating":54794,"itatively":54795,"çļĦæĦıè§ģ":54796,"Ġë¡":54797,"è¯īæ±Ĥ":54798,"Suba":54799,"éĸ¢ä¿Ĥ":54800,"×Ļ×ŀ×ķ×":54801,"Ġsupplementary":54802,"ĠкÑĤо":54803,"ĠFD":54804,"æĪIJä¸Ģ个":54805,"ĠWein":54806,"ĠÑŀ":54807,"ĠInner":54808,"minus":54809,"à¸Ĺà¸ĺิ":54810,"ĠDy":54811,"æ°´ç͵":54812,"è®ĵæĪij":54813,"_CON":54814,"ĠSecondly":54815,"ä»ĬåIJİçļĦ":54816,"ãĤĴæĮģ":54817,"érer":54818,"ozzáférés":54819,"ಸ":54820,"'I":54821,"é¦ĸåħĪæĺ¯":54822,"'av":54823,"Ġresh":54824,"ĠAnnie":54825,"ĠAnyone":54826,"æĶĴ":54827,"èĢģç¥ĸ":54828,"Ġszk":54829,"ÑĬÑĢ":54830,"Ċ":55163,"ä½ľåĩºäºĨ":55164,"Ġbolt":55165,".al":55166,"Jump":55167,"Ġela":55168,"Ġplanar":55169,"Teaching":55170,"Ġannoying":55171,"_sc":55172,"ä¸Ģåı·":55173,"ãģ«ãĤĪãĤĬ":55174,"(float":55175,".fl":55176,"Ġterc":55177,"ccion":55178,"å¹´å¹´":55179,"äº¤æĽ¿":55180,"以åıĬåħ¶ä»ĸ":55181,"Ġstagn":55182,"ĠROS":55183,"avg":55184,"ictionaries":55185,"ä»ħä»ħæĺ¯":55186,"ĠChad":55187,"fluence":55188,"ĠSongs":55189,"ĠاÙĦÙĪØ·ÙĨ":55190,"Sy":55191,"inkel":55192,"Ġinvariant":55193,"wait":55194,"åľ°å¯¹":55195,"åĪĨéIJĺ":55196,"åľ¨å·¥ä½ľ":55197,"åĪĨä¼ļ":55198,"çĤ³":55199,"æ·ļ":55200,"å²ij":55201,"{{\\":55202,"ä¸įå°½":55203,"西åĵ¥":55204,"Ġfrontal":55205,"ĠPhysiology":55206,"ĠФедеÑĢа":55207,"ĠAdventure":55208,"ĠØ£ØŃد":55209,"Ġeducación":55210,"ulgada":55211,"对çļĦ":55212,"à¹Ģà¸ŀิà¹Īม":55213,"_view":55214,"sie":55215,"åĴĮå®ĮåĸĦ":55216,"Ġcostume":55217,"Ġcontracted":55218,"Ġslowing":55219,"è¾ħ导åijĺ":55220,"å©¿":55221,"ĠTaxonID":55222,"Ġzij":55223,"Ġfreshman":55224,"Ġsymb":55225,"Equation":55226,"Ġbac":55227,"Ġinvert":55228,"ĠHof":55229,"设æ³ķ":55230,"ÏĦÎŃ":55231,"ĠLaser":55232,"...#":55233,"Sequence":55234,"Ġrabbits":55235,"ĠDifferential":55236,"ĠاÙĦدÙħ":55237,"é«ĺæ½®":55238,"Ġdepreciation":55239,".Scanner":55240,"ï¾":55241,"sein":55242,"ĠCod":55243,"åıĸå̼":55244,"hoff":55245,"atement":55246,"åħ¨æł¡":55247,"Bus":55248,"Poss":55249,"ĠdÃŃas":55250,"uben":55251,"åĪ©çĽĬçļĦ":55252,"ĠCause":55253,"æİĮ声":55254,"Ġflaws":55255,"åĩĨå¤ĩå·¥ä½ľ":55256,"Ġadmire":55257,"جع":55258,"ÅĤem":55259,"ĠÑĤакиÑħ":55260,"è¿Ļä¸ĭ":55261,"æĪij们å¿ħé¡»":55262,"å¹³æ°ij":55263,"Ġinquiries":55264,"åľ°éĹ®":55265,"èĢĮçŁ¥":55266,"Ġdeserved":55267,"åħ¬é¡·":55268,"ieme":55269,"(cl":55270,"[A":55271,"Ġpersone":55272,"æķ£çĥŃ":55273,"Ġà²ħ":55274,"ĠEnde":55275,"Ġsynchronization":55276,"/all":55277,"Ġdelen":55278,"第ä¸Ģ天":55279,"Ġpreferable":55280,"Ġautoimmune":55281,"ождениÑı":55282,"ourg":55283,"Ġunderstandable":55284,"è´¢çī©":55285,"Ġש×IJ×":55286,"纸ä¸Ĭ":55287,"Ġdiscounts":55288,"微信åı·":55289,"ä¾Ĩåΰ":55290,"_field":55291,"ĠInch":55292,"åı¦å¤ĸä¸Ģ个":55293,"Ġglor":55294,",the":55295,"bishop":55296,"åĨĬ":55297,"Ġjunk":55298,"ën":55299,"Ġসব":55300,"æģķ":55301,"Ġformas":55302,"榨":55303,"åı«å£°":55304,"çIJĨ论ä¸Ĭ":55305,"Ġelevate":55306,"說æĺİ":55307,"ĠBesøkt":55308,"Ġcircumference":55309,"éĬĢè¡Į":55310,"Ġhurried":55311,"ä¸įèĥľ":55312,"abar":55313,"Ġì½Ķ":55314,"ĠAustrian":55315,"ĠAviation":55316,"ëħ¸":55317,"KC":55318,"æĬ¤èĤ¤":55319,"ĠCoalition":55320,"_un":55321,"etting":55322,"çĶŁäº§åĴĮ":55323,"iatan":55324,"anical":55325,"ĠобоÑĢ":55326,"å¤įè®®":55327,"辩æĬ¤":55328,"anty":55329,"è¿Ľä¸ĢæŃ¥åĬłå¼º":55330,"=p":55331,"Ġasylum":55332,"ĠÑģказа":55333,"Ġpleas":55334,"ĠInitially":55335,"éģ¿å¼Ģ":55336,"Attributes":55337,"ĠFacility":55338,"zyk":55339,"Ġihren":55340,"Future":55341,"ä½łæĺ¯ä¸įæĺ¯":55342,"-card":55343,"arcer":55344,"ĠZimbabwe":55345,"Ġprogrammed":55346,"ĠEndocr":55347,"å½¼å¾Ĺ":55348,"à§įযাল":55349,"Ġfreshly":55350,"hões":55351,".display":55352,"ĠGibson":55353,"Failed":55354,"åıĤè°ĭ":55355,".equal":55356,"ĠTerritory":55357,"ä¸ĭ鼨":55358,"Former":55359,"-prov":55360,"èIJİ缩":55361,"íĭ°":55362,"ĠÙģÙĩ":55363,"èĬ±å¼Ģ":55364,"ãģªãĤĭ":55365,"ĠSupplementary":55366,"èªī为":55367,"æ²Ĥ":55368,"Û±Û¹":55369,"大åħ¨":55370,"czne":55371,"-consuming":55372,"enic":55373,"Ġyogurt":55374,"奮":55375,"ĠÑģоÑģ":55376,"untos":55377,"Ġcuerpo":55378,"Ġplastics":55379,"Ġairplane":55380,"_eq":55381,"Ġtraps":55382,"Ġarchitects":55383,"ĠαÏĢο":55384,"wartz":55385,"Ġoptimism":55386,"#ifndef":55387,"åĭģ":55388,"ÙĦاÙĭ":55389,"Ġcanonical":55390,"è¡Ģæµģ":55391,"ARNING":55392,"ĠStem":55393,"ĠClara":55394,"Ġtaxpayers":55395,"Ġdri":55396,"ç«ŀäºī对æīĭ":55397,"cka":55398,"ienced":55399,"-layer":55400,"Ġlenders":55401,"éķĸ":55402,"Prem":55403,"çļĦæ³ķ":55404,"ĠVs":55405,"æĪĸæľī":55406,"-square":55407,"otides":55408,"Ġ[],Ċ":55409,"ÑĩнаÑı":55410,"Ġgoodbye":55411,"ĠBerry":55412,"Ġsendo":55413,"éĻ·éĺ±":55414,"æĪijåħĪ":55415,"ĠYard":55416,"Ġкг":55417,"ÑĨÑĮ":55418,"ĠCoch":55419,"йÑģкой":55420,"дж":55421,"Ġ×Ķפ":55422,"ĠPall":55423,"Ġ×Ķ×ij×":55424,"ä¸įæĸŃæıIJåįĩ":55425,"æĪIJçĨŁçļĦ":55426,"-key":55427,"Numer":55428,"ĠtoString":55429,"ĠHits":55430,"ĠFruit":55431,"junct":55432,"ĠÏķ":55433,"Ġestrogen":55434,"ç§°èµŀ":55435,"ç»Ħç»ĩå¼Ģå±ķ":55436,"wr":55437,"ä¸Ģåΰ":55438,"éĩįè¦ģ讲è¯Ŀ":55439,"à§įà§°":55440,"ÙĥÙĪÙħ":55441,"æĪ·ç±į":55442,"åıĻäºĭ":55443,"ä½łåħĪ":55444,"è¿ĺç®Ĺ":55445,"çī©èģĶç½ij":55446,"éĢļè¿ĩäºĨ":55447,"facebook":55448,"]:ĊĊ":55449,"Ġchol":55450,"aina":55451,"åĬłä¹ĭ":55452,"ĠмаÑĤемаÑĤи":55453,"coal":55454,"ãģ©ãģ®":55455,"ĠCuban":55456,"ÏİνÏħμο":55457,"Ġbc":55458,"ĠChlor":55459,"ÅĤod":55460,"ÑĤелÑĮнаÑı":55461,"Ġપ":55462,"GRAM":55463,"ĠاÙĦذÙĬÙĨ":55464,"ĠGrim":55465,"å§Ĭ":55466,"èģĶåĤ¨":55467,"å¦ĩèģĶ":55468,"Ġuncovered":55469,"çļĦçģ«":55470,"tering":55471,"æ°´æµģ":55472,"Ġinsured":55473,"è¿ĻéĩĮæĺ¯":55474,"γκ":55475,"ä¸ĭæĿ¥äºĨ":55476,"Ġê°Ĵ":55477,"æīĭèħķ":55478,"麦åħĭ":55479,"Discover":55480,"ĠÑģвоÑİ":55481,"Ġiso":55482,"壹":55483,"ÑģÑĤава":55484,"aimana":55485,"ĠÑģооб":55486,"Phot":55487,"人æĥħ":55488,"ĠKM":55489,"èĥ½æľī":55490,"Õ¸ÖĢÕ":55491,"leness":55492,"è¡ĮæĶ¿å¤Ħç½ļ":55493,"Õ¸Õ²":55494,"[:,":55495,"ĠACS":55496,"áveis":55497,"åı¯è¡ĮæĢ§":55498,".u":55499,"课åIJİ":55500,"Ġmogelijk":55501,"Later":55502,"çļĦä¼ĺåĬ¿":55503,"ä¹ĭä¸ŃçļĦ":55504,"Ġeyel":55505,".*;ĊĊ":55506,"ĠPharmacol":55507,"bp":55508,"à¦ķà§įর":55509,"ĠTownship":55510,"-li":55511,"çļĦé¢ľèī²":55512,"ĠRit":55513,"Ġunjust":55514,"ĠNewcastle":55515,"ĠOriental":55516,"Ġbombs":55517,"Ġpts":55518,"ĠGir":55519,"ä¸įåIJĮæĦı":55520,"ĠTowards":55521,"owym":55522,"ĠVP":55523,"ÑĢÑĥеÑĤ":55524,"ĠÑĢеÑĪениÑı":55525,"æ³°å±±":55526,"-Re":55527,"}[]{":55528,"Ġconquer":55529,"-focused":55530,"compare":55531,"Ġbrowse":55532,"ĠCertification":55533,"ĠÐŁÐµÑĢе":55534,"Ġrainy":55535,"æĢ§çĸ¾çĹħ":55536,"éħįä»¶":55537,"é¾ļ":55538,"ĠìĹIJ":55539,"Ġtertiary":55540,"Ali":55541,"综ä¸Ĭ":55542,"Ġmono":55543,"åľ°åIJij":55544,"fert":55545,"转为":55546,"åī¯ä½ľç͍":55547,"å°¤åħ¶æĺ¯åľ¨":55548,"Ġpumpkin":55549,"\\\":":55550,"çļĦæĪIJåĬŁ":55551,"æŃ¼":55552,"Indeed":55553,"ãĤ¹ãĤ¿":55554,"ĠNAD":55555,"åĴĮä¸Ń":55556,"çŃīåĢĻ":55557,"Ġpredicate":55558,"ĠTap":55559,"Ġempt":55560,"ĠZheng":55561,"ĠTemp":55562,"ocese":55563,"-bar":55564,"oulos":55565,"Signature":55566,"ĠваÑģ":55567,"Ġsanctuary":55568,"ĠEco":55569,"æĪĸèĢħåħ¶ä»ĸ":55570,"ĠHumans":55571,".us":55572,"å¤ĸ人":55573,"åįķéĢīé¢ĺ":55574,"ä¼ĹçĶŁ":55575,"ĠMorrison":55576,"æĨij":55577,"ĠProvider":55578,"ĠCrus":55579,"Ġprópri":55580,"ĠαÏģÏĥενικÏĮ":55581,"åĴĮä»ĸçļĦ":55582,"Ġcoatings":55583,"rically":55584,"Ġonc":55585,"ában":55586,"Ġparalle":55587,"Brien":55588,"大èĩªçĦ¶":55589,"годнÑı":55590,"Ġrecalls":55591,"'n":55592,"ĠAgents":55593,"ooo":55594,"ĠاÙĦعربÙĬØ©":55595,"λί":55596,"Ġآخر":55597,"ĠPrinciple":55598,"ĠDT":55599,"à¸Ńะ":55600,"ĠBlair":55601,"TRUE":55602,"Ġznaj":55603,"ä½ĵåĨħçļĦ":55604,"ä»Ģä¹Īåij¢":55605,".Ed":55606,"ĠAttention":55607,"malink":55608,"å¸ĿåĽ½ä¸»ä¹ī":55609,"-str":55610,"Ġwhatsoever":55611,"Ġirony":55612,"ĠREST":55613,"宣è¨Ģ":55614,"Ġhectares":55615,"è¦ģ对":55616,"åħ¨åijĺ":55617,"Ġtonne":55618,"alon":55619,"èµ·åΰäºĨ":55620,"Ġdissip":55621,"ĠÑģвои":55622,"Ġhog":55623,"åıijå±ķåΰ":55624,"ãģĦãĤĭ":55625,"Ġprivately":55626,"Ġsaline":55627,"ĠPero":55628,"হার":55629,"Ġobjections":55630,"æĮ¥æīĭ":55631,"olytic":55632,"åĿļæĮģ以":55633,"ë³ij":55634,"ĠPhilippine":55635,"ĠCliff":55636,"Ġrested":55637,"é¢Ŀå®ļ":55638,"ĠÐĽÐ°":55639,"ĠDermat":55640,"人次":55641,"Ġberikut":55642,"Ġello":55643,"áĢĦáĢºáĢ":55644,"Enable":55645,"ĠÑģÑĤÑĢе":55646,"Ġvirtues":55647,"åĺ»åĺ»":55648,"akah":55649,"è¦ļ":55650,"AMPLE":55651,"ĠProcedures":55652,"ĠComprehension":55653,"tv":55654,"ĉend":55655,"-car":55656,"æ½Ķ":55657,"lace":55658,"ĠIRA":55659,"Ġrealism":55660,"-cor":55661,"Ġdeadlines":55662,"ĠChurchill":55663,"ĠDual":55664,"Ġhypothetical":55665,"\\limits":55666,"íĻľ":55667,"ĠÅ¡k":55668,"Ġatm":55669,")%":55670,")\\]":55671,"Ġbarrels":55672,"à¸łà¸¹":55673,"Ġpéri":55674,"å¾Ĺä½ı":55675,"Ġpermite":55676,"Ġmana":55677,"Ġkterá":55678,"Ġaffiliated":55679,"Ġ×ij×Ĵ":55680,"Measure":55681,"åĮĸå¦Ĩåĵģ":55682,"ĠOwer":55683,"вÑĭÑħ":55684,"åı©":55685,"ÓĻÑĢ":55686,"çļĦå®¶åºŃ":55687,"åŃIJå¼Ł":55688,"Ġaccustomed":55689,"æIJŃè½½":55690,"æŁłæª¬":55691,"ä»Ģä¹Īä¸ľè¥¿":55692,"Ġcircumstance":55693,"çIJĨä¼ļ":55694,"}},":55695,"âĢĻ)":55696,"ATER":55697,"/src":55698,"emory":55699,"åıĥèĪĩ":55700,"ÃĵN":55701,"Ġventures":55702,"pps":55703,"åİ®":55704,"å¤ĩ份":55705,"Ġmož":55706,"/Comments":55707,"idx":55708,"åŃIJåŃĻ":55709,"Ġscrib":55710,"éļıæīĭ":55711,"(config":55712,".What":55713,"Fast":55714,"æĸĩ人":55715,"Ġpoj":55716,"å®ĥåı¯ä»¥":55717,"建ç«ĭèµ·":55718,"åıªè¦ģä½ł":55719,"Ġодного":55720,"åĭīå¼·":55721,"Ġgraders":55722,"Ġ%,":55723,"ĠTorres":55724,"æĪijåĨį":55725,"Ġcredible":55726,"èĦ¾èĥĥ":55727,"Heap":55728,"Ġmois":55729,"æĭİ":55730,"Ġmethodological":55731,".comp":55732,"ĠCemetery":55733,"ĠÑģÑĢав":55734,"egal":55735,"commit":55736,"ĠÙĪØ§ÙĦع":55737,"NK":55738,"ï¼Į#":55739,"Ġvou":55740,"æĺ¯ä¸įèĥ½":55741,"ymers":55742,"èĬ±åįī":55743,".prototype":55744,"Hope":55745,"åī¯éĻ¢éķ¿":55746,"Ġracist":55747,"nutrition":55748,"ĠÙĩÙĨاÙĥ":55749,"çļĦå¹³åĿĩ":55750,"Ġexpectancy":55751,"à¸Ĥà¸Ļ":55752,"æıIJåΰçļĦ":55753,"ĠبÛĮÙħارÛĮ":55754,"Ġaggrav":55755,"ĠпÑĢинÑĨи":55756,"ç¹ģåįİ":55757,"Ġdiluted":55758,"ĠMang":55759,"ĠNou":55760,"æĪijåı¯":55761,"ĠجÙħÙĬع":55762,"wire":55763,"ocated":55764,"åĴĮéĿŀ":55765,"ĠRH":55766,"åı£è¯Ń":55767,"ĠAndrews":55768,"Mike":55769,"åĴĮè§£":55770,"Ġproduk":55771,"ÏĥÏĦα":55772,"Alignment":55773,"_object":55774,"still":55775,"oley":55776,"èµŀåIJĮ":55777,"è¨İè«ĸ":55778,"å¼Ģå§ĭçļĦ":55779,"(%":55780,"beiten":55781,"å®ĭ代":55782,"á½²":55783,"brief":55784,"ĠCategories":55785,"åĩºèī²çļĦ":55786,"Ġphilanthrop":55787,"('\\":55788,"Starting":55789,"çĢļ":55790,"å®ļçĤ¹":55791,"Ġremodel":55792,"Ġsolitary":55793,"éįĭ":55794,"-eyed":55795,"ĠLad":55796,"мена":55797,"Ġfertile":55798,"Ġdiscarded":55799,"å¢ĥçļĦ":55800,"å¾·éĩĮ":55801,"åħ¸ç¤¼":55802,"ĠArkiver":55803,"Ġrol":55804,"é¢ĦåħĪ":55805,"æĪijçļĦå¿ĥ":55806,"ĠÙĬست":55807,"ĠÑģÑĤои":55808,"Ġexperimentally":55809,"तà¥įत":55810,"Ġবিশà§įব":55811,"Ġseating":55812,"ractical":55813,"ĠSpelling":55814,"Ġwaved":55815,"Ġsolemn":55816,"Ġpills":55817,"ĠÙĪØ§ÙĤع":55818,"å¤įæĿĤ度":55819,"_USER":55820,"Ġaccreditation":55821,"{K":55822,"Ġmotif":55823,"èĸ¦":55824,"æ³īæ°´":55825,"æĪ¿éĹ´éĩĮ":55826,"Ġpolished":55827,"Ġavg":55828,"زÙĩ":55829,"iosa":55830,"Ġdeclines":55831,"WHO":55832,"Demo":55833,"fang":55834,"ĠSug":55835,"ĠبÙĬاÙĨات":55836,"ĠbÄĻdzie":55837,"ĠToyota":55838,"她åĢij":55839,"ĠLimit":55840,"ĠFee":55841,"æĪij们已ç»ı":55842,"netes":55843,"è¿Ļ个å°ı":55844,"ĠFischer":55845,"ĠZamb":55846,"Ġ׾×ķ":55847,"Ġipsum":55848,"idu":55849,"ĠScene":55850,"ĠIgG":55851,"å¹´èµ·":55852,"éĴ³":55853,"-temperature":55854,"å»¶å®ī":55855,"Ġreputable":55856,"abo":55857,"Ġarrog":55858,"使ç͍æĿĥ":55859,"ĠØ£ÙĪÙĦ":55860,"صÙĪØ±":55861,"Ġoccurrences":55862,"è£ģåĨ³":55863,"_width":55864,"dad":55865,"ĠاÙĦÙĦÙī":55866,"mittel":55867,"Ġerv":55868,"ĠMisc":55869,"ĠThy":55870,"Ġproprietary":55871,"ĠHipp":55872,"Ġdrums":55873,"ĠKeeping":55874,"Ġkidneys":55875,".email":55876,"Ġà¸Ħวาม":55877,"uned":55878,"aturation":55879,"thropoda":55880,"Ġmarsh":55881,"Ġunwavering":55882,"ĠMao":55883,"ĠKoch":55884,"åłķ":55885,"ĠاÙĦÙĬÙĪÙħ":55886,"Ġpoultry":55887,"à§ĩদ":55888,"åİ»åIJ§":55889,"åĭķçī©":55890,"æ¡ĮéĿ¢":55891,"Ġperpetu":55892,"ĠVC":55893,"Ġeducator":55894,"ç¨ĭåºıçļĦ":55895,"ández":55896,"èĥ°å²Ľ":55897,")'":55898,"ĠAllan":55899,"ĠðŁĻ":55900,"èµ·çĿĢ":55901,"ĠемÑĥ":55902,"Ġtao":55903,"Ġmug":55904,"ĠCASE":55905,"åħ±éĿĴ":55906,"èĤ¢ä½ĵ":55907,"Ġnoisy":55908,"-band":55909,"ợ":55910,"ĠChapel":55911,"Ġentrepreneurial":55912,"rée":55913,"zew":55914,"Ġpari":55915,"Ġneo":55916,"Ġrounding":55917,"Ġlethal":55918,"ĠتارÛĮ":55919,"য়া":55920,"-new":55921,"наÑĢод":55922,"ĠMySQL":55923,"ĠصÙĨ":55924,"ĠGreatest":55925,"ĠìķĦëĭĪ":55926,"Ġdelegates":55927,"Ġméth":55928,"Ġaccru":55929,"Fre":55930,"ifat":55931,"ç¬ĥ":55932,"NESS":55933,"ampton":55934,"ĠRaman":55935,"ç¬ijæĦı":55936,"ilion":55937,"åıĭæĥħ":55938,"uji":55939,"Ġmanifold":55940,"ç§ĥ":55941,"Õ¥Õ½":55942,"\"+":55943,"ĠDAY":55944,"Ġlocale":55945,"ĠDevil":55946,"_TO":55947,"Ġwiped":55948,"ĠTF":55949,"èĪĶ":55950,"å¤ļæł·æĢ§":55951,"(state":55952,"çļĦéĤ£äºĽ":55953,"ĠDEP":55954,"ä¸Ĭå±Ĥ":55955,"Ġpolynomials":55956,"ĠIngredients":55957,"æĸ°åŁİ":55958,"iset":55959,"Ġauditor":55960,"irmation":55961,"amat":55962,"ochen":55963,"(log":55964,"Ġasbestos":55965,"ĠOun":55966,"穹":55967,"æīĵæī°":55968,"ĠespecÃŃfic":55969,"ĠHearing":55970,"neapolis":55971,"Ġlur":55972,"Ġstos":55973,"ühren":55974,"ĠRegardless":55975,"ĠBurke":55976,"Ġperò":55977,"èĩªåı¤":55978,"Ġnortheast":55979,"amics":55980,"éłĺåŁŁ":55981,"Ġcompassionate":55982,"æŀĦæĪIJäºĨ":55983,"çϾéĩĮ":55984,"Ġpanc":55985,"ä¸Ģè¨Ģ":55986,"éģ²":55987,"Ġmartial":55988,"大å°ıçļĦ":55989,"Ġmemper":55990,"âĢĿãĢĤâĢľ":55991,"ائÙħ":55992,".exe":55993,"Ġhormonal":55994,"Ġë§Įëĵ¤":55995,"Ġtér":55996,"Ġviz":55997,"Ġ:)":55998,"èĤĿçĤİ":55999,"ĠÑĥÑģловиÑıÑħ":56000,"елÑĮ":56001,"Ġovertime":56002,"Ġnumerals":56003,"ĠOutcome":56004,"Ġà¤ķा":56005,"ducers":56006,"ĠÙĬجب":56007,"Ġwarriors":56008,"æįIJ款":56009,"åĴĮæĸĩåĮĸ":56010,"éĿ´":56011,"Ġimbalance":56012,"äºĶåĽĽ":56013,"éķ¿äºĨ":56014,"boys":56015,"墨西åĵ¥":56016,"ï¼ļ(":56017,"-cultural":56018,"Elo":56019,"éĹ®åĢĻ":56020,"ç®ĢåİĨ":56021,"³³³³³³³":56022,"Ci":56023,"à¸ģวà¹Īา":56024,"èŀºçº¹":56025,"valence":56026,"bola":56027,"Ġê²ĥìĿ´ëĭ¤":56028,"Ġsubstantive":56029,"ĢáĢ»":56030,"zik":56031,"çı¾åł´":56032,"Ġλεί":56033,"çķľçī§":56034,"buy":56035,"ctrine":56036,"çĦ¡è«ĸ":56037,"à¸Ľà¸£à¸°à¸Ĭ":56038,"èķ´åIJ«":56039,"ĠتضÙĬÙģ":56040,"Ġnursery":56041,"Ġpushes":56042,"ä»ĸ们éĥ½":56043,".group":56044,"มาà¸ĵ":56045,"ç¡®å®ļäºĨ":56046,"æīĵå¼ĢäºĨ":56047,"ranial":56048,"ÑĤек":56049,"train":56050,"æºħ":56051,"Ġcracking":56052,"ä¹Łæ²Ĵ":56053,"ãĥªãĤ¢":56054,"ĠJoined":56055,"ĠÑĢаÑģÑģка":56056,"大è±Ĩ":56057,"toire":56058,"ĠSensor":56059,"计ç®Ĺåħ¬å¼ı":56060,"à¹Ģà¸ĩ":56061,"åħĭçļĦ":56062,"Ġastronaut":56063,"Ġspear":56064,"ĠEPUB":56065,"æĥ³èµ·äºĨ":56066,"è·¨å¢ĥ":56067,"ĠDerek":56068,"ĠNiet":56069,"ĠArbeits":56070,"UPDATE":56071,"éĿŀ常好":56072,"å·¥ç¨ĭ建设":56073,"Ġfabulous":56074,"nor":56075,"iltr":56076,"ĠMH":56077,"ç»ħ":56078,"è§ģä»ĸ":56079,"Ġinherit":56080,"Ġdek":56081,"ĠBj":56082,"ipong":56083,"书æĪ¿":56084,"ussa":56085,"á»iji":56086,"(struct":56087,"Ġmating":56088,"Ġfreight":56089,"Ġarqu":56090,"äºĨä¸Ģ大":56091,"红å¤ĸ":56092,"æ¼ĶæĬĢ":56093,"×ķ׾×Ļ×Ŀ":56094,"Ġinbox":56095,"ãĢľ":56096,"-start":56097,"particular":56098,"felt":56099,"science":56100,"third":56101,"Ġgoat":56102,"ç»ĻåĩºäºĨ":56103,"æĻļé¥Ń":56104,"ĠتÙħاÙħ":56105,"hundert":56106,"æĵ´":56107,"Ġnationalism":56108,"LB":56109,"ÅĻenÃŃ":56110,"erent":56111,"ĠAce":56112,"Ġembedding":56113,"Ġcation":56114,"æŃ¯":56115,"åģī":56116,"è»Ĵ":56117,"Ġcooperate":56118,"-space":56119,"å¯ĵæĦı":56120,"æĹ¢å¾Ģ":56121,"Ġdefenses":56122,"çķĻåѦçĶŁ":56123,"è¯ĬçĸĹ":56124,"æĹłäººæľº":56125,"æīĢéľĢè¦ģçļĦ":56126,"reeNode":56127,"Ġretard":56128,"ĠÕº":56129,"座æ¤ħ":56130,"ĠاجتÙħاع":56131,"æľ¦":56132,"verts":56133,"okrat":56134,"å¾Ī强":56135,"ĠبأÙĨ":56136,"ĠVirus":56137,"Ġsuspects":56138,"ĠNixon":56139,"ĠCompensation":56140,"Ġunsafe":56141,"Ġthi":56142,"ĠComing":56143,"ĠSpread":56144,"Arthur":56145,"ĠMün":56146,"oble":56147,"亲è¿ij":56148,"ç»Īç»ĵ":56149,"æİĢèµ·":56150,"Ġaž":56151,"æĵįæİ§":56152,"Ġstaple":56153,"Ġutilizz":56154,"GAN":56155,"¢×ķת":56156,"izada":56157,"åĨ°éĽª":56158,"'])":56159,"Ġzad":56160,"Ġrecol":56161,"éĸ£":56162,"ĠFlood":56163,"Ġdrained":56164,"Ġveterinary":56165,"ĠREM":56166,"Ġity":56167,"Ġtutoring":56168,"ĠWarm":56169,"çĿ«":56170,"åĬĽæ±Ĥ":56171,"ĠPig":56172,"ectin":56173,"æĹ©çĤ¹":56174,"веÑĢÑģи":56175,"Ġtubular":56176,"Hozzáférés":56177,"Ġskies":56178,"Ġpassions":56179,"ĠÑĤела":56180,"ĠLynch":56181,"ä¸¥æł¼æĮīçħ§":56182,"Ġopioid":56183,"Ġchaotic":56184,"Ġnorthwest":56185,"ĠKamp":56186,"çĥŁèĬ±":56187,"å·¥ä¸ļåĮĸ":56188,"conditional":56189,"ÙĨاء":56190,"ĠклеÑĤ":56191,"Ġprognostic":56192,"owitz":56193,"ĠGed":56194,"Ġdetectors":56195,"ĠRepair":56196,"Ġpolygon":56197,"Ġжеле":56198,"æĬĹçĶŁç´ł":56199,"ì½Ķ":56200,"æĽ´è¦ģ":56201,"Usage":56202,"Ĺ×ĸ":56203,"Mail":56204,"auk":56205,"æİ¨å¼Ģ":56206,"à¸Īà¸Ī":56207,"Ġcapacitance":56208,"æĸ¼æĺ¯":56209,"æĹ¶æķĪ":56210,"çĿĢä½ł":56211,"RIB":56212,"usia":56213,"çĶµåľº":56214,"äºĮè¿Ľåζ":56215,"åĪĹ为":56216,"åħµåĬĽ":56217,"ĉself":56218,"ĠTOP":56219,"Ġsmartphones":56220,"ç¦ı建çľģ":56221,"Ġquotation":56222,"åıijåĩºçļĦ":56223,"interval":56224,"æł¹æľ¬æ²¡æľī":56225,"Ġà°¸":56226,"Making":56227,"è¦ģä¸įçĦ¶":56228,"ÙĪØ§Ø¨":56229,"éģĵä¸Ĭ":56230,"бÑĥÑĢ":56231,"à§ĭà¦ķ":56232,"ĠاÙĦعÙħ":56233,"Ġsymbolism":56234,"ĠLegislature":56235,"udence":56236,"share":56237,"åľŁè̳":56238,"Ġpleasing":56239,"æĭ·":56240,"ĠDept":56241,"æĬĢèīº":56242,"radas":56243,"åıįåºĶè¿ĩæĿ¥":56244,"Ġsod":56245,"Ġpacks":56246,"æģ©æł¼æĸ¯":56247,"elu":56248,"åıijçݰèĩªå·±":56249,"Ġelders":56250,"Ġlament":56251,"è¡°èĢģ":56252,"æĭįäºĨæĭį":56253,"roots":56254,"too":56255,"oyer":56256,"Ġexperimentation":56257,"Pron":56258,"ä¸Ńç¾İ":56259,"ä»¬åľ¨":56260,"ä¾®":56261,"Ġrestoring":56262,"Ġfatto":56263,"ĠTail":56264,"Ġorphan":56265,"å¤ĸ表":56266,"ÙĨداÙĨ":56267,"æĶ¶åħ¥çļĦ":56268,"ÑģÑĤвеннÑĭй":56269,"éĺŁä¼į建设":56270,"Ġchorus":56271,"Ġecu":56272,"Components":56273,"мини":56274,"à¥ģर":56275,"{pmatrix":56276,"ר×ķ":56277,"å±ĢçļĦ":56278,"ĠÙĦÙĥÙĨ":56279,"East":56280,"Ġresigned":56281,"è¿ĺåĮħæĭ¬":56282,"èĵ®":56283,"Large":56284,"ä¹Łæ²Ĵæľī":56285,"phyl":56286,"åIJĮä¸Ĭ":56287,"ê°IJ":56288,"Ġaltering":56289,"ĠнагÑĢÑĥз":56290,"ĠJoyce":56291,"模å¼ıçļĦ":56292,"æĹºçĽĽ":56293,"ĠнаÑģеÑĻ":56294,"Ġumbrella":56295,"ÙİÙĬ":56296,"éĹ®éĹ®":56297,"تÙħع":56298,".\"\"":56299,"Ġlegitimacy":56300,"çĢijå¸ĥ":56301,"ĠDeWalt":56302,"Jud":56303,"è¿ĺä¸įæĺ¯":56304,"éªĩ":56305,"boarding":56306,"ĉchar":56307,"è¿Ļå®¶ä¼Ļ":56308,"åΰä»ĸ":56309,"à´¯":56310,"Ġceremonies":56311,"åĢĺèĭ¥":56312,"ĠVideos":56313,"Ġendomet":56314,"ĠاÙĦظ":56315,"Ġhourly":56316,"ĠبÙĬÙĩا":56317,"Ġcottage":56318,"ĠLopez":56319,"ocz":56320,"ĠMarriage":56321,"ské":56322,"Ġtensile":56323,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤи":56324,"Ġfinden":56325,"νε":56326,"Smart":56327,"Ġ모ëĵł":56328,"çļĦ羣":56329,"åIJĮåѸ":56330,"Ġburnt":56331,"озÑıй":56332,"Ġcann":56333,"-full":56334,"Ġdecorative":56335,"Ġrook":56336,"Ġautobi":56337,"째":56338,"Ġeclipse":56339,"ÑĢин":56340,"åħĭæĭī":56341,"Unknown":56342,"ĠÙħÙĪØ§Ø¯":56343,"Ġspells":56344,"è¨ŃåĤĻ":56345,"omnia":56346,"owel":56347,"ukop":56348,"Ġ×Ļ×Ķ":56349,"ĠMarian":56350,"(set":56351,"è·ŁèijĹ":56352,"Ġearthquakes":56353,"à¸Ńุ":56354,"Ġrestrictive":56355,"omonas":56356,"çļĦåľŁåľ°":56357,"previous":56358,"ĠCastro":56359,"Ġleaks":56360,"å±İ":56361,"ĠгоÑĤов":56362,"Ġhappier":56363,"Ġnormative":56364,"éĵ¶åŃIJ":56365,"ĠMethodology":56366,"Ġbounce":56367,"ĠIE":56368,"Ġweer":56369,"å¸Ŀçİĭ":56370,"ĠÑģпеÑĨиали":56371,".Context":56372,"اÙĦÙī":56373,"AGES":56374,"تÙĪ":56375,"ĠÑĥменÑĮ":56376,"广å·ŀå¸Ĥ":56377,"Ġmeningkat":56378,"鸽":56379,"ãģłãģ£ãģŁ":56380,"âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ":56381,"Wall":56382,"æĺ¯æł¹æį®":56383,"ãģ§ãģ¯ãģªãģĦ":56384,"Ġsparked":56385,"Binding":56386,"Ġentra":56387,"ีà¹Īยà¸Ļ":56388,"omorphism":56389,"Ġguru":56390,"人éĸĵ":56391,"æģ¤":56392,"ĠVerlag":56393,".base":56394,"æĩīç͍":56395,"騰":56396,"à¸Ĭีวิà¸ķ":56397,"ĠλείÏĢει":56398,"-oper":56399,"StackTrace":56400,"Wi":56401,"说å®ŀè¯Ŀ":56402,"注æĦıçļĦæĺ¯":56403,"çļĦèĩªçĦ¶":56404,"ĠSecure":56405,"avoid":56406,"notations":56407,"Ġacknowledging":56408,"ĠChampion":56409,"panel":56410,"Ġnecrosis":56411,"Ġcrunch":56412,"Ġconveyed":56413,"teacher":56414,"Ġflask":56415,"ĠEurop":56416,"Ġpolymerase":56417,"bull":56418,"Ġcerca":56419,"éĤ£ä¸ªäºº":56420,"æĹłéĶ¡":56421,"å¢ŀéķ¿çİĩ":56422,"ĠAlbum":56423,"ruk":56424,"rentice":56425,"}^{*":56426,"Ġrewrite":56427,"ractor":56428,"æĮģçºĮ":56429,"åιéĤ£":56430,"ĠDN":56431,"Ġkits":56432,"éĩįè¦ģæĢ§":56433,"æŃĮåͱ":56434,"ิล":56435,"æĭī伸":56436,"Ġhoof":56437,"Ġaprendiz":56438,"ç¯ĩ竳":56439,"çłĶ讨ä¼ļ":56440,"Ġmatem":56441,"ÅĽnie":56442,"产ä¸ļçļĦ":56443,"Ġstreamline":56444,"-kasadpan":56445,"åı¯æł¹æį®":56446,"æĺİæľĿ":56447,"ä¸ĢæĬ¹":56448,"->_":56449,"ĠGiants":56450,"Ġants":56451,"ÛĮرÛĮ":56452,"Ġfulfil":56453,"æľ¬é¢Ĩ":56454,"é«ĺ大":56455,"Ġ׾×Ĵ":56456,"IJ×Ķ":56457,"USS":56458,"ĠNaCl":56459,"à¤ķà¥įष":56460,"akis":56461,"ĠGuar":56462,"ç¡®å®ŀæĺ¯":56463,"(Integer":56464,"tp":56465,"ĠAster":56466,"Ġà¦ľà¦¾à¦¨":56467,"ĠBCE":56468,"è᝿ĿIJ":56469,"âīł":56470,"éĭĴ":56471,"ĠTec":56472,"Ġclash":56473,"éĩį大çļĦ":56474,"æ¯ıåĢĭ":56475,"-place":56476,"ĠSymph":56477,"Ġsounding":56478,"åħĴåŃIJ":56479,"ĠTerminal":56480,"Ġdopamine":56481,"çŃ·åŃIJ":56482,"Ġfich":56483,".cs":56484,"ĠCypr":56485,"respective":56486,"syn":56487,"Ġditu":56488,"orsz":56489,"éģĵåħ·":56490,"Ġyuta":56491,"Ġcocaine":56492,"nj":56493,"ĠŽ":56494,"çĶ¢çĶŁ":56495,"Ġwaking":56496,"Ġchant":56497,"æŀĦæĪIJçļĦ":56498,"åĮħåIJ«äºĨ":56499,"Dans":56500,"yses":56501,"åİ¿æĶ¿åºľ":56502,"åѦä¼ļäºĨ":56503,"åij¼åIJ¸éģĵ":56504,"_params":56505,"Ġبزر":56506,"åŃŁåŃIJ":56507,"meas":56508,"ĠRadiation":56509,"hydrogen":56510,"çī©ä½ĵçļĦ":56511,"Org":56512,"Ġlad":56513,"INTER":56514,"Ġlinkage":56515,"åłĨ积":56516,"ĠKLIMA":56517,"ĠMira":56518,"research":56519,"èĩªåĭķ":56520,"jiang":56521,"å¢ŀè¿Ľ":56522,"åł°":56523,"/$":56524,"Ġש׳×":56525,",q":56526,"Ġblot":56527,"ĠÑĥÑģÑĤÑĢой":56528,"åIJĽä¸»":56529,"ĠAttack":56530,"¨àµįà´¨":56531,"Ġexpired":56532,"Ġ×ŀת":56533,"Ġquotient":56534,"öj":56535,"Ġheadaches":56536,"é£Łãģ¹":56537,"Ġliters":56538,"Ġfeder":56539,"éĢ®æįķ":56540,"intendo":56541,"Foreign":56542,"ĠобÑĬем":56543,"Ġamyl":56544,"ĠLatino":56545,"ĠпÑĢоÑĦеÑģÑģи":56546,"åỿĺ¯":56547,"ĠγÏħνα":56548,"cji":56549,"Ġdefer":56550,"ĠHood":56551,"bras":56552,"ä¼ļ议室":56553,"OX":56554,"ĠwÅĤ":56555,"çļĦæ²»çĸĹ":56556,"åı¤ä»Ĭ":56557,"Ġmetastasis":56558,"ê»ĺ":56559,"micro":56560,"ķáĢºáĢ":56561,"å;":56562,"ĠStudios":56563,"ĠMartinez":56564,"Ġmigrant":56565,"xd":56566,"ÑĩноÑģÑĤи":56567,"Ġtaxa":56568,"Ġdemonstrations":56569,"ĠاÙĦسر":56570,"åĢįæķ°":56571,"çļĦåİĭåĬĽ":56572,"Ġrehears":56573,"Ġlamps":56574,"utorial":56575,"游çİ©":56576,"è¾ķ":56577,"æľºç͵":56578,"æĬķä¿Ŀ":56579,"ÐŁÑĢо":56580,"ĠWITHOUT":56581,"ĠLIM":56582,"ç²Ł":56583,"转è¿ĩ":56584,"Ġbuds":56585,"åĥ¹å̼":56586,"ateurs":56587,"èĬĤçĤ¹çļĦ":56588,"Ġatrav":56589,"ült":56590,"ĠØ£ÙĬض":56591,"ĠCorinthians":56592,"Ġtraction":56593,"åģļåΰäºĨ":56594,"Ġberbagai":56595,"Ġcongenital":56596,"å¿ħä¸įåı¯":56597,"çĶ·åıĭ":56598,"Ġflashcards":56599,"ä»ĸçİ°åľ¨":56600,"Ġchunks":56601,"ĠÑģимпÑĤом":56602,"ITER":56603,"å·´æİĮ":56604,"ằ":56605,"Ġbites":56606,"ÑĪений":56607,"éĻ῏©":56608,"Ġprivat":56609,"å¯ĦçĶŁ":56610,"Ġduo":56611,"-ang":56612,"Ġpeanut":56613,"Ġmantle":56614,"ä¹Łä¸įç͍":56615,"Ġfaded":56616,"ellaneous":56617,"ĠYo":56618,"Ġsimples":56619,"åįĪé¤IJ":56620,"ĠصØŃ":56621,"æĹ¥å¸¸çĶŁæ´»ä¸Ń":56622,"æĸĩä½ĵ":56623,"ĠIntent":56624,"宾é¦Ĩ":56625,"ĠConsiderations":56626,"heits":56627,"Ġparen":56628,"uil":56629,".ĊĊĊĊ":56630,"åĪĨ管":56631,"åĨĽå®ĺ":56632,"å¾Įä¾Ĩ":56633,"ĠPersonality":56634,"atakan":56635,"ieke":56636,"Ġscanned":56637,"strings":56638,"-del":56639,"å¼ĢåıijåķĨ":56640,"Ġsten":56641,"ĠZarucchi":56642,"å·§åħĭåĬĽ":56643,"/${":56644,"ĠEar":56645,"KM":56646,"éĽĨèģļ":56647,"Ctrl":56648,"æĦŁãģĺ":56649,"ç²īæľ«":56650,"深深çļĦ":56651,"ĠIon":56652,"Ġgrupos":56653,"Ġbreathtaking":56654,"Ġdecentralized":56655,"ĠWebb":56656,"until":56657,"Ġsynerg":56658,"ĠHimself":56659,"Ġworms":56660,"idual":56661,"Ġhardness":56662,"ç¥ĸåħĪ":56663,"ĠBreaking":56664,"æŀ¢çº½":56665,"actly":56666,"ODIS":56667,"(--":56668,"/html":56669,"ìĽĮ":56670,"ĠPurs":56671,"ĠÕ«":56672,"ä¸ĵä¸ļåĮĸ":56673,"Ġdeepest":56674,"ç«¥è¯Ŀ":56675,"Ġalerts":56676,"inform":56677,"Ġdrank":56678,"ETS":56679,"éĮ²":56680,"Ġкомпа":56681,"Multiple":56682,"é¢Ŀ头":56683,"ĠAPP":56684,"Ġspotlight":56685,"Calculation":56686,"ública":56687,"ĠJudaism":56688,"çľĭä¸Ģä¸ĭ":56689,"é¦ĸæī¹":56690,"ueblo":56691,"kus":56692,"ÙħÙĬØ©":56693,"Ġetx":56694,"Ġurging":56695,"ĠÙĤدر":56696,"ĠεÏĦÏħμολογία":56697,"à¸ģลุà¹Īม":56698,"ĠBrandon":56699,"otros":56700,"িয়à§ĩ":56701,"Ġtxt":56702,"ĠBeta":56703,"åŃ¦è´¹":56704,"Ġнеза":56705,"idelberg":56706,"ĠAdmiral":56707,"ĠSquad":56708,"èįĶ":56709,"Ġcatalogue":56710,"lÃŃ":56711,"estre":56712,"åħļå»ºå·¥ä½ľ":56713,"Ġswallowed":56714,"Ġвозника":56715,"ĠSebastian":56716,"Ġsolubility":56717,"Sound":56718,"æīĭæĮģ":56719,"ĠGitHub":56720,"(ĊĊ":56721,"welt":56722,"éĢļçļĦ":56723,"å®£ä¼łéĥ¨":56724,"Ġrepeats":56725,"ĠFleet":56726,"Ġaunque":56727,"Ġstretches":56728,"两家":56729,"unga":56730,"åįİåĮĹ":56731,"ÑĢина":56732,"Ġhac":56733,"Ġtimeout":56734,"è§ĨåĬĽ":56735,"ĠHungarian":56736,"neur":56737,"å°Ĩ被":56738,"çī¹çĤ¹æĺ¯":56739,"æ¹¿åľ°":56740,"ÛĮÚ©ÛĮ":56741,"åIJĮçŃī":56742,"elsen":56743,"keleton":56744,"Ġwoven":56745,"ĠArsenal":56746,"Ġbehand":56747,"à¥įथ":56748,"ä¹°æĪ¿":56749,"Ġcontracting":56750,"iembre":56751,"詹å§Ĩæĸ¯":56752,"奪":56753,"äºĮåıī":56754,"çĹħåİŁ":56755,"ĠWhenever":56756,"__\":Ċ":56757,"æ¶ķ":56758,"çŁ¥éģĵèĩªå·±":56759,"Ġsilently":56760,"Continue":56761,"ĠSister":56762,"verbal":56763,"éĢĥè·ij":56764,"éĶĪéĴ¢":56765,"款项":56766,".CS":56767,"Ġconsultants":56768,"ĠVes":56769,"çłļ":56770,"Ġباع":56771,"å¤ĩåıĹ":56772,"ĠWalking":56773,"æĺİçıł":56774,"éĺ³åı°":56775,"ĠCertainly":56776,"ä¹Łç®Ĺæĺ¯":56777,"åѵåĮĸ":56778,"-organ":56779,"ĠPascal":56780,"帷":56781,"身穿":56782,"妥åĸĦ":56783,"FORMATION":56784,"Ġawa":56785,"Atl":56786,"WL":56787,"å¹³åĴĮ":56788,"马æĸ¯":56789,"-age":56790,"鼷éľĨ":56791,"æŃĮè¯į":56792,"Ġtitanium":56793,"ĠSanders":56794,"ĠRESULTS":56795,"åĨłçĬ¶çĹħæ¯Ĵ":56796,"(O":56797,"Ġanecd":56798,"æİĴè¡Į":56799,"è¡įçĶŁ":56800,"Bul":56801,"æĹłæĥħ":56802,"Ïĩε":56803,"superscriptðĿij":56804,"è®Ĭå¾Ĺ":56805,"éĴĵé±¼":56806,"妳":56807,".handle":56808,"Ġstudi":56809,"Ġmenc":56810,"å¾Ģå¾Ģæĺ¯":56811,"JE":56812,"ĠìĤ°":56813,"غراÙģ":56814,"à¹Ħมà¹ī":56815,"ivacy":56816,"amus":56817,"ĠÙĨÛĮاز":56818,"\"No":56819,"Ġimpul":56820,"uak":56821,"ä¼łåħ¥":56822,"ĠWiki":56823,"ĠBansa":56824,"ĠEucl":56825,"æ´±":56826,"èĢĮè¿ĩ":56827,"ÐĿÐŀ":56828,"ä»įçĦ¶æĺ¯":56829,"_q":56830,"ä¸Ń西":56831,"èħĮ":56832,"Ġpanor":56833,"ĠTemper":56834,"ĠNile":56835,"ÑĤам":56836,"饮éħĴ":56837,"ĠOste":56838,"°Ċ":56839,"......Ċ":56840,"Ġlá":56841,"оÑģÑĤ":56842,"Employ":56843,"Ġvictories":56844,"ĠÚĨÙĩ":56845,"ĠÑĦоÑĢма":56846,"details":56847,"ĠMons":56848,"емÑĭÑħ":56849,"åĨ°åĨ·":56850,"ĠгоÑĢод":56851,"ĠScholarship":56852,"å¢Ł":56853,"æ°´å¹³åĴĮ":56854,"Reducer":56855,"ĠاÛĮجاد":56856,"å¾Ģä¸Ĭ":56857,"amaan":56858,"æĸ¹ç¨ĭå¼ı":56859,"Ġaccommodations":56860,"Tur":56861,"Ġnegotiating":56862,"ĠenergÃŃa":56863,"Ķ׾":56864,"ĠквадÑĢа":56865,"ĠاÙģØ²Ø§ÛĮØ´":56866,"Ġ'%":56867,"Ġzb":56868,"åıĬ缸åħ³":56869,"轻声":56870,"çĶ»åĥı":56871,"Ġcuatro":56872,"ĠReduced":56873,"ä¸įåı¯éģ¿åħį":56874,"ĠÑģвоб":56875,"-ion":56876,"Ġwür":56877,"äºĨä¸Ģå®ļçļĦ":56878,"ève":56879,".repository":56880,"Ġtherapists":56881,"VL":56882,"lead":56883,"æķĻåѸ":56884,"Ġspeci":56885,"ĠSver":56886,"Ġkadaghan":56887,"ĠContribution":56888,"æĦıä¹īä¸ĬçļĦ":56889,"ĠPetersburg":56890,"Machine":56891,"ndon":56892,"ĠDys":56893,"è·¯è¿ĩ":56894,"à¸łà¸²à¸Ħ":56895,"Ġproyecto":56896,"ĠPenev":56897,"æľīåIJį":56898,"æīįåįİ":56899,"Ġjeans":56900,"æĿ¥è¡¨ç¤º":56901,"åIJĦåľ°çļĦ":56902,"Ġantioxidants":56903,"è¨ĵç·´":56904,"转åħ¥":56905,"ÑĢÑĥеÑĤÑģÑı":56906,"Selection":56907,"Ġà¦Ĩস":56908,"_VALUE":56909,"è¿Ļåľ¨":56910,"pek":56911,"Ġtaxpayer":56912,"cesso":56913,"ĠΧ":56914,"Ġ×ŀר×":56915,"éģģ":56916,"åīĸæŀIJ":56917,"ä½Ĩ对":56918,"Ġoperative":56919,"ãģŁãĤĬ":56920,"ucose":56921,"ĠQatar":56922,"Ġengineered":56923,"宽容":56924,"æ¯ĶæĪij":56925,"æ¯Ķè³½":56926,"åħ±åIJĮä½ĵ":56927,"pair":56928,"ĠÙĦÙĦØ£":56929,"Ġâī¡":56930,"(II":56931,"ä»ĸå°±æĺ¯":56932,"Ġdrastically":56933,"Äįet":56934,"oste":56935,"åĪĨæŀIJäºĨ":56936,"Ġfus":56937,"ĠLAN":56938,"ĠGamit":56939,"åΰæĪij":56940,"permalink":56941,"Individual":56942,"宽æĿ¾":56943,"Ġëį°ìĿ´íĦ°":56944,"å¦Ĥæĺ¯":56945,"ĠÙħاÙĩ":56946,"Ġconvictions":56947,"å¾Ī大ç¨ĭ度ä¸Ĭ":56948,"_z":56949,"ä¸įåĩºæĿ¥":56950,"强åĬ²":56951,"))))":56952,"çł´è§£":56953,"ĠLars":56954,"Ġ'''":56955,"iantes":56956,"Giya":56957,"Ġnež":56958,".CharField":56959,"cony":56960,"Ġzh":56961,"éĹ¨æ§Ľ":56962,"ADVERTISEMENT":56963,"Ġprecedent":56964,"Ġপারà§ĩ":56965,"Ġouv":56966,"¨":56967,"Ġinfarction":56968,"egan":56969,"åIJijåIJİ":56970,"Ġmata":56971,"ä¸įçα":56972,"ĠWebster":56973,"มห":56974,"Ġabused":56975,"ῶν":56976,"ç«ĻçĿĢ":56977,"åı¤æĢª":56978,"ĠMalcolm":56979,"Ġconfidentiality":56980,"ĠJenkins":56981,"Pulgada":56982,"ĠKnown":56983,"iked":56984,"åĨĻåŃĹ":56985,"عدد":56986,"rapeutics":56987,"ĠдоÑģÑĤаÑĤоÑĩно":56988,"MENTS":56989,"æľºåºĬ":56990,"Ġ#{":56991,"Ġsuperb":56992,"ÑĤÑĥÑĢе":56993,"ĠInstitut":56994,"Ġdissatisf":56995,"ĠEcuador":56996,"=âĪĴ":56997,"ká":56998,"zien":56999,"erst":57000,"Ġsr":57001,"STEP":57002,"ĠSaturn":57003,"æľīéĻIJ责任åħ¬åı¸":57004,"ĠLB":57005,"éĽĨåľĺ":57006,"YouTube":57007,"\"?":57008,"-zero":57009,"Ġenvis":57010,"åĽŀèIJ½":57011,"Chemistry":57012,"ukerken":57013,"-Sch":57014,"Ġabstraction":57015,"åŀ¢":57016,"çľ¼çľĭ":57017,"ĠÑĥд":57018,"ĠPagbu":57019,"åīĬå¼±":57020,"wau":57021,"fts":57022,"çļĦå°±":57023,"Ġscenery":57024,"å®Įå·¥":57025,"åij¨æģ©":57026,"å¤ĸå¥Ĺ":57027,"éļ¾æĢª":57028,"åĸĿäºĨ":57029,"ĠNieukerken":57030,")ÃĹ":57031,"ĠBottom":57032,"ä¹ŁæĹł":57033,"åĨĻçĿĢ":57034,"å²ģæĹ¶":57035,"_not":57036,"Ġamet":57037,"å¼ĢåıijçļĦ":57038,"ÙĨاÙĪÙĦ":57039,"IMARY":57040,"ĠColonial":57041,"ĠBritonhon":57042,"åħ¶ä¸ŃåĮħæĭ¬":57043,"ĠTroy":57044,"ĠPagbuok":57045,"ä¸įä½³":57046,"ĠRJ":57047,"zeug":57048,"åѦçĶŁä¼ļ":57049,"Ġfinale":57050,"æĻļäºĨ":57051,"建ç«ĭåģ¥åħ¨":57052,"ĠOperator":57053,"Radio":57054,"Ġaumento":57055,"ĠParehong":57056,"Ġpolicymakers":57057,"Ġresection":57058,"ĠColoring":57059,"ców":57060,"Ġhyster":57061,"ĠRaz":57062,"Ġupstairs":57063,"åĬŀæ¡Ī":57064,"yai":57065,"è¿Ļ让":57066,"æľ¬åıijæĺİ":57067,"èĭįçϽ":57068,"离å¿ĥ":57069,"Ġswear":57070,"ĠHamas":57071,"ĠεÏĢÏİνÏħμο":57072,"ĠFors":57073,"æĪĺçļĦ":57074,"å©¶":57075,"Ġpronouns":57076,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":57077,"æĤ¨å¥½":57078,"DUCT":57079,"Development":57080,"åĵģå°Ŀ":57081,"çĥ¨":57082,".readLine":57083,"Tinubdan":57084,"Kadaghan":57085,"EDAC":57086,"Ġmotivations":57087,"Ñĥма":57088,"èģĶåIJĪä¼ļ":57089,"Ġuitge":57090,"Ġشب":57091,"Ġpetit":57092,"ATIVE":57093,"çłĶç©¶èĢħ":57094,"çݰå®ŀçļĦ":57095,"ëĵ¤ìĿĦ":57096,"Kadaghanon":57097,"GY":57098,"ĠpÅĤ":57099,"帳":57100,"åľŁè̳åħ¶":57101,"Running":57102,"usta":57103,"Ġdistracted":57104,"awk":57105,"Ġdirective":57106,"å°Ķå¾·":57107,"ulant":57108,"æ±ĤèģĮ":57109,"æĺ¯ä¸Ģ项":57110,"潤":57111,"æĿĢæīĭ":57112,"Roskov":57113,"ĠSaysay":57114,"iminary":57115,"çIJĨ论åĴĮ":57116,"Ġincubated":57117,"æĥ³åĬŀæ³ķ":57118,"ä¸Ģç¬Ķ":57119,"à¹Ħวà¹ī":57120,"-Cl":57121,"夾":57122,"ĠYi":57123,"ienst":57124,"åĵºä¹³":57125,"ĠRussians":57126,"ĠFinnish":57127,"ĠEDU":57128,"Ġstacked":57129,"ä¹Łå¤ª":57130,"ä½łåºĶ该":57131,"椰":57132,"UTE":57133,"è°·æŃĮ":57134,"æľīæĦıæĢĿ":57135,"ä¸ĭ令":57136,"Ġmiserable":57137,"æŃ£æĸ¹å½¢":57138,"诫":57139,"Ġalum":57140,"ĠErik":57141,"ä¸İåIJ¦":57142,"Certain":57143,"Ġnúmeros":57144,"ĠTyr":57145,"Ġ((-":57146,"лоÑĢ":57147,"éĤ£å¼ł":57148,"Ġblamed":57149,"Ġlia":57150,"ä»ĸä»İ":57151,"Ġbuzz":57152,"obalt":57153,"Indones":57154,"建çŃijéĿ¢ç§¯":57155,"Ġenqu":57156,"ëĤľ":57157,"Ġshoots":57158,"Ġoffenders":57159,"å°Ĩèĩªå·±":57160,"ÙĬدة":57161,"ĠEthical":57162,"åĴ±åĢij":57163,"èijĹä½ľæĿĥ":57164,"Ġpreg":57165,"Ġsouthwest":57166,"Ġmonsters":57167,"åıįåIJij":57168,"带ç»Ļ":57169,"osaur":57170,"ĠPdf":57171,"cko":57172,"æīįè¡Į":57173,"Ġabbrevi":57174,"-conscious":57175,"=true":57176,"{z":57177,"ĠJet":57178,"ĠзавиÑģимоÑģÑĤи":57179,"ĠSchn":57180,"Ġonze":57181,"ĠMarÃŃa":57182,"ĠÐľÐ°ÑĢ":57183,"ĠLions":57184,"éħª":57185,"etters":57186,"ĠDia":57187,"å®ŀè®Ń":57188,"ĠUns":57189,"лÑĥÑĩ":57190,"Ġella":57191,"cheduled":57192,"Ġincubation":57193,"Ġsincere":57194,"ĠTRA":57195,"æĸ¹è¨Ģ":57196,"ÑĨин":57197,"Composite":57198,"Ñįн":57199,"ä¸įèĪį":57200,"Ġ×ŀ×Ĺ":57201,".Int":57202,"tf":57203,"äºĨè¿Ļ个":57204,"Ġguerre":57205,"游åĩ»":57206,"à§ĭধ":57207,"å¾Īå¥½åľ°":57208,"ĠÙĦÙĬ":57209,"ĠAngela":57210,"ĠPair":57211,"ĠdaÃŁ":57212,"èĨľçĤİ":57213,"Mask":57214,"mart":57215,"ĠBread":57216,"Ġshark":57217,"å°±åºĶ该":57218,"-ox":57219,"è·ijåİ»":57220,"Ġê°Ģìŀ¥":57221,"Ñĩем":57222,"ÑİÑīаÑı":57223,"ĠпоÑģÑĤÑĢо":57224,"Ġpomoc":57225,"Ġнаиболее":57226,"Ġreminis":57227,"à¸Ńีà¸ģ":57228,"ĠLauren":57229,"Islam":57230,"Ģë¡ľ":57231,"Ġmare":57232,"ĠвеÑģ":57233,"ÑģÑĤавиÑĤÑĮ":57234,"ĠиÑģкÑĥÑģ":57235,"Ġinad":57236,"Ġblending":57237,"Ġallen":57238,"æľĥæľī":57239,"è¯ī讼æ³ķ":57240,"Ġrappresent":57241,"çļĦè¡£æľį":57242,"åİ»æİī":57243,"Ġfosters":57244,"Bibli":57245,"å¿ħé¡»æĺ¯":57246,"調æķ´":57247,"(board":57248,"-index":57249,"Ġ*_":57250,".num":57251,"迹象":57252,"ného":57253,"Money":57254,"ĠSvens":57255,"ĠPoz":57256,"Phase":57257,"âĨIJ":57258,"Ġcreamy":57259,"called":57260,"anser":57261,"æĬĬè¿Ļ":57262,"]ï¼Į":57263,"ĠSally":57264,"�ĊĊ":57265,"Implement":57266,"Ġ(âĢĺ":57267,"æĦķ":57268,"Ġ''Ċ":57269,"Ġë¨":57270,"å¢ŀæĶ¶":57271,"纵横":57272,"å°ıé¾Ļ":57273,"ãģĵãģĵ":57274,"Notice":57275,"åħ»èĢģéĩij":57276,"ĠÑĢаÑģполож":57277,"\\prime":57278,"}),":57279,"[mid":57280,"Ġcue":57281,"温æ³ī":57282,"HSL":57283,"ĠJakarta":57284,"наÑĢÑĥ":57285,"Ġ×ijס":57286,"Ġporn":57287,"çµIJæ§ĭ":57288,"åѽ":57289,"天éģĵ":57290,"agra":57291,"Ġspans":57292,"settings":57293,"aval":57294,"ĠYemen":57295,"æķĻçīĪ":57296,"ĠModified":57297,"اØŃØ©":57298,"Ġसà¥ĩ":57299,"fik":57300,"дко":57301,"å¾Īæĺİæĺ¾":57302,"Ġerad":57303,"Ġdisappointing":57304,"ĠDoing":57305,"Ġevidenced":57306,"ĉv":57307,"ä¸įåIJ«":57308,"ĠNel":57309,"æĬ¥éĶĢ":57310,"æ¾İ":57311,"Ġmum":57312,"女主":57313,"两人çļĦ":57314,"Ġtempted":57315,"iator":57316,"asjon":57317,"æľīæľŁå¾ĴåĪij":57318,"Ġcompose":57319,"Ġcref":57320,"Ġcalculates":57321,"balanced":57322,"ĠOdd":57323,"è§£åīĸ":57324,"Ġstatutes":57325,"æŁIJ人":57326,"çĽĹçªĥ":57327,"ï¼Ĥ":57328,"Ġthereto":57329,"ÑģиÑı":57330,"within":57331,"¤×§":57332,"ĠGCD":57333,"被è¯Ħ为":57334,"teness":57335,"微信åħ¬ä¼Ĺåı·":57336,"Ġ>ĊĊ":57337,"Ġgrandparents":57338,"ĠTNF":57339,"ĠBudapest":57340,"Ġcantidad":57341,"Experience":57342,"ĠBrent":57343,"#pragma":57344,"ÅŃ":57345,"女æĢ§çļĦ":57346,"å¾Īå¤ļæĹ¶åĢĻ":57347,"æłĩå¿ĹçĿĢ":57348,"iez":57349,"metros":57350,"Ġaproxim":57351,"nest":57352,"íĿ":57353,"Ġ^{":57354,"Ġexile":57355,"èĥ¤":57356,"Ġseminar":57357,"å¸ĸåŃIJ":57358,"Ġeukary":57359,"ĠTill":57360,"reatening":57361,"åĨħæľī":57362,"ĠHed":57363,"-native":57364,"访è°Ī":57365,"ĠBiod":57366,"ĠGur":57367,"á½´":57368,"ç»ıæµİ社ä¼ļåıijå±ķ":57369,"éķ¿çĶŁ":57370,"èĦļæľ¬":57371,"Ġlemma":57372,"ĠExposure":57373,"ĠParameter":57374,"é¢Ŀ度":57375,"ĠAPPL":57376,"Ġundermine":57377,"çļĦåħ¨éĥ¨":57378,"ĠJesse":57379,"滾":57380,"ĠGlad":57381,"饮水":57382,"ĠìĭĿ":57383,"ĠPrinting":57384,"ĠGian":57385,"ä¼łæİĪ":57386,"æ¡ĤæŀĹ":57387,"answers":57388,".q":57389,"å¸ĪèµĦ":57390,"åĻ©":57391,"ĠBUT":57392,"年份":57393,"achten":57394,"Ġencode":57395,"Ġpsychologist":57396,"Ġmateria":57397,"⾨":57398,"Ġbrowsing":57399,"Ġabbreviation":57400,"åľ¨å±±":57401,"arman":57402,"ÄĽnÃŃ":57403,"Gender":57404,"}import":57405,"ĠгÑĢаж":57406,"ÙĪÙĦØ©":57407,"-val":57408,"Wow":57409,"িহ":57410,"åı¯èĥ½æľī":57411,"á¿Ĩ":57412,"andes":57413,"æŃ»åİ»":57414,"Ġmorale":57415,"inz":57416,"orkshire":57417,"_;":57418,"ĠFerd":57419,"æĹ©æĻļ":57420,"Ġcatches":57421,"Matt":57422,"enary":57423,"Ġdazu":57424,"ĠÑģÑĤол":57425,"Ġvocational":57426,"æĵģæľī":57427,"Ġbikes":57428,"ĠпÑĢименÑı":57429,"缸è¿ij":57430,"禧":57431,"ç«Ļç«ĭ":57432,"Ġನ":57433,"Ġdisciplinary":57434,"Ġdeprived":57435,"ä¸ĭåįĬ":57436,"è´¢æĶ¿éĥ¨":57437,"æŃ§è§Ĩ":57438,"æĪIJåĥı":57439,".am":57440,"åĬ¨èĥ½":57441,"ä½ĨåıĪ":57442,"骨é«ĵ":57443,"æŁĵèī²ä½ĵ":57444,"hp":57445,"Ġdd":57446,"æī§çħ§":57447,",,,,":57448,"ĠThesis":57449,"Ġviolate":57450,"ĠGraphics":57451,"categories":57452,"æĿ¥åģļ":57453,"ĠReach":57454,"ahi":57455,"伤亡":57456,"енной":57457,"ectetur":57458,"大便":57459,"Ġoverc":57460,"åIJ¸åıĸ":57461,"ĠInteresting":57462,"มืà¸Ń":57463,"Ġqt":57464,"å½ĵçĦ¶æĺ¯":57465,"åĪ·æĸ°":57466,"Ġclan":57467,"çİ©èĢį":57468,"ĠCounting":57469,"ĠìĨį":57470,"-rel":57471,"XL":57472,"ÄĮ":57473,"Ġequilib":57474,"ç»ıèIJ¥ç®¡çIJĨ":57475,"Ġconduit":57476,"otid":57477,"æ¸Ń":57478,",w":57479,"æį®æŃ¤":57480,"вал":57481,"åĨĻæĪIJ":57482,"Ġì²ĺ":57483,"Ġtrata":57484,"ĠΡ":57485,"æĪĸå¤ļ":57486,"Ġredundant":57487,"ĠGn":57488,"uciones":57489,"ĠLithuan":57490,"-red":57491,"åıijéŁ³":57492,"好ä¹ħ":57493,"æĬ¥åĪĬ":57494,"ĠAvailability":57495,"Ġpains":57496,"ryl":57497,"Ġtrajectories":57498,"_fe":57499,"petto":57500,"ä¸į稳å®ļ":57501,"ĠاÙĦÙħجرÙĩ":57502,"Ġcaratter":57503,"ĠGods":57504,"often":57505,"Ġbord":57506,"Ġbait":57507,"deck":57508,"äter":57509,"Ġdepressive":57510,"Ġphysi":57511,"大çģ«":57512,"åIJ¬åĬĽ":57513,"-wave":57514,"Ġzast":57515,"ĠاÙģØ±Ø§Ø¯":57516,"-base":57517,"ĠContinental":57518,"asal":57519,"æĺİç»Ĩ":57520,"å·²ç»ıå¼Ģå§ĭ":57521,"ðŁ¥":57522,"agt":57523,"phants":57524,"Ġpasswords":57525,"Ġartillery":57526,"ĠEld":57527,"çľĭæĪIJ":57528,"determ":57529,"-ab":57530,"enberg":57531,"ĠChes":57532,"Ġenchant":57533,"Ġcloudy":57534,"ĠArth":57535,"æµ·åºķ":57536,"满æĦı度":57537,"ä¸ĩå¹³æĸ¹ç±³":57538,"ĠTI":57539,"Ġheadlines":57540,"External":57541,"ĠزÙĨدگÛĮ":57542,"æĺİç¡®äºĨ":57543,"-icon":57544,"ĠBorg":57545,"Ġinterplay":57546,"merged":57547,"Ġmediation":57548,"éĽ£éģĵ":57549,"ĠMcGraw":57550,"ĠGmb":57551,"éĴ°":57552,"ĠDb":57553,"Ġunderp":57554,"Ġekonom":57555,"Ġ×ķ×Ķ":57556,"ĠCertified":57557,"ondere":57558,"ãģĬãĤĬ":57559,"_vector":57560,"ataset":57561,"Ġjard":57562,"Ġprintables":57563,"ĠDylan":57564,"ä¸įéĮ¯":57565,"æŃ»åĪij":57566,"两类":57567,"ĠNewsp":57568,"ĠMuk":57569,"Ġwhale":57570,"ouring":57571,"åħīçħ§":57572,"å®Įåħ¨æĺ¯":57573,"Temp":57574,"Ġmich":57575,"ĠCasc":57576,"issant":57577,"éŀł":57578,"leur":57579,"为åĩĨ":57580,"ä¸ĬåįĬ":57581,"è´¨éĩıåĴĮ":57582,"nika":57583,"Ġconduc":57584,"人çī©çļĦ":57585,"ĠChip":57586,"()){Ċ":57587,"åĮºåŁŁçļĦ":57588,"Ġlanes":57589,"好åIJĹ":57590,"Ġdisparities":57591,"æī³":57592,"ÅĦski":57593,"ĠPanama":57594,"Ġsow":57595,"äºĮæľĪ":57596,".init":57597,"ç§ijåѦçłĶç©¶":57598,"æ°´æºIJ":57599,"Ġreadable":57600,"çļĦä¸Ģ个éĩįè¦ģ":57601,"èĢĮä¸Ķåľ¨":57602,"临çķĮ":57603,"ĠCarp":57604,"追éĢIJ":57605,"Ġwelche":57606,"irma":57607,"åºĶèģĺ":57608,"Ġjudging":57609,"ĠاÙĦسÙħاÙĪÙĬÙĩ":57610,"ĠFry":57611,"Ġplumbing":57612,"°)":57613,"Ġscreened":57614,"Ġpouring":57615,"community":57616,"Earth":57617,"mc":57618,"Ġполез":57619,"æĪ¿éĸĵ":57620,"Ġмик":57621,"Sex":57622,"å¿ĥä¸Ĭ":57623,"})^{":57624,"ĠRBI":57625,"ĠÑģоÑĢ":57626,"Ġfurthermore":57627,"Ġlargo":57628,"Solutions":57629,"å°ı声":57630,"ĠArmed":57631,"ütz":57632,"Ġutmost":57633,"ĠScandin":57634,"èĢģçĪ·åŃIJ":57635,"Ġmitt":57636,"çļĦéĤ£æł·":57637,"/Square":57638,"ĠHypot":57639,"Vec":57640,"ä¸īæĿ¡":57641,"Ġasserts":57642,"ä¸įåı¯æĢĿè®®":57643,"+k":57644,"ĉres":57645,"ĠTale":57646,"ajax":57647,"antsay":57648,"ĠSoccer":57649,"å±į":57650,"æĬĬè¿ĻäºĽ":57651,"Claim":57652,"ĠпÑĢÑıм":57653,"åĤ³çµ±":57654,"ĠíĻĺ":57655,"çļĦæł¹æľ¬":57656,"Ġlone":57657,"Ġconstante":57658,"íĮIJ":57659,"_one":57660,"ترة":57661,"ĠAutism":57662,"å¦Ĥä»ĬçļĦ":57663,"Ġcafé":57664,"онÑĤа":57665,"è§Ħå®ļäºĨ":57666,"ĠTum":57667,"ÃŃveis":57668,"å¿«çĤ¹":57669,"æ¯ĶçļĦ":57670,"ĠдопÑĥ":57671,"Ġsecondo":57672,"ographically":57673,"itiv":57674,"osci":57675,"ĠKad":57676,"ĠKarn":57677,"ÑĩаÑģ":57678,"Ġbreasts":57679,"Ġìĸij":57680,"ĠEmerging":57681,"åķ§":57682,"ĠÑħаÑĢакÑĤеÑĢ":57683,")].":57684,".rand":57685,"Ġregain":57686,"ĠHelsinki":57687,"Ġελληνικά":57688,"Ele":57689,"äºĨåķĬ":57690,"esses":57691,"æĻĤæľŁ":57692,"Ġש×ij":57693,"ĠAMD":57694,"è³ĩè¨Ĭ":57695,"è§£åĨ³éĹ®é¢ĺçļĦ":57696,"Ġemergencies":57697,"ä¹Łæĺ¯ä¸Ģç§į":57698,"weak":57699,"ĠJanet":57700,"çļĦè¯Ĺ":57701,"ÑīÑĥ":57702,"èħ¹çĹĽ":57703,"ĠÙħÛĮÚ©ÙĨد":57704,"edical":57705,"story":57706,"ĠLever":57707,"Ġcigarettes":57708,"ĠDynasty":57709,"ĠTinipong":57710,"xiv":57711,"opin":57712,"Ġovercoming":57713,"èIJ¥æĶ¶":57714,"ĠHugo":57715,"Ġlut":57716,"çѱ":57717,"æ±Łå±±":57718,"å®ĺåĥļ":57719,"Ġпонима":57720,"ĠAging":57721,"åİĤæĪ¿":57722,"िल":57723,"çļĨæĺ¯":57724,"Ġperfor":57725,"æĸ°äº§åĵģ":57726,"Ġìĭ¬":57727,"Ġunacceptable":57728,"Ġimplicated":57729,"åĽ½éĻħåĮĸ":57730,"CCC":57731,"ĠTorah":57732,"zus":57733,"Ġtac":57734,"æīĢåij¨":57735,"rugu":57736,"奢ä¾Ī":57737,"Ġ(**":57738,"istory":57739,"ĠÑģÑĬ":57740,"ĠExamin":57741,"(long":57742,"ĠSAR":57743,"ifa":57744,"æ¦ĤåĨµ":57745,"Ġdistinguishing":57746,"ĠWSW":57747,"åŦ":57748,"Ġglobalization":57749,"Ġdrafted":57750,"æ¶Įçݰ":57751,"Too":57752,"-market":57753,"Ġsakop":57754,"akar":57755,"æľĽåİ»":57756,"ÑģÑĤана":57757,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ":57758,"æīĢ以æĪij们":57759,"avers":57760,"çµ¦ä½ł":57761,"Ġvault":57762,"ĠĠĠĠĠĠĠĊ":57763,"requent":57764,"helium":57765,"hering":57766,"riet":57767,"Ġhone":57768,"Ġskirt":57769,"ĠChristine":57770,")âĢľ":57771,"åĬ¨çī©çļĦ":57772,"åķ¼":57773,"æĺ¯ä¸Ģå®¶":57774,"课å¤ĸ":57775,"Ġforensic":57776,"gd":57777,"Õº":57778,"Ġstur":57779,"åİĨç»ı":57780,"Ġquestionnaires":57781,"ROW":57782,"ĠجÙħع":57783,"Ġmelalui":57784,"iculously":57785,"ķàµįà´ķ":57786,"æĪĸåľ¨":57787,"à¹īำ":57788,"Viewfinder":57789,"bies":57790,"yx":57791,"ĠNOW":57792,"伯çī¹":57793,"Ġvaccinated":57794,"Charl":57795,"Ġstandpoint":57796,"ĠENE":57797,"hew":57798,"ĠSignificance":57799,"ĠSociology":57800,"GW":57801,"ĠAlle":57802,"-but":57803,"广ä¹ī":57804,"COMP":57805,"æ°´èµĦæºIJ":57806,"Ġbeginnings":57807,"ĠCancel":57808,"Ġrut":57809,"Ġpresidency":57810,"Ġmemberikan":57811,"ĠDistrib":57812,"ĠSJ":57813,"-button":57814,"çļĦçIJĨçͱ":57815,"================================================================":57816,"ĠÙĪØ§ØŃد":57817,"ĠperÃŃodo":57818,"Ġmerchandise":57819,"HM":57820,"ä¸Ģè¯į":57821,"idean":57822,"ä¸ŃåĽ½å®¶":57823,"atoire":57824,"代表çĿĢ":57825,"ĠNano":57826,"Ġcurb":57827,"å®Įåħ¨åı¯ä»¥":57828,"æģ¶éŃĶ":57829,"Scene":57830,"ĠIls":57831,"ĠGazette":57832,"Ġহল":57833,"ĠLancet":57834,"agit":57835,"ä¹ĭä½Ļ":57836,"æīĭéĩĮçļĦ":57837,"Ġprepares":57838,"Ġchlorine":57839,":**ĊĊ":57840,"iali":57841,"িà¦Ĥ":57842,"IFIC":57843,"Ġverk":57844,"ĠAbuse":57845,"Ġfundraising":57846,"-terminal":57847,"ĠHamburg":57848,"ĠMell":57849,"累积":57850,"Ġattributable":57851,"Ġreciproc":57852,"Ġlipids":57853,"íĥĿ":57854,"é¡ĺæĦı":57855,"ĠTul":57856,"ä¸ļåĨħ":57857,"éĥ¨éĸĢ":57858,"ĠÑĩеÑĤÑĭÑĢе":57859,"Ġretinal":57860,"ضÙĬ":57861,"ĠDisability":57862,"Ġhurricane":57863,"fø":57864,"gren":57865,"åĽ¢éķ¿":57866,"设计åĴĮ":57867,"ĠCere":57868,"ÑĩаÑĤ":57869,"çļĦæĿIJæĸĻ":57870,"éªijåħµ":57871,"ä¸ĢåŃĹ":57872,"**[":57873,"Assign":57874,"åħ¬è¯ģ":57875,"-growing":57876,"{long":57877,"西åħ°":57878,"ĠCanon":57879,"ĠAngle":57880,"ĠCourts":57881,"inished":57882,"Ġtransistors":57883,"è¥Ł":57884,"ĠAPIs":57885,"Ġcubes":57886,"Ġhingga":57887,"庵":57888,"สัà¸ĩ":57889,"Ġ×IJ׾×":57890,"annica":57891,"ãģĵãģĨ":57892,"ائة":57893,"çļĦè§Ĵèī²":57894,"ëĭĺ":57895,"åĨĻåħ¥":57896,"ĠконÑģÑĤÑĢÑĥк":57897,"Ġmise":57898,"ĠKR":57899,"ĠÙħÙĩ":57900,"æŃ¥åħ¥":57901,"ĠÑĤакие":57902,"новним":57903,"phrag":57904,"ĠTanzania":57905,"Wind":57906,"Ġcsv":57907,"auv":57908,"Ġinfest":57909,"æľªæĽ¾":57910,"Ġвида":57911,"ĠAUT":57912,"常年":57913,"ĠMystery":57914,"Ġstad":57915,"occur":57916,"ĠOutside":57917,"Ġvaginal":57918,"สุà¸Ĥ":57919,"osocial":57920,"Ġken":57921,"Switch":57922,"idisciplinary":57923,"robe":57924,"Ġani":57925,"èĬį":57926,"пли":57927,"otoxic":57928,"æ¯į親":57929,"леннÑĭÑħ":57930,"Ġbroadband":57931,"èĤ¾èĦı":57932,"/licenses":57933,"Ġpermitting":57934,"Years":57935,"isans":57936,"åķĨåĵģçļĦ":57937,"ĠBOOK":57938,"quist":57939,"è°ĥåij³":57940,"Ġmeanwhile":57941,"æķijæµİ":57942,"Ġedible":57943,"ĠÑģÑĤÑĢои":57944,"ĠDear":57945,"工件":57946,"主ä½ĵçļĦ":57947,"Hum":57948,"èĵĿ天":57949,"Ġribbon":57950,"Ġprendre":57951,"Ġinhal":57952,"çļĦä¼łç»Ł":57953,"ä¸Ģ两":57954,"ĠUSE":57955,"Ġdeduction":57956,"Ġsponsors":57957,"Ġfueron":57958,"beth":57959,"ĠHeter":57960,"ĠClar":57961,"ĠVolks":57962,"tc":57963,"کتر":57964,"àµįà´¤":57965,"æ¼²":57966,"rida":57967,"ĠWHAT":57968,"æµģè¡ĮçļĦ":57969,"åĽĽçϾ":57970,"å¸ĮæľĽå¤§å®¶":57971,"ĠпÑĢог":57972,"Ġsadly":57973,"Ġkomen":57974,"Ġdeline":57975,"說äºĨ":57976,"ĠÑģим":57977,"ĠBases":57978,"оÑĢдина":57979,"TreeNode":57980,"æīįèĥ½å¤Ł":57981,"Ġsosten":57982,"Ġassemble":57983,"ĠPatricia":57984,"Transport":57985,"Ġperil":57986,"åĦ²":57987,"Ġslab":57988,"追溯":57989,"Practice":57990,"_right":57991,"Ġtendencies":57992,"Ġmx":57993,"engar":57994,"Ġsimmer":57995,"ĠcittÃł":57996,"ĠÑĢезÑĥлÑĮÑĤаÑĤе":57997,"ĠZw":57998,"Ġimpressions":57999,"éϤ以":58000,"Ġdiri":58001,"ĠSultan":58002,"ĠAudit":58003,"ିà¬":58004,"Ġfrowned":58005,"Ġinning":58006,"Ġbilling":58007,"Director":58008,"nai":58009,"allic":58010,"ydia":58011,"Ġíͼ":58012,"Å¡en":58013,"Ġkilometer":58014,"客æĪ·çļĦ":58015,"ç͵åĬ¨è½¦":58016,"秤":58017,"Bag":58018,"Ġkult":58019,"ĠÃİ":58020,"éĹ®é¢ĺä¸Ĭ":58021,"Ġfino":58022,"Ġfrog":58023,"ĠбоÑĢ":58024,"åįİ举":58025,"è·Łåīį":58026,"üller":58027,"ÑĭÑĤ":58028,"è¡ĢèĦī":58029,".tie":58030,"åħijæį¢":58031,"Ġpoblación":58032,"ĠAstroph":58033,"éĻķ西çľģ":58034,"ĠÐĶе":58035,".module":58036,"ä¸Ģå¹´çļĦ":58037,"Ġhelmet":58038,"Ġprotections":58039,"åħ¨çIJĥåĮĸ":58040,"Ġfug":58041,"ĠRams":58042,"å¼ºåĽ½":58043,"ĠMcCarthy":58044,"Ġoport":58045,"autres":58046,"ĠDieu":58047,"åĩ¸æĺ¾":58048,"_range":58049,"ĠProposition":58050,"çİĭå¦ĥ":58051,"æ²Ĵæĥ³åΰ":58052,"éĺµéĺµ":58053,"_u":58054,"ĠباسÙħ":58055,"Ġfunkc":58056,"ĠαÏħ":58057,"èĦ±èIJ½":58058,"+-+-":58059,"].[":58060,"ÃŃgen":58061,"(Request":58062,"ĠÙĦاعب":58063,"Ġmelanoma":58064,"Prefix":58065,"Sin":58066,"ĠHorm":58067,"é«ĺå±±":58068,"owaÅĤ":58069,"为éļ¾":58070,"Ġperiodo":58071,"èīºæľ¯çļĦ":58072,"Ġliquor":58073,"åľ°åİ»":58074,"ĠEdgar":58075,"ĠÐŀни":58076,".widget":58077,"Ġculp":58078,"è§Ħ竳åĪ¶åº¦":58079,"fair":58080,"Ġplur":58081,"clair":58082,"ĠWeiss":58083,"éģĶåΰ":58084,"å¬ī":58085,"Ġsnapshot":58086,"rish":58087,"Ġinactive":58088,"å½ĵéĢī":58089,"çļĦä¸Ģä½į":58090,"ĠìłIJ":58091,"à®°à¯ģà®":58092,"Ġping":58093,"ciu":58094,"ваÑĢи":58095,"Ġingl":58096,"åİŁæĿ¥æĺ¯":58097,"Ù쨶ÙĦ":58098,"åĪĩæĸŃ":58099,"ĠMarcel":58100,"å®īåħ¨æĦŁ":58101,"çIJĨ论çļĦ":58102,"Ġbev":58103,"ĠElli":58104,"Ġshiny":58105,".Serial":58106,"=int":58107,"stav":58108,"åł¡éķ¿":58109,"Ġunsuccessful":58110,"Ġлибо":58111,"Ġiterative":58112,"ĠLevi":58113,"Exists":58114,"Ġerm":58115,"çīĩåĮº":58116,"Ash":58117,"¡×§":58118,"Ġlalawigan":58119,"åıĪ说":58120,"ionales":58121,"uchy":58122,"éĴ±çļĦ":58123,"ĠExperts":58124,"ulière":58125,"Ġademás":58126,"Ġshareholder":58127,"_csv":58128,"ĠImproving":58129,"Ġtheoretically":58130,"ĠFight":58131,"æĭ¿äºĨ":58132,"তà§įয":58133,"Ġadmiration":58134,"blood":58135,"Ġpouvoir":58136,"\\item":58137,"Ġveil":58138,"Ġ».":58139,"ĠëĺIJëĬĶ":58140,"VB":58141,"ppings":58142,"Ġestat":58143,"ä»ĸ们ä¼ļ":58144,"ĠRelief":58145,"Ä±ÅŁ":58146,"âħ¡":58147,"ĠDrama":58148,"ĠKatie":58149,"Ġdemons":58150,"ÙĪÙĨÛĮ":58151,"IFE":58152,"owÄħ":58153,"â̦â̦âĢĿĊ":58154,"Ġfacile":58155,"Ġদà§ĩশ":58156,"Ġmoreover":58157,"ochromatic":58158,"bÄĽ":58159,"Ġschn":58160,"为ä»ĸ们":58161,"çݯæ¯Ķ":58162,"ศาสà¸ķรà¹Į":58163,"污æŁĵçī©":58164,"ĠÑĦинан":58165,"Sales":58166,"ĠChance":58167,"-met":58168,"Ġbaptism":58169,"ĠÑĥвелиÑĩи":58170,"uhi":58171,"YD":58172,"éĿŀè¦ģ":58173,"Ġvisions":58174,"ĠSaul":58175,"Ġpioneer":58176,"EDIT":58177,"Ġtheirs":58178,"çķĻå®Ī":58179,"åŃIJãģ©ãĤĤ":58180,"ä¸Ĭæ¼Ķ":58181,"Ġ{@":58182,"ãģĻãĤĭãģĵãģ¨ãģĮ":58183,"à§Ĥর":58184,"Ġloneliness":58185,"ĠSuggest":58186,"æľīåħ¶":58187,"大纲":58188,"вез":58189,"å¾Ĵå¼Ł":58190,"带åΰ":58191,"ãģ£ãģ¨":58192,"aliw":58193,"ĠczÅĤ":58194,"putation":58195,".description":58196,"à³Ĥ":58197,"Ġreversible":58198,"ä¸įèĩª":58199,"ĠNL":58200,"ä»ĸåį´":58201,"工夫":58202,"?\".":58203,"çľĭä½ł":58204,"éŁ³æ¨Ĥ":58205,"ĠJamaica":58206,"ĠPunjab":58207,"ÑĩеÑģкÑĥÑİ":58208,"Lim":58209,"tee":58210,"é£İäºij":58211,"Ġbonded":58212,"iston":58213,"ĠErnest":58214,"ç´¹ä»ĭ":58215,"ä¸ĵèģĮ":58216,"ujuan":58217,"åIJĪä½ľä¼Ļä¼´":58218,"Ġshocking":58219,"Ġempowered":58220,"çļĦçħ§çīĩ":58221,"Ġperox":58222,"ĠVere":58223,"åIJĪåͱ":58224,"ĠоÑĢганизаÑĨии":58225,"Ġfé":58226,"导èĩ´äºĨ":58227,"ĠMohammed":58228,"Ġici":58229,"owany":58230,"ä¸Ģ举":58231,"ä¸Ń对":58232,"ĠJE":58233,"å¼ĢéĺĶ":58234,"Ġgrading":58235,"razione":58236,"åĮĸçĸĹ":58237,"æĶ¿åħļ":58238,"Ġstuffed":58239,"æªĢ":58240,"ĠFreeman":58241,"Ġjamais":58242,"Ġmiejsc":58243,"ĠHear":58244,"-att":58245,"Ray":58246,"ĠOA":58247,"Ġан":58248,"åľ¨ä¸ĢæĹģ":58249,"å¨ģå»ī":58250,"ãĥĩãĤ£":58251,"ĠFelix":58252,"æīĢ以ä»ĸ":58253,"èĺij":58254,"orea":58255,"æīĢéķ¿":58256,"ĠмеÑģÑĤа":58257,"æ±ķ":58258,"çıŃéķ¿":58259,"ĠActual":58260,"enburg":58261,"è·³èĪŀ":58262,".request":58263,"another":58264,"é«ĺæĸ°æĬĢæľ¯":58265,"Ġacetyl":58266,"åİĨ代":58267,"ĠMask":58268,"ĠZu":58269,"Proxy":58270,"à¹Ĥà¸Ļ":58271,"Ġwidget":58272,"Ġanne":58273,"åζåīĤ":58274,"Ġdelic":58275,"ÑģкÑĥ":58276,"textbf":58277,"ä½Ĩè¿ĺæĺ¯":58278,"ç쵿°Ķ":58279,"Ġplatelet":58280,"æ·ĭæ¼ĵ":58281,"éĿ¢ä¸´çļĦ":58282,"Ġwrapper":58283,".Ass":58284,"øy":58285,"Academic":58286,"æłijåı¶":58287,"zte":58288,"ĠWS":58289,"å°İèĩ´":58290,"VENTION":58291,"æ¯ıä¸Ģ个人":58292,"Ġplanetary":58293,"punkt":58294,"ĠEMP":58295,"便åı¯":58296,"游è§Ī":58297,"åħ°å·ŀ":58298,"Ġtipos":58299,"ĠÑģÑĤоиÑĤ":58300,"ĠменÑĮÑĪе":58301,"inch":58302,"ç»Ħç»ĩåĴĮ":58303,"ĠÐŁÐ¾ÑįÑĤомÑĥ":58304,"Ġmöglich":58305,"Ġseulement":58306,"'int":58307,"ä¸įæĽ¾":58308,"æľĢ强":58309,"ĠâĪŀ":58310,"uncture":58311,"ÃŃtica":58312,"ĠParas":58313,"èϽçĦ¶æĺ¯":58314,"ä¿ĿæĮģçĿĢ":58315,"ĠSloven":58316,"UCT":58317,"anel":58318,"angement":58319,"åħ¨éĽĨ":58320,"slide":58321,"ĠاÙĦشر":58322,"ĠAthletic":58323,"ĠêµIJìľ¡":58324,"blocks":58325,"ĠáĥIJáĥ":58326,"ĠDok":58327,"Ġnowadays":58328,"ιÏĥÏĦ":58329,"éĢĤæĹ¶":58330,"ÑĢÑĥÑģ":58331,"åģľçķĻåľ¨":58332,"çĶ¢æ¥Ń":58333,"wach":58334,"æľĪèĩ³":58335,"Ġmasih":58336,"Ġorchestra":58337,"(first":58338,"éĿ©åij½çļĦ":58339,"_word":58340,"uctions":58341,"èĪªçıŃ":58342,"Ġcopying":58343,"Ġsiswa":58344,"tings":58345,"ä¸ĸ人":58346,"çĬ¶æħĭ":58347,"SEE":58348,"ĠmogÄħ":58349,"à´µ":58350,"Ġdismissal":58351,"ĠKitt":58352,"åĬ¨çĿĢ":58353,"åºĶ为":58354,"ÑĪÑĥ":58355,"ÙĤÙĬÙĤ":58356,"Ġoverhe":58357,"ĠUnique":58358,"engono":58359,"大æ¦Ĥæĺ¯":58360,"Ġsupplemented":58361,"Ġdecoration":58362,"æĺ¾å¾®":58363,"Ġwhistle":58364,"娩":58365,"Ġfrightened":58366,"Ġnig":58367,"ä¹į":58368,"Ġshipped":58369,"ä¸Ĭ天":58370,"ä¸ĸä¿Ĺ":58371,"硬çļĦ":58372,"Ġcontaminants":58373,"Ġdivergence":58374,"hamed":58375,"éĥ½ä¸įæķ¢":58376,"ạn":58377,"å¤ķéĺ³":58378,"née":58379,"ĠCNS":58380,"ÙĬÙĨا":58381,"Ñģпек":58382,"ĠJuliet":58383,"åĩ¦":58384,"ĠÕ¾":58385,"à¦Ńাবà§ĩ":58386,"Ġancestral":58387,"ìħĺ":58388,"coe":58389,"æŃ»èĢħ":58390,"cep":58391,"Ġparade":58392,"ä¸ĸçķĮæĿ¯":58393,"Ġcomfortably":58394,"bage":58395,"çļĦçľ¼":58396,"placed":58397,"æĽ²æĬĺ":58398,"ĠArkiverad":58399,"Sale":58400,"ayload":58401,"Ùĩاد":58402,"Ġheavenly":58403,"æIJĸéłŃ":58404,"ä¸įå±ij":58405,"leyball":58406,"Ġinvestigator":58407,"ĠRust":58408,"Ġmonopoly":58409,"Ġcss":58410,"æ¯ĭ":58411,"Ġensured":58412,"Ġtaxable":58413,"Ġratt":58414,"scher":58415,"-mon":58416,"-profile":58417,"ĠAnyway":58418,"ocytosis":58419,"è¿Ļåĩłå¤©":58420,"æ¯Ľå·¾":58421,"â̲-":58422,"Ġdroplets":58423,"Ġreprodu":58424,"Ġconfirming":58425,"带é¢Ĩä¸ĭ":58426,"æ©Łæ§ĭ":58427,"Sync":58428,"Ġsuis":58429,"ĠBruno":58430,"ãģĹãģ¾ãģĨ":58431,"åĹĵåŃIJ":58432,"æľĪä¸ŃæĹ¬":58433,"bbe":58434,"ĠHER":58435,"cester":58436,"ä½Ĩä»İ":58437,"å²IJ":58438,"){\\":58439,"éĥ½å¿ħé¡»":58440,"æ¸į":58441,"ĠPlastic":58442,"ĠâĪł":58443,"ĠRefuge":58444,"ĠImmigration":58445,"Ġstripped":58446,"ता":58447,"èĢĥè¯ģ":58448,"ĠدÙĩد":58449,"tymology":58450,"Ġhá»":58451,"Ġpalp":58452,"Ġpaz":58453,"ĠMam":58454,"å´Ń":58455,"'\\":58456,"ä¹ŁæĽ¾":58457,"å¦Ĥ表":58458,"Ġrefuses":58459,"ĠContinu":58460,"kb":58461,"åĴĮç»ıæµİ":58462,"è¿Ľåĩº":58463,"éĩįçĹĩ":58464,"Ġmigrate":58465,"ĠQUEST":58466,"ä½łéľĢè¦ģ":58467,"æĻ¦":58468,"ĠÑģÑĩеÑĤ":58469,"å¢ŀéĩı":58470,"ĠÙģØ±Ø¯":58471,"çŁŃè·¯":58472,"æ¸Ľå°ij":58473,"èĩªåĬ©":58474,"áll":58475,"æĮģèĤ¡":58476,"æļĤè¡Į":58477,"Ġempresas":58478,"Õ¹":58479,"æ¸ħæ¥ļåľ°":58480,")n":58481,"Cam":58482,"ãĢĤ)ĊĊ":58483,"ä¹ĭçī©":58484,"ĠCoul":58485,"Ġsegunda":58486,"Ġpuls":58487,"ĠÙħرÙĥز":58488,"åĴĮèĩªå·±":58489,"ových":58490,"æĵĴ":58491,"Ġskept":58492,"waukee":58493,"쪽":58494,"ĠXP":58495,"å·¡éĢ»":58496,"Ġformulations":58497,"สà¸Ńà¸ļ":58498,"ĠÑĤакое":58499,"Ġjourneys":58500,"Ġसà¤Ĥ":58501,"Ġshowcases":58502,"æľ¬åĽ½":58503,"ĠLogger":58504,"æĮ¤åİĭ":58505,"ĠتÙĪÙĦÛĮد":58506,"Ġmagyar":58507,"Ġencontrar":58508,"ãģ¨ãģĵãĤį":58509,"μm":58510,"çĹħ人çļĦ":58511,"è¿Ļä¸Ģ天":58512,"Ġmonkeys":58513,"×ķרת":58514,"ä¸ĢåĢĭ人":58515,"ointed":58516,"Ġgarner":58517,"Ġtutorials":58518,"çļĦåij¢":58519,"æĬĸéŁ³":58520,"Ġíĺ¸":58521,"Ġcomplication":58522,"Ġharbor":58523,"gior":58524,"ìĶ":58525,"ĠLok":58526,"ĠReserv":58527,"Ġpúblico":58528,"ãĤĴåıĹ":58529,"EGIN":58530,"Ġà¤ķà¥Ģ":58531,"çŃīåĨħ容":58532,"ĠIdea":58533,"orten":58534,"å¢ŀçĶŁ":58535,"boat":58536,"_pred":58537,"Ġì¹ĺ":58538,"kwargs":58539,"elsk":58540,"_CL":58541,"ªà¯įப":58542,"Ġmissiles":58543,"ĠHardy":58544,"ĠÙĬتÙĬÙħÙĩ":58545,"Ġgil":58546,"éĢłä»·":58547,"åĤij":58548,"Ġdoctr":58549,"Ġverschillende":58550,"ĠTIM":58551,"åıªæĢķ":58552,"åĪĹ举":58553,"AJ":58554,"Ġ׾×ĵ":58555,"ltre":58556,"ĠиÑģÑģледованиÑı":58557,"è¯ķè¡Į":58558,"Carl":58559,"Ġakin":58560,"Ġstabilization":58561,"Ġadsorb":58562,"ä¸ĬéĻIJ":58563,"ÑĢек":58564,"Weather":58565,"责任çļĦ":58566,"ĉscanf":58567,"å°±è¯Ĭ":58568,"ĠпÑĢепаÑĢа":58569,"Ġexponents":58570,"ĠкÑĢÑĥп":58571,"ĠتضÙĬÙģÙĦÙĩا":58572,"?)ĊĊ":58573,"ĠConstitutional":58574,",h":58575,"æľĢåħ·":58576,"临è¿ij":58577,"æ²īæĢĿ":58578,"Ġlien":58579,"irectional":58580,"icana":58581,"ï¼Įï¼Ī":58582,"Ġplague":58583,"à°¯":58584,"å¾ģéĽĨ":58585,"æĬĺåıł":58586,"ĠLicensed":58587,"Ġcrafting":58588,"_init":58589,"Ġnotamment":58590,"Ġàªķ":58591,",G":58592,"iben":58593,"款çļĦ":58594,"Ġunforgettable":58595,"nl":58596,"她è¿ĺ":58597,"Ġventric":58598,"fx":58599,"çıĢ":58600,"主ä¹īèĢħ":58601,"ĠÑģоÑģÑĤавлÑıеÑĤ":58602,"ç¼ĵåŃĺ":58603,"Õ¸Õ¶":58604,"Ġadaptability":58605,"Ġreservations":58606,",K":58607,"Ġeles":58608,"éĥ½æ²Ĵæľī":58609,"Ġaffid":58610,"ĠBuenos":58611,"anus":58612,"Ġect":58613,"Ġbizarre":58614,"ĠNil":58615,"serial":58616,"ÅĽl":58617,"è¯ļå®ŀ":58618,"Ġdescend":58619,"ĠScope":58620,"æµĵåİļ":58621,"Bud":58622,"_K":58623,"Ġfauna":58624,"éĢįéģ¥":58625,"éĿłçĿĢ":58626,"িà¦ķà§ĩ":58627,"ĠAMP":58628,"ĠPortfolio":58629,"olitics":58630,"Ġpus":58631,"çĿ¦":58632,"è¾ĥå·®":58633,"çĬ¯ç½ªçļĦ":58634,"饵":58635,"atsu":58636,"Ġдолжна":58637,"YZ":58638,"prite":58639,"à®¿à®Ł":58640,"ometown":58641,"éĻį临":58642,"ä½ľç͍ä¸ĭ":58643,"åŁºçĿ£æķĻ":58644,".int":58645,"ĠWake":58646,"ĠRecycl":58647,"æĮĤåľ¨":58648,"Ġcoag":58649,"-between":58650,"ÅĤoż":58651,"Ġriches":58652,"Ġbentuk":58653,"\\sigma":58654,"ĠVu":58655,"éķij":58656,"ropolis":58657,"æĺĨèĻ«":58658,"ĠHarmonic":58659,"ouc":58660,"åħ¶ä¸»è¦ģ":58661,"Ġmultivariate":58662,"Ġpunished":58663,"Ġmeses":58664,"Ġdesenvolvimento":58665,"Ġpk":58666,"ç»ĨåĮĸ":58667,"æ²¹èĦĤ":58668,"Ġpickup":58669,"Ġfibres":58670,"èª¿æŁ¥":58671,"ĠSCHOOL":58672,"umbar":58673,"Ġrebel":58674,"ĠNicole":58675,"Jeff":58676,"ά":58677,"ĠÙĦÙĩا":58678,"çͳè«ĭ":58679,"Ġpodstaw":58680,"secure":58681,"arz":58682,"Ġà¦ķম":58683,"çĺĭ":58684,"اÙ쨏":58685,"Ġinferred":58686,"Ġtactical":58687,"))/":58688,"å®ŀæĸ½ä¾ĭ":58689,"Texture":58690,"Ġripe":58691,"ĠоÑģнове":58692,"Wang":58693,"central":58694,"Ġextinct":58695,"ĠSpin":58696,"ĠAssuming":58697,"Ġicons":58698,"åħīè°±":58699,"éĤĦæ²Ĵ":58700,"ĠSuccessful":58701,"ĠHomer":58702,"ĠFur":58703,"æĺŁçIJĥ":58704,"è¶ģçĿĢ":58705,"Ġexpressive":58706,"ĠGardner":58707,"Ġstrom":58708,"好åķĬ":58709,"åħīæĺ¯":58710,"æĸ°çĶŁåĦ¿":58711,"è¿ĺåºĶ":58712,"-lo":58713,"-sex":58714,"Ġrotated":58715,"Ġaffirmed":58716,"δε":58717,"è¿Ļåĩłä¸ª":58718,"çīŁ":58719,".Aut":58720,"ç«ĭæĸ¹ç±³":58721,"Ġwährend":58722,"ä½ĨæĪij们":58723,"西æ¹ĸ":58724,"wel":58725,"Ġgarn":58726,"à¸Ħรà¸ĩ":58727,"'ai":58728,"Ġmetropolitan":58729,"åĨµä¸Ķ":58730,"æĢĢ念":58731,"Ġparked":58732,".ID":58733,"zahl":58734,"ĠtÅĻ":58735,"Ġteammates":58736,"ACA":58737,"Ġtremb":58738,"AGR":58739,"æĶ¾å¼ĥäºĨ":58740,"live":58741,"éĩī":58742,"Ġemple":58743,"ATP":58744,"å¾Ģäºĭ":58745,"ĠKhal":58746,"表æī¬":58747,"Ġcontinents":58748,"Ġseedlings":58749,"æĹ¶æľī":58750,"ä¾ĭåı¥":58751,"ĠEthernet":58752,"Ġstam":58753,"è¿Ļ天":58754,"以åIJİçļĦ":58755,"ĠFIGS":58756,"æĹ©æĹ©":58757,"ĠSTATE":58758,"ĠбеÑģп":58759,"Ġforthcoming":58760,"Ġsérie":58761,"Ġeman":58762,"Ġ];":58763,"Ġrewarded":58764,"`,`":58765,"åĻªéŁ³":58766,"CAP":58767,"Dar":58768,"ಮ":58769,"Ġbaru":58770,"ĠRED":58771,"issors":58772,"欣åĸľ":58773,"çĭ¡":58774,"个æĢ§åĮĸ":58775,"á̱á̏áĢ":58776,"geal":58777,"åįĩèµ·":58778,"[H":58779,"Ġlush":58780,"ä¸ĢåľĪ":58781,"Ġexpiration":58782,"Ġlst":58783,"ĠFS":58784,"ä¹ĻéĨĩ":58785,"Ġpillars":58786,".no":58787,"ä¸įéĢĤåIJĪ":58788,"ĠRac":58789,"à¹Ħà¸Ł":58790,"女çİĭ":58791,"Ġseals":58792,"æĪijåıijçݰ":58793,"Ġcanopy":58794,"ĠÏĦοÏħÏĤ":58795,"ä¹Łå¼Ģå§ĭ":58796,"ниÑĨ":58797,"Ġapocalyptic":58798,"ĠhÃł":58799,"ĠWORK":58800,"ĠLub":58801,"æ°ij宿":58802,"Ġlonging":58803,"éĩįè¦ģçļĦä½ľç͍":58804,"Ġjejich":58805,"ĠAzerba":58806,"ĠGerald":58807,"æĦıåIJij":58808,"åı·çº¿":58809,".method":58810,"åıĺå¾ĹæĽ´åĬł":58811,"Ġexposures":58812,"{longdiv":58813,"ĠпÑĢоиÑģÑħодиÑĤ":58814,"íķĺìĺĢëĭ¤":58815,"ĠCob":58816,"quisa":58817,"orest":58818,"åѦç¿Ĵ":58819,"Ġdevote":58820,"uve":58821,"ĠBrowser":58822,"-book":58823,"à¹īม":58824,"ç²¾å¯Ĩ":58825,"quoi":58826,"Ġpatrim":58827,"çĻ½è¡£":58828,"ĠбезопаÑģ":58829,"æĻĭåįĩ":58830,"Connor":58831,"Ġquartz":58832,"æľĢå°ij":58833,"æīĵæĸŃ":58834,"ÑĤелем":58835,"åı·çļĦ":58836,"Ġinnocence":58837,"æĸĩæĹħ":58838,"Ġ&Ċ":58839,"çĴIJ":58840,"èģĮä¸ļæĬĢæľ¯":58841,"ĠиÑģклÑİ":58842,".Status":58843,"Ġbanner":58844,"æĪijåı«":58845,"Ġdistort":58846,"æīĵå¾Ĺ":58847,"parable":58848,"Sud":58849,"sizeof":58850,"ĠWare":58851,"degener":58852,"Ġproductions":58853,"اÙĩØ´":58854,"Ġpillow":58855,"urbs":58856,"åĪĨæľŁ":58857,"Ġcorro":58858,"Eloquent":58859,"XR":58860,"}\\!":58861,"襲":58862,"ä¼ļè®®ä¸Ĭ":58863,"/env":58864,"ĠDos":58865,"çĦ¶å¤§":58866,"产ä¸ļåĮĸ":58867,"ĠRobertson":58868,"Past":58869,"ĠAnxiety":58870,"ĠÑĢÑĥков":58871,"çļĦæ°´å¹³":58872,"åıį驳":58873,"带走":58874,"ĠÑĥÑģи":58875,"Lalawigan":58876,"ĠROM":58877,"okens":58878,"Ġcertains":58879,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":58880,"ĠزباÙĨ":58881,"æ¯Ķçī¹å¸ģ":58882,"Ġmisf":58883,"ĠигÑĢа":58884,"measure":58885,"hoe":58886,"åĦĺ":58887,"ependencies":58888,"ĠبراÙĨÙĬÙĩ":58889,"转移åΰ":58890,"ĠSuk":58891,"Ġblends":58892,"ĠSubs":58893,"ĠFrankfurt":58894,"ĠÑģиÑģÑĤема":58895,"Ġmitochondria":58896,"Optional":58897,"************":58898,"说æĺİ书":58899,"Ġpinch":58900,"Ġsuperiority":58901,"Ġvip":58902,"tep":58903,"Ġkontrol":58904,"ĠDalam":58905,"æĹłè¯Ń":58906,"Ġgrapes":58907,"immun":58908,"åľ°åĿĹ":58909,"éĺ²æĻĴ":58910,"æ¥ŃåĭĻ":58911,"ÄĿ":58912,"ĠMyanmar":58913,"çĩĥæ²¹":58914,"ĠBris":58915,"åIJĥæĥĬ":58916,"ä¹IJåύ":58917,"å½ĴæĿ¥":58918,"Ġעש":58919,"Ġktóra":58920,"èİ«åIJįåħ¶":58921,"acje":58922,"åĽ½ä¼ļ":58923,"Ġcapillary":58924,"Ġtemptation":58925,"ĠLCD":58926,"Ġjurisdictions":58927,"Ġalguns":58928,"æį¢äºĨ":58929,"ĠInterpretation":58930,"à§įবর":58931,"ä¸ĢèĦļ":58932,"åľ¨å¤ĸéĿ¢":58933,".Object":58934,":e":58935,"itates":58936,"Ġgilt":58937,"Ġproclaim":58938,"ĠNice":58939,"ĠUND":58940,"Ġhandwriting":58941,"ĠÙħصرÙģ":58942,"ĠJL":58943,"è¿Ļä¸İ":58944,"prend":58945,"Ven":58946,"æĪĸ許":58947,"Ġsimplifying":58948,"èŁ²":58949,"Ġspokesperson":58950,"è¿Ļå¥Ĺ":58951,"åĤ¬åĮĸåīĤ":58952,"oreal":58953,"ä¹Łä¸įæĥ³":58954,"Ġfabricated":58955,"Ġmellitus":58956,"%(":58957,"rata":58958,"åĪĨå±Ĥ":58959,"åıĬ以ä¸Ĭ":58960,"åħīç͵":58961,"ĠاÙĦÙħÙĬÙĦ":58962,"MPa":58963,"/the":58964,"è̧":58965,"ĠFantasy":58966,"ESP":58967,"è£ħçļĦ":58968,"-Pacific":58969,"æĬĺèħ¾":58970,"Ġprzypad":58971,"kÄĻ":58972,"ĠHiro":58973,"룰":58974,"Ġsterile":58975,"Ġsank":58976,"Ġgep":58977,"odynamics":58978,"ikit":58979,"æİ¥èijĹ":58980,"Ġembryos":58981,"-Un":58982,"ĠYosh":58983,"æİ¨åĩºäºĨ":58984,"uas":58985,"Ġond":58986,"Ġsemantics":58987,"thus":58988,"ĠLeast":58989,".loc":58990,"æī¬å·ŀ":58991,"Ġprosecutor":58992,"Ġöver":58993,"ĠAuss":58994,"Ġultraviolet":58995,"Amazon":58996,"Balance":58997,"Ġnás":58998,"åijĹ":58999,"èµ·åĪĿ":59000,"Ġfriendships":59001,"ä¹Łä¸įåı¯èĥ½":59002,"carbon":59003,"qs":59004,"Ġorch":59005,"å¼Ģä¼ļ":59006,".code":59007,"Ġlabs":59008,"ë³Ģ":59009,"Ġnatu":59010,"Ġvarchar":59011,"Ġcontrasting":59012,"Ġmoderately":59013,"fal":59014,"ĠGenome":59015,"Ġsecre":59016,"ĠÑģвÑıзи":59017,"è½¬çľ¼":59018,"èĿĩ":59019,"witz":59020,"ĠStri":59021,"æ¸Ī":59022,"女åıĭ":59023,"shop":59024,"让å®ĥ":59025,"æī§çĿĢ":59026,"éné":59027,"Quantity":59028,"Ġplast":59029,"ĠLastly":59030,"UTHOR":59031,"Bra":59032,"margin":59033,"esan":59034,"áció":59035,"Ġgrinned":59036,"ĠÑģоÑģÑĤавлÑı":59037,"istema":59038,"Ġmanière":59039,"ĠÎĿÎŃα":59040,"Ġbob":59041,"Ġcolorectal":59042,"ussions":59043,"Ġнеде":59044,"ĠAshley":59045,"ĠÑģвойÑģÑĤва":59046,"Ġeccentric":59047,"oner":59048,"izards":59049,"Ġdisks":59050,"å°ıä¼Ļ":59051,"Sz":59052,"ĠSections":59053,"ä¸Ĭä¸Ģ个":59054,"Ġherd":59055,"orescence":59056,"ĠElim":59057,"çļĦä¿¡":59058,"ĠRid":59059,"Ġplut":59060,"è¡£çī©":59061,"ĠÐłÐ¤":59062,"ĠMate":59063,"Ġ/>ĊĊ":59064,"ANY":59065,"æ´ĭæ´ĭ":59066,"çĬ¯ç½ªå«Įçĸij人":59067,"ä¹ĭåĪĿ":59068,"ãĥ¼ãĤ·ãĥ§ãĥ³":59069,"ĠRocky":59070,"Ġcarbohydrate":59071,"Ġоколо":59072,":s":59073,"ĠDrew":59074,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":59075,"JV":59076,"åĵģå¾·":59077,"Ġquali":59078,"ĠArtists":59079,"饰æ¼Ķ":59080,"é¢ģåıij":59081,"Ġfry":59082,"Ġexpos":59083,"Ġrightly":59084,"ĠأخرÙī":59085,"াড়":59086,"_app":59087,"nb":59088,"mez":59089,"_element":59090,"Ġmercado":59091,"ĠдÑĢÑĥгом":59092,"(out":59093,"Ja":59094,"ĠEden":59095,"Stay":59096,"å¾Ģå¤ĸ":59097,"à¸ģลà¹Īาว":59098,"ĠMessages":59099,"ĠCharge":59100,"ç͵åı°":59101,"ãĤ¯ãĥĪ":59102,"Ġdunia":59103,"æľ¬èĥ½":59104,"å¹²éĥ¨èģĮå·¥":59105,"Ġεκ":59106,"=d":59107,"åĸĿæ°´":59108,"×ŀ×Ķ":59109,"ĠNashville":59110,"ĠRw":59111,"ĠSamples":59112,"éģ©åIJĪ":59113,"ĠHansen":59114,"енÑĤи":59115,"导游":59116,"otonin":59117,"ĠOrigins":59118,"æł¡æŃ£":59119,"Ġanalyzes":59120,"Ġvelocities":59121,"ĠHak":59122,"æĪijæĢİä¹Ī":59123,"×Ļ×Ķ×Ŀ":59124,"Ġtaller":59125,"Ġgroot":59126,"ĠOnes":59127,"Ġreste":59128,"Shop":59129,"ĠTigers":59130,"åľ¨åIJİ":59131,"Ġrider":59132,"太éϽ":59133,"ĠVolunteer":59134,"ä½łå°±æĺ¯":59135,"Ġwrapping":59136,"лиÑĩие":59137,"ĠClipboard":59138,"Loop":59139,"åĮ»çĸĹä¿ĿéĻ©":59140,"èŀºæłĵ":59141,"âľħ":59142,"Ġloudly":59143,"è´¨åľ°":59144,"-standing":59145,"ĠTat":59146,"æľīå¿ĥ":59147,"Ġnightmare":59148,"Ġnouveau":59149,"Ġkwadrado":59150,"-du":59151,"ĠKiss":59152,"ĠÃīt":59153,"éĢļ讯åijĺ":59154,"åľ¨åIJĮä¸Ģ":59155,"izia":59156,"ĠìĥĪ":59157,"pla":59158,"èѝ":59159,"å½ķåĥı":59160,"oglob":59161,"Ġlender":59162,"ração":59163,"Ġmarathon":59164,"ĠSerbia":59165,"Ġhunter":59166,"åIJĪè§Ħ":59167,"èĬĤçļĦ":59168,"liwo":59169,"λή":59170,"ÑĪиÑĤÑĮ":59171,"å®¶åºŃçļĦ":59172,"añ":59173,"Ġebook":59174,"มีà¸Ħวาม":59175,"ÑħодÑıÑĤ":59176,"fung":59177,"åIJĪå½±":59178,"缴è¾ĸ":59179,"Ġgrazing":59180,"æķ´ä½ĵçļĦ":59181,"åĥı个":59182,"èĥĮåıĽ":59183,"Ġà¦°à¦¾à¦ľ":59184,"ĉresult":59185,"-input":59186,"èįĢ":59187,"ĠNewman":59188,"ÑĤоÑĢии":59189,"ĠRelative":59190,"waż":59191,"èĿİ":59192,"Ġsiglo":59193,"Ġselections":59194,"Ġrestraint":59195,"Ġtai":59196,"Ġoptic":59197,"COMM":59198,"åįļå¼Ī":59199,"FIGURE":59200,"Capital":59201,"Fr":59202,"гла":59203,"å¸ĥé²ģ":59204,"æİ§ä»¶":59205,"different":59206,"hub":59207,"above":59208,"reta":59209,"ãĢįĊ":59210,".Rem":59211,"äºĭä»¶çļĦ":59212,"Ġunderscores":59213,"δÏģικÏĮ":59214,"Ġcozy":59215,"-sign":59216,"åįĹå®ģ":59217,"ĠFitness":59218,">>>>":59219,"åĵİåijĢ":59220,"jø":59221,"æĹ¥æľĪ":59222,"çī©è³ª":59223,"æģ¨ä¸įå¾Ĺ":59224,",g":59225,"Ġà¸ģà¹ĩ":59226,"Ġpledge":59227,"ä½łè¿Ļ个":59228,"è¾¾æłĩ":59229,"ĠاستخداÙħ":59230,"License":59231,"Diameter":59232,"oris":59233,"Ġocor":59234,"riks":59235,"権":59236,"åij¨äºĶ":59237,"_level":59238,"äºĨæĪijçļĦ":59239,"æľī以ä¸ĭ":59240,"bern":59241,"ä¸ĭè¡Į":59242,"ITIONS":59243,"Ġhora":59244,"Gate":59245,"Ġnieuwe":59246,"å¹¶åIJij":59247,"å¹´ä¸ŃåĽ½":59248,"æIJ¬è¿ģ":59249,"ä¾Ŀèµĸäºİ":59250,"ĠHopefully":59251,"Ġmall":59252,"Ġkada":59253,"æ¡Ķ":59254,"æ±īæĹı":59255,"Ġpelvic":59256,"ĠÑģÑĢедÑģÑĤв":59257,"å®īå¾½çľģ":59258,"ĠгеÑĢ":59259,"Ġverl":59260,"ĠнеболÑĮ":59261,"Ġhelicopter":59262,"ĠTER":59263,"Ġhumility":59264,"íݸ":59265,"Ġpunctuation":59266,"(âĪĴ":59267,"åħī纤":59268,".stream":59269,"è»ĭ":59270,"Travel":59271,"Ġdispose":59272,"æİ¨æµĭ":59273,"][\"":59274,"ĠCU":59275,"åľ¨åģļ":59276,"ĠGan":59277,"Ġpeppers":59278,"Ġspectrometry":59279,"love":59280,"æĬ¥éĢģ":59281,"Ġtrio":59282,"èģĭ":59283,"tenant":59284,"Ġhoy":59285,"ä¸Ģå¦Ĥ":59286,"ضÙĪ":59287,"æĮĤçīĮ":59288,"åij¨ä¸Ģ":59289,"ĠKörper":59290,"èĢ½è¯¯":59291,"Ġmaze":59292,"=\"\">":59293,"Ġestudo":59294,"ĠAgenda":59295,"ë°°":59296,"Ġcommissioner":59297,"éĴ»çٳ":59298,"İ":59299,"ĠCairo":59300,"ابط":59301,"å±¥èģĮ":59302,"QC":59303,"QS":59304,"Ġencyclopedia":59305,"-Mart":59306,"ĠEntwicklung":59307,"Upon":59308,"ĠExponent":59309,"Ġmuchas":59310,"Ġfounders":59311,"Ġgebruik":59312,"Shape":59313,"ĠSwan":59314,"âĢľĊĊ":59315,"apis":59316,"ĠGlenn":59317,"Ul":59318,"Ġsocks":59319,"Ġsclerosis":59320,"åħ¼èģĮ":59321,"Ġ×ĸ×ķ":59322,"ĠJenny":59323,"please":59324,"çŀħ":59325,"èµ¶å¿«":59326,"åĵĪåĵĪåĵĪåĵĪ":59327,"Ġturbulence":59328,"ĠKub":59329,"Ġpreoccup":59330,"åĸľåī§":59331,"Ġcircuitry":59332,"éĢļè´§èĨ¨èĥĢ":59333,"åĬ¡å®ŀ":59334,"楼çļĦ":59335,"hardt":59336,"è¿Ļéĥ¨åĪĨ":59337,"Ġdoorway":59338,"æ³¥åľŁ":59339,"èĨ³é£Ł":59340,"eye":59341,"ĠXiao":59342,"触æij¸":59343,"Ġmentioning":59344,"Ġgin":59345,"Ġclassmates":59346,"Ġdeterminant":59347,"uerdo":59348,"ĠBiblical":59349,"Ġcompetencies":59350,"Ped":59351,"Ġkad":59352,"hereum":59353,"æĮĩçͲ":59354,"ÙĨدÛĮ":59355,"ÃĥO":59356,"Ġdisposit":59357,"ĠHayes":59358,"ĠChapman":59359,"Ġtails":59360,"arro":59361,"اپ":59362,"ĠHep":59363,"åĴĮæĪij们":59364,"identified":59365,"æĤ¸":59366,"Compet":59367,"æĺ¾ç¤ºåĩº":59368,"ĠPenguin":59369,"idor":59370,"å¤įåıij":59371,"Ġalternatively":59372,"úsica":59373,"jonal":59374,"许ä¹ħ":59375,"cemia":59376,"Ġ'''Ċ":59377,"ĠVall":59378,"à°ª":59379,"Ġphotons":59380,"eners":59381,"redirect":59382,"Ġmonuments":59383,"æİ§åĪ¶ç³»ç»Ł":59384,"表éĿ¢ä¸Ĭ":59385,"äº§åľ°":59386,"ÑĢазÑĥ":59387,"debug":59388,"èĢħä¹Ł":59389,"ĠRegist":59390,"Emb":59391,"Ġseguir":59392,"ĠRangers":59393,"ĠHOW":59394,"rieve":59395,"(np":59396,"Ops":59397,"ĠCasa":59398,"ichia":59399,"çļĦä¸Ģèĩ´":59400,"Als":59401,"Ġpredecessor":59402,"大åıĶ":59403,"Ġrav":59404,"éĩijéĵ¶":59405,"Ġcakes":59406,"ĠGob":59407,"ä»İä¸Ģ个":59408,"æĸŃéĿ¢":59409,"ALT":59410,"Ġbede":59411,"Ġвозник":59412,"Ġhallway":59413,"Lect":59414,"ĠRica":59415,"车éģĵ":59416,"ĠÙĬس":59417,"æĻ®æŁ¥":59418,"åĴĮä¼ģä¸ļ":59419,"áĥĵ":59420,"转åΰ":59421,"ä¸įä»ħè¦ģ":59422,"èĶij":59423,"Ġìļ´":59424,"Ġassemblies":59425,"ĠEugene":59426,"uridad":59427,"ç»Ļä½łä»¬":59428,"æĸ½åĬł":59429,"æģ¢å¤įäºĨ":59430,"Ġwandering":59431,"Ġsage":59432,"Ġkot":59433,"opause":59434,"ĠBehav":59435,"Ìĥ":59436,"Ġtopped":59437,"-view":59438,"Ġbeb":59439,"ĠÙĪØ¶Ø¹":59440,"Ġcompetitor":59441,"Ġejerc":59442,"Altitude":59443,"-rays":59444,"ĠMau":59445,"åįģ个":59446,"زد":59447,"folios":59448,"Ġactividades":59449,"idin":59450,"æĪijä¸įä¼ļ":59451,"Ġesempio":59452,"ĠShir":59453,"æĿİçϽ":59454,"Ġscream":59455,"ç©į極":59456,"ĠTEM":59457,"ĠSenary":59458,"åζèį¯":59459,"çłº":59460,"é»ĦèĬ":59461,"Ġauc":59462,"ĠCanc":59463,"Ġcomed":59464,"ÎŃν":59465,"Raw":59466,"ULE":59467,"å¤Ń":59468,"ĠÑģпа":59469,"æ¸ħé£İ":59470,"Ġachieves":59471,"ÛĮÙĨÙĩ":59472,"å°ĸéĶIJ":59473,"áj":59474,"çĽ¸ä¼ł":59475,"裡éĿ¢":59476,"hee":59477,"åį¯":59478,"ยะ":59479,"Ġboosting":59480,"Ġinvoked":59481,"å¹¶èĥ½":59482,"Ġbarley":59483,"åĢ¡è®®":59484,"":59654,"çİ¥":59655,"ĠStats":59656,"åģļä¸įåΰ":59657,"Ġlongitude":59658,"æĪĺ线":59659,"æģ¶åĮĸ":59660,"ĠStanding":59661,"Ġnullable":59662,"躲éģ¿":59663,".check":59664,"ĠHaiti":59665,".item":59666,"Ġconject":59667,"Ġmultid":59668,"-vous":59669,"vb":59670,"à¸Ńืà¹Īà¸Ļ":59671,"Ġfamiliarity":59672,"å·¡å¯Ł":59673,"belief":59674,"production":59675,"Ġremembers":59676,"太é«ĺ":59677,"产ä¸ļç»ĵæŀĦ":59678,"åĮ»åѦéĻ¢":59679,"ĠÙĨظاÙħ":59680,"å¤ĸåľ¨":59681,"ĠGuild":59682,"Ġprogressively":59683,"WV":59684,"ĠздеÑģÑĮ":59685,"Ġtopological":59686,"Ġà¤ķà¥ĭ":59687,"-Time":59688,"对åѦçĶŁ":59689,"Ġzon":59690,"è¢ľ":59691,"ãĢĭ)ĊĊ":59692,"Split":59693,"allery":59694,"æķĻèģĮå·¥":59695,"Ġtopical":59696,"Blood":59697,"Ġhydroxide":59698,"Nick":59699,"atches":59700,"ĠGuo":59701,"Ġmassa":59702,"панÑģки":59703,"ovÄĽ":59704,"Ġzien":59705,"ĠChest":59706,"âĭ¯":59707,"èĢĮçĶŁ":59708,"ä¸Ģ年级":59709,"rox":59710,"æĪijéĹ®":59711,"æĥħæĦ¿":59712,"ituitary":59713,"ĠرÛĮ":59714,"Ġpitcher":59715,":B":59716,"åľ¨åħ¨":59717,"人å¤ļ":59718,"gett":59719,"èĢģ夫":59720,"è£ħç®±":59721,"èĻļå¼±":59722,"Ġaccredited":59723,"Pred":59724,"Ġmia":59725,"romes":59726,"ĠInvol":59727,"åIJĮèĥŀ":59728,"ratio":59729,"ĠGriffith":59730,"Ġhey":59731,"ophilic":59732,"Ġsandy":59733,"æīĢåѦ":59734,"Ġitiner":59735,"Ġ×Ķ׾×":59736,"Ġén":59737,"árias":59738,"ĠVigesimal":59739,"Ġteamwork":59740,"vano":59741,"åħĢ":59742,"swap":59743,"çľģå¸Ĥ":59744,"éĺ³åİ¿":59745,"å¾Īå¤ļ人éĥ½":59746,"ĠSandy":59747,"Positive":59748,"UIT":59749,"-Hill":59750,"ilin":59751,"Ġpertama":59752,"åħįçĸ«åĬĽ":59753,"cepts":59754,"خصص":59755,"Ľà§ģ":59756,"Ġroses":59757,"Ġindict":59758,"è°ĥåζ":59759,"hanced":59760,"Ġrepo":59761,"ĠÑĤол":59762,"brahim":59763,"å®īåħ¨ç®¡çIJĨ":59764,"Ġcomparatively":59765,"ãĢĭ)":59766,"parts":59767,"mode":59768,"对社ä¼ļ":59769,"ĠEditorial":59770,"æŃ£åĽłä¸º":59771,"Ġarchitectures":59772,"Ġgradients":59773,"ĠSpectrum":59774,"ĠWT":59775,"-inst":59776,"å̾åIJijäºİ":59777,"ÙĦت":59778,"è¯ķçĿĢ":59779,",E":59780,"idepress":59781,"Ġintertw":59782,"Ġinspections":59783,"ĠHoldings":59784,"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM":59785,"ĠSME":59786,"Ġgoss":59787,"Ġstray":59788,"åľ¨åŃ¦ä¹ł":59789,".pp":59790,"Hall":59791,"ä¸Ģæ³¢":59792,"plice":59793,"è½Ł":59794,"æĻ¾":59795,"Ġbelajar":59796,"ìĺĪ":59797,"à±ģà°²":59798,"poly":59799,"·¨":59800,"ä¸Ĭ车":59801,"ynn":59802,"ĠCharacterization":59803,"ĠWinn":59804,"ĠNgalan":59805,"мов":59806,"éĿ¢çĽ®":59807,"交éĻħ":59808,"Ġseorang":59809,"æĪij为":59810,"é£İæĥħ":59811,"Reset":59812,"Teacher":59813,"èĤ´":59814,"ĠMillennium":59815,"пеÑĢÑĮ":59816,"STAT":59817,"à¯įவ":59818,"-Pro":59819,"æĢ»è§īå¾Ĺ":59820,"ĠDebug":59821,"é³´":59822,"-core":59823,"éĿłè°±":59824,"éĸĭçϼ":59825,"æ³ķå¾ĭè§Ħå®ļ":59826,"ริà¸ĩ":59827,"ĠWalsh":59828,"Timer":59829,"ĠPLC":59830,"Ġendemic":59831,"à¹Ģà¸ł":59832,"abilidad":59833,"quisites":59834,"iahy":59835,"ç½ij绾çļĦ":59836,"ĠCombined":59837,"Ġevitar":59838,"æĹłéĿŀ":59839,"Ġnuances":59840,"á¹Ń":59841,"éłĺå°İ":59842,"æĸ°èĥ½æºIJ汽车":59843,"Grant":59844,"Ġvæ":59845,"çĦ¶åIJİå°Ĩ":59846,"Han":59847,"æµģä½ĵ":59848,"è¯ķ管":59849,"Ġadvertisements":59850,"_o":59851,"ä¾µåħ¥":59852,"âĤĤ":59853,"ència":59854,"Ġwipe":59855,"ĠProfession":59856,"æĥ³è¦ģçļĦ":59857,"ukum":59858,"Ġgeneralization":59859,"ĠLimits":59860,"ологиÑı":59861,"æķĻ師":59862,"讲çļĦ":59863,"该æĢİä¹Ī":59864,"æĭĺçķĻ":59865,"Ġdeprivation":59866,"ĠKend":59867,"à¹Ģรา":59868,"ĠâĹĨ":59869,"Ġcascade":59870,"ĠMama":59871,".Em":59872,"ÙĬاÙĭ":59873,"Ġneonatal":59874,"iesel":59875,"ä¹Łæĥ³":59876,"ĠQuinn":59877,"ĠInspector":59878,"èĦIJ":59879,"Ġbooked":59880,"Ġש×ij×":59881,"ä»ĸå¾Ī":59882,"pron":59883,"Ġpublicity":59884,"Ġµin":59885,".Find":59886,"ĠSetup":59887,"责任æĦŁ":59888,"ç¦ģæ¯Ĵ":59889,"Ġintravenous":59890,"Ġenclosure":59891,"ĠShan":59892,"Ġcombin":59893,"计åĪĴçĶŁèĤ²":59894,"åģľè½¦åľº":59895,"ĠÑģвоиÑħ":59896,"Ġrescued":59897,"ĠSophie":59898,"ĠÑĤÑĭÑģÑıÑĩ":59899,"çļĦéķ¿åº¦":59900,"åŁİä¸Ń":59901,"ĠMarkov":59902,"幸好":59903,"Ġclicked":59904,",J":59905,"çļĦä¹łæĥ¯":59906,"ĠHH":59907,"Inc":59908,"éĹ®çŃĶ":59909,"Ġastronomy":59910,"Gram":59911,"Ġleven":59912,"Ġuh":59913,"acha":59914,"Ġrunoff":59915,"มีà¸ģาร":59916,"Ġterrorists":59917,"ĠTribune":59918,"ĠWesley":59919,"+B":59920,"=\"@":59921,"gins":59922,"ĠCricket":59923,"ï¼Łï¼ģ":59924,"ĠGenerated":59925,"Ġtranslating":59926,"éĿ¢ä¸´çĿĢ":59927,"ibolana":59928,"ä¸įå°±æĺ¯":59929,"Ġchasing":59930,"()ĊĊĊ":59931,"ĠMK":59932,"ĠUng":59933,"Ġclassifier":59934,"consum":59935,"äºĨä¸Ģçķª":59936,"çݯå¢ĥåĴĮ":59937,"Ġконе":59938,"Ġdoctoral":59939,"ĠÙĪØ²":59940,"åįķ身":59941,"ĠConcrete":59942,"Ġágua":59943,"âĦĵ":59944,"Ġcathode":59945,"Ġinverted":59946,"çϽèıľ":59947,"Ġarticulate":59948,"çŁ¿å±±":59949,"slant":59950,"çĶľçĶľ":59951,"ãĤ²":59952,"åŤ":59953,"毫åħĭ":59954,";/":59955,"\"He":59956,"ermost":59957,"æŃ¢è¡Ģ":59958,".execute":59959,"ר×ĵ":59960,"èĥ¡åŃIJ":59961,"หาร":59962,"Ġnegotiated":59963,"Same":59964,"身åIJİçļĦ":59965,"Ġlegends":59966,"ĠCurr":59967,"hicles":59968,"ç»ıåİĨè¿ĩ":59969,"æĦı義":59970,"ymes":59971,"Ġfoundational":59972,"ĠRegression":59973,"Lead":59974,"ĠLomb":59975,"ãģ¾ãģĹãĤĩãģĨ":59976,"èĩªçĦ¶ç§ijåѦ":59977,"\\Support":59978,"utu":59979,"è¾Ń":59980,"eka":59981,"Ġwatershed":59982,"Ġstaging":59983,"åIJĪä½ľçļĦ":59984,"коÑĢ":59985,"åıįåĩ»":59986,"ĠAugustine":59987,"Ġfais":59988,"Whatever":59989,"对è¿ĻäºĽ":59990,"端åįĪ":59991,"Ġvinyl":59992,"æŃ£æĸ¹":59993,"-aged":59994,".form":59995,"大佬":59996,"Ġrelação":59997,"Ġpores":59998,"导管":59999,"è¯ķç͍":60000,"Ġfragmentation":60001,"Ġfurnish":60002,"ç´Ĭä¹±":60003,"æ²ĥå°Ķ":60004,"Ġlassen":60005,"arnya":60006,"å¤ļä¸ĩ":60007,"使èĢħ":60008,"ĠÑģÑĤÑĥд":60009,"èĪįä¸įå¾Ĺ":60010,"Ġlobe":60011,"çŁ¿çī©":60012,"nÄĽnÃŃ":60013,"Ġwichtig":60014,"æ·ĩ":60015,"ATING":60016,"ĠØ£ÙĩÙħ":60017,"ĠCooperation":60018,"ĠElliott":60019,")The":60020,"æİ¥å¤´":60021,"Ġhurts":60022,"讽åĪº":60023,"Ġtrês":60024,"ÑĤелÑĮÑģÑĤво":60025,"ĠPalestinians":60026,"Ġsoutheast":60027,"otimes":60028,"åįĹå±±":60029,"Ġbehavioural":60030,"ĠÅĽrod":60031,"ä¸ĢæľŁ":60032,"ĠKeys":60033,"ĠTriangular":60034,"uttering":60035,"MAP":60036,"business":60037,"verk":60038,"Ġquelle":60039,"åĪĨè¡Į":60040,"稼":60041,"ĠIty":60042,"Õ«Öģ":60043,"Ġcongressional":60044,"/com":60045,"ussian":60046,"ifter":60047,"ĠFraser":60048,"solete":60049,"Chen":60050,"Ġshocks":60051,"athi":60052,"ยุ":60053,"ĠMorocco":60054,"Ġsip":60055,"éb":60056,"çŁŃæĿ¿":60057,"ĠWikibolana":60058,"Ġhunters":60059,"zv":60060,"ÇIJ":60061,"ä¸ĸå®¶":60062,"ĠEdmund":60063,"(Node":60064,"Ġtk":60065,"grass":60066,"homme":60067,"Ġattraverso":60068,"(os":60069,"ĠÑĥз":60070,"ĠSpatial":60071,"æĿĤè´¨":60072,"ä½ĵå¤ĸ":60073,"ĠоÑĤÑģÑĥÑĤ":60074,"ABSTRACT":60075,"Ġcreditors":60076,"ĠInstruments":60077,"å¦Ĥæŀľè¦ģ":60078,"Ġpostpon":60079,"à¥įध":60080,"Ġdeclaring":60081,"çļĦåŃĹ":60082,"ĠCharts":60083,"æīĵéĸĭ":60084,"ĠNB":60085,"Ġerrone":60086,"Ġaccomplishment":60087,".author":60088,"Kh":60089,"Ġanisot":60090,"ĠCOLL":60091,"ĠÐĿов":60092,"ÐŃÑĤо":60093,"ستÙħ":60094,"奶èĮ¶":60095,"essi":60096,"èĩªå°Ĭ":60097,"é£İè²Į":60098,"ĠGujar":60099,"ĠHandle":60100,"ĠUltra":60101,"åĩłåĪĨéĴŁ":60102,"ĠÙĥÙĬÙģ":60103,"Ġjungle":60104,"ĠAway":60105,"ĠBlan":60106,"ĠParish":60107,"gart":60108,"Ġstall":60109,"Ġdiscontinu":60110,"ĠÑģÑĤаÑĢ":60111,"Ġknight":60112,"\\big":60113,"along":60114,"车身":60115,"豬":60116,"ä¸įåĨįæĺ¯":60117,"Ġprolifer":60118,"人æĿ¥":60119,"ä¸ĢèĪ¬ä¸º":60120,"ä¼¼ä¹İæĺ¯":60121,"Ġpairing":60122,"ĠзадаÑĩи":60123,"umo":60124,"adie":60125,"اÛĮÙĦ":60126,"åıĤä¸İèĢħ":60127,"Ġmanuscripts":60128,"æķ°çłģ":60129,"åĬ©æİ¨":60130,"кÑĥÑĢ":60131,"Ġblogging":60132,"ĠEvan":60133,"ziÄĩ":60134,"ĠModi":60135,"Ġspirituality":60136,"ĠزÙĬ":60137,"-ar":60138,"é«ĺ度çļĦ":60139,"Ġfurnished":60140,"Ġsteril":60141,"Ġrecombinant":60142,"ä½łæĿ¥":60143,"çIJĨè§£åĴĮ":60144,"ĠBuilt":60145,"Ġwastes":60146,"æĪijçα":60147,"ĠاÙĦرب":60148,"å®ŀè¯ģ":60149,"Ġopacity":60150,"Ġzag":60151,"каз":60152,"æł¸å¯¹":60153,"Ġstealing":60154,"wijs":60155,"ĠAuckland":60156,"Ġmicron":60157,"ĠOdys":60158,"Ġketika":60159,"Software":60160,"Ġimmobil":60161,"Ġবà§ĩশ":60162,"Ġrupture":60163,"Ġnavy":60164,"æĹ¶è¾°":60165,"где":60166,"æķĻèĤ²æ´»åĬ¨":60167,"åīįä¸ī":60168,"Ġcadre":60169,",I":60170,")\",":60171,"Ġerect":60172,"ĠgrÃ¶ÃŁ":60173,"èves":60174,"Ġ`${":60175,"ĠÐĶи":60176,"ëįĶ":60177,"ினà¯į":60178,".sql":60179,"ĠÙħÛĮÚ©ÙĨ":60180,"Ġcandles":60181,"ĠTsiahy":60182,"Ġvowels":60183,"sample":60184,"ĠOC":60185,"Ġtidal":60186,"DOM":60187,"jets":60188,"Ġcommenced":60189,"èıĬèĬ±":60190,"rossover":60191,"ĠUd":60192,"åıªåī©":60193,"Ġsistemas":60194,"enium":60195,"ĠRiley":60196,"urgery":60197,"ĠSkill":60198,"Operator":60199,"æģŃåĸľ":60200,";DR":60201,"æĬĹåĩ»":60202,"arana":60203,"罪çĬ¯":60204,"ophagus":60205,"Ġnutritious":60206,"Germany":60207,"Ġcrush":60208,"kW":60209,"å¤ĸå©Ĩ":60210,"马å°Ķ":60211,"\\beta":60212,"ĠCENT":60213,"æĸĩä»¶ä¸Ń":60214,"!.":60215,"\"(":60216,"çļĦæķ´ä½ĵ":60217,"ä¸įçĶĺ":60218,"Diagonal":60219,"ĠиÑģÑĤоÑĢии":60220,"å¸ĥå°Ķ":60221,"ĠMaar":60222,"åĬłåħ¥äºĨ":60223,"Ġconfl":60224,"Ġgiants":60225,"munition":60226,"Ġdistorted":60227,"åύåħ·":60228,"management":60229,"ĠÙģÙĤد":60230,"regate":60231,"नà¥įत":60232,"ĠGIS":60233,"æĪijç͍":60234,"çĿĢä»ĸçļĦ":60235,"омеÑĢ":60236,"-purpose":60237,"AGS":60238,"æľĢé«ĺ人æ°ijæ³ķéĻ¢":60239,"Ġnonsense":60240,"Ġpostal":60241,"æĹ¥å¤ľ":60242,"æĶ¶çĽĺ":60243,"Ġcelebrity":60244,"ése":60245,"è¯ķæİ¢":60246,"èse":60247,"Ġlokal":60248,"åĬij":60249,"大æľī":60250,"ĠполÑĮзова":60251,"å¿ħä¿®":60252,"Ġì§ģ":60253,"ç¨įç¨į":60254,"بÙĤ":60255,"çļĦä¸ĢæĿ¡":60256,"货车":60257,"processor":60258,"ĠMaurice":60259,"Australia":60260,"িনà§įতà§ģ":60261,"altern":60262,"åħ¥ä½ı":60263,"лÑıÑĤÑĮ":60264,"åĪijç½ļ":60265,"ĠÚ¯ÙģØª":60266,"Ġseva":60267,"è½®åĽŀ":60268,"çľ¨çľ¼":60269,"Ġinsecurity":60270,"channel":60271,"éļIJçŀĴ":60272,"Vi":60273,"æĴ©":60274,"Ġpredominant":60275,"INC":60276,"æĥħåĨµè¿Ľè¡Į":60277,"SCI":60278,"ĠDesigner":60279,"\\!\\":60280,"å°ĨæĪIJ为":60281,"Ġmisconduct":60282,"Ġsandwic":60283,"æķijæ²»":60284,"Ġusefulness":60285,"Ġहà¥ĭ":60286,"ĠForsch":60287,"ĠTrek":60288,"ĠSabb":60289,"à³Ĭ":60290,"Ġguideline":60291,"ĠاÙĦداÙĬرÙĩ":60292,"Ġstole":60293,"建æĿIJ":60294,"åĪĨæŃ§":60295,"鼶éĥ¨ä»¶":60296,"ĠاÙĦÙĨاس":60297,".ext":60298,"ารà¹Į":60299,"ãģįãģŁ":60300,"â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦â̦":60301,"ç½·":60302,"ĠMyers":60303,"Ġdébut":60304,"çļĦåıįåºĶ":60305,"Ġbackpack":60306,"Ġlinearly":60307,"ĠÙĦØŃ":60308,"Ġwrestling":60309,"æ¤ŃåľĨ":60310,"ĠÙĨÙĤØ·Ùĩ":60311,"ĠGmbH":60312,"é«ĺäºĮ":60313,"loan":60314,"Ġconditioned":60315,"ĠOpening":60316,"âĤ¬âĦ¢":60317,"emis":60318,"é¦ĸå±Ĭ":60319,"któ":60320,"ä¸Ģ缴åΰ":60321,"ëıĦë¡Ŀ":60322,"=${":60323,"Ġwre":60324,"λά":60325,"Ġnaturale":60326,"Ġmdl":60327,"å°Ĩ她":60328,"===":60329,"ĠGlobe":60330,"Ġambitions":60331,"Ġ׼×ļ":60332,"Ġguessed":60333,"ĠBrady":60334,"abric":60335,"Ġwhales":60336,"unnel":60337,"ĠOwner":60338,"align":60339,"leigh":60340,"Ġunos":60341,"è½»éĩį":60342,"Fixed":60343,"ä¸Ģä¾§":60344,"è¿Ļç¯ĩæĸĩ竳":60345,"Ġsentiments":60346,"Ġobservable":60347,"çĬ¶æĢģä¸ĭ":60348,"klore":60349,"Ġapartments":60350,"Gy":60351,"erb":60352,"æĮĽ":60353,"Ġasteroid":60354,";)":60355,"-assisted":60356,"ä¸ĢéĹª":60357,"нной":60358,"Apply":60359,"详解":60360,"Preview":60361,"ĠESL":60362,"auff":60363,"ividad":60364,"Ġ):":60365,"à¹ģà¸Ĥ":60366,"Ġrgba":60367,"mot":60368,"Ġech":60369,"Ġefter":60370,"Ġtreasures":60371,"æĩ¿":60372,"Package":60373,"ĠRare":60374,"Ġincarcer":60375,"楼ä¸ĭ":60376,"Ġbamboo":60377,"ä¸Ģè§ģ":60378,"ĠпÑĢедлож":60379,"åĮºåĴĮ":60380,"åħīè¾ī":60381,"ĠCluster":60382,"ĠBrock":60383,"ï¼IJï¼IJ":60384,"ä¸ļä½Ļ":60385,"×Ļ×IJ×":60386,"upon":60387,"ĠAdrian":60388,"Ġembryonic":60389,"åĭµ":60390,"è¯įåħ¸":60391,"){ĊĊ":60392,"-Col":60393,"æĹ¶éĴŁ":60394,"Ġringing":60395,"ãİ¡":60396,"æĸ¹åı¯":60397,"好æľĭåıĭ":60398,"河水":60399,"aea":60400,"ĠØ«ÙĦاث":60401,"دÙĬدة":60402,"ĠGOP":60403,"åIJ¸åħ¥":60404,"Ġdeparted":60405,"ÑĢп":60406,"ĠJag":60407,"ä»ĸç͍":60408,"Ġexceptionally":60409,"éĤ£éº½":60410,"Ġhypers":60411,"Ġmultiplex":60412,"Ġthresholds":60413,"ĠMatters":60414,"phosph":60415,"inaire":60416,"_EX":60417,"Ġcured":60418,"Nut":60419,"Ġexpressly":60420,"èĤ¥æĸĻ":60421,"Saint":60422,"Ġhele":60423,"ä»ĸéĥ½":60424,"åĵģåij³":60425,"央è§Ĩ":60426,"ĠEPS":60427,"å®ļæĢ§":60428,"ĠHumanities":60429,"Joseph":60430,"âĢĶĊ":60431,"éĿ¢åīįçļĦ":60432,"Ġlainnya":60433,"ĠAmbassador":60434,"ĠFurn":60435,"Ġکر":60436,"Ġcontroversies":60437,"lisitry":60438,"路人":60439,"Ġmaturation":60440,"ìϏ":60441,"ĠChocolate":60442,"山顶":60443,"ALTH":60444,"Ġinvestigación":60445,"Ġbrasile":60446,"ĠÑĢÑıд":60447,"ĠNoun":60448,"èݺ":60449,"è§ĤéŁ³":60450,"第ä¸Ģä½į":60451,"Ġsnippets":60452,"erte":60453,"antically":60454,"à´®":60455,"ĠAUD":60456,"atech":60457,"æľĢå°ıçļĦ":60458,"ĠUpdates":60459,"зÑĭваеÑĤ":60460,"Fall":60461,"ĠWidget":60462,"ĠPodcast":60463,"MOD":60464,"implementation":60465,"å²Ķ":60466,"Ġ×ij×§":60467,"å½ĵå¹´çļĦ":60468,"æ³Ħéľ²":60469,"ĠÑħÑĥд":60470,"åĽ°éļ¾çļĦ":60471,"ï½ŀï½ŀ":60472,"ĠKashmir":60473,"était":60474,"Ġheadline":60475,"ç®Ģè¦ģ":60476,"æłijä¸Ĭ":60477,"ä¸Ŀ绸":60478,"èĻļ空":60479,"湿润":60480,"Ġà¹Ģà¸ŀราะ":60481,"å°Ĩè¦ģ":60482,"éĹªç͵":60483,"Ġnadika":60484,"è·Łåľ¨":60485,"çĻ«çĹ":60486,"交æİ¥":60487,"ĠÙĨÙĩ":60488,"ĠмеÑħани":60489,"dbc":60490,"夯å®ŀ":60491,"MED":60492,"æĦıè¦ĭ":60493,"åıĸåIJij":60494,"çķĻç»Ļ":60495,"bolic":60496,"ĠHands":60497,"Ġtranscripts":60498,"Ġsebelum":60499,"ĠPada":60500,"Ġsociedad":60501,"åĭĥåĭĥ":60502,"ĠмакÑģима":60503,"Ġalcoholic":60504,"èī²è°ĥ":60505,"ìĸ¸":60506,".draw":60507,"ä¹īè¯į":60508,"ĠSharon":60509,"embangan":60510,"à§ĩয়":60511,"çıłå®Ŀ":60512,"خطط":60513,"mind":60514,"contents":60515,"_height":60516,"Ġrebounds":60517,"ĠпозволÑıеÑĤ":60518,"ĠYorkshire":60519,"黯":60520,"è¿Ļæł·çļĦè¯Ŀ":60521,"ĠSubsequently":60522,"åħIJ":60523,"åı¯è§Ĩ":60524,"æľ¬æ³ķ":60525,"è¡ĮæĶ¿ç®¡çIJĨ":60526,"%@":60527,"说çļĦæĺ¯":60528,"å¸ĪçļĦ":60529,"Cut":60530,"空åĨĽ":60531,"Ġgrill":60532,"åħ³ç³»åΰ":60533,"Ġattacker":60534,"çĪ¶äº²çļĦ":60535,"西éŨ":60536,"ĠÑģовеÑĤ":60537,"åĻĹ":60538,"æľĥèѰ":60539,"æī¹éĩı":60540,"ÑĪее":60541,"並æ²Ĵæľī":60542,"Ġlangs":60543,"changes":60544,"大éĥ½":60545,"å¼Ĥåľ°":60546,"Ġprogresses":60547,"æĸ°æĹ¶ä»£ä¸ŃåĽ½çī¹èī²ç¤¾ä¼ļ主ä¹ī":60548,"Wat":60549,"åºĶæĮī":60550,"Ù쨹":60551,"Ġmilitar":60552,"-seconds":60553,"Ġmungkin":60554,"å¾Īä½İ":60555,"лаÑĢ":60556,"Ġyielding":60557,"arem":60558,"ادÙī":60559,"ĠRising":60560,"Ġjer":60561,"ĠIntellectual":60562,"Steve":60563,"Ġê°Ģì§Ģ":60564,"\\Eloquent":60565,"ipot":60566,"Ġtraverse":60567,"Ġentwick":60568,"ç»Ħç»ĩéĥ¨":60569,"ĠDocumentation":60570,"-room":60571,"Ord":60572,"Ġmunicipalities":60573,"Ġnalukop":60574,"dek":60575,"ĠسÙĪ":60576,"é¢ģå¥ĸ":60577,"ĠNobody":60578,"otoxicity":60579,"è¿Ļæł·çļĦ人":60580,"atl":60581,"ĠAnim":60582,"주ìĿĺ":60583,"kommen":60584,"åĬ¨æ¼«":60585,"-router":60586,"hadow":60587,"ĠJude":60588,"Ġformulate":60589,"Ġmentre":60590,"Anyway":60591,"students":60592,"resso":60593,"åĢŁè´·":60594,"æ´Ĺ涤":60595,"çľ¾äºº":60596,"ä¹ĭäºİ":60597,"Stats":60598,"à¸Ĺี":60599,"亮度":60600,"å°ı说ç½ij":60601,"ére":60602,"Staff":60603,"_link":60604,"ĠPhotography":60605,"éħ±æ²¹":60606,"çļĦåľ¨":60607,"çļĦèģĶç³»":60608,"è¿ĺè®°å¾Ĺ":60609,"âm":60610,"Fibonacci":60611,"Bron":60612,"Ġdeber":60613,"มà¸Ļ":60614,"ĠInjury":60615,"ollower":60616,"Ġdescended":60617,"ĠCisco":60618,"ĠEva":60619,"arke":60620,"æ¸ħæ¾Ī":60621,"Ġتخ":60622,"ç®Ģæĺĵ":60623,"èįīèİĵ":60624,"_label":60625,"à¹Ģà¸Ķียว":60626,"Ġprophets":60627,"лле":60628,"头æĻķ":60629,"ระà¸Ķัà¸ļ":60630,"èı²å¾ĭ":60631,"Ġscholarships":60632,"Ġ\"(":60633,"impact":60634,"ĠBibliography":60635,"ĠÑĥÑĢовенÑĮ":60636,"XP":60637,"Ġoutbreaks":60638,"使ç͍èĢħ":60639,"Ġdomest":60640,"Ġpenetrate":60641,"wiki":60642,"entukan":60643,"ĠNAME":60644,"Ġetter":60645,"wang":60646,"infect":60647,"ĠTYPE":60648,"Ġ**(":60649,"ĠÑħÑĢа":60650,"æ½Ľ":60651,"Ġdestined":60652,"Ġallo":60653,"Ġblunt":60654,"åĬ¡å¿ħ":60655,"å¹³æĸ¹åħ¬éĩĮ":60656,"fra":60657,"个个":60658,"è¿ĩæĿ¥äºĨ":60659,"Ġdelves":60660,"اÙĨت":60661,"åĨľå®¶":60662,"ãĤĮãģ¦":60663,"ĠRhodes":60664,"_ids":60665,"ĠThem":60666,"iales":60667,"Ġmempun":60668,"Ġboiler":60669,"åĺĢ":60670,"ĠHistoria":60671,"Fil":60672,"definition":60673,"ĠÑģÑĥб":60674,".Control":60675,"\"}Ċ":60676,"Ġthrill":60677,"Ġcorrelate":60678,"ĠGuidance":60679,"æĬķ身":60680,"¤×ĺ":60681,"اÙĦÙħÙĬÙĦ":60682,"ĠìľĦíķľ":60683,"ä¸ĸ代":60684,"åĪĻ为":60685,"èıł":60686,"Ġairports":60687,"×Ļ×ij×Ķ":60688,"Ġawaiting":60689,"ĠGrow":60690,"Ġtril":60691,"è¯¥é¡¹çĽ®":60692,"奶ç²ī":60693,"ä½ľåĵģçļĦ":60694,"ispers":60695,"моÑĤÑĢи":60696,"viv":60697,"Ġfoc":60698,"ĠEj":60699,"Ġquem":60700,"对ä»ĸçļĦ":60701,"çĥŁèįī":60702,"çĻ«çĹ«":60703,"(ex":60704,"von":60705,"ĠMV":60706,"ĠHang":60707,"èĬĭ":60708,"ĠspoÅĤ":60709,"_query":60710,"ĠNFT":60711,"à¸Ĺีà¹Īสุà¸Ķ":60712,"+C":60713,"ãĤĴä½ľ":60714,"-pass":60715,"å¥Ĺè·¯":60716,"Ġpsic":60717,"Ġpierws":60718,"Ġstab":60719,"Ġtestosterone":60720,"æİ§åĪ¶åľ¨":60721,"ĠAutomatic":60722,"иÑĢа":60723,"Ġusa":60724,"è¦ģåĿļæĮģ":60725,"æīĢæľīåζ":60726,"éĻį鼨":60727,"æĽ´å¤ļçļĦæĺ¯":60728,"Ġoriginating":60729,"ipper":60730,"ĠIMF":60731,"è¶Ĭåıij":60732,"çĺ«":60733,"å¦ĩ人":60734,"ĠWyoming":60735,"ĠíĻĶ":60736,"tot":60737,"ĠVAT":60738,"Ġafterward":60739,"arger":60740,"åĸľå¥½":60741,"æĽ¾ç¶ĵ":60742,"æĪĺæĸĹåĬĽ":60743,"igit":60744,"åİĮæģ¶":60745,"/ex":60746,"ĠYug":60747,"rato":60748,"ccoli":60749,"Ġverso":60750,"Alpha":60751,"-flow":60752,"ĠParticular":60753,"ä¸ŃçļĦä¸Ģ个":60754,"ichever":60755,"ĠBenedict":60756,"Ġcompetitiveness":60757,"Ġжела":60758,"ĠCres":60759,"Ñģков":60760,"ãĤıãģĽ":60761,"Ġaerobic":60762,"=y":60763,"abh":60764,"ĠFeld":60765,"енного":60766,"ickness":60767,"ytu":60768,"Ġkommt":60769,"æĬļåħ»":60770,"Ġfungus":60771,"åĮ»æĬ¤äººåijĺ":60772,"åįķä»·":60773,"-bearing":60774,"Ġìĭľìŀij":60775,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ":60776,"Ġà¸ĸ":60777,"icable":60778,"ĠHEL":60779,"Ġpossibile":60780,"æĪĢ":60781,"ĠسÙĬ":60782,"Img":60783,"ازÙĩ":60784,"Pointer":60785,"Ġzod":60786,"æ±ĩç¼ĸ":60787,"Ġвоздей":60788,"æĹłå¿§":60789,"yla":60790,"amu":60791,"ĠRaven":60792,"Ġusar":60793,"Ġrains":60794,"çŁ³èĨı":60795,"éĻ·åħ¥äºĨ":60796,"Mex":60797,"æł¸æŁ¥":60798,"\"ï¼Į":60799,"ĠmÃŃst":60800,"åĽ½äºº":60801,"hyp":60802,"ä¸Ģä»¶äºĭ":60803,"ĠTranscript":60804,")-(":60805,"ØŃدث":60806,"ĠíļĮ":60807,"çļĦä¸Ģ大":60808,"榻":60809,"åģ¥åº·åıijå±ķ":60810,"ĠSurgical":60811,"KY":60812,"æİ¨èĸ¦":60813,"ĠCanadians":60814,"itin":60815,"cone":60816,"_dist":60817,"Ġintimacy":60818,"Ġrehiyon":60819,"лого":60820,"ünst":60821,"Ġnep":60822,"Ġ×IJ×Ĺ×ĵ":60823,"æł¹æľ¬ä¸Ĭ":60824,"ä»ħ为":60825,"Ġoficial":60826,"ĠPAT":60827,"ä»»åĬ¡çļĦ":60828,"ĠاسÙĦاÙħ":60829,"举åįĹäºļ":60830,"æĦŁè¬Ŀ":60831,"rances":60832,"ĠSlav":60833,"对è¿Ļ个":60834,"lichkeit":60835,"Patient":60836,"atis":60837,"åĩĿåĽº":60838,"×Ļת×Ķ":60839,"æľĹ读":60840,"è¯Ľ":60841,"âĢĿ.Ċ":60842,"جا":60843,"à¸Ĺำà¸ĩาà¸Ļ":60844,"ĠNumerical":60845,"aplenty":60846,"Ġtempat":60847,"lioma":60848,"ĠBasis":60849,"anganese":60850,"§×¦":60851,"ಾà²Ĺ":60852,"Hay":60853,"sometimes":60854,"Ñĩника":60855,"Ġtradu":60856,"ç¾²":60857,"ித":60858,"ãģ«ãģĬãģĦãģ¦":60859,"å¹¢":60860,"deep":60861,"Ġnumbersaplenty":60862,"ìł¸":60863,"ë¨":60864,"Ġbukan":60865,"çĽijçĿ£æ£ĢæŁ¥":60866,"Ġê²ĥìĿĦ":60867,"ãĤĨ":60868,"è¿Ħä»Ĭ":60869,"growth":60870,"ĠMakes":60871,"çŁ¥è¯ĨåĪĨåŃIJ":60872,"ĠInitialize":60873,"half":60874,"utet":60875,"ä»ĸ便":60876,"ä¹Łè®©":60877,"ĠYun":60878,"åĬŁæ³ķ":60879,"meno":60880,"çIJĨ论ä¸İ":60881,"ÑĤÑĭй":60882,"ಲà³įಲ":60883,"Ġë¡ľ":60884,"Ġinaccurate":60885,"Ġmengen":60886,"åĪĨåŃIJçļĦ":60887,"Ġmoeten":60888,"Ġmempunyai":60889,"deal":60890,"Ġconcerts":60891,"arine":60892,"éĩįåıł":60893,"æ·±åİļ":60894,"à¸ļà¸Ĺ":60895,"èī¯å¿ĥ":60896,"-ban":60897,"é£İ湿":60898,"Ñĩик":60899,"/bootstrap":60900,"åĬ¨åĬĽåѦ":60901,"éĿ¢åħ·":60902,"ophagy":60903,"èĥ½æĬĬ":60904,"ÑĨов":60905,"-mode":60906,"以ä¸ĭæĺ¯":60907,"æĬ½åıĸ":60908,"ERSON":60909,"人大代表":60910,"Ġnouvelle":60911,"Ġnab":60912,"å½Ŀ":60913,"Bu":60914,"Ġhats":60915,"å¿ĥè·³":60916,"åħ¨è¿ĩç¨ĭ":60917,"Ġplayful":60918,"ĠCompute":60919,"ĠSkip":60920,"stown":60921,"ĠBuk":60922,"ä½Ĩä»ĸ们":60923,"ĠShot":60924,"ÑĨиÑĺе":60925,"Ġáreas":60926,"ĠBeit":60927,".printStackTrace":60928,"ĠMeasuring":60929,"大æĦı":60930,"ä¸ĵè¾ij":60931,"Ġvaak":60932,"á»ĩu":60933,"å®ŀäºĭæ±Ĥ":60934,"itious":60935,"ÑĢиÑģ":60936,"Ġrainbow":60937,"åħīæºIJ":60938,"Ens":60939,"Ġà¦ķà¦¾à¦ľ":60940,"ĠResponsibilities":60941,"ãģŀ":60942,"åľ¨æµ·":60943,"ĠNV":60944,"èµĦéĩijçļĦ":60945,"Ġscalable":60946,"Ġbambini":60947,"Ġspice":60948,"éĽı":60949,"Ġsolver":60950,"ustainable":60951,"hana":60952,"rios":60953,"级åĪ«çļĦ":60954,"æĵĭ":60955,"屬æĸ¼":60956,"ä¸ĸç´Ģ":60957,"даниÑı":60958,"ĠRobot":60959,"åı²æĸĻ":60960,"ĠCoverage":60961,"Ġfamille":60962,"åį«çĶŁéĹ´":60963,"Ġпедаг":60964,"è´«åĽ°æĪ·":60965,"essential":60966,"Ġcontend":60967,"å¤©çľŁ":60968,"ĠÚ©ÙĪ":60969,"Ġë³µ":60970,"Ġkleine":60971,"Dur":60972,"Ñħо":60973,"mani":60974,"ä¼ĹæīĢåij¨":60975,"олÑĮзÑĥ":60976,"Ġappraisal":60977,"Ġconstructions":60978,"ĠPresidential":60979,"ĠUniversities":60980,"রà§įà¦ķ":60981,"ĠTank":60982,"çݰ任":60983,"åζè£ģ":60984,"æİ¨æĸŃ":60985,"票æĪ¿":60986,"Ġantigens":60987,"Ġinfin":60988,"Ġdegeneration":60989,"æ¯ı天éĥ½":60990,"orbid":60991,"äºijåįĹçľģ":60992,"èģĨåIJ¬":60993,"ĠSAS":60994,"Ġstrategically":60995,"oce":60996,"insk":60997,"è´¨éĩı管çIJĨ":60998,"_queue":60999,"ĠMorm":61000,"Ġpoised":61001,"Ġneedles":61002,"Steps":61003,"Ġastronomical":61004,"ĠFi":61005,"Ġseinen":61006,"åĩºéĻ¢":61007,"ĠBurton":61008,"ãģ¹ãģ¦":61009,"Diff":61010,"horn":61011,"otin":61012,"Ġseize":61013,"ĠSentences":61014,"cuts":61015,"ä¸ĢæĹł":61016,"ĠOrders":61017,"ĠPetroleum":61018,"Ġprimeiro":61019,"ĠATM":61020,"Fraction":61021,"Stage":61022,"ĠThoughts":61023,"pee":61024,"Ġruined":61025,"Ġparasite":61026,"bay":61027,"ĉĠĠĠĠĠĠĠ":61028,"Ġsolvents":61029,"Ľá̽":61030,"ĠTL":61031,"Ġtawo":61032,"æĮ¯èį¡":61033,"openhagen":61034,"ĠReplies":61035,"Ġsubdivision":61036,"ĠTens":61037,"ĠBorrow":61038,"ogang":61039,"äºĮåŃĹ":61040,"å®Ľå¦Ĥ":61041,"Ġvod":61042,"æĹ¶æ®µ":61043,"Ġsumala":61044,"ĠSepar":61045,"ĠSpa":61046,"温å·ŀ":61047,"Ġinjections":61048,"ĠاÙĦرئÙĬس":61049,"ĠÑĥпоÑĤÑĢеб":61050,"ĠÙĪØ³ÙĦÙħ":61051,"mor":61052,"Äı":61053,"Ġاد":61054,"ÑĢоз":61055,"æĹ¶èĬĤ":61056,"çĶ¨åľ¨":61057,"дии":61058,"linux":61059,"å°±åĮ»":61060,"åıĪä¼ļ":61061,"è¾ĥ强":61062,"Ġcollectors":61063,"lander":61064,"ãĥ³ãģ®":61065,"à¦ĥ":61066,"èĥ§":61067,"声åĵį":61068,"Ġfibrobl":61069,"èºį":61070,"ĠIndo":61071,"åŁŁåIJį":61072,"åı¬å¼ĢäºĨ":61073,"è®ĬæĪIJ":61074,"}'":61075,"enzie":61076,"åīįæīĢæľª":61077,"ĠartÃŃculo":61078,"ÙİØ¨":61079,"æİ¥çº³":61080,"taÅĤ":61081,"ependant":61082,"çľĭçĿĢä»ĸ":61083,"硬度":61084,"ĠJulius":61085,"ç®Ģ便":61086,"Ġanatomical":61087,"ĠÂłĠÂłĠÂłĠÂł":61088,"Ġperennial":61089,"Ġflap":61090,"lein":61091,"ãģľ":61092,"ĠEfficient":61093,"éķ¿æĸ¹":61094,"transform":61095,".Net":61096,"ĠOccupational":61097,"ä¸Ń使ç͍":61098,"è¡ĢçļĦ":61099,"ฤษ":61100,"throws":61101,"ç»ĵæŀľçļĦ":61102,"Ġcemetery":61103,"ĠDG":61104,"åģļå®Į":61105,"retched":61106,"éĢłå°±":61107,"Ġsurprises":61108,"Ġpersecution":61109,"Ġcompulsory":61110,"ä¹ĵ":61111,"ĠBinding":61112,"ાàªĤ":61113,"ĠкаждÑĭй":61114,"Ġfalta":61115,"ĠHonda":61116,"Ġfunk":61117,"ographed":61118,"bildung":61119,"Ġxy":61120,"ÙģØªÙĩ":61121,"ĠMultip":61122,"Studies":61123,"çļĦ主人":61124,"Ġkunt":61125,"othalam":61126,"è¿IJæ²³":61127,"éªģ":61128,"ĠBetty":61129,"ĠLyon":61130,"Ġshedding":61131,"åĺĺ":61132,"åĩłä¹İæĺ¯":61133,"çļĦä¸Ńå¿ĥ":61134,"ielsen":61135,"æĺŁç©º":61136,"Band":61137,"ĠPapa":61138,"éĢģä¸Ĭ":61139,"KG":61140,"Ġargc":61141,"Ġaccelerating":61142,"ĠHurricane":61143,"platform":61144,"ä¸Ĭèħº":61145,"ĠÙħغ":61146,"Ġcrus":61147,"Ġcollaborations":61148,"Ġpronoun":61149,"ĠÑĢаÑģÑħод":61150,"âħł":61151,"Warning":61152,"ĠLodge":61153,"ómo":61154,"ĠÙĩÙĨ":61155,"Ġpsychologists":61156,"мÑĭÑģ":61157,"Lines":61158,"bash":61159,"hör":61160,"ratt":61161,"Ġmovable":61162,"Ġমানà§ģষ":61163,"è¿Ľä¸ĢæŃ¥æıIJé«ĺ":61164,"bred":61165,"igion":61166,"Ġstran":61167,"thermal":61168,"Ġmissionary":61169,"ĠRecommendations":61170,"Harry":61171,"ĠbyÅĤo":61172,"Ġ\"<<":61173,"å¹´çĶŁ":61174,"ä¸ĭå±±":61175,"пиÑģ":61176,"ä¸ĢæĥĬ":61177,"ãĤĪãģı":61178,"Ġdiversification":61179,"丣":61180,"peak":61181,"atorium":61182,"足足":61183,"ĠìĤ´":61184,"ĠGenerator":61185,"Kah":61186,"æģ¯çļĦ":61187,"Ġpresumed":61188,"advisor":61189,"ම":61190,"Ġvigorous":61191,"esia":61192,"ä½Ķ":61193,"еннÑĭй":61194,"Ġprofiling":61195,"urgy":61196,"Ġdeclares":61197,"ç»ĺåĽ¾":61198,"ieur":61199,"åħ±åIJĮçļĦ":61200,"chemia":61201,"ĠCoastal":61202,"Ġcoerc":61203,"ĠIntro":61204,"à¸ŀล":61205,"ĠPemb":61206,"Ġunl":61207,"ĠKP":61208,"åĺĹ":61209,"ĠجاÙĨ":61210,"()));Ċ":61211,"Ġignition":61212,"หà¸Ļà¹īาà¸":61213,"Ġveterinarian":61214,"çļĦåľ°ä½į":61215,"Ġzem":61216,"Ġaxios":61217,"ĠProfit":61218,"账款":61219,"Ġkhông":61220,"åŀ£":61221,"åIJĥåΰ":61222,"Quiz":61223,"ä¸ºæľŁ":61224,"ĠMorph":61225,"Ġpunk":61226,"è¿Ļæīįæĺ¯":61227,"isser":61228,"åĨįçݰ":61229,"æij¸äºĨ":61230,"ç¾İæĻ¯":61231,"Ġdocumento":61232,"ĠγÏħναικείο":61233,"Ġmaximizing":61234,"Ġcoment":61235,"ĠMoral":61236,"fac":61237,"ага":61238,"éŀĺ":61239,"rocytes":61240,"pra":61241,"è¿Ļæĺ¯ä»Ģä¹Ī":61242,"inguish":61243,"Break":61244,"è̦åIJĪ":61245,"Hom":61246,"Ġsaddle":61247,"ĠTil":61248,"ousel":61249,"رÙĬد":61250,"à¹ģรà¸ĩ":61251,"潮湿":61252,"Ġtis":61253,"ĠTeh":61254,"Ġcharacterised":61255,"大家好":61256,"ajÄħce":61257,"Ġcries":61258,"Ġwolves":61259,"ĠSanto":61260,"ĠSharp":61261,"æłijæŀĿ":61262,"ĠÙħÙĪØ§ÙĤع":61263,"'l":61264,"ritical":61265,"Ġbenefited":61266,"Ġfiz":61267,"Ġreminding":61268,"æİīçļĦ":61269,"éĺ´æĢ§":61270,"ĠProvides":61271,"ĠGul":61272,"å°ıå§ijå¨ĺ":61273,"éĤ£å¹´":61274,"æľŁçĽ¼":61275,"Ġadmired":61276,"-action":61277,"æīĵåĮħ":61278,"ä½łæĪij":61279,"å¾ĹçĽĬ":61280,"æµģæĦŁ":61281,"åľ£äºº":61282,"ifera":61283,"Ġbasil":61284,"DAO":61285,"Ġconfrontation":61286,"ĠINTRODUCTION":61287,"æ´¼":61288,"Joe":61289,"æµĭéªĮ":61290,"Ġíħ":61291,"çĦ¦æĢ¥":61292,"丰å¯Įå¤ļ彩":61293,"ĠRhode":61294,"ÑĢеÑĤÑĮ":61295,"ujo":61296,";;;;":61297,"ĠLINEAR":61298,"اÙĨÙĪ":61299,"ĠاÙĦتÙĪØ§ØµÙĦ":61300,"*(-":61301,"Ġtabs":61302,"ovy":61303,"æīĵ好":61304,"Ø´ÛĮ":61305,"åĪĢåħ·":61306,"ĠBufferedReader":61307,"λÎŃ":61308,"ĠWorker":61309,"å¿«éĢŁåıijå±ķ":61310,"variable":61311,"fera":61312,"Ġrepay":61313,"åŁİåł¡":61314,"तà¥Ģ":61315,"ĠÑĢаÑģпÑĢоÑģÑĤÑĢан":61316,"ĠICC":61317,"ĠNN":61318,"天线":61319,"æŀģ度":61320,"Ġrebels":61321,"ëŀij":61322,"Ġwhisper":61323,"ĠпÑĢоÑģÑĤÑĢан":61324,"ĠDunn":61325,"ĠاÙĦدÙĬÙĨ":61326,"(output":61327,"æľĥåħĴ":61328,"ĠMajesty":61329,".objects":61330,"ĠTW":61331,"Ġwag":61332,"osal":61333,"ĠInequ":61334,"Ġ/Ċ":61335,"اÙĦÙĥ":61336,"é£İæľº":61337,"Ġbald":61338,"Ġcoordinating":61339,"èłķ":61340,"ĠEyes":61341,"Ġrasp":61342,"Ġoutdated":61343,"Ġتست":61344,"à¸ļัà¸Ļ":61345,"REG":61346,"ĠдвижениÑı":61347,"-green":61348,"à±ģà°¨":61349,"tong":61350,"ŀáĢ":61351,".str":61352,"Ġpasse":61353,"'.Ċ":61354,"Kag":61355,"widet":61356,"éĿ³":61357,"çĸ¯æĭī":61358,"ç»ıæµİåıijå±ķçļĦ":61359,"Ãło":61360,"Ġprofessions":61361,"å¸Ĥå̼":61362,"åħ¶ä¸Ńæľī":61363,"åĩºç§Łè½¦":61364,"ĠاÙĦØ£ØŃÙħر":61365,"ç´°èĥŀ":61366,"è¼Ŀ":61367,"СÑĤа":61368,"éģĩåΰçļĦ":61369,"Extra":61370,"udio":61371,"ĠSeoul":61372,"Ġloci":61373,"åįİ侨":61374,"Ġendured":61375,"ZE":61376,"osting":61377,"巨人":61378,"ÙijØ©":61379,"Ġsupplemental":61380,"CES":61381,"æľºçIJĨ":61382,"實çı¾":61383,"èĩªçͱçļĦ":61384,"转åıĺ为":61385,"Ġdisagreement":61386,"ĠAlgorithms":61387,"Ġpaperwork":61388,"Ġsqueezed":61389,"RH":61390,"_rate":61391,"æĹ¶èĢĮ":61392,"Ñĥл":61393,"Ġниз":61394,"ç¾İèģĶåĤ¨":61395,"è´§è¿IJ":61396,"ãģ¨ãģªãĤĭ":61397,"Ġtiger":61398,"ä¼ļ社":61399,"Ġsignifies":61400,"Ġredis":61401,"cmd":61402,"fest":61403,"Ġlays":61404,"chal":61405,"ĠTheoretical":61406,"åĴĮæĿİ":61407,"éĤĦåľ¨":61408,"ä»Ĭ天æĺ¯":61409,"è¹Ļ":61410,"ĠCONCLUS":61411,"Ġgouvern":61412,"Ġpriced":61413,"Ġsering":61414,"Ġechoed":61415,"Ġsupplementation":61416,"-q":61417,"seven":61418,"çłĶç©¶åijĺ":61419,"æĻļä¼ļ":61420,"è¶ĭäºİ":61421,"Ġcorrelates":61422,"Ġpreserv":61423,"à¸Ńล":61424,"æĬĢæľ¯åĪĽæĸ°":61425,"สà¸Ńà¸ĩ":61426,"ĠпаÑĤ":61427,"Ġcoincidence":61428,"pliance":61429,"ĠInstitutes":61430,"Ġhomeschool":61431,"åĪĨåĪ¥":61432,"ĠíĹ":61433,"ĠBirthday":61434,"Ġê²°ê³¼":61435,"ftig":61436,"Ġamor":61437,"Ġretina":61438,"uchs":61439,"ਤ":61440,"ĠнеобÑħодим":61441,"atang":61442,"天åIJİ":61443,"Ġبسبب":61444,"à¸Ľà¸ģ":61445,"kv":61446,"Inside":61447,"Liber":61448,"ĠDw":61449,"åħ¬æĸĩ":61450,"åĿŀ":61451,"æĨĤ":61452,"navbar":61453,"çĤ½":61454,"nec":61455,"åIJijä¸ĬçļĦ":61456,"Ġgroundbreaking":61457,"ĠBillboard":61458,"åĵªæĢķæĺ¯":61459,"ĠOmega":61460,"widetilde":61461,"Ġcipher":61462,"ĠCats":61463,"Ġstub":61464,"arto":61465,"ĠاطÙĦ":61466,"è¾ĥéķ¿":61467,"-cal":61468,"à¥įप":61469,"ĠTradition":61470,"Ġheavens":61471,"à§Ģত":61472,"Ġê²Ģ":61473,"ĠSherman":61474,"Ġkabanay":61475,"Ġarsen":61476,"Ġpiles":61477,"ĠتÙĦÙĥ":61478,"ĠÕį":61479,"Ġ׼×ŀ×ķ":61480,"Ġmientras":61481,"ĠHers":61482,"æĪijä¸įèĥ½":61483,"好åĩł":61484,"ĠWitt":61485,"-di":61486,"historic":61487,"åIJµæŀ¶":61488,"âĬĻ":61489,"Ġinland":61490,"çļĦç®Ĭ":61491,"одей":61492,"åįļè§Ī":61493,"ç¹ŀ":61494,"doms":61495,"Ġmoderation":61496,"Ġsurrend":61497,"Ġcommunist":61498,"Ġconsiste":61499,"ĠACE":61500,"Ġengag":61501,"ĠÙħاÙĨÙĨد":61502,"lea":61503,"ĠMare":61504,"ĠHockey":61505,"æ»ĩ":61506,"Ġboarding":61507,"ä¸¥æł¼æī§è¡Į":61508,"cases":61509,"posts":61510,"Ġrenamed":61511,"大èħ¿":61512,"æŃ£ç»ı":61513,"ĠQi":61514,"à¥įà¤ķ":61515,"erala":61516,"á̱á̬áĢ":61517,"Ġbrow":61518,"ppling":61519,":ĊĊĊ":61520,"identity":61521,"éĢĻä½į":61522,"Ġmarriages":61523,"Ġmanagerial":61524,"çŃīé¢ĨåŁŁ":61525,"oporosis":61526,"веÑĢÑģиÑĤеÑĤ":61527,"either":61528,"ĠHeather":61529,"Ġreceivers":61530,"ĠcaÅĤ":61531,"Ġnmi":61532,"Ġcontrasts":61533,"æijĨæīĭ":61534,"Ġcereal":61535,"å®¶ç͍":61536,"Metadata":61537,"hé":61538,"raduate":61539,"ifth":61540,"ĠOD":61541,"-fund":61542,"ddot":61543,"à¸Ľà¸ı":61544,"大å§IJ":61545,"ä¸ĭæ°´":61546,"llo":61547,"æ¸ħæĻ°çļĦ":61548,"Ġbroadcasting":61549,"ĠMATLAB":61550,"æľīèī²":61551,"Ġoccupations":61552,"Walk":61553,"ä¸į缸åIJĮ":61554,"å®īå®ģ":61555,"Ġ})ĊĊ":61556,"æŃ¤ç§į":61557,"Ġavons":61558,"åĶ®åIJİ":61559,"Ġvoluntarily":61560,"Protocol":61561,"çIJ¢ç£¨":61562,"Ġsull":61563,"ärt":61564,"失è°ĥ":61565,"popular":61566,"ĠZiel":61567,"æĬ¬æīĭ":61568,"ĠNOTE":61569,"\\{\\":61570,"]){Ċ":61571,"entuk":61572,"Ġkop":61573,"Ġkrit":61574,"Ġoutro":61575,"Ġzas":61576,"æīĵæī«":61577,"skiej":61578,"æ·±åħ¥å¼Ģå±ķ":61579,"ĠFach":61580,"ipel":61581,"使ä»ĸ们":61582,"çĥŃå¿ĥ":61583,"atalog":61584,"Ġsuspend":61585,"Ġneurotrans":61586,"éĥ¨åĴĮ":61587,"åķĥ":61588,"åı£å¤´":61589,"Viet":61590,"æķĸ":61591,"Ġissuance":61592,"ì±ħ":61593,"ĠLent":61594,"人åĢij":61595,"ĠElig":61596,"},{":61597,"è¡°éĢĢ":61598,"hua":61599,"ķáĢ":61600,"rund":61601,"ĠChuck":61602,"Ġmanifested":61603,"åĪĨ辨çİĩ":61604,"Migration":61605,"inho":61606,"...âĢĿ":61607,"害ç¾ŀ":61608,".now":61609,"ÑĥлÑĮ":61610,"ĠAnonymous":61611,"ัà¸Īà¸Ī":61612,"æĬķæľº":61613,"ĠWestminster":61614,"Ġdashboard":61615,"ĠPon":61616,"ensa":61617,"èĩªå·±ä¹Ł":61618,"ðĿĹ":61619,"Ġcomplaining":61620,"-threatening":61621,"大å¹ħ度":61622,"ĠHidden":61623,"iesiÄħ":61624,"ød":61625,"ĠRichards":61626,"åIJ»åIJĪ":61627,"ĠвеÑĤ":61628,"ĠSynonyms":61629,"Dark":61630,"ÃŃz":61631,"Ġconscient":61632,"-dep":61633,"à½Ĥ":61634,"çŃīå½¢å¼ı":61635,"Ġretains":61636,"è§ĤçļĦ":61637,"èĢĮ对":61638,"说çļĦè¯Ŀ":61639,"æ°ijåĬŀ":61640,"entu":61641,"ĠInquiry":61642,"ä¸ĭæĹ¬":61643,"é«ĺæĸ°åĮº":61644,"å®īçĦ¶":61645,"ราย":61646,"Ġmarital":61647,"Ġসহ":61648,"ĠможеÑĤе":61649,"ĠRoc":61650,"ĠGD":61651,"ä¹ĭåĪĨ":61652,"代ä¼ļ":61653,"Ġhumano":61654,"ĠизменениÑı":61655,"ĠEh":61656,"Ġbland":61657,"主æĴŃ":61658,"éĿĴéĵľ":61659,"Ġ%>Ċ":61660,"ĠاÙĦØ£Ùħر":61661,"Ġflavour":61662,"/#":61663,"SPJ":61664,"Ġশিà¦ķà§įষ":61665,"Ġassigning":61666,"å«Įå¼ĥ":61667,"ĠInstitutional":61668,"Autor":61669,"ĠShore":61670,"ĠXXX":61671,"ĠIntermediate":61672,"ä¸įçķĻ":61673,"ĠHeights":61674,"itoring":61675,"Ġmarkedly":61676,"妥åįı":61677,"quality":61678,"é«ĺå°ļ":61679,"æĸ¯åį¡":61680,"Ġíģ¬":61681,"ãģĵãģ¨ãģ§":61682,"Ġ---|---|---|---":61683,"å¾¹åºķ":61684,"ĠVoy":61685,"à¤¾à¤ľ":61686,"Ġadministrat":61687,"Ġverbose":61688,"ĠOfficers":61689,"ä¸ŀ缸":61690,"dos":61691,"ĠMU":61692,"ä¸Ģ亮":61693,"ĠDone":61694,"oprop":61695,"indic":61696,"éĿĴçĿ":61697,"Ġhumanos":61698,"à¹ģมà¹Ī":61699,"Ġidol":61700,"ç²¾ç¥ŀæĸĩæĺİ":61701,"COUNT":61702,"uale":61703,"SUV":61704,"Ġtapestry":61705,"ĠOrchestra":61706,"}f":61707,"ĠzaÄį":61708,"Ġadolescence":61709,"ابت":61710,"æ³ķå¾ĭçļĦ":61711,"Divide":61712,"Ġlagi":61713,"unami":61714,"Ġrus":61715,"ĠdalÅ¡ÃŃ":61716,"Ġrall":61717,"Ġflor":61718,"ĠکاÙĩØ´":61719,"ĠMusical":61720,"Ġkomt":61721,"çļĦåIJĹ":61722,"ĠLig":61723,"ĠOL":61724,"creens":61725,"Ġcontacting":61726,"Ġstylish":61727,"ĠCyprus":61728,"ETER":61729,"LEX":61730,"EPA":61731,"=%":61732,"Ġwiki":61733,"[D":61734,"μη":61735,"ĠDigest":61736,"ĠâĤ¹":61737,"_":61738,"Ġdenominators":61739,"ĠCarson":61740,"åįĬå²Ľ":61741,"Ġmotorcycle":61742,"Ġkahenera":61743,"bons":61744,"ç»ıè´¸":61745,"ĠоÑģи":61746,"=$(":61747,"Ġдвига":61748,"ç¥Ŀè´º":61749,"Une":61750,"ä¸ºéĽ¶":61751,"ussia":61752,"Û±Û°":61753,"eby":61754,"âĢ¢âĢ¢":61755,"ĠPersons":61756,",V":61757,"gement":61758,"kun":61759,"å¸ĤåľºçĽij管":61760,"Äįen":61761,"Ġprzep":61762,"Ġpendidikan":61763,"Ġgihulagway":61764,"asuk":61765,"ĠÑĤогда":61766,"Ãļ":61767,"Ġconverge":61768,"ĠXbox":61769,"leuk":61770,"ĠRehabilitation":61771,"ĠÚĨÙĨد":61772,"ĠRSS":61773,"Ġchr":61774,"Ġ->Ċ":61775,"Ġplayoff":61776,"-tr":61777,"arang":61778,"Ġbins":61779,"olysis":61780,"Ġendeavors":61781,"YL":61782,"ein":61783,"nv":61784,"chim":61785,"ieni":61786,"(dp":61787,"åł´åIJĪãģ¯":61788,"Ġignorant":61789,"YT":61790,"Ġashamed":61791,"ĠFB":61792,"äºĮåįģäºĶ":61793,"ĠÑĢавен":61794,"ĠIncluded":61795,"ignon":61796,"ĠRevol":61797,"èĮ¯":61798,"çļĦéĩįè¦ģç»ĦæĪIJéĥ¨åĪĨ":61799,"Ġamplified":61800,"Ġlakh":61801,"æŀ¸":61802,"Ġframing":61803,"ĠColeman":61804,"×ģ":61805,"Ġfrente":61806,"è¦ģçľĭ":61807,"åĮĸçŁ³":61808,"éĢļå¾Ģ":61809,"Ġsmells":61810,"ä½ĻåIJį":61811,"Ġendorsed":61812,"Ġhatch":61813,"Ġcontractual":61814,"Ġadjustable":61815,"Ġresignation":61816,"å¯ĦåŃĺåύ":61817,"åĽŀæĥ³":61818,"ÙĢÙĢÙĢÙĢ":61819,"Ġunemployed":61820,"ĠCrawford":61821,"PCR":61822,"votes":61823,"ophe":61824,"Ġnoises":61825,"Ġcostumes":61826,"ĠÙĩÛĮ":61827,"ĠEmbed":61828,"ĠнедоÑģÑĤа":61829,"ĠSodium":61830,"utong":61831,"å°±æĪIJäºĨ":61832,"çϾåĪĨæ¯Ķ":61833,"Cet":61834,"æĪij们åİ»":61835,"è´Ńç½®":61836,"çļĦéĿŀ":61837,"urent":61838,"缺åı£":61839,"亦åı¯":61840,"Ġrhythms":61841,"Ġeternity":61842,"soon":61843,"æ¼¾":61844,"payment":61845,"rino":61846,"Ġsupplément":61847,"ĠVatican":61848,"Ġalias":61849,"ĠKes":61850,"imenti":61851,"åĪĿ次":61852,"তà§įর":61853,"ipzig":61854,"ĠAlg":61855,"ĠTournament":61856,"Ġlymphoma":61857,"ĠWinston":61858,"ület":61859,"Como":61860,"Ġmethanol":61861,"Ġembarrassed":61862,"rière":61863,"åħļæł¡":61864,"æ¯Ķè¾ĥå¤ļ":61865,"Ġprogressed":61866,"Ġeliminates":61867,".utils":61868,"alom":61869,"ĠоÑģÑĤ":61870,"visual":61871,"褪":61872,"odon":61873,"ä¸įå®Į":61874,"uropa":61875,"series":61876,"çı¾è±¡":61877,"ĠSurvival":61878,"_sp":61879,"ĠLima":61880,"Ġspun":61881,"ãģªãģĭãģ£ãģŁ":61882,"-west":61883,"好åĥıæĺ¯":61884,"à»":61885,"计æĹ¶":61886,"èµ°çĿĢ":61887,"Ġmieux":61888,"Debug":61889,"ĠLudwig":61890,"Ġhou":61891,"ç¶ľ":61892,"ĠNYC":61893,"éĤ£ä¸ĢåĪ»":61894,"Observ":61895,"ĠOakland":61896,"(to":61897,"Ġambassador":61898,"Ġlabelled":61899,"XN":61900,"ÐŀÑĤвеÑĤ":61901,"ä¸ŃéĺŁ":61902,"ç½µ":61903,"éĺ²æ±Ľ":61904,"Ġargv":61905,"åĵªæľī":61906,"Ġ-.":61907,"é£Ļ":61908,"Ġpathogenic":61909,"Effective":61910,"妨ç¢į":61911,"Ġì¢ĭ":61912,"irse":61913,"Ġatop":61914,"ĠĊĊĊ":61915,"Ġactivism":61916,"ITAL":61917,"å®ĹçļĦ":61918,"tis":61919,"byter":61920,"é¢Ĩ导åĴĮ":61921,"为ä»Ģä¹Īä¸į":61922,"Õ¾Õ¡Õ®":61923,"!\");Ċ":61924,"ĠTao":61925,"ç»ļ":61926,"ÑĤнÑĭÑħ":61927,"ÏĦÏħ":61928,"Ġsock":61929,"Ġabord":61930,"ovsky":61931,"ĠClaude":61932,"è¾¹ä¸Ĭ":61933,"ĠcaracterÃŃsticas":61934,"dehyde":61935,"کس":61936,"ĠShim":61937,"Ġmultinational":61938,"à§ģব":61939,"Ġclimates":61940,"ĠSylv":61941,"ä»İè¿Ļ个":61942,"åijĬè¯īè®°èĢħ":61943,"ätz":61944,"Cover":61945,"\\;":61946,"å¹Ĥ":61947,"æīĢè¦ģ":61948,"Ġumum":61949,"اجÙĩ":61950,"Ġadvocated":61951,"Boolean":61952,"Ġadditives":61953,"ĠMercedes":61954,"åīĽæīį":61955,"¦":61956,"Ġoud":61957,"Ġischemic":61958,"acea":61959,"ĠNET":61960,"_sort":61961,"ĠSaid":61962,"+d":61963,"รัà¸ģษ":61964,"Ġexhaustion":61965,"Ġdisrupted":61966,"æĪijåİ¿":61967,"çļĦ大åŀĭ":61968,"ÉĻl":61969,"asti":61970,"ĠнайÑĤи":61971,"ĠAdri":61972,"è¡ĮæĶ¿éĥ¨éŨ":61973,"çµIJåIJĪ":61974,"åħļåı²åŃ¦ä¹łæķĻèĤ²":61975,"ĠBasketball":61976,"mv":61977,"ĠHag":61978,"Ġcane":61979,"ĠÑģÑĤанови":61980,"ĠÑģÑĤоÑĢон":61981,"Ġaroma":61982,"å¤įåį°ä»¶":61983,"çļĦ形象":61984,"istine":61985,"ĠEins":61986,"под":61987,"æīĵèµ¢":61988,"çĽIJéħ¸":61989,"Ġspinach":61990,"Ġcil":61991,"Ġdific":61992,"Republic":61993,"rypto":61994,"ĉe":61995,"åĩºåİ»äºĨ":61996,"è¿ĺä¸įå¦Ĥ":61997,"并为":61998,"ĠÙ쨶":61999,"SEC":62000,"athe":62001,"Ġspawn":62002,"åį³åľ¨":62003,"Ġreinforcing":62004,"Ġcasualties":62005,"ĠCay":62006,"-saving":62007,"ĠDefic":62008,"Ġharmless":62009,"cache":62010,"Ġauthoritative":62011,"çļĦèģ²éٳ":62012,"ffield":62013,"üs":62014,"bh":62015,"ĉg":62016,"Ġvicious":62017,"åĪĺå¤ĩ":62018,"Â¥":62019,"ologische":62020,"å¤ĸåįĸ":62021,"letcher":62022,"Ġdevised":62023,")+(":62024,"ĠDamage":62025,"å¾Īåĸľæ¬¢":62026,"Ġthroughput":62027,"ĠCalories":62028,"ĠCz":62029,"ĠDum":62030,"ä¼´ä¾£":62031,"Affiliations":62032,"ĠOxygen":62033,"Ġunused":62034,"ĠLeslie":62035,"éĥ½å¿«":62036,"ĠÐĴе":62037,"Ġcompletes":62038,"Ġseparates":62039,"ĠChev":62040,"æĬĵä½ıäºĨ":62041,"ĠInstant":62042,"ĠEnhance":62043,"Ġspé":62044,"Ġplayoffs":62045,"Ġideally":62046,"ASP":62047,"ĠInflation":62048,"Ġemitting":62049,"åı¸å¾Ĵ":62050,"áĥĽ":62051,"Ġattendees":62052,"\"[":62053,"ĠRET":62054,"ĠNed":62055,"ä¹ĭåŃIJ":62056,"ĠاÙĦاÙħ":62057,"à¹Ģริà¹Īม":62058,"æĭĸå»¶":62059,"subseteq":62060,"icillin":62061,"Ġsharks":62062,"uvant":62063,"売":62064,"çªģåĩ»":62065,"\"That":62066,"ÑĢаÑħ":62067,"éĴĽ":62068,"ĠAlleg":62069,"æ·±è¿ľ":62070,"æ£Ģçĸ«":62071,"أة":62072,"×ķ×ij×Ķ":62073,"åĬ¹æŀľ":62074,"ä¸İ管çIJĨ":62075,"Ġáll":62076,"ĠPill":62077,"åIJijæĿ¥":62078,"lavery":62079,"ĠÑĨен":62080,"æ·±æ·±åľ°":62081,"ä¸įèĢIJ":62082,"åζåĨ·":62083,"kerja":62084,"Ġweil":62085,"ĠAndreas":62086,"ĠPRES":62087,"à©°":62088,"éģİäºĨ":62089,"诱åıij":62090,"lew":62091,"ĠHip":62092,"Ġreacts":62093,"Ġhydrocarbon":62094,"çļĦéĥ½æĺ¯":62095,"ĠÙ쨱Ùĩ":62096,"æŃ¦èĢħ":62097,"ĠCorner":62098,"uttered":62099,"NM":62100,"ĠÃĩ":62101,"å°Ĩ以":62102,"Ġনি":62103,"ãĤ¿ãĤ¤":62104,"+j":62105,"rw":62106,"Ġcompliant":62107,".Service":62108,"omencl":62109,"éļIJèͽ":62110,"-channel":62111,"Ġbothered":62112,"_pre":62113,"ĠBears":62114,"æĸ¹å·®":62115,"JavaScript":62116,"ä¹ĭæ³ķ":62117,"Ġdissolve":62118,"Ġprism":62119,"æīĵéĢļ":62120,"åħ³éĶ®åŃĹ":62121,"ĠíĨµíķ´":62122,"ppa":62123,"िव":62124,"ĠâĹĭ":62125,"Ġcommissions":62126,"åı£ç¢ij":62127,"宫é¢Ī":62128,"ĠKazakh":62129,"Ġ({Ċ":62130,"Ġscav":62131,"éĩĩ纳":62132,"Ġgeop":62133,"ç¯Ĩ":62134,"鲤":62135,"Ġinformational":62136,"Ġanomaly":62137,"åĺ²ç¬ij":62138,"järvi":62139,"è¯Ħè®®":62140,"Ther":62141,"Vit":62142,"ĠMIC":62143,"è¿ĽçļĦ":62144,"èg":62145,"-MS":62146,">/":62147,"Ġemoc":62148,"ĠGuerra":62149,"Ġgrote":62150,"ĠMinute":62151,"Ġvisuals":62152,"ĠSilicon":62153,"Ġwhith":62154,",:":62155,"ĠSites":62156,"security":62157,"大åIJĮ":62158,"umped":62159,"ĠØŃÙħ":62160,"_post":62161,"Ġshutdown":62162,"yel":62163,"ĠPermanent":62164,"Ġmanners":62165,"èĬ±æľµ":62166,"è¿IJåĬ¨ä¼ļ":62167,"Answered":62168,"ĠComputers":62169,"çģ°å°ĺ":62170,"ä¸ī年级":62171,"ĠMittel":62172,"plastic":62173,"以ä¿Ŀè¯ģ":62174,"Ġpropositions":62175,".readline":62176,".ãĢIJ":62177,"ÙģÙĩ":62178,"Ġconfession":62179,"ĠسÙħا":62180,"亦æľī":62181,"åģļ好äºĨ":62182,"Ġnoteworthy":62183,"._Ċ":62184,"ĠOrg":62185,"çľģçķ¥":62186,"第äºĮç§į":62187,"fluor":62188,"ĠNortheast":62189,"yb":62190,"anders":62191,"ussed":62192,"Ġmethylation":62193,"}t":62194,"çļĦè¦ģ":62195,"æĹ¥çħ§":62196,"å¤ĸåķĨ":62197,"лее":62198,"ĠgiÃł":62199,"ĠPrix":62200,"Ġимп":62201,"æīĭå¥Ĺ":62202,"åłªç§°":62203,"________________________________________________________________":62204,"éºĴéºŁ":62205,"äºĨä¸ŃåĽ½":62206,"çĨł":62207,"ibernate":62208,"Ġisotope":62209,"ĠRag":62210,"idez":62211,"è°ĥçIJĨ":62212,"约å®ļçļĦ":62213,"!).":62214,"çϽäºĨ":62215,"ĠGastroenter":62216,"Ġtect":62217,"æŁļ":62218,"ĠHeidelberg":62219,"Ø´Ùħ":62220,"erne":62221,"ãĢĩ":62222,"ä¸Ńåŀĭ":62223,"æľĢçŁŃ":62224,"ĠFundamental":62225,"Ġrheumat":62226,"åij¨æģ©æĿ¥":62227,"以赴":62228,"Ġquizzes":62229,"Ġdenen":62230,"Ġcondensation":62231,"ĠPCB":62232,"ÚĺÙĩ":62233,"emper":62234,"Ġmotifs":62235,"ĠZenith":62236,"ADC":62237,"ĠOlder":62238,"Ĺ×Ķ":62239,"ĠFolk":62240,"åºĶæĺ¯":62241,"Ġpossibilit":62242,"çłĶ究表æĺİ":62243,"èİ«æĸ¯ç§ij":62244,"ĠGameObject":62245,"ouss":62246,"Ġzeb":62247,"楼ä¸Ĭ":62248,"æŁĶåĴĮ":62249,"çijŀåħ¸":62250,"âĢĿ?":62251,"Ġhealed":62252,"æŀĦçŃij":62253,"ĠÙħÙĨت":62254,".factory":62255,"Ġplateau":62256,"Ġpragmatic":62257,"Ġnets":62258,"ógica":62259,"Earlier":62260,"Kaliwatan":62261,"ĠSomal":62262,"ĠTales":62263,"abogon":62264,"ä¹IJéĺŁ":62265,"Ġswung":62266,"ĠRegiment":62267,"Ġnahimut":62268,"خذ":62269,".blogspot":62270,"åĴĮä¸ŃåĽ½":62271,"Ġquestionable":62272,"Slice":62273,"Ġquint":62274,"صÙĪØµ":62275,"举起":62276,"Extension":62277,"Ġ---------":62278,"Ġ®":62279,"Ġidentific":62280,"ĠConse":62281,"åĪĽè®¾":62282,"ç»Ĩå¿ĥ":62283,"-chain":62284,"accharide":62285,"ĠWade":62286,"Ġмаг":62287,"Ġunlawful":62288,"Ġdome":62289,"ä¸į许":62290,"ecu":62291,"Ġaltre":62292,"Ġabsorbing":62293,"aternity":62294,"ĠBatman":62295,"Ġnahimutangan":62296,"EEK":62297,"/bash":62298,"afi":62299,"æģIJæħĮ":62300,"æ²®":62301,"oyo":62302,"ĠدرÙħاÙĨ":62303,"å°ģè£ħ":62304,"Shell":62305,"ĠاÙĦبØŃ":62306,"Ġprohibition":62307,"Ġpeasant":62308,"ĠCeltic":62309,"Ġstakes":62310,"ĠÙĢ":62311,"ÙĬاÙĨ":62312,"ĠClement":62313,"å·¥æ¥Ń":62314,"cki":62315,"ël":62316,"大åı«":62317,"ä¸īçŃīå¥ĸ":62318,"åIJ¬è¯Ŀ":62319,"оÑĤÑĭ":62320,"ulares":62321,"éĿłåľ¨":62322,"Overview":62323,"以为æĺ¯":62324,"ĠاÙĦØ¥ÙĨجÙĦÙĬزÙĬØ©":62325,">The":62326,"ertz":62327,"ä¹łé¢ĺ":62328,"Ġëıħ":62329,"ĠPrediction":62330,"vh":62331,"ĠCMS":62332,"Ġuniversally":62333,"owering":62334,"ĠAfricans":62335,"Ġê°ĢëĬ¥":62336,"Ġwrink":62337,"ĠGothic":62338,"Ġtucked":62339,"chia":62340,"metro":62341,"IJ×Ļ×Ŀ":62342,"æķ°åĴĮ":62343,"æĬ½åĩº":62344,"æ¯ķä¸ļäºİ":62345,"æĿ¥å¾ĹåıĬ":62346,"ĠFGC":62347,"onderd":62348,"Ġflooded":62349,"ĠAlexandria":62350,"_ERROR":62351,"Ġnasty":62352,"ĠKaiser":62353,"主æķĻ":62354,"олеÑĤ":62355,"ĠзаÑĤем":62356,"Ġcrab":62357,"-table":62358,"Ġsurfact":62359,"prisingly":62360,"åıĹä¸įäºĨ":62361,"çļĦ人åı£":62362,"اتÙĩ":62363,"Ġembody":62364,"ÐŀÑģ":62365,"ãģŁãĤģãģ®":62366,"æľīçIJĨ":62367,"就走":62368,"转æĬĺ":62369,"æŀģèĩ´":62370,"Ġdomination":62371,"èĴ¼":62372,"Ġвели":62373,"å¸IJæĪ·":62374,"ĠÄijiá»ĥm":62375,"Ġinstitu":62376,"ð٤":62377,"Hal":62378,"Ġsess":62379,"çļĦä¸ĢçĶŁ":62380,"ĠBasil":62381,"Ġreacted":62382,"olución":62383,".Google":62384,"Division":62385,"Salary":62386,"obbies":62387,"Ġhalls":62388,"ĠBelle":62389,"ibili":62390,"åħ¬éĸĭ":62391,"à¸ŀà¸ļ":62392,"ãĤ¹ãĤ¯":62393,"ĠTECH":62394,"ÑĤиÑģÑĤи":62395,"微波":62396,"umping":62397,"Ġmalware":62398,"ÑĭÑħа":62399,"Ġcolonization":62400,"jvu":62401,"lj":62402,"Ġevidently":62403,"èĩ»":62404,"Ġpec":62405,"Ġatravés":62406,"Ġbp":62407,"ä¸įå®Įåħ¨":62408,"ÑĪении":62409,"Ġswallow":62410,"Ġresonate":62411,"Ġবিà¦Ń":62412,"ηγÎŃÏĤ":62413,"âĪĩ":62414,"å¸Ĥåľºç«ŀäºī":62415,"ĠChester":62416,"mable":62417,"Ġsut":62418,"Ġintrac":62419,"Ġ׼×ĵ×Ļ":62420,"Foundation":62421,"dots":62422,"ä½ıæīĢ":62423,"ĠاÙĦØ£ÙĨ":62424,"ãģijãĤĮãģ°":62425,"fahr":62426,"æľīæ¯Ĵ":62427,"Ġinfiltr":62428,"Ġmille":62429,"èIJ½åı¶":62430,"åĨ²éĶĭ":62431,"zeichnet":62432,"mask":62433,"heiten":62434,"Ġ-*-":62435,"æ²Ļ滩":62436,"Repo":62437,"Ġfondament":62438,"ĠLé":62439,"ĠDesc":62440,"ĠÑĢежи":62441,"Ġwetlands":62442,"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\":62443,"Ġkasadpan":62444,"è°¬":62445,"administ":62446,"å°ıçIJĥ":62447,"çαæĬ¤":62448,"è¯Ńè¨ĢçļĦ":62449,"ç¬ijçĿĢ说":62450,"ĠNue":62451,"ÈĽi":62452,"åľ¨ç©ºä¸Ń":62453,"çŃīåIJĮ":62454,"รà¹Īวม":62455,".sci":62456,"ĠProviding":62457,"ĠмÑĭÑĪ":62458,"饺åŃIJ":62459,"idences":62460,"ë¸Į":62461,"äºĨå¤ļå°ij":62462,"ĠÑĥÑģлÑĥ":62463,"ĠHuss":62464,"ĠLuft":62465,"æİ©çĽĸ":62466,"Ġà¸Ĥà¸Ńà¸ĩ":62467,"Ġappel":62468,"athic":62469,"è¿ĺ羣æĺ¯":62470,"Ġerected":62471,"åĿıæŃ»":62472,"Ġhydration":62473,"Ġgroupe":62474,"ĠWear":62475,"å°±çĽ´æİ¥":62476,"巨头":62477,"Ġbinomial":62478,"æĪijå¦Ī":62479,"ĠProble":62480,"åĻ¢":62481,"ĠByte":62482,"ĠGarcÃŃa":62483,"大米":62484,"åı¯ä»¥åĪĨ为":62485,"åĨįå¤ļ":62486,"/python":62487,"OFF":62488,"Те":62489,"ä¸įä¸Ģèĩ´":62490,"িà¦Ĺ":62491,"Ġhomeostasis":62492,"Ġinvestigates":62493,"ĠÑģопÑĢоÑĤив":62494,"ä¸įå¿į":62495,"éĢIJä¸Ģ":62496,"空æ°Ķä¸Ń":62497,"ÏģοÏĤ":62498,"åĪ¹è½¦":62499,"ĠCristo":62500,"Ġkali":62501,"åĽŀåįĩ":62502,"Ġoportun":62503,"ĠLys":62504,"acting":62505,"ĠStroke":62506,"ualmente":62507,"å®īå¾·":62508,"ĠNth":62509,"æķ¬ä¸ļ":62510,"ä»ĸè¿ĺæĺ¯":62511,"بÙĪØ¯":62512,"Ġsubordinate":62513,"-than":62514,"ĠMissing":62515,"ĠاÙĦÙ쨱":62516,"åĿIJæłĩç³»":62517,"Ġabruptly":62518,".contrib":62519,"ĠتÙĦ":62520,"/square":62521,"å¡«æĬ¥":62522,"#SPJ":62523,"ĠUSDA":62524,"\"/>":62525,"ĠÙħÙħÚ©ÙĨ":62526,"Ġpopped":62527,"Ġdeben":62528,"Ġvz":62529,"Imagine":62530,"ĠPoisson":62531,"éĻĩ":62532,"èī°è¾Ľ":62533,"=C":62534,"wag":62535,"æĴŃåĩº":62536,"åºĶå½ĵåľ¨":62537,"ི":62538,"ĠëĤĺíĥĢ":62539,"Ġfren":62540,"æīĵæŀ¶":62541,"Ġlimestone":62542,"è¼Ľ":62543,"第åħŃ竳":62544,"Ġupgrades":62545,"ä¿Ŀç½Ĺ":62546,"ĠÑĩеÑĤ":62547,"اشت":62548,"-The":62549,"ĠGUI":62550,"ĠÙĪÙĥاÙĨ":62551,"doing":62552,"éĮ¦":62553,"æ¯į亲çļĦ":62554,"Luke":62555,"average":62556,"número":62557,"ĠTaj":62558,"Ġrude":62559,"æĹłæķĮ":62560,"Ġopted":62561,"fj":62562,"ĠProjekt":62563,"æµ·çļĦ":62564,"matically":62565,"Ġ-----------------":62566,"èĩªå·±åİ»":62567,"Ġprotagon":62568,"Ġcousins":62569,"Ġinertia":62570,"Ãŀ":62571,"å¿ĥåĬĽ":62572,"马路":62573,"Ġacquisitions":62574,"Ġancestor":62575,"-it":62576,"èιä¸Ĭ":62577,"ç¥ĿæĦ¿":62578,"Ġhá»įc":62579,"ĠMoll":62580,"äº®çĽ¸":62581,"ĠUniversidade":62582,"Ġvibrations":62583,"ĠArmenian":62584,"Ġdissemination":62585,"Ġdiffered":62586,"ĠStatements":62587,"à¹Ģรียà¸Ļรูà¹ī":62588,"tod":62589,"Ġtomar":62590,"Ġstool":62591,"çľģ份":62592,"ĠResponsibility":62593,"azzo":62594,"ç´§è¿«":62595,"Linear":62596,"Ġcelebrates":62597,"Fractions":62598,"POSE":62599,"åĩĭ":62600,"è¿Ļ群":62601,"Ġturtles":62602,"ĠDirections":62603,"å¦ŀ":62604,"两éĥ¨åĪĨ":62605,"consult":62606,"ĠЧÑĤо":62607,"Sha":62608,"ĠTissue":62609,"à§Ģà§Ł":62610,"Ġemig":62611,"çļĦç¾İ好":62612,"Greg":62613,"两æīĭ":62614,"Ġactuator":62615,"ÊĬ":62616,"Ġpyt":62617,"çļĦçĤ¹":62618,"óp":62619,"cientos":62620,"ĠAnything":62621,"çļĦç͵":62622,"Ġ׾צ":62623,"Ġdrones":62624,"ĠOTHER":62625,"çĿĢå°ı":62626,"è¿ĺä¸įéĶĻ":62627,"çĥĥ":62628,"жении":62629,"ĠØ¢Ùħ":62630,"èĤ¾ä¸Ĭèħº":62631,"Ġespacio":62632,"Ġà¸Ľà¸£à¸°":62633,"ĠNintendo":62634,"اتÛĮ":62635,"خاص":62636,"$-":62637,"ä¸įéĶĪéĴ¢":62638,"-sl":62639,"åĨ·æ°´":62640,"ç·ı":62641,"Ġinsightful":62642,"Ġcrianças":62643,"ĠLcom":62644,"ĠëĪ":62645,"æĬķ产":62646,"ĠAngels":62647,"ä¸Ģ带ä¸Ģè·¯":62648,"å°±æĿ¥":62649,"Ġsobie":62650,"äºĶæĺŁ":62651,"åĮ»çĶŁçļĦ":62652,"ĠHeinrich":62653,"åĸµ":62654,")!":62655,"ÑģÑĤвÑĥÑİ":62656,"ĠíĻķìĿ¸":62657,"Tools":62658,"asaki":62659,"åī§éĻ¢":62660,"Ġpoorer":62661,"Interview":62662,"åıĤèĢĥçŃĶæ¡Ī":62663,"ĠBard":62664,"оÑĢÑĭ":62665,"Ġcataly":62666,"Hol":62667,"Anchor":62668,"åıªèĥ½æĺ¯":62669,"Ġconjugate":62670,"reference":62671,"åĽłä¸ºå¥¹":62672,"梦ä¸Ń":62673,"åªĴä½ĵçļĦ":62674,"Ġsanitation":62675,"Ġboleh":62676,"ĠEEG":62677,"Ġindica":62678,"নà§įতà§įর":62679,"åħ«ä¸ª":62680,"ĠEpic":62681,"Genus":62682,"Ġpelos":62683,"Hu":62684,"ä¸Ģè·¯ä¸Ĭ":62685,"ä¾ĨæºIJ":62686,"ç®±åŃIJ":62687,"à¶ļ":62688,"Ġshouting":62689,"Ġniv":62690,"é¦Ļèķī":62691,"æľīäººåľ¨":62692,"kÄħ":62693,"ĠSwe":62694,"çĬ¯äºĨ":62695,"ĠÑĢÑı":62696,"rex":62697,"Ġmest":62698,"é¢IJ":62699,"ÐłÐIJ":62700,"ï¼ħï¼Į":62701,"Ġsposób":62702,"univers":62703,"eus":62704,"ĠSeth":62705,"oints":62706,"çϾåIJĪ":62707,".Ct":62708,"追åĬł":62709,"XV":62710,"Ġtes":62711,"Ġprofoundly":62712,"主人åħ¬":62713,"æĺ¯ä¸įä¼ļ":62714,"å°±èĥ½å¤Ł":62715,"åºĹçļĦ":62716,"åºŃéĻ¢":62717,"ĠبرÙĨاÙħÙĩ":62718,"Ġrien":62719,"Pb":62720,"ĠIstanbul":62721,"å¹¼ç¨ļ":62722,"ĠRailroad":62723,"ocin":62724,"èĮ¸":62725,"à§ģদà§įধ":62726,"鼻è¦ĸ":62727,"ografÃŃa":62728,"obacteria":62729,"å¿ĥçĹħ":62730,"ä¸ĢçĽ´ä»¥æĿ¥":62731,"ĠFellowship":62732,"-yl":62733,"/key":62734,"uple":62735,"atom":62736,"çļĦåĽĽ":62737,"ĠWit":62738,"å°Ħåĩ»":62739,"ÙĴÙĨ":62740,"aryngeal":62741,"\"There":62742,"await":62743,"ĠArtikel":62744,"ĠThomson":62745,"ĠFrauen":62746,")A":62747,"-aut":62748,"éµ":62749,"angible":62750,"åIJİ人":62751,"æĪij们认为":62752,"Ġacademy":62753,"ĠGenerate":62754,"ÑĤÑĭÑħ":62755,"ĠMicrobiology":62756,"सà¤Ĥ":62757,"à¸Ľà¸£à¸°à¸ģà¸Ńà¸ļ":62758,"ĠShipping":62759,"à¹ģà¸Ĺ":62760,"Ġvalence":62761,"Ġmasterpiece":62762,"à®±à¯įà®±":62763,"Ġintermittent":62764,"=k":62765,"ãĤīãģĦ":62766,"æ°ijæĹıçļĦ":62767,"èµĦ产çļĦ":62768,"ĠĠĠĠĉ":62769,"段èIJ½":62770,"tained":62771,"请æķĻ":62772,"\"}":62773,"ĠMOS":62774,"ÑĤика":62775,"ĠعدÙħ":62776,"Ġsunrise":62777,"ĠÑĢаÑģÑģÑĤоÑı":62778,"ÄįenÃŃ":62779,"å®ŀç͍æĸ°åŀĭ":62780,"å¿ĺè®°äºĨ":62781,"-sidlakan":62782,"ĉdef":62783,"ĠBri":62784,"emiah":62785,"Ġagile":62786,"ç¾İå¾·":62787,"ĠÙĤدÙħ":62788,"Ġheroic":62789,"ĠCave":62790,"endi":62791,"ĠVisa":62792,"Ġtelecommunications":62793,"żyw":62794,"Ġdishon":62795,":D":62796,"wedge":62797,"åľ¨ä¸ĢäºĽ":62798,"йд":62799,"ĠãĢĤâĢĿĊĊ":62800,"Ġgenerative":62801,"মন":62802,"]);":62803,"ä¸Ńè¿Ľè¡Į":62804,"Ġdistraction":62805,"idium":62806,"åıijç»Ļ":62807,"Ġaggregates":62808,"Ġcompleto":62809,"Ġhors":62810,"ë³´ëĭ¤":62811,"Ġexplanatory":62812,"Ġyelled":62813,"èĢĮ对äºİ":62814,"ĠAlert":62815,"红楼":62816,"为äºĨ让":62817,"Ġhorizontally":62818,"Wikipedia":62819,"rass":62820,"æĶ¿çŃĸåĴĮ":62821,"à§įযা":62822,"Ġspectacle":62823,"ĠHimal":62824,"ĠNAS":62825,"满åĪĨ":62826,"terms":62827,"åıijæĶ¹":62828,"èĢĮå®ļ":62829,"ãĥĴ":62830,"Ġnozzle":62831,"thought":62832,"ĠLaur":62833,"wier":62834,".back":62835,"Ġberm":62836,"ĠGap":62837,"é©Ń":62838,"Ġrestructuring":62839,"Ġvegetarian":62840,"mium":62841,"Ġ****":62842,"ĠParm":62843,"Ġmodifier":62844,"äºĮçļĦ":62845,"ĠIndic":62846,"Õ¡Õ¦":62847,"pis":62848,"æľīåĩłä¸ª":62849,"覽":62850,"Ġinteres":62851,"æĶ¾ç͵":62852,"ÙİØ¹":62853,"Ġsinking":62854,"Adapt":62855,"éĢģè¾¾":62856,"Ġ×ŀ×Ĵ":62857,"ĠPrec":62858,"Ġhypoc":62859,"Ġligands":62860,"ĠMETHODS":62861,"Ġlegg":62862,"never":62863,"Ġedema":62864,"Converter":62865,"éĢĤ度":62866,"Ġìķ½":62867,"ĠGPT":62868,"Ġinvoice":62869,"åĨħ容åĴĮ":62870,"ÐŁÐ¾Ð´":62871,"ĠFormal":62872,"Ġintersect":62873,"èѬå¦Ĥ":62874,"zat":62875,"Ġbilingual":62876,"é«ĺé¢ij":62877,"åħļæĶ¯éĥ¨ä¹¦è®°":62878,"Ġfaçon":62879,"оÑģа":62880,"Ġbattlefield":62881,"Ġìłķë³´":62882,"ĠXVIII":62883,"åľ¨æ²¡æľī":62884,"ĠWan":62885,"Ġagro":62886,"بط":62887,"à¸Ķำ":62888,"ĠHutch":62889,"à¸Ħม":62890,"Ġelongated":62891,".cc":62892,"æ·º":62893,"ç»Ħè£ħ":62894,"éĵ®":62895,"numbers":62896,"ĠTub":62897,"ĠGeological":62898,"Ġvitality":62899,"ripemd":62900,"Ġpromotional":62901,"ĠCandidate":62902,"ĠVed":62903,"æĹłå¤Ħ":62904,"åıĹä¼Ĺ":62905,"第åįģåĽĽ":62906,"amia":62907,"าà¸ĺ":62908,"å±Ĭåħ¨åĽ½":62909,"ĠTikTok":62910,"Ġlange":62911,"å¤§éĽ¨":62912,"åħ¬éĩĮçļĦ":62913,"æµ·åŁŁ":62914,"ä¹īåĬ¡æķĻèĤ²":62915,"ĠElectricity":62916,"_or":62917,"ignore":62918,"enging":62919,"ÙģÛĮ":62920,"×ij×Ļ":62921,"Ġheir":62922,"ä¸įä½İäºİ":62923,"æ·Ĩ":62924,"èħ®":62925,"ł×ĺ":62926,"ĠObservable":62927,"ĠPollution":62928,"_column":62929,"çĽİ":62930,"Invent":62931,"Ġ!!":62932,"ĠпеÑĢемен":62933,"天涯":62934,"ĠدÙĪÙĨ":62935,"ä¸Ģå®ļèĥ½":62936,"ä¹Łè®¸æĺ¯":62937,"statement":62938,"ĠSheriff":62939,"fat":62940,"Ġconcl":62941,"æĿ¿ä¹¦":62942,"ĠMcN":62943,"æľīæīĢä¸įåIJĮ":62944,"Ġkamu":62945,"ĠнаÑģелениÑı":62946,"-Th":62947,"renched":62948,"åħµåĽ¢":62949,"manship":62950,"Ġunite":62951,"è¡ĮæĺŁ":62952,"ÙĩÙIJ":62953,"Ġpeek":62954,"สà¸Ńà¸Ļ":62955,"绿èī²çļĦ":62956,"åı¯æĢķçļĦ":62957,".aspx":62958,"没åħ³ç³»":62959,"ĠBeer":62960,"Mu":62961,"enog":62962,"Ġaffective":62963,"_pair":62964,"Ġinserting":62965,"Ġmailing":62966,"ĠWeg":62967,"为ä¸ŃåĽ½":62968,"Ġagosto":62969,"çĬģ":62970,"Ġowning":62971,"ĠPlaza":62972,"Ġalcuni":62973,"-border":62974,"(filename":62975,"(auto":62976,"ĠGiant":62977,"Ġbyla":62978,"é«ĺçŃīåŃ¦æł¡":62979,"ĠZoo":62980,"ĠCurrency":62981,"ĠSubtraction":62982,"Ġmalf":62983,"ĠRais":62984,"åIJ¬ä¼Ĺ":62985,"Ġintervene":62986,"ĠBott":62987,"stituted":62988,"ĠCalif":62989,"ĠCinema":62990,"Mom":62991,"æĥ°":62992,"ĠKai":62993,"æľ¬åĵģ":62994,"Ġ+ĊĊ":62995,"ajan":62996,"ĠGreene":62997,"Ġprincipalmente":62998,"Ġeyebrows":62999,"âce":63000,"Ġteaspoons":63001,"Originally":63002,"ĠMER":63003,"ĠPurch":63004,"ĠداÙĨشگاÙĩ":63005,"Say":63006,"impro":63007,"Ġroasted":63008,"efits":63009,"Ġinfrast":63010,"Ġimaginative":63011,"Ġalve":63012,"Ġjelly":63013,"Ġâīł":63014,"ĠSubl":63015,"quelle":63016,"ä»İ头":63017,"Ġsklearn":63018,"ĠAdvisor":63019,"Ġdemographics":63020,"干活":63021,"Ġforeigners":63022,"临åºĬ表çݰ":63023,"Ġiodine":63024,"สà¸Ķà¸ĩ":63025,"è¡Ģæłĵ":63026,"Ġতাদà§ĩর":63027,"Ġinclination":63028,"rities":63029,"ĠAda":63030,"à¸ļาà¸ĩ":63031,"ĠMinneapolis":63032,"Ġমহ":63033,"è´´è¿ij":63034,"ãģ¡ãĤĥ":63035,"ĠPilot":63036,"ĠwspóÅĤ":63037,"æĢ»æĬķèµĦ":63038,"zenÃŃ":63039,"Ġrevise":63040,"沸èħ¾":63041,"ä»İä¸ļ人åijĺ":63042,"åºĶ以":63043,"ĠShannon":63044,"ĉnew":63045,"Ġgev":63046,"ĠDetermination":63047,"Ġspam":63048,"Dead":63049,"Ġconcluding":63050,"ĠاÙĦات":63051,"éģĵ士":63052,"-colored":63053,"!'":63054,"ĠBoss":63055,"å¼ĢåĬŀ":63056,"complex":63057,"计ç®ĹçļĦ":63058,"اصر":63059,"å°ıåIJĥ":63060,"ç¾İåĨĽ":63061,"ç¾İåĽ½äºº":63062,"pipe":63063,"ĠLuxemb":63064,"+m":63065,"ĠNerv":63066,"comings":63067,"-sk":63068,"ä¸ŃçļĦåºĶç͍":63069,"ĠнеÑģ":63070,"ĠAdvantages":63071,"çļĦæľĢå°ı":63072,"audio":63073,"Ġclimat":63074,"Ġasymmetric":63075,"Chain":63076,"æĸĩåĮĸ建设":63077,"ronics":63078,"ÑĢоÑģа":63079,"åĪĩåīĬ":63080,"Ġ׾×ij×":63081,"Ġnalista":63082,"\\Controllers":63083,"Ġrecombination":63084,"ä¸įæĦ§":63085,"Ġdecimeters":63086,"Ġopenness":63087,"LinkedList":63088,"ĠLG":63089,"ivered":63090,".We":63091,"Ġtua":63092,".sleep":63093,"ĠâĨIJ":63094,"åħ¨åĬĽä»¥èµ´":63095,"åīįåĪĹèħº":63096,"ìĿ´ëĤĺ":63097,"Ġvols":63098,"ثار":63099,"ÑĪиб":63100,"Ġbreastfeeding":63101,"æľīè¶£çļĦ":63102,"ĠвозможноÑģÑĤÑĮ":63103,"ahabogang":63104,"Ġundesirable":63105,"ä¸Ĭå±±":63106,"ÑģÑĤей":63107,"åģļä¸ĢäºĽ":63108,"个åŃIJ":63109,"bernetes":63110,"Å¡ÃŃm":63111,"Ġqualquer":63112,"Rot":63113,"ĉl":63114,"çļĦä½İ":63115,"Ġmatemat":63116,"ĠHAVE":63117,"ลà¸Ķ":63118,"Ġmembantu":63119,")\"Ċ":63120,"Ġinicial":63121,"严å¯Ĩ":63122,"continuous":63123,"Ġcrashed":63124,"ÑĪаеÑĤ":63125,"á»Ŀi":63126,"åħļé£İå»īæĶ¿":63127,"Sab":63128,"ĠLum":63129,"åħĪå°Ĩ":63130,"Heart":63131,"éĢļçŁ¥ä¹¦":63132,"ĠWidth":63133,"Ġpracy":63134,"ĠLoading":63135,"Ġmoss":63136,"客ä½ĵ":63137,"ר×ĺ":63138,"FI":63139,"czenia":63140,"åľ¨å¥¹çļĦ":63141,"à¸ľà¹Īาà¸Ļ":63142,"Ġmuestra":63143,"Ġail":63144,"çݰ货":63145,"Ġadviser":63146,"ĠPenal":63147,"ä¿Ŀåħ¨":63148,"Ġsynaptic":63149,"ĠФÑĢан":63150,"gend":63151,"ĠGrades":63152,"ARA":63153,"rai":63154,"äºĨä¸ī":63155,"ĠHaut":63156,"åį±éļª":63157,"å¼Ħå¾Ĺ":63158,"æİ©é¥°":63159,"ä»ĸäºĨ":63160,"è¯ĿéŁ³":63161,"æĶ¶åΰäºĨ":63162,"åĨľåİĨ":63163,"Ġprobation":63164,")],":63165,"ĠKön":63166,"à¯Ĩய":63167,"Ġexerted":63168,"çıij":63169,"ĠSpart":63170,"Ġlayered":63171,"Ġapologize":63172,"Ġinterpolation":63173,"ĠMou":63174,"çŁ¿äº§":63175,"å°±åľ¨äºİ":63176,"recogn":63177,"Rub":63178,"ĠGm":63179,"ĠÑģамоÑģÑĤоÑı":63180,"Ġnj":63181,"ĠGU":63182,"ĠProgressive":63183,"åIJ¬åIJ¬":63184,"à¸Īà¸Ļ":63185,"Ġmicrobes":63186,"å±ıèͽ":63187,"-existing":63188,"ĠBend":63189,"åĪĻåľ¨":63190,"-lasting":63191,"ä¸Ģæŀļ":63192,"ÎŃÏģ":63193,"াদà§ĩশ":63194,".params":63195,"ÑģпÑĥбли":63196,"çİĽä¸½":63197,"Ġ×ŀ×Ļ":63198,"Ġbombing":63199,"Ġcp":63200,"arqu":63201,"ĠLOVE":63202,"Ġselects":63203,"Ġbranding":63204,"Ġ×Ķ×ŀש×":63205,"rounded":63206,"å¹²æĹ±":63207,"ĠغÛĮر":63208,"Ġpersuasive":63209,"Dire":63210,"ĠnÃ¥":63211,"(\".":63212,"Ġfetus":63213,"èĺijèıĩ":63214,"-tech":63215,"ĠDeze":63216,"èijĹçļĦ":63217,"(props":63218,"ç»ıéªĮåĴĮ":63219,"ĠRevista":63220,"ĠLibraries":63221,"ä½ķ以":63222,"Ġrealizes":63223,"ä¸ĩåĪĨ":63224,"Unique":63225,"å¿ĻçĿĢ":63226,"Ġubang":63227,"Audio":63228,"ZW":63229,"ijah":63230,"à¹Ģหà¸ķุ":63231,"ĠStarted":63232,"rologic":63233,"åΰäºĨä¸Ģ":63234,"Ġgov":63235,"Ġkim":63236,"thews":63237,"Ġà¦ķিনà§įতà§ģ":63238,"åĪ©ç͍çİĩ":63239,"éĺ»æĮ¡":63240,"Tokenizer":63241,"Ġscarcity":63242,"Federal":63243,"æ»ķ":63244,"Ġreferendum":63245,"Marg":63246,"_fl":63247,"ensors":63248,"å¤ļæĸ¹":63249,"-loop":63250,"å·¥ç¨ĭçļĦ":63251,"Technology":63252,"Ġвека":63253,"Ġcounselor":63254,"Ġeagerly":63255,"Native":63256,"xj":63257,"åıĤèĢĥæĸĩçĮ®":63258,"ĠPatch":63259,"Articles":63260,"說å®Į":63261,"ĠÑĪпанÑģки":63262,"Ġocup":63263,"Ġsewage":63264,"ĠTampa":63265,"chl":63266,"åľĨéĶ¥":63267,"Ġweakly":63268,"Ġprzeci":63269,"Ġtrif":63270,"óln":63271,"Ġstriving":63272,"çĺŁ":63273,"æĹģéĤĬ":63274,"Ġbiomedical":63275,"cimiento":63276,"زÙĬد":63277,"ĠBelarus":63278,"ÅĤoÅĽci":63279,"åĩĨç¡®çļĦ":63280,"/:":63281,"istä":63282,"ĠJill":63283,"Ġdieta":63284,".servlet":63285,"éĤ£äºĽäºº":63286,"/SEDAC":63287,"aqu":63288,"ĠHond":63289,"-dev":63290,"Linux":63291,"ĠBulgaria":63292,"Ġsuburban":63293,"饪":63294,"áĥ¢":63295,"æŃ¦æľ¯":63296,"Ġurb":63297,"ĠاÙĦساعÙĬÙĩ":63298,"ĠENG":63299,"Ġoscillator":63300,"ÑĤой":63301,"åľ¨åħ¨çIJĥ":63302,"Ġguessing":63303,"Ġreliably":63304,"kami":63305,"تبر":63306,"ä¹ĭä½ľ":63307,".input":63308,"èĬ¥":63309,"ĠPCI":63310,"ä¸Ĭéĥ¨":63311,"ordable":63312,"Mil":63313,"ĠDro":63314,"éĺĪ":63315,"Ġdistinctions":63316,"欣æħ°":63317,"ĠUhr":63318,"Ġanalges":63319,"ĠBoeing":63320,"Ġשע":63321,"ÑĻено":63322,"à¸Ļà¹īà¸Ńย":63323,"hours":63324,"ĠDW":63325,"areth":63326,"ĠLaunch":63327,"çĴ§":63328,"Ġbatting":63329,"meaning":63330,"Ġà¦ķারণ":63331,"ĠÑĦон":63332,"ĠCircular":63333,"Ġkicking":63334,"Ġinception":63335,"æħij":63336,"Ġতাহ":63337,"nton":63338,"Ġdispat":63339,"壽":63340,"rosine":63341,"ĠLegislative":63342,"è¿Ľé£Ł":63343,"èī²ç´ł":63344,"Äįas":63345,"Ġreviewers":63346,"è¿Ļä¹Īå¤ļå¹´":63347,"ĠÐIJлекÑģ":63348,"мÑĭм":63349,"ÐŀÐĴ":63350,"Performance":63351,"Õ²":63352,"ãģµ":63353,"ereotype":63354,"Ġcredential":63355,"\"\"\"":63356,"FK":63357,"åѦ士":63358,"ĠØ´ÙĥÙĦ":63359,"assis":63360,"åĨĽåĽ¢":63361,"Ġtanan":63362,".ComponentModel":63363,"ĠDust":63364,"ÙĩÙı":63365,"ãĥ¤":63366,"西æĸ¯":63367,"éľĩåĬ¨":63368,"CONT":63369,"Ġplaster":63370,"Ġroce":63371,"Ġinfiltration":63372,"Ġvezes":63373,"Õ¡Õ°":63374,"èįīåľ°":63375,"Ġgihabogon":63376,"Sources":63377,"ĠACL":63378,"å½·":63379,"å½ĵåľ°çļĦ":63380,"çĨĬçĮ«":63381,"ĠØ¢ÙĦ":63382,"ĠBrill":63383,"Ġalgunas":63384,"ĠмоменÑĤ":63385,",â̦,":63386,"æŃ³":63387,"ç¾ģ":63388,"ĠFlowers":63389,"ĠListening":63390,"Ġdiagnoses":63391,"ãģĵãģ¨ãĤĤ":63392,"ĠPakistani":63393,"estro":63394,"éľ§":63395,"ĠÑĨиÑĦ":63396,"Ġterrifying":63397,"çŀ³åŃĶ":63398,"è¯Ńä¹ī":63399,"åĮ»éĻ¢çļĦ":63400,"ephal":63401,"Ġfootsteps":63402,"ãģĵãĤĮãģ¯":63403,"Ġzpůso":63404,"Ġinsane":63405,"Ġдиаг":63406,"é¢ľèī²çļĦ":63407,"Ġderives":63408,"ÑĤелÑĮнÑĭм":63409,"ãĥ¡ãĥª":63410,"Finding":63411,"天èĬ±":63412,"çĶŁäº§çº¿":63413,"Ġairlines":63414,"ĠSect":63415,"æ¸ħçν":63416,"ĠGupta":63417,"ascar":63418,"Ġcomforting":63419,"-gen":63420,"ÛĮÛĮر":63421,"ç½ijèĨľ":63422,"Ġmétodo":63423,"ĠOutline":63424,"æĶ¾å°ĦæĢ§":63425,"人ãģ®":63426,"é¦ĸ个":63427,"亲æīĭ":63428,"æĺ¯åIJ¦åŃĺåľ¨":63429,"Ġvillain":63430,"èĥļèĥİ":63431,"]>":63432,"rity":63433,"patibility":63434,"åł´æīĢ":63435,"Ġstabilize":63436,"ằng":63437,"hta":63438,"émat":63439,"éģŀ":63440,"ĠStability":63441,"ĠWeak":63442,"ä¸ĸçķĮ大æĪĺ":63443,"گرÛĮ":63444,"ĠØ´ÙĪÙĨد":63445,"orton":63446,"çĥĻ":63447,"äss":63448,"ĠPlural":63449,"ç§°ä½ľ":63450,"æĭ¿æĿ¥":63451,"å¨ĺåŃIJ":63452,"ĠзавиÑģиÑĤ":63453,"rides":63454,"è¿Ļä¹Īåģļ":63455,"Ġquantification":63456,"ünf":63457,"åĽłä¸ºä½ł":63458,"ĠQuinary":63459,"Ñģии":63460,"_hash":63461,"ĠÑģледÑĥÑİ":63462,"μÎŃν":63463,"å¸ĤåľºéľĢæ±Ĥ":63464,"kos":63465,"Ġ\\(-\\)":63466,"Ġscarcely":63467,"Ù¡":63468,"èĢĮä¸Ķæĺ¯":63469,"зии":63470,"orphous":63471,"Court":63472,"Ġaust":63473,"essive":63474,"Ġcolleg":63475,"åħ±åIJĮåĬªåĬĽ":63476,"Ġbourgeois":63477,"vider":63478,"Ġusable":63479,"Ġextrap":63480,"Ġcosting":63481,"åįĬ个æľĪ":63482,"+\\,":63483,"Ġfascinated":63484,"Ġaerosol":63485,"always":63486,"ĠWTO":63487,"Ġkost":63488,"æĸ°è½¦":63489,"ĠResil":63490,"[])":63491,"åĩĨç¡®åľ°":63492,"Ġscalability":63493,"Encoding":63494,"ĠSiber":63495,"Ġproclaimed":63496,"-pressure":63497,"ĠSignificant":63498,"çļĦ身ä¸Ĭ":63499,"Ġmettre":63500,"Ġinfinitely":63501,"åIJ¬å®Į":63502,"ìŀĪ":63503,"å°ģéĶģ":63504,"ĠCryptographic":63505,"accharides":63506,"ĠAgen":63507,"ä¸Ģ家人":63508,"Ġnewcom":63509,"Ġgenerosity":63510,"å¼łåĬĽ":63511,"Ġì²´":63512,"Ġcontradictory":63513,"çļĦåĦ¿åŃIJ":63514,"abas":63515,"éĹ®é¢ĺåĴĮ":63516,"Ġpaints":63517,"hil":63518,"Ġmalt":63519,"staff":63520,"åIJİ代":63521,"æĻĮ":63522,"åİŁåĪĻä¸Ĭ":63523,"èĥ¸éĥ¨":63524,"ĠSixty":63525,"Guard":63526,"ĠAthletics":63527,"Ġdiligence":63528,"RED":63529,"\\'":63530,"Ġعشر":63531,"orna":63532,"ÑĸлÑĮ":63533,"å¹¶ä¸Ķåľ¨":63534,"å£ĵåĬĽ":63535,"ahar":63536,"ÑİÑīим":63537,"Ġfossils":63538,"isé":63539,"oteca":63540,"ĠFerm":63541,"ä½Ĩ没æľī":63542,"åĻ´":63543,"è§Ĥåħī":63544,"ĠJohnston":63545,"ĠгÑĢÑĥппÑĭ":63546,"Fred":63547,"\\mu":63548,"ĠKatherine":63549,"ĠÙħÙħا":63550,"è¯ķæł·":63551,"ÑģÑĤвоваÑĤÑĮ":63552,"Ġatheros":63553,"Ġlandmarks":63554,"çĴĩ":63555,"inguished":63556,"Ġalleles":63557,"ĠInfection":63558,"é¢ĩæľī":63559,"Rew":63560,"ÑĪÑĤа":63561,"γÏģαÏĨ":63562,"Ġmisery":63563,"ĠSAM":63564,"è±ļ":63565,"åħ·æľīä¸Ģå®ļçļĦ":63566,"Ġatrial":63567,"å°ıæĿ¿":63568,"ç¬ĶçĶ»":63569,"chrome":63570,"aculture":63571,"åľ°å¤Ħ":63572,"èĢĮä¸įèĥ½":63573,"çŁŃ缺":63574,"ĠCambodia":63575,"à¹Ģà¸Īà¹īาà¸":63576,"ÙĥÙĪ":63577,"Ġpointers":63578,"ICLE":63579,"Scan":63580,"_valid":63581,"cola":63582,"Ġpropriet":63583,"Mind":63584,"ĠMum":63585,"æ²¹çļĦ":63586,"ĠÑĨенÑĤÑĢа":63587,"ĠобÑĭÑĩно":63588,"LU":63589,"Ġeh":63590,"hension":63591,"ç¥ĸçζ":63592,"ாத":63593,"ĠÑĥÑĢовнÑı":63594,"ÙĦÙħاÙĨ":63595,"Ġmultifaceted":63596,"ayette":63597,"èĻı":63598,"è¿Ļä¸Ģå¹ķ":63599,"èģĮä¸ļéģĵå¾·":63600,"ĠBraun":63601,"Ю":63602,"ĠâĢĿ,":63603,"ÏĮν":63604,"oooo":63605,"Ġsermon":63606,"Ġverv":63607,"使æĪij们":63608,"ĠÑħол":63609,"订éĺħ":63610,"California":63611,"ಾರ":63612,"nim":63613,"ucion":63614,"ĠZwe":63615,"å¦Ĥä½ķåľ¨":63616,"(matrix":63617,"(+":63618,"åģļçļĦäºĭæĥħ":63619,"oxin":63620,"ĠAmpl":63621,"æķĮ人çļĦ":63622,"Ġmendapat":63623,"Ġcouncils":63624,"ĠLOC":63625,"ĠSeek":63626,".style":63627,"æĿ¯åŃIJ":63628,"ĠлÑİди":63629,"/sp":63630,"Ġì¡":63631,"ĠгоÑĢода":63632,"criptive":63633,"................................................":63634,"Ġtranscriptional":63635,"ĉset":63636,"åŀĥåľ¾åĪĨç±»":63637,"orate":63638,"emary":63639,"Ġammunition":63640,"Ġmatplotlib":63641,"ä¹Ŀåįģ":63642,"ĠпÑĢода":63643,"ĠDirective":63644,"ĠSit":63645,"æĪ®":63646,"è¡ĮåĪĹ":63647,"Ðĺн":63648,"Ġpyram":63649,"Ġadvocating":63650,"ĠDana":63651,"ético":63652,"ĠлеÑĩениÑı":63653,"à¸Ľà¸±à¸įห":63654,"à¸ij":63655,"çĶ¨è½¦":63656,"æĿİæŁIJ":63657,"érique":63658,"íħĮ":63659,"]].":63660,"çłĶç©¶æĪIJæŀľ":63661,"Ġsque":63662,"Ġadjud":63663,"æł©":63664,"Ġoutright":63665,"alsa":63666,"æ¤įçī©çļĦ":63667,"åħ·ä½ĵæĥħåĨµ":63668,"ĠFuneral":63669,"çľĭè§ģäºĨ":63670,"ĠBloomberg":63671,"ĠëĨĴ":63672,"åΰæŃ¤":63673,"æĤ£åĦ¿":63674,"åıĤåĬłä¼ļè®®":63675,"ĠاÙĦجسÙħ":63676,"Ġpeasants":63677,"ĠÎłÎ·Î³ÎŃÏĤ":63678,"ĠQué":63679,"ĠIa":63680,"ĠCET":63681,"覧":63682,"Ġdashed":63683,"Ġalunos":63684,"Ġcontested":63685,"Ġspicy":63686,"å¤ĸä¾§":63687,"-Term":63688,"Ġdụ":63689,"ĠCable":63690,"ĠPJ":63691,"çĥŃè¡Ģ":63692,"åįİ缼":63693,"è¿İæĿ¥äºĨ":63694,"วิà¸Ĭ":63695,"ĠComplexity":63696,"Roll":63697,"tax":63698,"моÑĢ":63699,"conditions":63700,"æ®Ĩ":63701,"ROS":63702,"ĠHighly":63703,"ائÙĩ":63704,"Ġnatura":63705,"Ġhely":63706,"å±±åİ¿":63707,"失踪":63708,"Ġspreadsheet":63709,"-operative":63710,"Ġapplicability":63711,"Ġabolition":63712,"Ġvue":63713,"iris":63714,"ĠDIR":63715,"æĿĥåĪ©è¦ģæ±Ĥ":63716,"usep":63717,"çļĦä¸ĢåįĬ":63718,"ROC":63719,"Ġdrawbacks":63720,"সল":63721,"åŃĺåĤ¨åύ":63722,"çĥ¹é¥ª":63723,"æľīå¾Ī大çļĦ":63724,"qv":63725,"à¸ľà¸¥à¸´à¸ķ":63726,"-Ar":63727,"onies":63728,"ĠFrag":63729,"Ġsampai":63730,"ensin":63731,"Ġبزرگ":63732,"IPO":63733,"Ġselama":63734,"(âĢľ":63735,"Ġui":63736,"Ġ%)":63737,"åıij表çļĦ":63738,"éļĨéĩį":63739,".security":63740,"Ġmaneuver":63741,"_search":63742,"Ġblows":63743,"åı¯ä»¥åİ»":63744,"کرد":63745,"INES":63746,"ĠLevy":63747,"çĮ©":63748,"alkyl":63749,"given":63750,"Ġdocker":63751,"ĠGENER":63752,"Ġresides":63753,"çķ¥æľī":63754,"å̼å¾Ĺä¸ĢæıIJ":63755,"Ġpicnic":63756,"Structure":63757,"ĠXIII":63758,"ابات":63759,"ä¹ĺ以":63760,"ᱣ":63761,"????????":63762,"缸åĬł":63763,"æĤ¬æµ®":63764,"вÑĭм":63765,"缴è¾ĸå¸Ĥ":63766,"ĠUber":63767,"ä¹ĭ声":63768,"çĹī":63769,"Ïĥο":63770,"декÑģ":63771,"å¿«éĢŁçļĦ":63772,"oise":63773,"Ġย":63774,"Ġ(~":63775,"ĠGao":63776,"Isa":63777,"æµĵæµĵ":63778,"HU":63779,"ÚĪ":63780,"íͼ":63781,"æľįåĬ¡åijĺ":63782,"ĠNewsletter":63783,"ĠPolymer":63784,"ĠSes":63785,"æĸ°åªĴä½ĵ":63786,"社ä¼ļä¿ĿéĻ©":63787,"PPT":63788,"تاÙĨ":63789,"ĠTeen":63790,"Ġmilhões":63791,"Ġpastoral":63792,"Ġহিস":63793,"åħ³æ³¨çļĦ":63794,"Ġnunca":63795,"Ġcatastrophic":63796,"Bound":63797,"jah":63798,"Ġwagon":63799,"ĠCry":63800,"åĴĮçĶŁæ´»":63801,"èıģ":63802,"Ġinternship":63803,"åħ¶å®ŀå°±æĺ¯":63804,"×Ļ׾×ķ":63805,"itles":63806,"Ġpleaded":63807,"ĠResponseEntity":63808,"elajaran":63809,"åĽ¾ä¸º":63810,"Ġorganizers":63811,"åĪĨå¸ĥå¼ı":63812,"MAC":63813,"Ġmacht":63814,"Ġdehydration":63815,"ĠLon":63816,"Ġcondensed":63817,"ĠSteam":63818,"Ġtemperate":63819,"ĠAcquisition":63820,"Ġdécl":63821,"ĠASD":63822,"ληθ":63823,"çļĸ":63824,"Ġasse":63825,"æŃ£å̼":63826,"à°¡":63827,"缮æłĩæĺ¯":63828,"Ġwereld":63829,"çŁ¿çŁ³":63830,"ļáŀ":63831,"æĹ¶æĬ¥":63832,"viol":63833,"Ġentitle":63834,"æİ¨éĢģ":63835,"mathit":63836,"ÐŁÐ°":63837,"Ġοι":63838,"ĠмеÑĤал":63839,"irin":63840,"ĠGriffin":63841,"sr":63842,"ĠMedian":63843,"Headers":63844,"çļĦæ¦Ĥçİĩ":63845,"Ġlis":63846,"ÑĪаÑĤÑĮ":63847,"æĺ¯äººç±»":63848,"Ġdepended":63849,"ĠHighlights":63850,"AQ":63851,"åľ¨åħ¶ä¸Ń":63852,"Ġpetrol":63853,"ĠMillionen":63854,"åĨħç§ij":63855,"ĠاÙĦتØŃ":63856,"Remote":63857,"'O":63858,"Ġshuttle":63859,"Ġscrat":63860,"ĠassertEquals":63861,"ĠпÑĢовод":63862,"Ġtamp":63863,"ĠÕĦ":63864,"svg":63865,"滥ç͍":63866,"uscript":63867,"æİ¨å¯¼":63868,"ĠCHE":63869,"çļĦä¸Ģç³»åĪĹ":63870,"Ġantip":63871,"ersonal":63872,"åĮĹ京大åѦ":63873,"elaide":63874,"ĠNoble":63875,"ufficiency":63876,"æĹłåIJį":63877,"èĻĶ":63878,"Ġduas":63879,"ĠParadise":63880,"hof":63881,"coli":63882,"NAM":63883,",\\]":63884,"ä¸ĢæĹģçļĦ":63885,"iedy":63886,"Ġoriginalet":63887,"_files":63888,"Ġcambi":63889,"DOWNLOAD":63890,"Ġ'';Ċ":63891,"å®īæĬļ":63892,"åŁºå»º":63893,"леннÑĭе":63894,"Ġ$$\\":63895,"Hours":63896,"çļĦè¿IJåĬ¨":63897,"racts":63898,"åĦŁ":63899,"Ġmicrobiota":63900,"Biology":63901,"give":63902,"kj":63903,"ĠMPC":63904,"åħ¥æĪ·":63905,"ĠAlang":63906,"绳åŃIJ":63907,"otrans":63908,"ä¸į足以":63909,"ÚĺÛĮ":63910,")d":63911,".Item":63912,"åĽŀåIJĪ":63913,"è¿Ļç§įæĸ¹å¼ı":63914,"simple":63915,"positories":63916,"ĉw":63917,"adin":63918,"ä¿®åīª":63919,"ĠExperiments":63920,"ihil":63921,"Ġirrespective":63922,"é¡¶å°ĸ":63923,"Ġmasalah":63924,"ĠAutor":63925,"Ġmiracles":63926,"çIJĨç§ij":63927,"èĢĮä»ĸ":63928,"-get":63929,"Ġbricks":63930,"Ġformatted":63931,"Ġhistogram":63932,"Ġcitrus":63933,"Ġvoltages":63934,"Ġandroidx":63935,"Ġ×Ķר×IJש":63936,"å¼ĢæľĹ":63937,"ç»ĵèĬĤ":63938,"æİ¥çº¿":63939,"Ġdebated":63940,"uem":63941,"è·Łä¸Ĭ":63942,"Ġstringent":63943,"æŃ¢çĹĽ":63944,"æŃĮèĪŀ":63945,"ĠاÙĦسÙĦ":63946,"çĥ¦èºģ":63947,"Ġphosphat":63948,"ĠVT":63949,"ÑĤей":63950,"ä¼ģä¸ļ管çIJĨ":63951,"Ġoxides":63952,"Ġimposs":63953,"çĵ¶é¢Ī":63954,"Ġlymphocytes":63955,"ĠDuck":63956,"Ġì§Ī":63957,"è¦ı模":63958,"ĠCollaborative":63959,"åľ¨åIJĦ":63960,"Ñĩное":63961,"ricos":63962,"ĠCommit":63963,"Along":63964,"Ġtornado":63965,"Ġuterus":63966,"/met":63967,"Ġfreezer":63968,"Ġдека":63969,"Ġconsulted":63970,"ĠشدÙĨ":63971,"Father":63972,"Ġinic":63973,"Ġlaat":63974,"æĸ½èĤ¥":63975,"Ġjeopard":63976,"åıijå¸ĥä¼ļ":63977,"ĠдÑĢÑĥгой":63978,"database":63979,"Ġbeard":63980,"Ġelusive":63981,"èĢģåĮĸ":63982,"åIJĪä¼Ļ人":63983,"Ġcounters":63984,"çļĦåĬªåĬĽ":63985,"ãĢįãĢģãĢĮ":63986,"æ²¹çͰ":63987,"ĠExtended":63988,"åĮĪ奴":63989,"\"_":63990,"è¨ĺå¾Ĺ":63991,"-net":63992,"ĠWikis":63993,"Ġinfantry":63994,"æij§æ¯ģ":63995,"Ġaantal":63996,"ĠBacon":63997,"åĽ½ä¼ģ":63998,"åıijçĥ§":63999,"EMS":64000,"Ġbrowsers":64001,"Ġmnož":64002,"Ġapex":64003,"ä¸Ģ个æĸ°çļĦ":64004,"Exit":64005,"Ġnaive":64006,"Ġmorally":64007,"zeniu":64008,"bench":64009,"æĢĴçģ«":64010,"ĠÑĤÑıже":64011,"ĠPlains":64012,"ĠPediatric":64013,"ĠмаÑĤеÑĢиала":64014,"Ġfeu":64015,"ä¼ĺå¼Ĥ":64016,"åľ°è¡¨":64017,"Ġlookup":64018,"åħ«çϾ":64019,"ĠAcids":64020,"å¹³åı°çļĦ":64021,"Ġapprentices":64022,"Ec":64023,"è¿ĩ失":64024,"URS":64025,"Ġevaluates":64026,"Formula":64027,"ä¹Ĵä¹ĵ":64028,"绥":64029,"htra":64030,"身å¤Ħ":64031,"Allow":64032,"ĠDevelopmental":64033,"ĠObservatory":64034,"}}\\]":64035,"Ġreservoirs":64036,"èĢ³æľº":64037,"ĠFör":64038,"æŀĹçļĦ":64039,"Ġplasticity":64040,"_addr":64041,"Failure":64042,"å°±ç͍":64043,"Ġinsists":64044,"âε":64045,"ç§»åĭķ":64046,"ಪ":64047,"won":64048,"è¼ķè¼ķ":64049,"-screen":64050,"主è¦ģçͱ":64051,"ĠاÙĦسÙħاÙĪÙī":64052,"ĠPlat":64053,"population":64054,"\\Model":64055,"åľ¨ä¸ĸçķĮ":64056,"Ġdetached":64057,"ĠDeutsche":64058,"Ġconstrued":64059,"容积":64060,"uye":64061,"ĠØ·ÙĪØ±":64062,"Ġneutroph":64063,"ĠLunar":64064,"Ġmorte":64065,"ĠвеÑĢоÑıÑĤ":64066,"Way":64067,"ĠPablo":64068,"æķıéĶIJ":64069,"obar":64070,".Se":64071,"Medicine":64072,"ĠDell":64073,"ä¸Ĭåı°":64074,"ä½ľæĽ²":64075,"ਨ":64076,"ĠFunktion":64077,"تبار":64078,"第ä¸Ģåįĥ":64079,"غÙħ":64080,"ĠDurham":64081,"ÑģÑĥÑĢ":64082,"Ġrefinement":64083,"Ġpresidente":64084,"ĠÑĪколÑĮ":64085,"Ġmorn":64086,"ĠEnforcement":64087,"ä¸īè§Ĵå½¢çļĦ":64088,"миниÑģÑĤÑĢа":64089,"باد":64090,"èµ°è¿ij":64091,"ĠGenes":64092,"Ġnostro":64093,"=\"${":64094,"ĠSlides":64095,"å±Ĥ次çļĦ":64096,"Ġsuoi":64097,"Ġholdings":64098,"ÙĪØ¬Ø¯":64099,"ĠVelocity":64100,"Ġétaient":64101,"Ġerfol":64102,"ĠPhosph":64103,"×ij×Ķ":64104,"Ġscenic":64105,"Ġdipole":64106,"失èIJ½":64107,"à§ĭà¦Ł":64108,"enna":64109,"æ¹ĸæ³Ĭ":64110,"Ġì§ij":64111,"å±±èĦī":64112,"ĠпоÑģе":64113,"è³Ģ":64114,"ç©¿æ¢Ń":64115,"Ġprincipales":64116,"ģı":64117,"æļijæľŁ":64118,"flammation":64119,"à¹ģà¸ģà¹Ī":64120,"à§ģà¦ķ":64121,"ĠÑĢод":64122,"ĠCotton":64123,"ç½°":64124,"Ġportraits":64125,"Ġà¦ķথ":64126,"rimin":64127,"Ġdealers":64128,"æĬķåħ¥åΰ":64129,"æīŃ转":64130,"Ġusu":64131,"åĩºåľº":64132,"Ġpillar":64133,"orative":64134,"ĠSql":64135,"å¾Ĭ":64136,"åĴĮ大家":64137,"天æĸĩ":64138,"Ġbehold":64139,"æ´Ľåħĭ":64140,"ĠMargin":64141,"ÙĪØ³Øª":64142,"Ġdenn":64143,"ĠRELATED":64144,"marked":64145,"Ġ×ijר×":64146,"åī¥åīĬ":64147,"agation":64148,"Ġafforded":64149,"yj":64150,"说åĩºæĿ¥":64151,"ynth":64152,"Ġpasser":64153,"æķijçģ¾":64154,"Ġley":64155,"Ġabras":64156,"éĵ¬":64157,"æĺķ":64158,"Ġ+++":64159,"æīĺ管":64160,"åѦéĻ¢çļĦ":64161,"àŃĩ":64162,"ĠTibet":64163,"æĹ¶æĹ¶":64164,"æľ¬æĬ¥":64165,"ĠArbit":64166,"Ġvenom":64167,"Ġtariff":64168,"ĠDAT":64169,"lecting":64170,"åIJijå·¦":64171,"ĠOncology":64172,"ÙĪÙĨا":64173,".file":64174,"ĉĊĉĊ":64175,"counting":64176,"Install":64177,"Ġdew":64178,"Ġretailer":64179,"èĩ³åħ³éĩįè¦ģ":64180,"ĠCum":64181,"ĠChamp":64182,"æĿijåŃIJ":64183,"lige":64184,"ĠÑģоÑħÑĢа":64185,"ĠBehaviour":64186,"Ġsymptomatic":64187,"linked":64188,"yards":64189,"Ġvib":64190,"åľ¨èģĮ":64191,"rias":64192,"à¹ģà¸ŀ":64193,"Ġodp":64194,"selector":64195,"{q":64196,"å¹¶æľī":64197,"èµĦæľ¬å¸Ĥåľº":64198,"ä¼ļæĺ¯":64199,"å¹¶äºİ":64200,"виÑĤÑĮ":64201,"Ġubiquitous":64202,"çϽç³ĸ":64203,"Ġrefractive":64204,"hler":64205,"ĠGin":64206,"Ù쨏":64207,"ĠSchwartz":64208,"Generated":64209,"lou":64210,"anst":64211,"ĠCock":64212,"æľīçļĦæĺ¯":64213,"Ġjeunes":64214,"ĠPastor":64215,"éĿĵ":64216,"çľĭæĪij":64217,"åı«ä»ĸ":64218,"à½ij":64219,"Ġ×IJ×ij׾":64220,"çĪ±äºº":64221,"REL":64222,"ĠRegistry":64223,"订ç«ĭ":64224,"ÑĩеÑģком":64225,"ĠÑģÑĤепени":64226,"åħ±é¸£":64227,"éĩĩç͍çļĦ":64228,"Ġrigor":64229,"Catalan":64230,"Ġrevol":64231,"ĠبÙĪØ¯Ùĩ":64232,"Ġà¦ķà¦¿à¦Ľà§ģ":64233,"Ġpid":64234,"ä¸Ĭæĺł":64235,"Ġclergy":64236,"缴è§ī":64237,"ãģķãĤī":64238,"-directed":64239,"Ġrů":64240,"为å®ŀçݰ":64241,"è¿Ļä»¶":64242,"aleb":64243,"ĠPride":64244,"Ġmuttered":64245,"qli":64246,"}).":64247,"è½½èį·":64248,"çŁŃè¯Ń":64249,"ĠLDL":64250,"Ġendpoints":64251,"ĠLagos":64252,"abili":64253,"Ġmeille":64254,"Ġthinkers":64255,"िष":64256,"ĠStreng":64257,"leases":64258,"ĠBea":64259,"ĠÑģвоего":64260,"HV":64261,"ĠÙĬØ´":64262,"/local":64263,"Ġà¦ħনà§ĩà¦ķ":64264,"ä¸įçŃīå¼ı":64265,"å·¥æľŁ":64266,"logger":64267,"åIJ¯ç͍":64268,"Ġhooked":64269,"ĠSounds":64270,"ĠLaf":64271,"گذ":64272,"ĠÑĪиÑĢок":64273,"ĠExhibit":64274,"ĠÙĪÙĤت":64275,"spection":64276,"é¥Ńèıľ":64277,"forth":64278,"çαä¸Ĭ":64279,"Ġlaunches":64280,"Ġwholesale":64281,"Ġcura":64282,"çŃīæ´»åĬ¨":64283,"Ġunreasonable":64284,"Ġdecode":64285,"Õ¡Õ²":64286,"Ġextracting":64287,"Ġflourish":64288,"+r":64289,"Ġterme":64290,"ä¸ĵä¸ļçŁ¥è¯Ĩ":64291,"Ġquantified":64292,"opoulos":64293,"款å¼ı":64294,"並ä¸į":64295,"-An":64296,"ä¸Ģåħĥ":64297,"ĠÐłÑĥ":64298,"ĠاÙĨساÙĨ":64299,"Ġসà§įথ":64300,"è¼Ķ":64301,"åºŁæ°´":64302,"ĠÙĩÙħÙĩ":64303,"âĢº":64304,"Ġconserve":64305,"lightenment":64306,"Ġsubstitu":64307,"å·¥åĮł":64308,"Ġbreaches":64309,"ĠApproaches":64310,"Ġbury":64311,"çĦ¶åIJİç͍":64312,"Ġmessenger":64313,"Ġtransitional":64314,"Ġdiversos":64315,"ĠFuk":64316,"ä¹Łéĥ½æĺ¯":64317,"Jean":64318,"ĠTracking":64319,"ĠAdj":64320,"retch":64321,"æĻºåķĨ":64322,"Behavior":64323,"ĠDavidson":64324,"!(\"":64325,"â̤":64326,"ãĢĭãĢĤĊ":64327,"ĠاÙĦÙħØ·ÙĦع":64328,"åIJ¯èĴĻ":64329,"ĠJap":64330,"Ġположи":64331,"æĤª":64332,"Ġgenotypes":64333,"_function":64334,"á̝áĢķáĢºáĢ":64335,"ĠاÙĤتص":64336,"Bear":64337,"ä¼ļåıijçĶŁ":64338,"ĠEnergie":64339,"urai":64340,"ĠÐIJн":64341,"DEBUG":64342,"Ġ'-'":64343,"KEN":64344,"ĠClosing":64345,"ĠBronze":64346,"ä¸ĢåłĨ":64347,"ĠtrÄĥm":64348,"æĸĩæ¡Ī":64349,"ĠApplying":64350,"ĠDetail":64351,"America":64352,"ĠCtrl":64353,"ĠOVER":64354,"å¤ļåįĬ":64355,"çļĦèĥ½éĩı":64356,"æĬĬä»ĸ们":64357,"posites":64358,"é»Ħæĺı":64359,"hmen":64360,"çļĦ羣å®ŀ":64361,"ĠThema":64362,"壬":64363,"ัà¸ŀ":64364,"ä¸ĩäºĭ":64365,"ÑģÑĸ":64366,"æ²»åĽ½":64367,"Ġأش":64368,"XC":64369,"Ġaujourd":64370,"ĠPoverty":64371,"Ġconcess":64372,"ĠDaten":64373,"ĠLau":64374,"è§ģçĬ¶":64375,"Ġglowing":64376,"Ġeryth":64377,"Ġexclaimed":64378,"ĠStark":64379,"Ñħой":64380,"Ġlightness":64381,"Ġpuluh":64382,"Ġchalk":64383,"Ñĭн":64384,"Ġdisse":64385,"ĠHypertension":64386,"ĠBros":64387,"è¿ĽéŨ":64388,"jest":64389,"ĠWheel":64390,"Ġcoloured":64391,"Ġtestify":64392,"Ġأد":64393,"è¿ĻäºĽä¸ľè¥¿":64394,"ĠOriginally":64395,"ĠSituation":64396,"ĠCAM":64397,"assembly":64398,"ÏĦαÏĤ":64399,"èº²åľ¨":64400,"Ġcocoa":64401,"Ġld":64402,"Ġvys":64403,"Ġíģ´":64404,"Ġabnorm":64405,"å®¶ç͵":64406,"èĢĮæĪIJçļĦ":64407,"ĠReve":64408,"Ġ×Ķ×ĺ":64409,"Ġtotient":64410,"micos":64411,"åı¤èĢģçļĦ":64412,".beans":64413,"ĠÑĤка":64414,"å¦Ĥä¸ĭåĽ¾":64415,"ĠCOND":64416,"æĻŁ":64417,"ä¾§éĩį":64418,"Expected":64419,"ĠпоÑĤомÑĥ":64420,"ĠÑĢавна":64421,"gele":64422,"ĠResort":64423,"èrent":64424,"综èīº":64425,"means":64426,"ä¸įä»ħåı¯ä»¥":64427,"êµIJìľ¡":64428,"à¥įà¤ļ":64429,"Researchers":64430,"ä¸Ģç§įæĺ¯":64431,"TreeLabel":64432,"ĠSnap":64433,"Ġкаждого":64434,"Ġnickname":64435,"Quality":64436,"nets":64437,"åĴĮä»ĸ们":64438,"æ³ķæ¡Ī":64439,"Ġbuz":64440,"ç«ĻçļĦ":64441,"ĠSymphony":64442,"Ġsubscriber":64443,"Ġfishes":64444,"Ġthematic":64445,"åĿļä¿¡":64446,"Ġdiamonds":64447,"Ġbash":64448,"æģª":64449,"ĠResponsible":64450,"-operation":64451,"gary":64452,"ç»Ĭ":64453,"ĠERA":64454,"itecture":64455,"'am":64456,"?_":64457,"ĠبررسÛĮ":64458,"ä»ĵåĤ¨":64459,"äºĨä¸ĢåľĪ":64460,"å®ĮåĸĦçļĦ":64461,"Ġgihapon":64462,"лим":64463,"Ġsubunit":64464,".\",Ċ":64465,"Ġfacets":64466,"ÙĪØ¯Ø©":64467,"Ġmanipulated":64468,"_response":64469,"íĻ©":64470,"ĠØŃÙĬاتÙĩ":64471,"Ġsze":64472,"åŁł":64473,"宫殿":64474,"ĠNeuroscience":64475,"POR":64476,"zcz":64477,"æīĢ产çĶŁçļĦ":64478,"åħ¬ç͍":64479,"ĠÙħرب":64480,"Ġfloral":64481,"ç»ĵçŁ³":64482,"Ġeconomical":64483,"critical":64484,"Ġsalute":64485,"ëıħ":64486,"íģ":64487,"меÑĤÑĮ":64488,"å¼łå¼Ģ":64489,"Ġnanot":64490,"nuts":64491,"é¢Ħ示":64492,"Ġанг":64493,"ĠÏĦὸ":64494,"èĮĥåĽ´åĨħçļĦ":64495,"Ġavenues":64496,"éĢĻæĻĤ":64497,"Ġwaveform":64498,"ĠÑĤеÑħни":64499,"Ġamerican":64500,"ivores":64501,"缸ä½į":64502,"ĠCasey":64503,"Ġcocktail":64504,"Ġinterpreter":64505,"çijľä¼½":64506,"ĠGuatemala":64507,"Ġlowercase":64508,"çºłç¼ł":64509,"å¤ļæł·çļĦ":64510,"ĠRegarding":64511,"ĠKane":64512,"Ġprefers":64513,"Ġshrubs":64514,"ĠHIST":64515,"liable":64516,"å®Ŀçİī":64517,"ĠÐŁÐ¾Ð»":64518,"Ġprowad":64519,"ĠksiÄħż":64520,"antung":64521,"çºĤ":64522,"éĵ¶æ²³":64523,"/String":64524,"ĠTeng":64525,"åįĩéĻį":64526,"Ibid":64527,"lash":64528,"Ġarb":64529,"åħµåύ":64530,"ĠصÙĦÙī":64531,"Ġimmersed":64532,"Ġincluso":64533,"famil":64534,"Ġcomplic":64535,"å¾®åĪĨ":64536,"Ġmarzo":64537,"дом":64538,"зÑĮ":64539,"à°Ĥ":64540,"è§Ĩå¯Ł":64541,"Ãły":64542,"éĿĴçĿIJ":64543,"ĠسÙħاÙĪÙī":64544,"_MAX":64545,"å®ŀåĬ¡":64546,"è¦ĭåΰ":64547,"arik":64548,"agos":64549,"ç§Ĩ":64550,"æİ¨åĩºçļĦ":64551,"-main":64552,"Ġsensations":64553,".stereotype":64554,"ĠIdeal":64555,"Folder":64556,"...âĢĿĊĊ":64557,"æĢ»åħ±":64558,"Ġcourty":64559,"èīºäºº":64560,"ĠSeeing":64561,"Ġcosas":64562,"ĠاÙĦØŃÙĬاة":64563,"transaction":64564,"Ġoscillations":64565,"Ġoutf":64566,"éķ¿å¾ģ":64567,"radle":64568,"ä¸ĢæŃ¥æŃ¥":64569,"Ġelemento":64570,"æĸ°é¢ĸ":64571,"èĩªæĿ¥":64572,"æ³¢åħ°":64573,"Imm":64574,"Autowired":64575,"chus":64576,"iry":64577,"Ġרק":64578,"ĠÙħطاÙĦ":64579,"Ik":64580,"anu":64581,"ÙĪÙĬر":64582,"been":64583,".location":64584,"Ġincapable":64585,"ĠPlasma":64586,"ĉĉĉĉĉĉĉĉĉ":64587,"Ġhommes":64588,"Ġpitched":64589,"pected":64590,"ellations":64591,"è¾Ľåĭ¤":64592,"ĠاÙĦجÙħ":64593,"Ġìĭľê°Ħ":64594,"ä½ĵä¼ļåΰ":64595,"urger":64596,"æĮŁ":64597,"羣æĥħ":64598,"Ġhydrolysis":64599,"ä¸Ģç¢Ĺ":64600,"èī¦":64601,"æĢ¥éľĢ":64602,"Ġvacant":64603,"ÑĽÐ°":64604,"ĠÑĢабоÑĤÑĥ":64605,"ĠElekt":64606,"çļĦ第":64607,"ç»°":64608,"èIJĬ":64609,"å¡ijæĢ§":64610,"Ġeinfach":64611,"Ġeconomist":64612,"ĠAnalog":64613,"æĤłä¹ħ":64614,"Hebrew":64615,"ĠSigma":64616,"Ġricht":64617,"ĠÙħÙĨÙĩ":64618,"éĢģåİ»":64619,"ĠEquilateral":64620,"å·¥åħ·æłı":64621,"ĠNutr":64622,"éro":64623,"intah":64624,"Ġparce":64625,"æĦٿ̧":64626,"onometry":64627,"èĩªçĦ¶çķĮ":64628,"ĠToxic":64629,"é«ĺç§ijæĬĢ":64630,"æĭĽçīĮ":64631,"ĠÐIJл":64632,"ĠRecip":64633,"Ġprofessionally":64634,"æŃ´":64635,"Ġslender":64636,"Ġinve":64637,"ĠINFORMATION":64638,"æ··æ·Ĩ":64639,"iennent":64640,"ĠCW":64641,"æĬĬ人":64642,"ÑĤего":64643,"FFER":64644,"رÙĪÙĩ":64645,"Ġdilution":64646,"Accept":64647,"Ġcohesion":64648,"ĠTort":64649,"ä¼ļéĢłæĪIJ":64650,"åĨħåĪĨæ³Į":64651,"ä½İè°ĥ":64652,"ĠRG":64653,"æ°´åľŁ":64654,"èī¯ãģĦ":64655,"Neill":64656,"Ġsuprem":64657,"Cancel":64658,"Ġgh":64659,"竣æĺ¯":64660,"éĹŃä¸Ĭ":64661,"?v":64662,"æĸ°æĺ¥":64663,"ĠÙħتر":64664,"ört":64665,"ĠMonetary":64666,"è·³è·ĥ":64667,"æīĵæĭĽåij¼":64668,"Ġdrawer":64669,"ĠMelissa":64670,"Ġà¸Ľà¸µ":64671,"ĠInsecta":64672,"ĠاÙĦرÙĬاض":64673,"ĠForbes":64674,"ãģ¨ãģĻãĤĭ":64675,"ĠповеÑĢÑħноÑģÑĤи":64676,"Kom":64677,"ĠTod":64678,"æİ¥è¿ŀ":64679,"Ġestruct":64680,"åĶIJæľĿ":64681,"ä¸¥æł¼èIJ½å®ŀ":64682,"ĠпаÑĨи":64683,"Ġpouvez":64684,"ĠFabric":64685,"itur":64686,"ĠStrom":64687,"éĵ¶è¡ĮåŃĺæ¬¾":64688,"िस":64689,"ÙĪÙħÛĮ":64690,"ruitment":64691,"Johnson":64692,"Ġreprésent":64693,"Ġempowers":64694,"CEPT":64695,"裡çļĦ":64696,"ĠHazard":64697,"ĠContributions":64698,"ersche":64699,"FFFFFF":64700,"ĠEDT":64701,"Ġxs":64702,"ãģªãģ®ãģ§":64703,"اعدÙĩ":64704,"Ġbelangrijk":64705,"çļĦå®ŀåĬĽ":64706,"efic":64707,",NULL":64708,"Ever":64709,"ären":64710,"Ġslim":64711,"è¿ĻäºĽå¹´":64712,"ĠاÙĦØ¥ÙĨساÙĨ":64713,"Ġcamin":64714,"éģıåζ":64715,"XD":64716,"Ready":64717,"ĠWhatsApp":64718,"æħĺ":64719,"çİ°åľ¨å·²ç»ı":64720,"ettle":64721,"Ġgarment":64722,"ĠDocker":64723,"è¡Į车":64724,"Ġsweetness":64725,"ĠWarsz":64726,"Ġcoincide":64727,"availability":64728,"Ġfinals":64729,"_EN":64730,"Ġmidd":64731,"ĠLabs":64732,"اعÙĬØ©":64733,"scripts":64734,"Ñĥп":64735,"èµĦäº§è´ŁåĢº":64736,"Ġξε":64737,"è¡Ĩ":64738,"没æĶ¶":64739,"åIJ¬èµ·æĿ¥":64740,"Ġadenine":64741,"Ġsubmarine":64742,"å°Ĩèĩªå·±çļĦ":64743,"Ġpolype":64744,"ä¹Ķæ²»":64745,"èĥ°å²Ľç´ł":64746,"Ġinne":64747,"uties":64748,"ĠPione":64749,"ĠìĥĿê°ģ":64750,"arXiv":64751,"Ġoppos":64752,"Ġfluency":64753,"Ġdancers":64754,"ĠFam":64755,"ensemble":64756,"Dif":64757,"ربÙĬØ©":64758,"ĠÙĩÙħÚĨÙĨÛĮÙĨ":64759,"YW":64760,"Ġbunk":64761,"çľĭ她":64762,"æĺİæ¸ħ":64763,"èĢģ人家":64764,"èĦĬé«ĵ":64765,"usepackage":64766,"Political":64767,"ĠdÃŃ":64768,"ĠePub":64769,"reset":64770,"Ġcomics":64771,"Ġtribunal":64772,"夺åıĸ":64773,"åľ¨æŃ¤åŁºç¡Ģä¸Ĭ":64774,"adors":64775,"èĮī":64776,"第ä¸Ģæī¹":64777,"å·¥ç¨ĭé¡¹çĽ®":64778,"ãĥ¼ãĥĹ":64779,"Ġpakig":64780,"\"If":64781,".stdin":64782,"ĠÑıв":64783,"HK":64784,"èĥ½çľĭåΰ":64785,"Ġimmersion":64786,"à¥ĩन":64787,"Ġcohesive":64788,"Ġaureus":64789,"Ġrecht":64790,"ĠполÑĥÑĩиÑĤÑĮ":64791,"ausing":64792,"оÑĢÑı":64793,"Ġmonks":64794,"Ġniż":64795,"Ġexhibitions":64796,"Ġsyllabus":64797,"оÑĤа":64798,"ÑĤной":64799,"Ġleagues":64800,"è®¾åľ¨":64801,"Å¡enÃŃ":64802,"æľīèĩªå·±çļĦ":64803,"ĠпÑĢоÑĨеÑģÑģа":64804,"ĠProductions":64805,"åij¨éķ¿":64806,"Ġsensed":64807,"ãĤīãģ®":64808,"種é¡ŀ":64809,"Ġhearings":64810,"ahanay":64811,"Ġconquered":64812,"é¦ĸä½į":64813,"à²Ĺಳ":64814,"bingkil":64815,"}//":64816,"座çļĦ":64817,"ä¸įä½ıäºĨ":64818,"Ġclarification":64819,"ĠBMW":64820,"exists":64821,"头çĸ¼":64822,"ائر":64823,"Ġinteracts":64824,"Ġfixtures":64825,"others":64826,"Ġtackling":64827,".Message":64828,"omyc":64829,"hetically":64830,"Ġmessy":64831,"ëĤ¨":64832,"åı¯éĩĩç͍":64833,"Ġgoats":64834,"Ġqueer":64835,"çľ¼çIJĥ":64836,"Ġ×ij׳":64837,"çļĦå·¥ä½ľäººåijĺ":64838,"ĠMemphis":64839,"åĮķ":64840,"ç§§":64841,"é¢ĨåľŁ":64842,"çݯç»ķ":64843,"ĠLiang":64844,"ológico":64845,"Ġastonishing":64846,"Ġpakigbingkil":64847,"thora":64848,"éķĩåİĭ":64849,"Ġbutterflies":64850,"ĠÑĢабоÑĤе":64851,"ĠStrip":64852,"áct":64853,"æ¤įæłª":64854,"Ġnueva":64855,"RIC":64856,"Ġeux":64857,"Ġgait":64858,"çĻľ":64859,"วà¸Ī":64860,"è²§":64861,"ä¸ĭéĿ¢æĺ¯":64862,"Verified":64863,",%":64864,"Gar":64865,"iglia":64866,"ĠXxxxx":64867,"Ġsentencing":64868,"磺":64869,"èģĺ请":64870,"Ġgranular":64871,"ĠнаÑħодиÑĤÑģÑı":64872,"è¿Ļ对äºİ":64873,"åIJİä¸ĸ":64874,"-ser":64875,"åħīæ³½":64876,"èį¯çī©çļĦ":64877,".Json":64878,"Ġcovariance":64879,"çļĦ代表":64880,"è¿Ļæł·ä¸ĢæĿ¥":64881,"Angle":64882,"Ġricher":64883,"pancy":64884,"íķ´ìķ¼":64885,"ĠTypical":64886,"Ġundis":64887,"Ġоказа":64888,"ĠPharm":64889,"èĮĥåĽ´çļĦ":64890,"rul":64891,"reso":64892,"رج":64893,"è½¼":64894,"åıªä¸º":64895,"ymen":64896,"Ġrainforest":64897,"ä¸įä¸Ģæł·çļĦ":64898,"רץ":64899,"妹åŃIJ":64900,"Ju":64901,"äºĶåħŃ":64902,"Ġshrub":64903,"ĠDrake":64904,"kter":64905,"urf":64906,"Ġcomer":64907,"ZD":64908,"Ġколе":64909,"æĪ¿éŨ":64910,"cznie":64911,"åĮºåŁŁåĨħ":64912,"girl":64913,"Ġtêm":64914,"includes":64915,"Ġpaved":64916,"Ġgren":64917,"ĠLeeds":64918,"ä¸Ĭä¹Ł":64919,"Ġstatist":64920,"ĠEmpty":64921,"Ġredox":64922,"Ġloosely":64923,"NotNull":64924,"Ġtoxin":64925,"Ġpues":64926,"Ġsalinity":64927,"++Ċ":64928,"ĠÑģÑĤандаÑĢ":64929,"_position":64930,"æĶ¾äºĨ":64931,"Ġtriggering":64932,"ĠRuntime":64933,"Ġmyös":64934,"éĩį伤":64935,"æºĿ":64936,"Ġtutte":64937,"émon":64938,"Ġwasting":64939,"ç»ĵæŀľè¡¨æĺİ":64940,"åıªèĥ½åľ¨":64941,"..............................":64942,"ä¹ĭæĪĺ":64943,"ÅĻen":64944,"Ġestimator":64945,"Ġmanageable":64946,"Georg":64947,"Ġconceal":64948,"è¬Ģ":64949,"ĠFacilities":64950,"ĠInclusion":64951,"ç¨İé¢Ŀ":64952,"æľĢ大å̼":64953,"Ġimplantation":64954,"ifice":64955,"Ġplatinum":64956,"åĩłåįĥ":64957,"(\"":64992,"ä¸įåıªæĺ¯":64993,"ĠChancellor":64994,"èĸ¬":64995,"Ġelegance":64996,"Ġtg":64997,"ÑĪка":64998,"Ġgeg":64999,"å®īå¨ľ":65000,".sup":65001,"cientists":65002,"ĠкоÑĤоÑĢом":65003,"ederation":65004,"ä»Ģä¹Īäºĭæĥħ":65005,"ĠCroatia":65006,"ĠBlank":65007,"æ¼ĶåĮĸ":65008,"Ġscripture":65009,"ĠSpiritual":65010,"å°Ĩé¢Ĩ":65011,"شتÙĩ":65012,"Ġcabbage":65013,"تÙģØ§Ø¹":65014,"'al":65015,"Ġnomb":65016,"ĠCoven":65017,"赦":65018,"iena":65019,"-model":65020,"ĠPatel":65021,"èµĭå̼":65022,"æIJ¬è¿IJ":65023,"ĠBelgian":65024,"ocar":65025,"åĮºæĶ¿åºľ":65026,"keyword":65027,"Ġanthropology":65028,"ĠTelesc":65029,"iola":65030,"zeros":65031,"ĠÑĥÑĩаÑģÑĤи":65032,"Joined":65033,"Ġbinnen":65034,"ĠDion":65035,"Subt":65036,"é»ĺå¥ij":65037,"à©ĭ":65038,"çļĦåĽŀ":65039,"ĠDag":65040,"æĪij便":65041,"-file":65042,"Ġplantation":65043,"ĠاÙĦأش":65044,"éĢļ常æĺ¯":65045,"subscriptðĿľ":65046,"ĠÐŁÐ¾Ñģле":65047,"å®ĥä¸İ":65048,"éĢIJå¹´":65049,"æĺ¾ç¤ºåύ":65050,"ĠMiy":65051,"Ġlei":65052,"åįĥéĩij":65053,"Ġammonium":65054,"\\lambda":65055,"Ġprizes":65056,"ł×ĵ":65057,"Ġhass":65058,"ĠGMAT":65059,"缪":65060,"Ġcleavage":65061,"æĬ¤åį«":65062,"ĠíĴ":65063,"éĩįåºĨå¸Ĥ":65064,".):":65065,"åĨħä¾§":65066,"常æľī":65067,"ducation":65068,"guna":65069,"ĠLenn":65070,"æŁij":65071,"åĽłçβ":65072,"upuncture":65073,"Ġvolts":65074,"Commit":65075,"æ¸ħæ¥ļäºĨ":65076,"ĠAntarctic":65077,"èµ·ãģĵ":65078,"åŁİå¢Ļ":65079,"à¸ķà¸Ļ":65080,"ç¥ŀç»ıç³»ç»Ł":65081,"æ±Łè¥¿çľģ":65082,"ತà³įತ":65083,"ĠSSE":65084,"phalt":65085,"Ġ'{":65086,"Ġкод":65087,"åıijæĮ¥äºĨ":65088,"ĠWisdom":65089,"åĴĶ":65090,"Ġscars":65091,"ัส":65092,"ä¸ĩä½Ļ":65093,"ाà¤ķ":65094,"Ġfraudulent":65095,"าà¸Ĺีà¹Ī":65096,"Sup":65097,"kas":65098,"Ġoggi":65099,"-fat":65100,"ENGTH":65101,"(next":65102,"Ub":65103,"Ġagli":65104,"ä¸įå°ij人":65105,"Explain":65106,"æ³Ħæ¼ı":65107,"umers":65108,"ieves":65109,"æĢ¥åī§":65110,"Ġhemoglobin":65111,"ĠProzent":65112,"ĠвÑĭпÑĥ":65113,"ĠInstitutions":65114,"ĠØ®ÙĪØ§Ùĩد":65115,"Ġhundre":65116,"iquement":65117,"Ġcalend":65118,"abin":65119,"-dist":65120,"ĠÙĬؤ":65121,"sudo":65122,"没éĹ®é¢ĺ":65123,"Ġ@@":65124,"ĠManagers":65125,"ĠInternacional":65126,"Ġepistem":65127,"ĠNaOH":65128,"ãģ«ãģªãĤĬ":65129,"Ġilluminated":65130,"Ġباعث":65131,"ä½ľæ¥Ń":65132,"âĢĵĊĊ":65133,"波形":65134,"ãĥķãĤ£":65135,"ĠLeonardo":65136,"åijľåijľ":65137,".register":65138,"ĵ°":65139,"Ġqua":65140,"æĽ´æ·±":65141,"线åĴĮ":65142,"Wild":65143,"çĶŁå§ľ":65144,"Ġstakeholder":65145,"åħīçĽĺ":65146,"ViewById":65147,"tub":65148,"andering":65149,"ÙħÙĦÙĥ":65150,"æľ¬æľĪ":65151,"ajes":65152,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":65153,"XI":65154,"Ġwears":65155,"æĻī":65156,"æŃ£ç¢º":65157,".call":65158,"Ġproudly":65159,"Ġrobotics":65160,"&B":65161,"Ġration":65162,"ĠÙħدار":65163,"貸":65164,"êmes":65165,"urtle":65166,"ectar":65167,"æ³ķåħ°":65168,"ĠSheets":65169,"Ġsympath":65170,"Ich":65171,".sum":65172,"arthritis":65173,"ĠAbsolute":65174,"Ġplains":65175,"Ġmilestones":65176,"ĠDivis":65177,"ĠÑĺÑĥ":65178,"åĨ»ç»ĵ":65179,"ĠÑįлекÑĤÑĢи":65180,"Ġmodalities":65181,"Ġpersuaded":65182,"æ²Ļåıijä¸Ĭ":65183,"Camera":65184,"ĠDEM":65185,"ujÃŃcÃŃ":65186,"éº»æľ¨":65187,"ĠìĿ´ìļ©":65188,"ä¸įå°ijäºİ":65189,"pk":65190,"èµĺ":65191,"Ġtrouver":65192,"ĠGoods":65193,"碱æĢ§":65194,"onset":65195,"ĠBeau":65196,"Ġúltimo":65197,"ĠJensen":65198,"Ġ$('#":65199,"Ġbeliever":65200,"峨":65201,"à³Ģ":65202,"Ġbadge":65203,"éĻĨåĨĽ":65204,"Ġдев":65205,"çŃīä»ĸ":65206,"Ġpodob":65207,"cco":65208,"Ġsignalling":65209,"urous":65210,"ĠPTSD":65211,"ĠRapp":65212,"积æŀģå¼Ģå±ķ":65213,"éŃĶ鬼":65214,"æĪijå°±ä¸į":65215,"ioa":65216,"æĭīä¸ģ":65217,"ÑĬек":65218,"ĠRoberto":65219,"ç®Ń头":65220,"ëĦ¤":65221,"éĻĮçĶŁäºº":65222,"Ġsturdy":65223,"(json":65224,"/%":65225,"|l":65226,"Ñģм":65227,"æĪij羣":65228,"èĢħ为":65229,"ä»·æł¼çļĦ":65230,"Minimum":65231,"negie":65232,"ĠkHz":65233,"opian":65234,"éĢī项åį¡":65235,"Ġweighs":65236,"Virtual":65237,"ĠbÃłi":65238,"ĠMud":65239,"ĠBian":65240,"ále":65241,"ĠPhyt":65242,"Ġunfolding":65243,"Ġhusbands":65244,"Ġwied":65245,"ĠUIT":65246,"æ·ĭå·´ç»ĵ":65247,"Mouse":65248,"éĴ±åĮħ":65249,"éĥ½æĥ³":65250,"Ġ_ĊĊ":65251,"æīĭ表":65252,"IImage":65253,"à¯įள":65254,"Ġproofs":65255,"æºĢ":65256,"çļĦ好å¤Ħ":65257,"_types":65258,"ĠGBP":65259,"Ġanime":65260,"ĠGuru":65261,"è¡ĢæµĨ":65262,"æ¡ĮåŃIJä¸Ĭ":65263,"anneer":65264,"informatics":65265,"æľĢéķ¿":65266,"ĠGreens":65267,"Ġплан":65268,"Ġluxurious":65269,"Kahenera":65270,"ãĥķãĤ©":65271,"Ġreminiscent":65272,"ept":65273,"Ġdort":65274,"Ġgobier":65275,"ĠChu":65276,"ĠzwiÄħz":65277,"াà¦Ń":65278,"ACTION":65279,"astrous":65280,"没æľīæĥ³åΰ":65281,"ĠеÑij":65282,"æħİéĩį":65283,"åĶij":65284,"رÙĬÙĤØ©":65285,"åī¥å¤º":65286,"ĠSeite":65287,"ĠAppropri":65288,"æ°ij主åħļ":65289,")**ĊĊ":65290,"_block":65291,"mé":65292,"ĠBesch":65293,"ĠHerman":65294,"Ġscrews":65295,"Ġvlast":65296,"Ġfrecu":65297,"Ġblooms":65298,"Scal":65299,"phins":65300,"ç½ijçĤ¹":65301,"ĠBiotechnology":65302,"Ġkra":65303,"Ġ-----------------------------------------------------------------":65304,"éķ¿è¾Ī":65305,"ĠCODE":65306,"Ġligament":65307,"ãĤīãĤĮãģŁ":65308,"ĠScreening":65309,"Ġeuropé":65310,"èı²å¾ĭ宾":65311,"ickers":65312,"è¿ijåĩłå¹´":65313,".gl":65314,"å¥ĸåѦéĩij":65315,"é̲åİ»":65316,"ĠSans":65317,"Ġsmo":65318,"计ç®Ĺåĩº":65319,"質éĩı":65320,"Ġrez":65321,"å·³":65322,"èµ°åΰäºĨ":65323,"ĠúÄį":65324,"Ġਦ":65325,"Ġtheatrical":65326,"Ġregex":65327,"Ġconclus":65328,"æ±ĩ票":65329,"æĹ§çļĦ":65330,"æĸĩæĺİçļĦ":65331,"Maximum":65332,"Ġpolymerization":65333,"_logic":65334,"Ġunexpl":65335,"åħ¬ç«ĭ":65336,"åĽ½å®¶å®īåħ¨":65337,"ç»Ŀ大éĥ¨åĪĨ":65338,"ĠÙĪØ§ÙĦتÙĬ":65339,"ISTS":65340,"(cur":65341,")a":65342,"äºĨä¸Ĭåİ»":65343,"ĠLah":65344,"secret":65345,"-log":65346,"ĠTaliban":65347,"Ġconcealed":65348,"峡谷":65349,"ĠÙĪØ§ÙĦØŃ":65350,"Ġmattress":65351,"lv":65352,"Ġmam":65353,"Ġcommune":65354,"æĤ£çĹħ":65355,"Ġಹ":65356,"Pt":65357,"astical":65358,"Ġraid":65359,"Ġered":65360,"ĠعÙħر":65361,"кими":65362,"Ġeats":65363,"å¾Ļ":65364,"åħ¶éĹ´":65365,"×Ļס×ĺ":65366,"awy":65367,"/Y":65368,"=self":65369,"Messages":65370,"ĠDenomin":65371,"ĠعÙĦÙĬÙĩا":65372,"Ġrugby":65373,"æ¦Ħ":65374,"ĠоÑĤлиÑĩа":65375,"<":65376,"Ġcen":65377,"Ġblev":65378,"ä¸ŃåIJ«æľī":65379,"×ķÖ¼":65380,"ĠدÙĬ":65381,"äººä»¬åľ¨":65382,"çĶŁæĢģç³»ç»Ł":65383,"Ġdisputed":65384,"Ġparadise":65385,"-ext":65386,"Ġcompressor":65387,"åĩıåİ»":65388,"Ġpredator":65389,"ĠUntuk":65390,"..........................................":65391,"(Z":65392,"è¿ĺæĮº":65393,"ĠMilwaukee":65394,"Ham":65395,"ĠWissenschaft":65396,"æŃ£è¦ģ":65397,"Ġmesure":65398,"æľŁä¸Ń":65399,"Ġfacto":65400,"stitut":65401,"èĩªç§ģ":65402,"Ġsynonym":65403,"BG":65404,"Ġsurgeons":65405,"å¼ĢçĿĢ":65406,"üm":65407,"ifestyle":65408,"ĠÑĢоз":65409,"æijĦåĥı头":65410,"buf":65411,"ĠRevolutionary":65412,"Ġcerebro":65413,"çļĦåħīèĬĴ":65414,"ëĦĪ":65415,"Ġépoca":65416,"ĠKum":65417,"ĠÙħخطط":65418,"iritual":65419,"Ġdiaphrag":65420,"Statistics":65421,"åĸĥåĸĥ":65422,"Ġevacuation":65423,"Ġfilament":65424,"Ġproposing":65425,"åĪĽæĸ°åĪĽä¸ļ":65426,"Ġfixes":65427,"ä¸Ń说":65428,"ĠJorge":65429,"Ġabril":65430,"Ġterribly":65431,"ĠOverflow":65432,"Ġবিষ":65433,"ĠWatt":65434,"Ġmilliliters":65435,"Ġpromotions":65436,"Sports":65437,"rÃł":65438,"Ġ-*":65439,"ÙĬاÙħ":65440,"缴å±ŀ":65441,"éĢīçļĦ":65442,"Ġrepaired":65443,"èĥĮåĮħ":65444,"éĺµæ³ķ":65445,"第ä¸Ģç§į":65446,"ĠVoltage":65447,"cool":65448,"Ġenvi":65449,"/problems":65450,"anic":65451,"Ġisinstance":65452,"license":65453,"麥":65454,"iscus":65455,"ĠSandra":65456,"fires":65457,"editor":65458,"ĠCoc":65459,"irling":65460,"Ġparcel":65461,"é£İä¿Ĺ":65462,"arnation":65463,"ĠLeading":65464,"Ġloi":65465,"Ġavoids":65466,"lters":65467,"ä¸ĵéŨçļĦ":65468,"nx":65469,"Ġimminent":65470,"ÑįÑĢ":65471,"Ġmastering":65472,"ĠKnights":65473,"itoneal":65474,"Ġsott":65475,"åı¯ä¿¡":65476,"azzi":65477,"ĠArms":65478,"Ġতা":65479,"Ġthanked":65480,"ĠNGC":65481,"ensit":65482,"=='":65483,"нÑıеÑĤÑģÑı":65484,"éĴĪ对æĢ§":65485,"/inch":65486,"èĪĮ头":65487,"ĠPrices":65488,"ĠWise":65489,"Ġdislike":65490,"Ġrd":65491,"æĹ¶éĴĪ":65492,"Ġvulgar":65493,"æĦĪåIJĪ":65494,"Ġpioneering":65495,"_json":65496,"ciones":65497,"Ġbuoy":65498,"-dose":65499,"ä¹Łæĺ¯å¦ĤæŃ¤":65500,"Ġtrustee":65501,"Cook":65502,"A":65503,"Ġturbines":65504,"æľī为":65505,"äºĭä¸ļçļĦ":65506,"Ġ\\({}^{\\":65507,"stem":65508,"-member":65509,"å¾®éĩı":65510,"ĠTrauma":65511,"Ġnarrowed":65512,"approved":65513,"Ġdez":65514,"Ġzomb":65515,"ç¾Ł":65516,"Colors":65517,"Ġscreamed":65518,"balls":65519,"ĠÑģÑĥÑīеÑģÑĤв":65520,"/cl":65521,"ordination":65522,"Ġactin":65523,"(){":65524,"æ¯ıéļĶ":65525,"Ġfacult":65526,"ä¹ĭå¹´":65527,"ä¸įè¦ģåĨį":65528,"ĠMusik":65529,"æĭľè®¿":65530,"ĠÑĥÑģловиÑı":65531,"Ġferv":65532,"ĠSask":65533,"Ġadversely":65534,".For":65535,"_def":65536,"åīį端":65537,"ĠMonroe":65538,"Ġhurting":65539,"Ġiterator":65540,"lp":65541,"±Ħ":65542,"Ġobec":65543,"å°Ĩ对":65544,"Ġdecimeter":65545,"ĠÙģØ§Ø±":65546,"èİ·åĪ©":65547,"ĠÙĪØ£ÙĨ":65548,"Ġнадо":65549,"ĠØ£ÙĦÙģ":65550,"Ġannouncing":65551,"éķ¿æľŁçļĦ":65552,"åŃķèĤ²":65553,"精彩çļĦ":65554,"OE":65555,"aÅĤ":65556,"è¿ĺä¸įå¤Ł":65557,"Ġscare":65558,"Ġlaughs":65559,"èĩºçģ£":65560,"æĺ¯å¥½":65561,"æĽ³":65562,"çŃīä¸Ģç³»åĪĹ":65563,"Ġampere":65564,"ä¿Ŀè¯ģäºĨ":65565,"ש×ķת":65566,"absolute":65567,"Ġinfringement":65568,"Ġrecharge":65569,"åıijè§ī":65570,"çłĶç©¶ä¸Ń":65571,"åĪĩçīĩ":65572,"visit":65573,"Ġprohibit":65574,"åħ¬äº¤è½¦":65575,"ĠPhotoshop":65576,"FAIL":65577,"åľ¨ç¬¬":65578,"Ġexig":65579,"Ġprodukt":65580,"Ġdemocrat":65581,"æĵ¬":65582,"даÑİÑĤ":65583,"ĠIdentifier":65584,"ĠParticipation":65585,"ĠWL":65586,"Ġпл":65587,"æī¾äºĨ":65588,"-centric":65589,"}a":65590,"ĠÑģÑĤен":65591,"à¹Ģà¸ĩิà¸Ļ":65592,"'ab":65593,"ĠHog":65594,"ĠAbbey":65595,"عارÙģ":65596,"Ġinsignificant":65597,"çļĦéĢļ":65598,"åľ¨æĹ¥æľ¬":65599,"Ġunic":65600,"æ´»è¡Ģ":65601,"åı²è®°":65602,"çŀ©":65603,"Shift":65604,"表达äºĨ":65605,"è¿ĩéĶĻ":65606,"Ġunusually":65607,"Ġneighbouring":65608,"à¦Ĺà§įরহ":65609,"红ç»Ĩèĥŀ":65610,"饮ç͍":65611,"纲è¦ģ":65612,"ĠÙħدر":65613,"ĠÑĤеоÑĢи":65614,"feed":65615,"ĠÕ¸ÖĤ":65616,"Male":65617,"Ġunterschied":65618,"Ġforesee":65619,"(Object":65620,"gf":65621,"Ġgown":65622,"vex":65623,"å±Ĩ":65624,"Ġprinci":65625,"ijf":65626,"мин":65627,"ĠGeo":65628,"ĠCoin":65629,"\"In":65630,"chner":65631,"Represent":65632,"Ġexported":65633,"Ġpartes":65634,"è¾ĥå¤ļçļĦ":65635,"earcher":65636,"å¹³æķ´":65637,"Ġwatts":65638,"Ġmushroom":65639,"ĠSays":65640,"éĻįèĩ³":65641,"áºŃt":65642,"æĺ¯éľĢè¦ģ":65643,"äºĮ楼":65644,"å¤įæ´»":65645,"æĿ¡ä»¶åĴĮ":65646,"Ġindexed":65647,"ĠBash":65648,"åīįä¸Ģ":65649,"æ²»çĹħ":65650,"è¿Ļ个è¯į":65651,"eday":65652,"chini":65653,"ĠGö":65654,"女åħĴ":65655,"åºıåı·":65656,"ĠâĢľ[":65657,"ä¸ľæµ·":65658,"Õ¡Õ¤":65659,"çļĦå°ıä¼Ļä¼´":65660,"ĠEquality":65661,"Ġcylinders":65662,"åľ¨ç½ijä¸Ĭ":65663,"ä¸ĵ注äºİ":65664,"Hyp":65665,"ĠSorry":65666,"æ¥ĵ":65667,"زة":65668,"ĠBarrett":65669,"Ġuniqueness":65670,"Ġethyl":65671,"æī©å»º":65672,"Ġportrayal":65673,"GIS":65674,"plings":65675,"رÙħز":65676,"oclonal":65677,"ÏĨο":65678,"Ġsecretly":65679,"Estado":65680,"Ġrubbed":65681,"Ġwyp":65682,"çľĭéĩį":65683,"ç©¿åĪº":65684,"åĪĨ娩":65685,"Ġblindness":65686,"ĠتغÛĮÛĮر":65687,"":66902,"Ġζ":66903,"ç¿»äºĨ":66904,"就让":66905,"pointer":66906,"ĠHerr":66907,"ĠMetric":66908,"ÑģÑĤойÑĩи":66909,".\",":66910,"Ġedific":66911,"paralle":66912,"ĠRespect":66913,"Ġgenocide":66914,"æij¸ç´¢":66915,"Ġaffirmative":66916,"ĠÑĥзна":66917,"ç»ıéĶĢåķĨ":66918,"@Component":66919,"TN":66920,"Ġsepsis":66921,"éĩįåIJĪ":66922,"åĸĤåħ»":66923,"ðŁIJ":66924,"è¿Ļä¹Ī大":66925,"åIJĮæł·æĺ¯":66926,"ĠMessiah":66927,"ĠÙĬد":66928,"åıijçݰéĹ®é¢ĺ":66929,"meter":66930,"ĠKurd":66931,"åĨ·åĵ¼":66932,"ìķł":66933,"æĮĤçĿĢ":66934,".addEventListener":66935,"Ġداشت":66936,"Ġming":66937,"ĠIbn":66938,"åı¯ä¹IJ":66939,"ĠKul":66940,"Ġresumed":66941,"ĠاÙĦا":66942,"è¯Ħ级":66943,"ริม":66944,"æ±²åıĸ":66945,"zba":66946,"opens":66947,"Ġপà§įরà¦ķ":66948,"ĠaquÃŃ":66949,"åįģåĽĽäºĶ":66950,"å±ķçݰäºĨ":66951,"ĠSchol":66952,"haul":66953,"ĠSocorro":66954,"Ġoltre":66955,"æĿ¥ç¡®å®ļ":66956,"Ġhombre":66957,"Disclaimer":66958,"æĸĩéĽĨ":66959,"׾×ĵ":66960,"Ġfiguring":66961,"ÙĪÙĥبÙĩ":66962,"Ġintricacies":66963,"wiÄĻks":66964,"cx":66965,"Ġrost":66966,"waÄĩ":66967,"Ġestudiantes":66968,"Ġinmates":66969,"ieden":66970,"没æľīåĬŀæ³ķ":66971,"è³ĵ":66972,"Ġdisposable":66973,"Ġdisruptions":66974,"Ġadversity":66975,"utung":66976,"ppi":66977,"ĠÃĪ":66978,"............":66979,"çİ©æ³ķ":66980,"άÏģ":66981,"ĠHamlet":66982,"othelioma":66983,"ĠConsultant":66984,"ĠвÑĤоÑĢой":66985,"dag":66986,"Ġbewe":66987,"è°ĥåĴĮ":66988,"å¸ĤåľºèIJ¥éĶĢ":66989,"æĢķæĺ¯":66990,"ابÙĩ":66991,"ĠÑıн":66992,"à¸ķà¹Īาà¸ĩà¹Ĩ":66993,"ópez":66994,"ĠDisaster":66995,"ĠRecall":66996,"Ġhäuf":66997,"ĠÑĢазвиÑĤие":66998,"çķıæĥ§":66999,"Ġaluminium":67000,"ĠAnalytical":67001,"Ġ(>":67002,"缴åįĩ":67003,"ĠActor":67004,"ĠPEN":67005,"ĠاÙĦعدÙĬد":67006,"Ġವ":67007,"ĠкÑĢови":67008,"çĿĢçľ¼çĿĽ":67009,"èĵĿèī²çļĦ":67010,"ê¸Ģ":67011,"è¿IJç®Ĺ符":67012,"anian":67013,"Ġoutpatient":67014,"举åĬŀçļĦ":67015,">();ĊĊ":67016,"Ġstamps":67017,"ĠMöglich":67018,"iec":67019,"Ġmega":67020,"åĴĮæĹł":67021,"Ġdisob":67022,"ĠAndré":67023,"认è¯ĨçļĦ":67024,"Twitter":67025,"ä¸įå¿ħè¦ģçļĦ":67026,"Ġoffenses":67027,"Formatter":67028,"ĠCustomers":67029,"ocup":67030,"ĠOu":67031,"otech":67032,"Ġgenital":67033,"Ġ×ij׼׾":67034,"ابÛĮ":67035,"鬼åŃIJ":67036,"ĠPreliminary":67037,"ад":67038,"ĠJH":67039,"Platform":67040,"Ġમ":67041,"Ñģив":67042,"Ľ×ļ":67043,"褶":67044,"ä¾ĿçĦ¶æĺ¯":67045,"Ġbekan":67046,"åı¯æ¯Ķ":67047,"å¸Ī大":67048,"åĬĽéĩıçļĦ":67049,"ĠاÙĦÙħتØŃدة":67050,"éķ¿æķĪ":67051,"åĩıåħį":67052,"Ġ׾ק":67053,"Ġterrified":67054,"æĶ¾åĩº":67055,"伸åĩºæīĭ":67056,".Use":67057,"Ġhadde":67058,"ĠLei":67059,"ĠEvil":67060,"rogate":67061,"ĠDeriv":67062,"ãģ«ãģĤãĤĭ":67063,"หวà¹Īาà¸ĩ":67064,"×Ļ׾×ĵ":67065,"uploads":67066,"åİ»åĮ»éĻ¢":67067,"åħĪç͍":67068,"çļĦç²¾":67069,"ãģŁãģł":67070,"å̻ç»ıçIJĨ":67071,"Ġactividad":67072,"ãģ¼":67073,"onga":67074,"ĠReasons":67075,"æ²¹æ°Ķ":67076,"以åıĬ对":67077,"Ġcorrectness":67078,"ĠбÑĭÑģÑĤÑĢо":67079,"Ġkaum":67080,"ëŀ¨":67081,"ĠCron":67082,"ĠCopenhagen":67083,"ĠBASE":67084,"èĩªåıij":67085,"Ġ×ķת":67086,"Ġtandem":67087,"è¶Ĭä¾Ĩè¶Ĭ":67088,"Ip":67089,"Ġsniff":67090,"è¬Ŀè¬Ŀ":67091,"_action":67092,"åľ¨è¥¿":67093,"vertical":67094,"æĿijèIJ½":67095,"请èģĶç³»":67096,"ĠSPEC":67097,"ĠPythag":67098,"--)Ċ":67099,"Ġtratt":67100,"端æŃ£":67101,"ĠMae":67102,"Ġelast":67103,"encil":67104,"sterne":67105,"橱":67106,"ฯ":67107,"åĮĸçī©":67108,"Ġbiodegrad":67109,"ĠMonica":67110,"dish":67111,"ihu":67112,"罪çļĦ":67113,"ои":67114,"Ġconcom":67115,"Ġautonom":67116,"çļĦä¸ĢåĢĭ":67117,"åİĭå®ŀ":67118,"Ġdragging":67119,"=\"#\">":67120,"Ġhorns":67121,"Ġkemudian":67122,"\\Response":67123,"åıĪåIJį":67124,"ä»Ģä¹Īåľ°æĸ¹":67125,"ĠвиÑĤами":67126,"Ġestablishments":67127,"ÐłÐ°Ð·":67128,"_read":67129,"ĊĠĠĊ":67130,"宦":67131,"åĴĻ":67132,"æĢĿ绪":67133,"尽头":67134,"ĠоÑĤÑĢа":67135,"ĠAnglic":67136,"æĬĹæĭĴ":67137,"Ġtariffs":67138,"æĢ»ç®Ĺ":67139,"ä»ĬåĽŀ":67140,"ç¬ijèijĹ":67141,"������":67142,"ĠاÙĦØ£ÙħرÙĬÙĥ":67143,"(...":67144,"Ġnationality":67145,"]#Ċ":67146,"ç»ĺæľ¬":67147,"éĶ£":67148,"Ġfearful":67149,"Ġplugins":67150,"anter":67151,"ĠmÃŃn":67152,"stated":67153,"Ġslugs":67154,"ĠCRM":67155,"Ġżycia":67156,"YN":67157,"ifolia":67158,"ĠÅĻÃŃ":67159,"Ġtych":67160,"Ġevenings":67161,"å¸ĤåľºåĮĸ":67162,"-management":67163,"ĠDing":67164,"è´¹ç͍çļĦ":67165,"ĠDante":67166,"овÑĭÑħ":67167,"asked":67168,"çļĦå°ı说":67169,"/minute":67170,"毫æĹłçĸijéĹ®":67171,"ãģ»ãģ©":67172,"rovers":67173,"ç¾Į":67174,"ĠتÙĪØ³Ø·":67175,"ä¸ĵå¿ĥ":67176,"äºĨä¸ĢæĬĬ":67177,".front":67178,"ĠCongressional":67179,"ĠCAST":67180,"Queen":67181,"Ġmenstrual":67182,"ctive":67183,"ç¶ĵ常":67184,"ĠãĢİ":67185,"åĩºè·¯":67186,"Ġmonopol":67187,"ĠRisks":67188,"Ġsewer":67189,"çķ¥å¾®":67190,"å¥ĩçļĦ":67191,"Ġfingerprint":67192,"ĠпÑĢоблемÑĭ":67193,"è¶ķç·Ĭ":67194,"ĠHI":67195,"Ġ&.":67196,"ç»ĻæĪijçļĦ":67197,"Complex":67198,"থম":67199,"ĠMiranda":67200,"çºĶ":67201,"å¥Ķé©°":67202,"à°¿à°Ĥà°":67203,"ĠStephanie":67204,"Ġaa":67205,"åĵĨ":67206,"Ġdecorations":67207,"znam":67208,"Ġstatues":67209,"Ġdynamical":67210,"åŃĻä¸Ńå±±":67211,"Ġrepertoire":67212,"Ġvalleys":67213,"éĢłå½±":67214,"å¦Ĥæŀľèĥ½":67215,"ECTOR":67216,"-prot":67217,"å¾ĴæŃ¥":67218,"Processor":67219,"olerant":67220,"/'":67221,"Ġmans":67222,"åĽŀè´Ń":67223,"ÙĤس":67224,"ãģĹãģªãģĦ":67225,"Ġkinda":67226,"æľĢè¿ijçļĦ":67227,"Ġפע":67228,"ä¸ĭæĦıè¯Ĩ":67229,"次ãģ®":67230,"æĪij们æľī":67231,"èĮ¬":67232,"飻":67233,"ĠBiophys":67234,"奶油":67235,"Ġìĸ¸":67236,"éıĪ":67237,"马æĿ¥è¥¿äºļ":67238,"ĠExponentiation":67239,"Ġdisastrous":67240,"æ¼³":67241,"abit":67242,"çϾèĬ±":67243,"ágina":67244,"å¾ĺå¾Ĭ":67245,"\\Facades":67246,"éĹ´æĸŃ":67247,"å®Ŀ马":67248,"Ġlexical":67249,"ĠÏĢεÏģι":67250,"Serv":67251,"é£Łç®¡":67252,"ĠAlk":67253,"Õ¥Õ²":67254,"åĴĮåĪĨæŀIJ":67255,"iky":67256,"æ¯Ļ":67257,"製éĢł":67258,"Ġpyl":67259,"çļĦå¿ħè¦ģ":67260,"çľŁå¥½":67261,"æĹħç¨ĭ":67262,"Ġthicker":67263,"ierten":67264,"Ġdaring":67265,"Ġstric":67266,"Ġ×ŀ×ij":67267,"Ġbesoin":67268,"çĮľæĥ³":67269,"è·¯çͱåύ":67270,"Ġnephew":67271,"Ġspécial":67272,"èĢĮè¿ĻäºĽ":67273,"åıĹåĬĽ":67274,"à¸Ħà¹Ī":67275,"à¥ĩव":67276,"Ġattracts":67277,"memory":67278,"Ġerk":67279,"à¸ķว":67280,"æµĵ缩":67281,"×Ļש×Ķ":67282,"ĠpossÃŃvel":67283,"ĠаÑĤ":67284,"'e":67285,"xter":67286,"ãģ§ãģĤãĤĬ":67287,"Ġperceptual":67288,"ĠKerala":67289,"ricas":67290,"失常":67291,"ĠCheese":67292,"游æĪıä¸Ń":67293,"ĠmayorÃŃa":67294,"ĠSacred":67295,"functions":67296,"symbol":67297,"以æ±Ĥ":67298,"æŃ¤åľ°":67299,"azure":67300,"çŃĶ辩":67301,"Ġجر":67302,"Ġfantas":67303,"iket":67304,"Ġ'_":67305,"æĤ¼":67306,"å»¶æľŁ":67307,"åѤç«ĭ":67308,"ĠFriedman":67309,";\">Ċ":67310,"ĠSina":67311,"pei":67312,"Ġweg":67313,"ç¥ŀéĢļ":67314,"ĠAngular":67315,"é²ľèī³":67316,"سبة":67317,"ĉdouble":67318,"Ġtt":67319,"ĠHollow":67320,"è¿ijè§Ĩ":67321,"'T":67322,"åĨķ":67323,"以èĩ³":67324,"Ġdrastic":67325,"aldi":67326,"æŀ¸æĿŀ":67327,"åĴĮè¦ģæ±Ĥ":67328,"Ġarticulated":67329,"æĪIJæľ¬çļĦ":67330,"nels":67331,"ä¸Ĭçļ®":67332,"Ġabusive":67333,"änder":67334,"_password":67335,"ĠSuddenly":67336,"èĢĮ为":67337,"åīįåİ»":67338,"æīĵæŃ»":67339,"Ġbask":67340,"Ġchemically":67341,"æĪĺäºīçļĦ":67342,"Ġpemer":67343,"Lang":67344,"Navigation":67345,"Ġboxing":67346,"Ġhydrophobic":67347,"ĠeCollection":67348,"åIJİæĿ¥çļĦ":67349,"æĹħé¦Ĩ":67350,"Ġhostility":67351,"æĮĤéĴ©":67352,"ĠÑĩаÑģов":67353,"å¿ĥèĦıçĹħ":67354,"Guid":67355,"беÑĢ":67356,"àµĩ":67357,"à¹Ģลย":67358,"Ġsafeguarding":67359,"Ġà¹Ģมืà¹Īà¸Ń":67360,"inker":67361,"éħįåģ¶":67362,"Ġ×ij×Ķ":67363,"ä¸įåģľçļĦ":67364,"Ġpavement":67365,"Ġmanure":67366,"Ġendif":67367,"Ġsituación":67368,"Ġbirths":67369,"èĢĹè´¹":67370,"ĠELSE":67371,"太çĽij":67372,"ĠMuscle":67373,"è´«ç©·":67374,".From":67375,"Ġbuah":67376,"çĥŃ度":67377,"Ġcaste":67378,"èĤłèĥĥ":67379,"cad":67380,"æĥĬ人":67381,"engineering":67382,"osk":67383,"æĪij们æĿ¥":67384,"ĠмÑı":67385,"æµ·æĬ¥":67386,"Ġkein":67387,"èIJ½æĪ·":67388,"remember":67389,"èĽŁ":67390,"Ġunequal":67391,"ç®Ģ缴æĺ¯":67392,"Serialize":67393,"'i":67394,"Touch":67395,"Ġnatives":67396,"elde":67397,"ĠESA":67398,"ç»ĵæŀľæĺ¯":67399,".event":67400,"çīĽä»Ķ":67401,"ĠBasically":67402,"çĨ¬å¤ľ":67403,"represented":67404,"Purpose":67405,"nier":67406,"ĠPinterest":67407,"Ġfertilization":67408,"çģ«è½¦ç«Ļ":67409,".div":67410,"Ġdetects":67411,"Ġwaiver":67412,"ĠMachines":67413,"-independent":67414,"Ġuczni":67415,"Ġantidepress":67416,"åħ¬åħ±åį«çĶŁ":67417,"è£ĻåŃIJ":67418,"رشÙĬÙģ":67419,"Ġattenuation":67420,"Dutch":67421,"就读":67422,"ĠкоÑģ":67423,"å¹²èѦ":67424,"群çļĦ":67425,"ĠÑĢели":67426,"Ġત":67427,"ĠLEG":67428,"ractive":67429,"ICI":67430,"Ġstimulates":67431,"ĠÑĤÑĢÑĥда":67432,"Ġalém":67433,"ĠLact":67434,"ĠHeil":67435,"Ġimmortal":67436,"齡":67437,"Ġkommen":67438,"änge":67439,"ಿಸ":67440,"à¸ģรม":67441,"chend":67442,"æĻļå¹´":67443,"ĠNorfolk":67444,"�����":67445,"Ġjunto":67446,"ĠSAL":67447,"å¹¶ä¸įä¼ļ":67448,"(arg":67449,"Bern":67450,"Ġct":67451,"ĠfÅij":67452,"oger":67453,"çģ«åĬĽ":67454,"èĤ¯å®ļçļĦ":67455,"ÙIJÙĦ":67456,"Ġfisheries":67457,"ÙĩÙĨ":67458,"Ġsubl":67459,"াথà§ĩ":67460,"Ġbeforehand":67461,"åIJĥå¾Ĺ":67462,"Ġartific":67463,"ĠManila":67464,"ãģĿãģĵ":67465,"ĠPromotion":67466,"Ġrek":67467,"å½ĵå½Ĵ":67468,"ä¿ĿèŃī":67469,"åĨ²çĿĢ":67470,"Ġ×IJ׾×IJ":67471,"}}\\,":67472,"æ°§åĮĸçī©":67473,"èĨĿçĽĸ":67474,"Taylor":67475,"Ġluego":67476,"Ġstarters":67477,"åı¤éķĩ":67478,"éĶħä¸Ń":67479,"à¸ģลาà¸ĩ":67480,"practice":67481,"æī¼":67482,"ç§įçļĦ":67483,"åĨľæľº":67484,"åģı离":67485,"aic":67486,"ãĢĤ\"ĊĊ":67487,"ĠSap":67488,"ersen":67489,"ÑİÑīее":67490,"ä»ĺè´¹":67491,"Execution":67492,"Ġ\",\"":67493,"ĠQR":67494,"taire":67495,"æī§è¡ĮçļĦ":67496,"æĨ¨":67497,"×ķ׾×ķ×Ĵ":67498,"Ġ'/'":67499,"ĠLights":67500,"åĪ»èĭ¦":67501,"ĠØŃÙĦ":67502,"Ġstaffing":67503,"éģ¥è¿ľ":67504,"ož":67505,"Ġbeiden":67506,"åľ¨æŁIJäºĽ":67507,"ÙĨÙĬÙĨ":67508,"ÏĦÏģ":67509,"Ġbuilders":67510,"/journal":67511,"ĠASCII":67512,"VIS":67513,"Ġmetamorph":67514,"اÛĮÙĩ":67515,"alaman":67516,"åIJīåĪ©":67517,"ĠPropTypes":67518,"`.`":67519,"产ä¸ļåĽŃ":67520,"ĠLibya":67521,"Ġmulticultural":67522,"ĠBaldwin":67523,"ÙĪØ±ÙĬ":67524,"课åłĤä¸Ĭ":67525,"éĺĻ":67526,"åŃĹåı·":67527,"Ġtechnician":67528,"鼻èħ¦":67529,"è¨ĺäºĭ":67530,"çIJĨäºĭä¼ļ":67531,"ì»":67532,"Ġedad":67533,"ç§ijå¹»":67534,"Ġmitigating":67535,"Ġpancreas":67536,"FX":67537,"èĩªç§°":67538,"Ġszám":67539,"ä¼Ĭæĸ¯åħ°":67540,"تÙĬÙĨ":67541,"ĠÑĥÑĩеÑĤ":67542,"å®ļä¹ī为":67543,"Ġferry":67544,"клад":67545,"Magn":67546,"æĸ°ä¸Ģ代":67547,"rieben":67548,"_form":67549,"arab":67550,"æĥļ":67551,"æ¯ĶåĪĨ":67552,"ĠGateway":67553,"fahren":67554,"ÑĢиÑĦ":67555,"åħĥæĹ¦":67556,"Constant":67557,"ĠAcknowledgements":67558,"-ag":67559,"unas":67560,"åĪĨä¹ĭ":67561,"çĿĢå®ŀ":67562,"IRC":67563,".button":67564,"çļĦ空":67565,"ĠAAA":67566,"otom":67567,"سÙĬÙĨ":67568,"æĶ¾åģĩ":67569,"çĻ¼è¡¨":67570,"次æĹ¥":67571,"山谷":67572,"容纳":67573,"转åŀĭåįĩ级":67574,"ĠBoris":67575,"Western":67576,"çļĦè¿Ļ个":67577,"Ġshattered":67578,"Ġpervasive":67579,"å¼Ģä¸ļ":67580,"Ġcaptive":67581,"Ġsynonymous":67582,"å¯ĤéĿĻ":67583,"Ġordinance":67584,")+\\":67585,"alore":67586,"تارÙĬØ®":67587,"Ġ_âĢľ":67588,"éŁ³åĵį":67589,"请大家":67590,"Ġcardboard":67591,"Ġflawed":67592,"Ai":67593,"ĠFres":67594,"inescence":67595,"ĠExit":67596,"游记":67597,"ä»ĭæĦı":67598,"LEMENT":67599,"è¿·ä¿¡":67600,"è°ķ":67601,"èĢħãģ®":67602,"Ġactivating":67603,"çģ«èĬ±":67604,"å®Ŀè´µçļĦ":67605,"Ġchiefly":67606,"ĠоÑģобенно":67607,"ä¸į以":67608,"ĠChim":67609,"éĤ£éĤĬ":67610,"é¦ĻçļĦ":67611,"Agent":67612,"RAL":67613,"Tour":67614,"ä¸Ģèĩī":67615,"agus":67616,"åģĩè£ħ":67617,"èĦĬæŁ±":67618,"lasses":67619,"acey":67620,"ĠGiving":67621,"éϤæ³ķ":67622,"Ġsupervisors":67623,"Ġélèves":67624,".Close":67625,"Ãİ":67626,"æŀĦæĢĿ":67627,"ना":67628,"planes":67629,"xb":67630,"Ġмл":67631,"åĩºè®©":67632,"Ġstride":67633,"Ġcertifications":67634,"ĠاÙĦÙĤدÙħ":67635,"æīĵåį°æľº":67636,"Ġcryptocurrencies":67637,"ĠBarr":67638,"à¸ģà¹Īà¸Ńà¸Ļ":67639,"Ġportfolios":67640,"ĠÐļи":67641,"Dispatch":67642,"Ġthu":67643,"Ġinsol":67644,"ivering":67645,"éĩĬä¹ī":67646,"åĵ²åѦ家":67647,"áºŃn":67648,"Ġjetzt":67649,"Ġfácil":67650,"Ġtrình":67651,"轻度":67652,"Feedback":67653,"Ġperiphery":67654,"ĠDominican":67655,"Ġtau":67656,"åIJĦåİ¿":67657,"èĥ¡æ¤Ĵ":67658,"/watch":67659,"Ġswinging":67660,"Ġtheolog":67661,"ä¹Łè¶ĬæĿ¥è¶Ĭ":67662,"ixing":67663,"ĠIsh":67664,"Ġobserves":67665,"Ġανα":67666,"Err":67667,"ানà§ĩর":67668,"ç¥Ī祷":67669,"ãĢĤãĢijĊĊ":67670,"ä¸Ńæµ·":67671,"Ġrecognizable":67672,"èĪĪè¶£":67673,"Must":67674,"Ġreflux":67675,"åħ¨éĿ¢åıijå±ķ":67676,"/js":67677,"å¢ĻéĿ¢":67678,"ĠEncourage":67679,"ðŁijī":67680,"_edge":67681,"ĠBake":67682,"ijken":67683,"Asia":67684,"Ġurg":67685,"Uh":67686,"ä»ĸ为":67687,"稳åģ¥":67688,"ĠSinger":67689,"ĠпоÑģледова":67690,"Õ«Õ½":67691,"!!!!!!!!":67692,")f":67693,"èĭ±éķij":67694,"Ġprix":67695,"Ġ×IJ×Ļף":67696,"èĮ¯èĭĵ":67697,"Ġï¬Ĥ":67698,"ï¼İï¼Ī":67699,"ĠاÙĦخاص":67700,"\\phi":67701,"Ìį":67702,"Ġتط":67703,"_equal":67704,"ĊĊ":68373,"æ¸ħåįİ大åѦ":68374,"åĮĸæĪIJ":68375,"ografie":68376,"ĠHumph":68377,"gil":68378,"jus":68379,"ningar":68380,"ç»ŃèĪª":68381,"ĠобнаÑĢÑĥ":68382,"çģµåĬĽ":68383,"ĠTomorrow":68384,"ĠSatisf":68385,"æ·¬":68386,"åŁºæķ°":68387,"ĠMaritime":68388,"Ġà¦ħà¦Ń":68389,"宿主":68390,"ié":68391,"Ġhust":68392,"åľ§":68393,"产å¦ĩ":68394,"è´¯éĢļ":68395,"ä»İ严治":68396,"Ġcalf":68397,"ä¹IJäºİ":68398,"Ġswings":68399,"Ġfellows":68400,"Ġworkbook":68401,"è¯ŃçļĦ":68402,"è¨ĢçļĦ":68403,"读äºĨ":68404,":{":68405,"大å¸Ŀ":68406,"Ġcrawl":68407,"Talk":68408,"çľ¼çľ¸":68409,"çļĦæ°Ķæ°Ľ":68410,"bill":68411,"culture":68412,"ä¼ļç»Ļ":68413,"åħ¨ç¤¾ä¼ļ":68414,"Ġantique":68415,"Ġspecialization":68416,"ĠÑĢазнÑĭÑħ":68417,"ĠÑĦоÑĢмÑĭ":68418,"bility":68419,"oty":68420,"ĠPiano":68421,"人社":68422,"ĠDeck":68423,"Ġsumm":68424,"عÙĦÙĪÙħات":68425,"subscriptðĿIJ":68426,"Ġmươi":68427,"ä¹ŁåºĶ该":68428,"Scanner":68429,"Ġrobbery":68430,"éĩĩåıĸäºĨ":68431,"èĥĥèĤł":68432,"ĠÄįi":68433,"-row":68434,"åħ¶çī¹å¾ģ":68435,"éķ¿çĽ¸":68436,"缴æİ¥å½±åĵį":68437,"Ġhypothesized":68438,"ĠReeves":68439,"Ġadorable":68440,"é²ľæĺİçļĦ":68441,"Ġnuanced":68442,"身åīį":68443,"ĠEcho":68444,"ä¾ĽéľĢ":68445,"æī¿ç§Ł":68446,"游æĪıçļĦ":68447,"Ġclarified":68448,"caster":68449,"peace":68450,"ä¸ĭåĨĮ":68451,"ä½łå®¶":68452,"Ġconsciously":68453,"æ²īçļĦ":68454,"Ġfemme":68455,"ä¸į论æĺ¯":68456,".btn":68457,"ĠBiz":68458,"ĠHK":68459,"à¸Ĭà¹Īวà¸ĩ":68460,"è¯ģæĺİäºĨ":68461,"Ġluggage":68462,"Ġcytokine":68463,"ologue":68464,"Always":68465,"ĠPierce":68466,"-word":68467,"Ġsebag":68468,"Patients":68469,"伪éĢł":68470,"ì¦Ī":68471,"bos":68472,"ĠRomantic":68473,"Ġlegislators":68474,"ĠSubtract":68475,"ĠFlying":68476,"cyj":68477,"erger":68478,"æ¤į被":68479,"openia":68480,"Ġmonastery":68481,"æ·¼":68482,"Ġ\\((\\":68483,"ĠبدÙĪÙĨ":68484,"ç¨İåĬ¡æľºåħ³":68485,"åİŁä»¶":68486,"åı£åı·":68487,"Ġsomatic":68488,"å½ķåζ":68489,"Ġ×IJ×ļ":68490,"Ġbrakes":68491,"Ġsofa":68492,"Ġeval":68493,"ĠEntom":68494,"ä»ĩæģ¨":68495,"æ·µ":68496,"æĶ¾è¿Ľ":68497,"sequent":68498,"ĠAdventures":68499,"æ¶Īæķ£":68500,"à®ķà¯į":68501,"-info":68502,"ĠÑĢеÑģÑĥÑĢ":68503,"âĸª":68504,"åĸĿèĮ¶":68505,"çĽIJæ°´":68506,"Psi":68507,"Ġtrench":68508,"Ġunin":68509,"åѦåīį":68510,"Ġminder":68511,"doctor":68512,"gester":68513,".IOException":68514,"Aj":68515,"Ġunres":68516,"æĿ¥è®¿":68517,"undi":68518,"nahme":68519,"Done":68520,"undefined":68521,"Ġsupporter":68522,"声称":68523,"æı¡ä½ı":68524,"è¿Ķè¿ĺ":68525,"ULD":68526,"alms":68527,"主æĿĥ":68528,"ä¸įåIJĮçļĦæĺ¯":68529,"çIJĨ解为":68530,"èĥ½éĩıçļĦ":68531,"Ġbearings":68532,"รัà¸IJ":68533,"ĠByzantine":68534,"PHP":68535,"ç±³åħ°":68536,"Appendix":68537,"ÑģÑĤÑĥпи":68538,"Ru":68539,".Reg":68540,"Ġdances":68541,"èĩªå¼º":68542,"æķ°åįĥ":68543,"èħ±":68544,"Ġcondiciones":68545,"Ġbullets":68546,"FileName":68547,"ZT":68548,"Ġcapacidad":68549,"æĵ¾":68550,"ambil":68551,"ĠÎŃνα":68552,"Ġhoch":68553,"Ġparti":68554,"Ġperseverance":68555,"Ġnont":68556,"ĠTac":68557,"Ġ}),Ċ":68558,"æŃ£èĥ½éĩı":68559,"ä¿¡èªī":68560,"åįģæĿ¡":68561,"é»ijå¤ľ":68562,"ĠعÙĨدÙħا":68563,"ĠCALL":68564,"ĠпÑĢеп":68565,"ĠاÙĦØ´ÙĬ":68566,"ç·Ĭå¼µ":68567,"æ¾Ħæ¸ħ":68568,"à¶±à·Ĭ":68569,"åħ¬åŃĻ":68570,"×ķ×Ļ×ķת":68571,"åĨĻåľ¨":68572,"Ġransom":68573,"Ġtournaments":68574,"RAW":68575,"ĉdata":68576,"èĮĹ":68577,"Ġmenus":68578,"ç¼ĸç»ĩ":68579,"ç§ijæĬĢ大åѦ":68580,"ĠControls":68581,"çļĦ人类":68582,"ãĤ¹ãģ®":68583,"Party":68584,";ãĢĬ":68585,"}.\\]ĊĊ":68586,"Ġlbf":68587,"Ġij":68588,"æĪij们è¿ĺ":68589,"Ġsocialism":68590,"ĠMagyar":68591,"basic":68592,"Ġdreamed":68593,"Ġ×Ľ×ª":68594,"ĠAssessing":68595,"==\"":68596,"Ġnud":68597,"pecified":68598,"ä¸ī个人":68599,"é·":68600,"iono":68601,"ï¼ģï¼Ī":68602,"Ġpais":68603,"Authentication":68604,"ĠCosm":68605,"ĠTibetan":68606,"Ġprophecy":68607,"ä¹Łåı«":68608,"ĠÑĢези":68609,"abilidade":68610,"dif":68611,"Ġ\"\"\"ĊĊ":68612,"æ¶ĤæĬ¹":68613,"\";Ċ":69399,"Had":69400,"以太":69401,"Ġdira":69402,"è§£æ³ķ":69403,"åIJĦæĹı":69404,"Ġsmarter":69405,"viewport":69406,"貫":69407,"ĠAspects":69408,"Korean":69409,"ĠMd":69410,"Ġkph":69411,"åħ¶äºĮ":69412,"两ç»Ħ":69413,"çįĦ":69414,"æģ¢å¾©":69415,"áĥIJáĥłáĥ":69416,"деÑĤÑĮ":69417,"PART":69418,"Ġnx":69419,"ĠSung":69420,"ĠFax":69421,"åı¯å°±":69422,"ĠÑģÑĢеди":69423,"æ¹ĸ人":69424,"Ġnecesario":69425,".const":69426,"éĹ»åIJį":69427,"Ġ×¢×ij":69428,"Ġpoisonous":69429,"Ġpog":69430,"ĠCara":69431,"Ġanton":69432,"ĠDates":69433,"ĠAlto":69434,"ÑĤаÑĨии":69435,"约åįł":69436,"features":69437,"è³ĩæľ¬":69438,"Ġponder":69439,"æĪijçľĭåΰ":69440,"å°Ĩå®ĥ":69441,"æŀģåħ·":69442,"CLC":69443,"ĠDU":69444,"Ġcorrective":69445,"Ġinducing":69446,"ĠتعاÙĦÙī":69447,".Inter":69448,"éľ¾":69449,"ĠмÑĥж":69450,"çī©ä¸ļ管çIJĨ":69451,"Ġnoc":69452,"Ġquota":69453,"ç¤ģ":69454,"ä»Ģä¹Īåı«":69455,"åķĨè´¸":69456,"ĠIntra":69457,"-east":69458,"ĠCake":69459,"ĠNão":69460,"è¿Ļç¬Ķ":69461,"ĠSheffield":69462,"vig":69463,"äºĨæĪij们":69464,"Ġweary":69465,"åĩºæ¼Ķ":69466,"ημο":69467,"ÑļÑĥ":69468,"Ġmenyebabkan":69469,"å±ĢéĻIJæĢ§":69470,"piration":69471,"ï¼īãĢĭ":69472,"Seq":69473,"ĠDefendants":69474,"à«įય":69475,"Ġläs":69476,"plac":69477,"ÙĪØ¯Ùĩ":69478,"Ñĸн":69479,"æķijåij½":69480,"Ġcategorical":69481,"Ġancestry":69482,"Dal":69483,"çļĦåįķ":69484,"å¦Ĥåľ¨":69485,"Ġamusement":69486,"çϽçİī":69487,"å¹»çģ¯":69488,"æľīä¸įåIJĮçļĦ":69489,"ĠÑģобÑĭ":69490,"æ¥ŀ":69491,"ç¿Ł":69492,"ĠEvening":69493,"ĠSUMMARY":69494,"KW":69495,"åĴĮåŃ©åŃIJ":69496,"ÙĪÛĮد":69497,"ĠCentimeter":69498,"helf":69499,"Ġsued":69500,"è¿ĽçIJĥ":69501,"ä¸Ģ书":69502,"大èĤł":69503,"çŃīéĥ¨éŨ":69504,"åħħè¶³çļĦ":69505,"/U":69506,"Dit":69507,"æıIJçĤ¼":69508,"Ġprofund":69509,"çϻ山":69510,"à¸ĵà¸ij":69511,"Ġmiracul":69512,"ÛĨ":69513,"åħĥ宵":69514,"çłĶç©¶ä¼ļ":69515,"ä½įç½®çļĦ":69516,"ĠполноÑģÑĤÑĮÑİ":69517,"à¹ģรà¸ģ":69518,"çIJĨ工大åѦ":69519,"çŁŃæĹ¶éĹ´åĨħ":69520,"ĠPrecision":69521,"ĠپرÙĪ":69522,"ĠvÄĽt":69523,"igner":69524,"tsch":69525,"çıĤ":69526,"à´Ł":69527,"Ġsuperconduct":69528,"è°ģçŁ¥":69529,"ĠâĨĴĊĊ":69530,"Ġpopulasyon":69531,"ĠGPA":69532,"æĪij们æĬĬ":69533,"å½ĵäºĭ人çļĦ":69534,"Ġhemorrhage":69535,"Ġili":69536,"ä¸Ĭåı¤":69537,"ĠпÑĢоÑģ":69538,"asionally":69539,"agl":69540,"Ġjets":69541,"åĵģæł¼":69542,"ĠSettlement":69543,"Recommended":69544,"æĪĺåIJİ":69545,"Ġpresenta":69546,"çłĶç©¶äºĨ":69547,"åħŃçϾ":69548,"ĠпÑĢиваÑĤ":69549,"æ¡Īä»¶çļĦ":69550,"Ġreefs":69551,"Ġï¼İ":69552,"Ġgg":69553,"Ġobsession":69554,"Ġpals":69555,"åĩºä¹İ":69556,"Ġetching":69557,"èµŀèµı":69558,"åĪĽä¸ļæĿ¿":69559,".ad":69560,"SOL":69561,"headers":69562,"Так":69563,"Ġorganisational":69564,"_delete":69565,"Ġbude":69566,"Ġbawah":69567,"ç»®":69568,"åįĬæľĪ":69569,"Ġaccessory":69570,"é̲ä¸ĢæŃ¥":69571,"Ġarab":69572,"ãĢĤï¼īĊĊ":69573,"ä¸įä½ľ":69574,"æĪIJ績":69575,"åĽŀè°ĥ":69576,"اÙĦع":69577,"ίν":69578,"UNG":69579,"èµĭèĥ½":69580,"Drug":69581,"quick":69582,"Ġresiding":69583,"oyl":69584,"ä¸ĢèĪ¬äºº":69585,"è°ģçŁ¥éģĵ":69586,"Ġbowls":69587,"ĠKaplan":69588,"Ġcaves":69589,"çļĦæĢ§æł¼":69590,"ç읿³¡":69591,"Ġпомога":69592,".Controls":69593,"äºļ马éĢĬ":69594,"Ġtasted":69595,"ĠCaf":69596,"ä¸Ģæľµ":69597,"ç»ĵèĤł":69598,"äº²çľ¼":69599,"ĠHarvest":69600,"ĠSalem":69601,"{cases":69602,"Routes":69603,"ĠDio":69604,"åľ°åĪ©":69605,"Ġescrib":69606,"ĠÏĦε":69607,".route":69608,"Ġinferences":69609,"ĠPAC":69610,"Ġdreaming":69611,"accessible":69612,"Fn":69613,"-taking":69614,"Ġ×ķ׾×IJ":69615,"å¥łå®ļ":69616,"gado":69617,"ĠAircraft":69618,"æĺ¯å®Įåħ¨":69619,"ä¹ĭæ¯Ķ":69620,"æĸĩç§ij":69621,"计åħ¥":69622,"Ġgraf":69623,"Ġrepayment":69624,"ĠÑĦай":69625,"OTE":69626,"Ġpertain":69627,"念念":69628,"Ġà¦¬à¦Ľ":69629,"Ġviolating":69630,"åıij表äºĨ":69631,"çłĶ究人åijĺ":69632,"ĠпÑĢедо":69633,"Ġreimbursement":69634,"inig":69635,"ĠScout":69636,"ĠPerl":69637,"çŃijçī¢":69638,"particularly":69639,"ä¸Ģåij³":69640,"ĠGST":69641,"Ġshelters":69642,"Ġfunção":69643,"Ġepigen":69644,"ĠопÑĢеделÑıеÑĤÑģÑı":69645,"Ġfichiers":69646,"á¾":69647,"Ġtolerated":69648,"çϽéĵ¶":69649,"ĠDigits":69650,"ĠBangkok":69651,"Ġnesting":69652,"çļĦçŁ³":69653,"ĠTG":69654,"ĠQur":69655,"Ġfireplace":69656,"Ġrugged":69657,"amientos":69658,"ĠRash":69659,"ÙĪØ§ÙħÙĦ":69660,"Ġpotencial":69661,"Shared":69662,"éĴĪçģ¸":69663,"ĠVerbs":69664,"Ġcuad":69665,"mie":69666,"Ġamorphous":69667,"Ġobras":69668,"ĠÐŁÑĢед":69669,"åİĨåı²æĸĩåĮĸ":69670,"Ġmosquitoes":69671,"à¹Ģลืà¸Ńà¸ģ":69672,"ĠEVER":69673,"éĥ½å¾Ĺ":69674,"Ġopener":69675,"ĠDonna":69676,"Ġionization":69677,"浸润":69678,"emas":69679,"ĠFrage":69680,"æĶ¾åѦ":69681,",Ċ":70197,"ÑĢиÑĨа":70198,"Ġdonner":70199,"ר×Ļ×ļ":70200,"ĠпоÑģколÑĮкÑĥ":70201,"ĠMalay":70202,"inence":70203,"Ġbate":70204,"имÑĥ":70205,"ĠBUS":70206,"abella":70207,"ermis":70208,"æ°Ķå¾Ĺ":70209,"ĠбÑĢо":70210,"cemic":70211,"Ġà¹Ģà¸Ĥ":70212,"åľ¨éĩĮéĿ¢":70213,"ramos":70214,"Ġrelapse":70215,"Ġcols":70216,"-determ":70217,"åħŃ年级":70218,"-story":70219,"ĠBoat":70220,"åѸéĻ¢":70221,"ĠÑģоедин":70222,"सà¥įत":70223,"Diagn":70224,"车轮":70225,"μαÏĦα":70226,"ĠMongol":70227,"ĢáĢ»á̱á̏áĢ":70228,"Abb":70229,"ĠGaming":70230,"éŵ":70231,"Ġdetained":70232,"ĠзаÑĤÑĢа":70233,"Ġseminars":70234,"ĠChef":70235,"Ġsuperficie":70236,"Ġsä":70237,"ĠEQU":70238,"диÑı":70239,"nicos":70240,"(lambda":70241,"ÏĬ":70242,"Ġrails":70243,"ĠRetirement":70244,"踹":70245,"Translation":70246,"ÑĦоÑĢми":70247,"ĠShopping":70248,"oos":70249,"enthal":70250,"ĠDynam":70251,"Ġconsom":70252,"客车":70253,"èΰéĺŁ":70254,"ĠÑĥÑĩиÑĤÑĭ":70255,"ĠPly":70256,"oxygen":70257,"ATS":70258,"ĠMegan":70259,"ĠToward":70260,"Arabic":70261,"Portuguese":70262,"Ġbritt":70263,"Ġthym":70264,"quarter":70265,"åĩºåįĸ":70266,"â̲,":70267,"-_":70268,"çļĦåIJĮ":70269,"iatrist":70270,"Ġповед":70271,"ĠCommentary":70272,"ĠHVAC":70273,"PU":70274,"chens":70275,"eming":70276,"å·¥ä½ľæĹ¶":70277,"اÙĩÙħ":70278,"Ġpalav":70279,"äºĮåįģå¹´":70280,"æĪ°é¬¥":70281,"hetti":70282,"说èĩªå·±":70283,"é¦Ĵ":70284,"áŀĢ":70285,"Ġdownloads":70286,"æĿľçĶ«":70287,"èIJ¥ä¸ļæĶ¶åħ¥":70288,"worms":70289,"Ġhose":70290,"å¼łæŁIJ":70291,"ĠÑģÑĤав":70292,"ä¸įå¾Ĺè¶ħè¿ĩ":70293,"Ġrivalry":70294,"ĠпомоÑīи":70295,"èĦijæµ·ä¸Ń":70296,"ĠScul":70297,"peated":70298,"à¹Ĭ":70299,"ATO":70300,"Labels":70301,"ounty":70302,"éķ¿æĸ¹å½¢":70303,"建äºİ":70304,"æŃ¤é¡¹":70305,"åIJĥèĭ¦":70306,"ĠEdwin":70307,"Åijs":70308,"Ġmijn":70309,"Ġlatch":70310,"enerate":70311,"Ġdistractions":70312,"λικά":70313,"Agric":70314,"ĠÑģооÑĤвеÑĤÑģÑĤвии":70315,"Ġthrom":70316,"åľ¨è¢«":70317,"cyt":70318,"median":70319,"å¢ŀåĬłåΰ":70320,"档次":70321,"ĠWellness":70322,"distance":70323,"ĠCars":70324,"herty":70325,"ä¹Łå·²ç»ı":70326,"Ġslipping":70327,"ĠDisabilities":70328,"Ġinforming":70329,"ëIJľëĭ¤":70330,"åģļ大":70331,"еÑĤо":70332,"ĠEnlightenment":70333,"ĠпÑĢибÑĭ":70334,"ĠÐĴели":70335,"accuracy":70336,"Popular":70337,"oltre":70338,"ÙİØ©":70339,"ĠMetall":70340,"ĠMalta":70341,"åīĩæĺ¯":70342,"ä¸Ńåıijçݰ":70343,"ä½łè¦ģæĺ¯":70344,"Ñģий":70345,"ĠHousehold":70346,"ĠCHECK":70347,"òn":70348,"çļĦåįķä½į":70349,"Ġstunned":70350,"Ġalley":70351,"å¹¶æł¹æį®":70352,"INF":70353,"ç»Ĩå¾®":70354,"ä¸įçŁ¥æīĢ":70355,"Ġentertained":70356,"å°ıå¼Ł":70357,"жде":70358,"带宽":70359,"calc":70360,"Ġmortar":70361,"ĠÑĤÑĢеÑĥголÑĮ":70362,"Living":70363,"å¥ļ":70364,".substring":70365,"ĠKilometers":70366,"æħ·æħ¨":70367,"åĨĽåĮº":70368,"ยาย":70369,"Ġкакой":70370,"ĠطرÙĬÙĤ":70371,"ï¬ĥ":70372,"Ġupgrading":70373,"ĠDul":70374,"Ġcomputations":70375,"ĠTwelve":70376,"íİĺìĿ´ì§Ģ":70377,"Yellow":70378,"Ġashes":70379,"ĠDATE":70380,"ĠNG":70381,"æĽ²éĿ¢":70382,"Ġconcentrating":70383,"ĠVerd":70384,".number":70385,"åį±éĻ©çļĦ":70386,"Ġbranching":70387,"ĠAlbany":70388,"nombre":70389,"åı½":70390,"Ġmatéri":70391,"embrance":70392,"Ġživot":70393,"ĠMohammad":70394,"à¹Ģà¸ģีà¹Īยวà¸ģัà¸ļ":70395,"ĠHU":70396,"Ġcongrat":70397,"ĠVest":70398,"ç©ºæł¼":70399,"à¸Ĭืà¹Īà¸Ń":70400,"ĠKO":70401,"Ġ}).":70402,"ĠÙģÙĦ":70403,"Quote":70404,"](/":70405,"ëĿ¼ê³ł":70406,"obacterium":70407,"(iii":70408,"ĠWrong":70409,"åŃ¦æ´¾":70410,"Ġtemperament":70411,"Errors":70412,"á̝áĢķáĢºáĢħ":70413,"åİ»è¿ĩ":70414,"æŃ»äºİ":70415,"éĺ³æĺİ":70416,"å®¶éķ¿ä»¬":70417,"ĠBuilder":70418,"祺":70419,"æ°Ķæµģ":70420,"Ġaquest":70421,"ĠAudi":70422,"Ġspikes":70423,"åħ·é«Ķ":70424,"islation":70425,"ombo":70426,"ä¼ļæĬĬ":70427,"Ġcostru":70428,"Conference":70429,"éĵ¶è¡Įåį¡":70430,"клон":70431,"ĠYas":70432,"äºĮ年级":70433,"è¿Ľè¡Įæ¯Ķè¾ĥ":70434,"ĠFortune":70435,"Ġtempting":70436,"Ġsack":70437,"åĽ²":70438,"åIJĪãĤıãģĽ":70439,"å¼ķæµģ":70440,"梦幻":70441,"麻çħ©":70442,"Ġcourtyard":70443,"icamente":70444,"ocarcinoma":70445,"ĠRey":70446,"Ġphương":70447,"ände":70448,"ĠHoffman":70449,"ä½łåĨį":70450,"好åIJĥçļĦ":70451,"å¹¶åĪĹ":70452,"年代åĪĿ":70453,"ĠÎŃÏĩ":70454,"ä¸ŃåѦçĶŁ":70455,"Ġinvo":70456,"ĠAgosto":70457,"Ġmystical":70458,"辨åĪ«":70459,"Ġannoyed":70460,"Ġaper":70461,"Ġmots":70462,"Ġlions":70463,"人æĿĥ":70464,"تÙĤ":70465,"Ġdistancing":70466,"Ġkunst":70467,"ĠGCSE":70468,"pared":70469,"odb":70470,"èį¯åīĤ":70471,"çͰéĩİ":70472,"å¦Īå¦ĪçļĦ":70473,"Ġfueled":70474,"Ġgranite":70475,"Ġroyalty":70476,"enties":70477,"ĠLt":70478,"æ³¢éķ¿":70479,"OTAL":70480,"æµĩæ°´":70481,"Ġhypoxia":70482,"Permission":70483,"ĠShapes":70484,"ĠMyc":70485,"Ġtanpa":70486,"Ġbonne":70487,"Ġdiscovers":70488,"HEAD":70489,"ĠاÙĦأع":70490,"Ġfreq":70491,"ĠAmin":70492,"ĠاÙĦأد":70493,"tanler":70494,"oarthritis":70495,"Ġkb":70496,"apen":70497,"ĠVOL":70498,"åı¯ä»¥å¾Ĺåΰ":70499,"ä¸ĩåĨĨ":70500,"ระหวà¹Īาà¸ĩ":70501,"Training":70502,"imps":70503,"æľ¬éĩij":70504,"ĠDiane":70505,"ribe":70506,"她ä¸į":70507,"ç«ĻäºĨèµ·æĿ¥":70508,"åĩĨç¡®æĢ§":70509,"-minus":70510,"æĢ»æľī":70511,"elenium":70512,"Ġspontaneously":70513,"çŁ¥åIJį度":70514,"ĠÅĽwiat":70515,"emoc":70516,"Ġacordo":70517,"Ġmaid":70518,"ĠAntarctica":70519,"ĠfÃŃsica":70520,"rollment":70521,"ĠInvestors":70522,"ĠPassion":70523,"jala":70524,"animal":70525,"ĠMilit":70526,"å¤ļéĩį":70527,"eback":70528,"åªĴé«Ķ":70529,"finite":70530,"éĺĢéŨ":70531,"JM":70532,"ĠPPT":70533,"ĠHegel":70534,"çĤ¸å¼¹":70535,"/get":70536,"Ġpies":70537,"ä¸Ĭåįĥ":70538,"å¦Ĥå®ŀ":70539,"å¤ĸ壳":70540,"çıłåŃIJ":70541,"éĢīåĩº":70542,"nyder":70543,"Ġ?>":70544,"Ġadaptable":70545,"Ġà°ħ":70546,"ĠArchaeology":70547,"\"<<":70548,"anship":70549,"å¦ĵ":70550,"èĩ´åij½":70551,"çͳè¯ī":70552,"èį·èĬ±":70553,"Ġtors":70554,"ĠABS":70555,"è¡ĮèĢħ":70556,"ĠAnimation":70557,"Ġverz":70558,"Ġarbitr":70559,";-":70560,"Va":70561,"ĠThir":70562,"主åĭķ":70563,"åįĹå®ĭ":70564,"Ġethic":70565,"à¸ķà¸Ńà¸Ļ":70566,"æĬµå¾¡":70567,"Ġattendant":70568,"REC":70569,"ĠиÑĤ":70570,"Ġdeductions":70571,"ĠRespondent":70572,"_stdio":70573,"Ġwitnessing":70574,"mars":70575,"åıĤä¿Ŀ":70576,"Ġterb":70577,"stehen":70578,"ĠPenny":70579,"Ġstellen":70580,"ĠRetro":70581,"ĠPaula":70582,"Ġpipelines":70583,"ĠConcord":70584,"ĠBü":70585,"okol":70586,"å¤ļè°¢":70587,"Ġtrout":70588,"Ġtermasuk":70589,"æĢ§è´¨çļĦ":70590,"æĺ¯æĮĩåľ¨":70591,"ĠCLASS":70592,"Inject":70593,"åĪĩåı£":70594,"ç²ĺè´´":70595,"Ġwarrants":70596,"Digit":70597,"æ¾İæ¹ĥ":70598,"Ġostat":70599,"ĠCanter":70600,"ĠÑįÑĤим":70601,"Ġmelanch":70602,"æ¯ĶåĪ©":70603,"çĪĨçł´":70604,"Õ¸ÖĤÕ©ÕµÕ¡Õ¶":70605,"ĠÑĥÑĢавнениÑı":70606,"Ġbovine":70607,"cza":70608,"Ġlept":70609,"Ġmonarchy":70610,"Ġtenemos":70611,"менÑĤÑĭ":70612,"ĠÙħدÛĮر":70613,"Ġmourning":70614,"ĠJW":70615,"Ġarriv":70616,"ìŀIJê°Ģ":70617,"ĠOperational":70618,"Ġrenders":70619,"Ġdetectable":70620,"ĠPLAN":70621,"Ġë²ķ":70622,"ĢáĢ»á̱á̏áĢĽá̽":70623,"Ġquin":70624,"ERIC":70625,"ĠTiO":70626,"ĠPrentice":70627,"ĠWI":70628,"Ġrespecto":70629,"Ġcleanup":70630,"ôm":70631,"ĠAnnex":70632,"å°±ä¸įè¦ģ":70633,"åŃIJæłij":70634,"漫éķ¿":70635,"人æīįçļĦ":70636,"åı¯éĿłçļĦ":70637,"ç¶ŃæĮģ":70638,"éģĵ人":70639,"çͱæĿ¥":70640,"Ġwarns":70641,"ĠLinguistics":70642,"leave":70643,"çľ¼çļ®":70644,"ceral":70645,"åĵªä¸Ģ个":70646,"å¾IJå·ŀ":70647,"Ġprosperous":70648,"´ī":70649,"Ġsupermarket":70650,"_func":70651,"çĿ¡äºĨ":70652,"ĠSingular":70653,"=device":70654,"ĠMatching":70655,"ĠInvalid":70656,"Ġpratic":70657,"åĢĴéľī":70658,"çĸijä¼¼":70659,"Ġmolten":70660,"Ġstrained":70661,"×ķר×ķת":70662,"}}\\),":70663,"ĠCompanion":70664,"ĠHabitat":70665,"rath":70666,"antwort":70667,"å¿ĥäºĭ":70668,"Ġnewton":70669,"åĢĴåľ¨":70670,"Ġutilizar":70671,"odend":70672,"Ġ<>":70673,"reno":70674,"åıįæĺłåĩº":70675,"................................................................................................................................":70676,"ç²¾éĢļ":70677,"åĨĻå¾Ĺ":70678,"çͰéĹ´":70679,"é̲ä¾Ĩ":70680,"Ġobsessed":70681,"Iron":70682,"æĪŁ":70683,"-stop":70684,"å½ĵåīįçļĦ":70685,"漫éķ¿çļĦ":70686,"Ġdegraded":70687,"Ġбибли":70688,"åͤéĨĴ":70689,"ĠEck":70690,"ĠLal":70691,"æĪijå¿ĥéĩĮ":70692,"éĤ£ä»½":70693,"æ·±åIJ¸":70694,"迫使":70695,"Ġapar":70696,"æĹ¶ä¸įæĹ¶":70697,"fetch":70698,"arit":70699,"ĠmÃ¥":70700,"å¿ĥç¥ŀ":70701,"اÙĨس":70702,"uckle":70703,"èĮ«çĦ¶":70704,"avir":70705,"Ġbushes":70706,"à´¨":70707,"Shipping":70708,"Ġoccupies":70709,"Ġderechos":70710,"åı¯åı£":70711,"á»Ń":70712,"Ġcommanding":70713,"æķ²éŨ":70714,"ç¯Ħåľį":70715,"ĠAnalyze":70716,"Ġsosial":70717,"buffer":70718,"çī¹å¼ĤæĢ§":70719,"Ġdetailing":70720,"Ġsplash":70721,"á̬áĢ¡á̝áĢķáĢºáĢħ":70722,"ĠIvy":70723,"ä¸Ĭéĥ½":70724,"Ġtrud":70725,"è¨Ĺ":70726,"ĠداخÙĦ":70727,"äºĨä¸Ģè·³":70728,"echa":70729,"гани":70730,"Ġcaption":70731,"Ġtagged":70732,"\"])Ċ":70733,"Ki":70734,"-sw":70735,"åĺĨ":70736,"Ġwisely":70737,"ĠGyne":70738,"è¾Ļ":70739,"Ġzoning":70740,"Ġslit":70741,"ĠاÙĦأرض":70742,"-reported":70743,"è¾Ĩ车":70744,"Ġlouder":70745,"ece":70746,"anity":70747,"使åĬ²":70748,"ÑģкÑĥÑİ":70749,"ĠReson":70750,"Ġtrustworthy":70751,"è¿Łçĸij":70752,"turn":70753,"¯¯":70754,"ĠNinety":70755,"_RO":70756,"Ġরাà¦ĸ":70757,"Ġwheelchair":70758,"顯çĦ¶":70759,"(@\"":70760,"ÑıвлениÑı":70761,"vw":70762,"Äķ":70763,"太好äºĨ":70764,"Ġdocs":70765,"ĢáĢ»á̱á̏áĢĽá̽á̬áĢ¡á̝áĢķáĢºáĢħ":70766,"çļĦæľĢåIJİ":70767,"ä¸į符":70768,"ielding":70769,"+H":70770,"åħļæĢ»æĶ¯":70771,"ACTER":70772,"çŃĽæŁ¥":70773,"ĠConversation":70774,"apun":70775,"Ġfebr":70776,"ĠEsther":70777,"_email":70778,"kiego":70779,"Ġdang":70780,"ĠbÄĽ":70781,"ÙĬاء":70782,"chev":70783,"æĸ¯å¡Ķ":70784,"ĠÙĤÙĬÙħØ©":70785,"Ġcompensated":70786,"ĠReferanser":70787,"ĠMeasurements":70788,"è¾¾ä¸įåΰ":70789,"ĠпÑĢиводи":70790,"/AIDS":70791,"indsay":70792,"éĸ¢æķ°":70793,"ĠSterling":70794,"gene":70795,"gling":70796,"ĠTruck":70797,"è¿Ļä¸Ģ个":70798,"Ġ×Ļצ":70799,"åºĨ幸":70800,"Ġcytoplasm":70801,"Ġstrawberries":70802,"divided":70803,"ĠCFR":70804,"Than":70805,"ligt":70806,"ĠÑģиÑģÑĤеме":70807,"æĮĩçĤ¹":70808,"ATES":70809,"colors":70810,"ä¸ī个æĸ¹éĿ¢":70811,"ĠÚĨÛĮ":70812,"åĩºå¸Ńä¼ļè®®":70813,"ä¸ĢåĨį":70814,"éĹ®äºĨ":70815,"ĠLambert":70816,"Ġbrushed":70817,"ĠкоÑįÑĦÑĦиÑĨиенÑĤ":70818,"Ġcál":70819,"Ġstaged":70820,"è¿Ļéĥ½æĺ¯":70821,"ĠآزÙħ":70822,"à§Ĥপ":70823,"ĠBrigade":70824,"åºĶ符åIJĪ":70825,"ĠкÑĢеди":70826,"ĠAtom":70827,"`,Ċ":70828,"ĠFIT":70829,"activated":70830,"åİĤçļĦ":70831,"Ġinfert":70832,"OutputStream":70833,"Çİn":70834,".microsoft":70835,"опÑĢиÑı":70836,"çļĦç¥ŀèī²":70837,"ìĪľ":70838,"Ġartifact":70839,"cine":70840,"ÌĦ":70841,"Ġnhi":70842,"Ġgarments":70843,"ä¸įèī¯åıįåºĶ":70844,",u":70845,"isance":70846,"个大":70847,"hedron":70848,"Äģr":70849,"=âĢĿ":70850,"åı¯è¡ĮçļĦ":70851,"ÙħاÙħ":70852,"Ġdaytime":70853,"ืà¸Ļ":70854,"èĴ¸é¦ı":70855,"رÙĥ":70856,"å°ijåħĪ":70857,"Ġtextiles":70858,"Ġescaping":70859,"Ġê´Ģ볨":70860,"AML":70861,"ç§ŁæĪ¿":70862,"ĠRestoration":70863,"Ġkok":70864,"Ġsteroids":70865,"!Ċ":71178,"çľĭä¸įåĩº":71179,"çĽ¸ä¼´":71180,"ĠHealing":71181,"æĹłè§Ĩ":71182,"ίδ":71183,"éĶĢåĶ®æĶ¶åħ¥":71184,"ä¸Ģçŀ¬éĹ´":71185,"ĠвÑĭÑħод":71186,"Ġexecutable":71187,"ĠReflection":71188,"æ»ŀåIJİ":71189,"ĠRugby":71190,"Ġyourselves":71191,"æľ¬å±Ĭ":71192,"åIJ¦åīĩ":71193,"è¿Ļä¹Ī大çļĦ":71194,"éģĵè·¯ä¸Ĭ":71195,"ĠNutrients":71196,"ĠAutomotive":71197,"ĠChambers":71198,"åı°çļĦ":71199,"ικÎŃÏĤ":71200,"ĠLaurent":71201,"Flex":71202,"Ġank":71203,"ĠLance":71204,"Ġdrills":71205,"Ġconnective":71206,"æľĭåıĭçļĦ":71207,"MIT":71208,"wand":71209,"ĠDOS":71210,"ä¸ĭåİ»äºĨ":71211,"ä½łæĺ¯åIJ¦":71212,"-bo":71213,"ĠاÙĦأرشÙĬÙģ":71214,"å®ŀéĻħä¸Ĭæĺ¯":71215,"ë¯Ģë¡ľ":71216,"Ġcommencement":71217,"æ©Ħæ¦Ħ":71218,"ç͍工":71219,"ĠдиÑģ":71220,"arching":71221,"ĠÐłÐ°Ñģ":71222,"Ġscrub":71223,"ĠÑĥнивеÑĢÑģиÑĤеÑĤ":71224,"ozygous":71225,"Ġ(«":71226,"ĠWP":71227,"è¿Ļå°Ĩ":71228,"eeks":71229,"çħ§äº®":71230,"Already":71231,"éģ¿åŃķ":71232,"Ġpetite":71233,"Ġuterine":71234,"olina":71235,"ãĤĭãģĵãģ¨ãģĮ":71236,"à±Ĭ":71237,"unningham":71238,"çŁ¢éĩı":71239,"factor":71240,"ĠPerc":71241,"**.:":71242,"ĠManifest":71243,"Ġcheckout":71244,"ĠRomance":71245,"utas":71246,"Ġjoka":71247,"Ġdisconnected":71248,"Ġchewing":71249,"Ġskup":71250,"ัม":71251,"éģįå¸ĥ":71252,"ĠBool":71253,"ihar":71254,"ĠÓ©":71255,"ĠFees":71256,"æĪijè¿Ļ个":71257,"åıĺæĢģ":71258,"åѦçĶŁå¯¹":71259,"è³ĩéĩij":71260,"综ä¸ĬæīĢè¿°":71261,"ÑĥÑĩи":71262,"Ġexperi":71263,"auge":71264,"Ġexplode":71265,"Õ¥Öģ":71266,"Ġorally":71267,"allon":71268,"平平":71269,"èĩªçĦ¶èĢĮ":71270,"diagn":71271,"ĠFundamentals":71272,"é¢ĦæĸĻ":71273,"ÙĪÙĨت":71274,"è°ĵä¹ĭ":71275,"ocumented":71276,".valueOf":71277,"Zhang":71278,"åIJİå°±":71279,"å¾Īæľīåı¯èĥ½":71280,"ĠогÑĢаниÑĩе":71281,"ulia":71282,"бÑĢе":71283,"Ġconveniently":71284,"ÖīĊĊ":71285,"Ġskeptical":71286,"åIJİ天":71287,"Ġerase":71288,"_PRO":71289,"ÛĮÙħÛĮ":71290,"ĠSacramento":71291,"arithms":71292,"Ġbells":71293,"ĠStrait":71294,"Ġ%}ĊĊ":71295,"æĪIJåĬŁåľ°":71296,"èĪªè¡Į":71297,"å¼Ģåı£éģĵ":71298,"Ġlapar":71299,"çŁ¥æĥħ":71300,"ismatic":71301,"adaan":71302,"Exchange":71303,"Ġcathedral":71304,"æľīæīĢ帮åĬ©":71305,"ĠBaltic":71306,"Õ¡ÕµÕ«Õ¶":71307,"Ġinici":71308,"çļĦå¹´":71309,"ĠNIH":71310,"-hu":71311,"ĠÑħоÑĤÑı":71312,"Ġdinosaur":71313,"à¸ķà¹īà¸Ńà¸ĩà¸ģาร":71314,":`":71315,"æĮĩå°ĸ":71316,"ĠÐļаÑĢ":71317,"\"But":71318,"çļĦäºĶ":71319,"Ġtransgender":71320,"æīĢ以æīį":71321,"Ġpolling":71322,"æijĩæĻĥ":71323,"ĠâϦ":71324,"æĺ¯çĽ®åīį":71325,"Ġдене":71326,"éĿĴéĿĴ":71327,"VALUES":71328,"çļĦ计åĪĴ":71329,"Ġexponentially":71330,"å®īä¿Ŀ":71331,"اÙĦØ«":71332,"ারà§ĩ":71333,"Ġذات":71334,"ISSION":71335,".select":71336,"æĹłæķ°çļĦ":71337,"Ġdelinqu":71338,"-built":71339,"Ġserpent":71340,"Ġbowling":71341,"çļĦæľĢæĸ°":71342,"Identify":71343,"lekt":71344,"ĠDanger":71345,"æĪijå½ĵæĹ¶":71346,"ÛĮØ·":71347,"ĠGN":71348,"Ġunpaid":71349,"Ġspeculative":71350,"Throw":71351,"Ġslammed":71352,"åĬ¿å¿ħ":71353,"Ġneurodegener":71354,"onica":71355,"reduce":71356,"berty":71357,"ikus":71358,"å«¡":71359,"DEN":71360,"çļĦç±»åŀĭ":71361,"ä¸Ģ竳":71362,"Ġmeest":71363,"ä¸¤åľ°":71364,"Ġhelium":71365,"Ġunsere":71366,"ĠMovies":71367,"\"fmt":71368,"ÑĩÑĭ":71369,"åĨįæľī":71370,"ategoria":71371,"':'":71372,"åı¯ä»¥å¯¹":71373,"æł¹æį®åľ°":71374,"缮æłĩåĴĮ":71375,"GCF":71376,"[C":71377,"å°Ĩè¿ĻäºĽ":71378,"çıŃç»Ħ":71379,"æ°¸éģł":71380,"ĠJuli":71381,"Easy":71382,"åĮĸ身":71383,"å®Įå¤ĩ":71384,"-carbon":71385,"Ġзапа":71386,"ĠSyntax":71387,"ĠоÑħ":71388,"Ġdoubling":71389,"åĵįäºĨ":71390,"Ġnationale":71391,"ĠسازÙħاÙĨ":71392,"_up":71393,"ĠAkadem":71394,"_J":71395,"çļĦå±ĢéĿ¢":71396,"éģ·":71397,"ĠëĿ¼":71398,"Ġdép":71399,"è¿IJèIJ¥åķĨ":71400,"åŃĺéĩı":71401,"Ġfrightening":71402,"ĠnenÃŃ":71403,"adia":71404,"æ³ķ令":71405,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":71406,"(expected":71407,"wm":71408,"çĤ¹æĺ¯":71409,"Ġsingers":71410,"μÏĮ":71411,"/jquery":71412,"lasse":71413,"Ġmouths":71414,"ĠÑĢÑĭн":71415,"Ġadministering":71416,"Ġregiment":71417,"ĠاÙĦÙħÙĦ":71418,"ĠÚ©ÙĦÛĮ":71419,"ĉtemp":71420,"Seb":71421,"Wellington":71422,"缸符":71423,"Ġimpatient":71424,"levard":71425,"Police":71426,",##":71427,"Ġproduz":71428,"ĠCharacters":71429,"àµįà´Ł":71430,"tb":71431,"Ġcasing":71432,"Ġsludge":71433,"רס":71434,"ĠÙĪØ§ÙĦد":71435,"ĠEliot":71436,"ĠÑĦинанÑģов":71437,"Ġthức":71438,"à¸ļà¸Ħ":71439,"owska":71440,"æ¯ı个æľĪ":71441,"Ġyouths":71442,"Ġmente":71443,"大ç¥ŀ":71444,"åIJįçīĩ":71445,"ä½ľç͍äºİ":71446,"Ġfascination":71447,"ĠLamp":71448,"colon":71449,"Ġalarming":71450,"ĠاÙĦاجتÙħاعÙī":71451,"é£Ħ":71452,"ä¿Ŀæļĸ":71453,"-you":71454,"ĠÑģÑĥмма":71455,"Hop":71456,"cpy":71457,"ĠtÃŃnh":71458,"度éĩı":71459,"ç»ĦåĪĨ":71460,"Ġgoverno":71461,"Ġforeground":71462,"ĠREVIEW":71463,"å¹´ä¸Ń":71464,"æĹ¥ç¨ĭ":71465,"รà¹Īาà¸ĩ":71466,"ç®Ĺæ³ķçļĦ":71467,"è¿ĻæĶ¯":71468,"ellants":71469,"ibia":71470,"æĭįæĭį":71471,"ĠÐijи":71472,"èIJ¥ä¸ļæī§çħ§":71473,"Ġsymmetrical":71474,"Ġpistol":71475,"ĠFilipino":71476,"Rules":71477,"Ġlest":71478,"Ġwildly":71479,"ĠCalculating":71480,"otus":71481,"ĠBey":71482,"ĠEarlier":71483,"performance":71484,"åºķèķ´":71485,"å®ŀéĻħéĹ®é¢ĺ":71486,"ĠاÙĦتج":71487,"EGF":71488,"第åįģäºĶ":71489,"Ġabc":71490,"×ķ×¥":71491,"çľ¼çľ¶":71492,"èĭ±åĭĩ":71493,"帮ä»ĸ":71494,"ELS":71495,"cled":71496,"å¤ļæĺ¯":71497,"ä¸ĢåĽ¢":71498,"cycles":71499,"Õ¡Õº":71500,"çļĦ大éŨ":71501,"ĠاÙĦاعتد":71502,"Ġmúsica":71503,"ELF":71504,"Ġstacks":71505,"åı¯çͱ":71506,".Search":71507,"ĠMarl":71508,"Ġfelony":71509,"enched":71510,"erset":71511,"izados":71512,"éĹ«":71513,"æĸ°è¯¾":71514,"çł¥":71515,"ĠÑĥлÑĥÑĩ":71516,"Ġhoog":71517,"Ġnicotine":71518,"xo":71519,"~/":71520,"еви":71521,"她对":71522,"ÅĻej":71523,"ĠÐĶж":71524,"ä¸įç¡®å®ļæĢ§":71525,"ä¸Ģ模":71526,"Ġclam":71527,"ä¹ĭæĦŁ":71528,"ĠNeo":71529,"åľ£ç»ı":71530,"Ġrookie":71531,"åħħåĪĨèĤ¯å®ļ":71532,"ĠÑıнва":71533,"æľīç͍çļĦ":71534,"为å¥ijæľº":71535,"eczy":71536,"innitus":71537,"Ġexperimenting":71538,"ÙijÙIJ":71539,"Ġprosecutors":71540,"korzyst":71541,".os":71542,"Ġverde":71543,"*}":71544,"ĠTing":71545,"ä»İæł¹æľ¬ä¸Ĭ":71546,"æĺ¾åį¡":71547,"Ġcorrecting":71548,"Ġcavalry":71549,"Ġchords":71550,"Ġmismatch":71551,"ĠجدÛĮد":71552,"gap":71553,"ĠNSC":71554,"лÑĭй":71555,"åij¦":71556,"åĩºçĶŁäºİ":71557,"Schedule":71558,"isers":71559,"ulmonary":71560,"aharan":71561,"(\"./":71562,"Ġಬ":71563,"ĠHandling":71564,"ĠSquares":71565,"ĠобÑĥÑĩениÑı":71566,"ramient":71567,"äºĮæŀģ管":71568,"é¦ĸéĢī":71569,"åħ´èĩ´":71570,"Hydro":71571,"Sport":71572,"Ġdeque":71573,"Ġclassics":71574,"åħħæĸ¥":71575,"joining":71576,"Ġibn":71577,"Ġtilted":71578,"Ġwizard":71579,"Ġzie":71580,"ç͵æĬ¥":71581,"åįĹçĵľ":71582,"æĽ´å¤ļåľ°":71583,"å¤ĸåĽ½äºº":71584,"Principal":71585,"ĢáĢ»á̱á̏áĢĽá̽á̬áĢ¡á̝áĢķáĢºáĢħá̝":71586,"ĠConsolid":71587,"ILY":71588,"Generally":71589,"Ġcelebrities":71590,"-regulated":71591,"acjÄĻ":71592,"Ġtransgenic":71593,"Ġsamt":71594,"ĠElena":71595,"uju":71596,"ĠGeneric":71597,"åıªè¦ģæľī":71598,"çļĦè¶ĭåĬ¿":71599,"Ġsurtout":71600,"æĢ»å·¥ä¼ļ":71601,"ĠجاÛĮ":71602,"elte":71603,"×Ļ×Ļף":71604,"Align":71605,".getName":71606,"Ġà¦ķার":71607,"èįīæľ¨":71608,"ÑĤÑĭе":71609,"ĠConsultado":71610,"URRENT":71611,"princ":71612,"æĦŁå®ĺ":71613,"æİ¨ç§»":71614,"ذÙĬ":71615,"çͲéĨĽ":71616,"èģĬèģĬ":71617,"ä»ĸä¸İ":71618,"åŁİåİ¿":71619,"æ³¢çļĦ":71620,"å°ĩè»į":71621,"configuration":71622,"ĠArabs":71623,"stag":71624,"ĠCerv":71625,"Ġdetox":71626,"×Ļ׾×ķת":71627,"ĠPIN":71628,"ĠVale":71629,"جات":71630,"ĠMast":71631,"çī½":71632,"Ġ×ĺ×":71633,"Ġdebit":71634,"Ġethylene":71635,"Ġdissipation":71636,"ĠÑģпи":71637,"ÑĩиÑĤаÑĤÑĮ":71638,"Kap":71639,"举äºļ":71640,"ÙĴر":71641,"Loss":71642,"etas":71643,"ĠSPE":71644,"家常":71645,"åıĹè¿ĩ":71646,"Ġresto":71647,"REM":71648,"ĠBasel":71649,"ĠEssex":71650,"寺åºĻ":71651,"enchymal":71652,"Ġcomerc":71653,"ĠKuwait":71654,"è¿Ļ次çļĦ":71655,"<":72284,"Ġterug":72285,"Ġ×ķש×":72286,"æİ¨èįIJçļĦ":72287,"ĠQuébec":72288,"é«ĺå°Ķ":72289,"ĠRex":72290,"axon":72291,"å®ĥèĥ½":72292,"ĠAdvertisement":72293,"社ä¼ļåѦ":72294,"/match":72295,"Ġprofessionalism":72296,"æµ®åĬ¨":72297,"饥饿":72298,"/equivalent":72299,"ĠMys":72300,"ä¸Ģæĭ³":72301,"ultats":72302,"ĠGeology":72303,"åı«äºº":72304,"éĴ»çłĶ":72305,"ĠвоÑģпиÑĤа":72306,"ĠLorenzo":72307,"Ġsibling":72308,"ikir":72309,"æ¤įåħ¥":72310,"ĠSeminar":72311,"ĠSitu":72312,"æıIJåIJį":72313,"ç®Ģ约":72314,"é£ŀéĢŁ":72315,"æľ¨æĿ¿":72316,"ĠЧа":72317,"ĠSUR":72318,"Ġunsett":72319,"'eau":72320,"_var":72321,"ĠSTART":72322,"Ġpumped":72323,"ĠOpposition":72324,"?...ĊĊ":72325,"endu":72326,"è£ħæľī":72327,"ĉĉĉĉĊ":72328,"ĠmmHg":72329,"Ġdifférentes":72330,"âteau":72331,"ĠÑĥÑĤвеÑĢ":72332,"Ġgeology":72333,"å²Ĺä½įä¸Ĭ":72334,"maybe":72335,"'=>'":72336,"ãĢĸ":72337,"ĠTrag":72338,"ĠMongo":72339,"çİĸ":72340,"ĠKudos":72341,"à°Ł":72342,"à¸Ľà¸±à¸Īà¸Ī":72343,"âĶľ":72344,"éĸĢåı£":72345,"Ġpública":72346,"ä¸İä¼Ĺ":72347,"Ġprescribe":72348,"uttgart":72349,"Ġroughness":72350,"Ġpolymorphism":72351,"-country":72352,"ĠRwanda":72353,"ĠmA":72354,"å¿ĥæĦı":72355,"Ġevol":72356,"çĶµçº¿":72357,"ĠEngaging":72358,"ĠگرÙĪÙĩ":72359,"ĠKeynes":72360,"Features":72361,"ĠANOVA":72362,"ĠWitness":72363,"ége":72364,"bung":72365,"¼áĢ":72366,"çļĦåı¦ä¸Ģ":72367,"ä½µ":72368,"为她":72369,"æĿijå§Ķä¼ļ":72370,"éĻIJé¢Ŀ":72371,"ĉtry":72372,"Ġgratuit":72373,"Õ¥ÖĢÕ¨":72374,"/img":72375,">:":72376,"Ġbiting":72377,"iesen":72378,"Ġunilateral":72379,"Ġlasers":72380,"å®Īæ³ķ":72381,"ä¿ĿéĻ©äºº":72382,"Ġredundancy":72383,"ĠÑģовеÑĢÑĪен":72384,"çļĦéŁ³ä¹IJ":72385,"ĠDairy":72386,"ikers":72387,"æĹłçŁ¥":72388,"ç͵平":72389,"Ġpersists":72390,"Ġequiv":72391,"åħĭéļĨ":72392,"رÛĮÙĤ":72393,"иÑģаÑĤÑĮ":72394,"Fit":72395,"Ġcrossover":72396,"Ġincompet":72397,"алов":72398,"Ġconte":72399,"Ġacquainted":72400,"ĠاÙĦسÙĦاÙħ":72401,"Ġresisted":72402,"aon":72403,"çļĦæŃ£ç¡®":72404,"iché":72405,"éĩį度":72406,"ĠComfort":72407,"èģĶæīĭ":72408,"ĠAmber":72409,"ĠCalgary":72410,"çĤºä½ķ":72411,"URAL":72412,"æľºæŀĦåĴĮ":72413,"agrams":72414,"èľľèľĤ":72415,"Ġsmokers":72416,"çļĦè§£éĩĬ":72417,"^{+":72418,"Ġtopography":72419,"одаÑĢÑı":72420,"ĠQualifications":72421,"RON":72422,"jian":72423,"çļĦæĻĤéĸĵ":72424,"å쬦":72425,"èĭ±åľĭ":72426,"Ġlenker":72427,"Ġdiversas":72428,"Ġinfatti":72429,"çĮĿ":72430,"è²»ç͍":72431,"ĠHapit":72432,"äºĭäºĭ":72433,"éĢıè§Ĩ":72434,"éĴ¢æĿIJ":72435,"Ġroofs":72436,"Ġlumbar":72437,"Ġpractise":72438,".Cross":72439,"ç´¢æĢ§":72440,"ĠAustralians":72441,"ĠвзÑĢоÑģ":72442,"ĠMole":72443,"ĠLiqu":72444,"órm":72445,"æµĭç®Ĺ":72446,"Ġniem":72447,"å®Įæķ´æĢ§":72448,"ivit":72449,"Ġformative":72450,"-sum":72451,"丧尸":72452,"ICAgICAgICAgICAg":72453,"Lem":72454,"ä¸ĢæĹı":72455,"()))":72456,"æķ°æį®éĽĨ":72457,"éĩijèŀįæľįåĬ¡":72458,"ĠAlberto":72459,"ĠWARRANTIES":72460,"tool":72461,"çݺ":72462,"åħ¨è¦ĨçĽĸ":72463,"çķĪ":72464,"ä¼ģä¸ļåıijå±ķ":72465,"ĠFlexible":72466,"LowerCase":72467,"/blob":72468,"Ġmeningkatkan":72469,"ä¸įå¤į":72470,"Ġefficiencies":72471,"Ġát":72472,"cció":72473,"Ġplethora":72474,"Blueprint":72475,"Ġreptiles":72476,"Ġacclaimed":72477,"Stephen":72478,"λοÏĤ":72479,"oplankton":72480,"ĠAcknowledgments":72481,"Ġجزء":72482,"åĩıæİĴ":72483,"Ġgraphite":72484,"ведение":72485,"auen":72486,"Ġlifecycle":72487,"ÑĢÑĥÑİÑĤ":72488,"Ġphotographers":72489,"modified":72490,"Ġblogger":72491,"æł¹æľ¬å°±":72492,"Ġnostra":72493,"Ġquir":72494,"æŃ£çĽ´":72495,"Ġgalleries":72496,"ĠInfantry":72497,"\"\\":72498,"Ġdung":72499,"æĺĩ":72500,"ĠOok":72501,"ĠKuh":72502,"çݯçIJĥ":72503,"æ¦ķ":72504,"æ½ĩæ´Ĵ":72505,"帷å¹ķ":72506,"ĠKell":72507,"ä¾ĭé¢ĺ":72508,"Ġembarrassing":72509,"Ġgebruikt":72510,"ikin":72511,"Ġprincipio":72512,"Twenty":72513,"ĠwiÄĻc":72514,"having":72515,"ĠSain":72516,"estamps":72517,"åĴĨ":72518,"ä¸ĭ乡":72519,"kaar":72520,".eu":72521,"ظÙħØ©":72522,"major":72523,"ĠÑģмеÑĢ":72524,"Ġà¤ĸ":72525,"åĵªè£¡":72526,"Ġপাত":72527,"×ķ׼׾":72528,"ampu":72529,"dfs":72530,"ĠÐijÑĥ":72531,"ederb":72532,"åIJĪæł¼çļĦ":72533,"ĠRabbi":72534,"ĠFitzgerald":72535,"å°±çľĭåΰ":72536,"ecip":72537,"ĠкапиÑĤа":72538,"ĠинÑĦек":72539,"iformes":72540,"ĠCorrection":72541,"{h":72542,"ä»·éĴ±":72543,"æİ¨è¿Ł":72544,"ALTER":72545,"ROSS":72546,"ä¹Łä¸į好":72547,"ÙĬÙ쨩":72548,"第ä¸ĥ竳":72549,"Ò±":72550,"usch":72551,"Ġalright":72552,"resident":72553,"Ġcontinual":72554,"ãģĹãģ¦ãģĦãģŁ":72555,"ĠZeus":72556,"ĠMutual":72557,"ĠHä":72558,"Ġokres":72559,"ĠMcKin":72560,"(typeof":72561,"åİ»çľĭçľĭ":72562,"à¸Ķิà¸Ļ":72563,"elas":72564,"åĨĹ":72565,"æĪij们ä»İ":72566,"Anim":72567,"Ġà¦ķি":72568,"_filter":72569,"slug":72570,"Cas":72571,"Fair":72572,"×£":72573,"edere":72574,"metadata":72575,"Ġcrossword":72576,"ĠاÙĦÙĤد":72577,"éĹªéĹª":72578,"Ġcaregiver":72579,"Ġtearing":72580,"æĺ¯ä»ĸçļĦ":72581,"ä¸ºåĽ½å®¶":72582,"ä¹Łè¨±":72583,"Ġbuys":72584,"Alice":72585,"é¥ŃåIJİ":72586,"ĠBrexit":72587,"æĽ¾ç»ıçļĦ":72588,"åѦçĶŁçļĦåŃ¦ä¹ł":72589,"Ġparece":72590,"æīĢ带æĿ¥çļĦ":72591,"åĨĽè®Ń":72592,"èĢģå¸ĪåĴĮ":72593,"lasting":72594,"Ġaquarium":72595,"nahmen":72596,"èĩ³å°Ĭ":72597,"Ġwary":72598,"Ġrond":72599,"ä½łè¯´çļĦ":72600,"海峡":72601,"Ġcutoff":72602,"èİ«éĿŀ":72603,"Ġexhaustive":72604,"à°¿à°¨":72605,"ĠSelbst":72606,"tero":72607,"ĠRAD":72608,"oreg":72609,"physical":72610,"çľĭåľ¨":72611,"hopping":72612,"Ġ×IJשר":72613,"ùng":72614,"backgroundColor":72615,"ĠокÑĢÑĥжа":72616,"ĠTrigonometric":72617,"progress":72618,"温室":72619,"éĢīæĭ©æĢ§":72620,"ĠIsraelites":72621,"Ġwarranted":72622,"ĠROI":72623,"onation":72624,"ãĤĴãģ¤":72625,"ĠاÙĦÙħØ®":72626,"nÄĽjÅ¡ÃŃ":72627,"ждениÑı":72628,"Ġdivergent":72629,"Ġfors":72630,"åĽĽçº§":72631,"ارت":72632,"å·®ä¸įå¤ļäºĨ":72633,"ziÄĻki":72634,"Ġinforms":72635,"¶ĊĊ":72636,"Ġlorsque":72637,"DG":72638,"pples":72639,"为çͱ":72640,"à¤ħ":72641,"çĶŁäº§ä¼ģä¸ļ":72642,"ä¸Ľä¹¦":72643,"åѦéķ¿":72644,"è¿ĩåī©":72645,"çŃīå¤ļ个":72646,"åı¯ä»¥è¢«":72647,"Ġdiscs":72648,"à¨ķ":72649,"Ġoccupancy":72650,"Ġhydrated":72651,"Ġdictators":72652,"yyyy":72653,"éĺIJéĩĬ":72654,"Ġpharmacological":72655,"ĠðĿIJ´":72656,"-breaking":72657,"wl":72658,"Ġslack":72659,"Ġdati":72660,"ĠÙĤسÙħ":72661,"ĠмаÑĪи":72662,"ĠباÙĦÙħ":72663,"ë©Ķ":72664,"ìĺ¨":72665,"ĠMorton":72666,"ĠCherry":72667,"VEN":72668,"ĠاÙĦÙĴ":72669,"consciously":72670,"ë©´ìĦľ":72671,"Ġpyro":72672,"ĠDud":72673,"ély":72674,"Ġprů":72675,"约ä¼ļ":72676,"ĠкÑĥлÑĮÑĤÑĥÑĢÑĭ":72677,"ĠBibcode":72678,"çļĦèĦ¸ä¸Ĭ":72679,"ĠMight":72680,"obody":72681,"Ġبط":72682,"ç§»åΰ":72683,"æĿ¾å¼Ģ":72684,"æł¹æľ¬ä¸į":72685,"ĠBreakfast":72686,"ĠDivers":72687,"Ġhemod":72688,"ä»ĸãģ®":72689,"ĠKIND":72690,"iencias":72691,"åĽĽæµ·":72692,"Choice":72693,"ÉĻs":72694,"ĠÑģай":72695,"ndan":72696,"ĠNina":72697,"ĠDemo":72698,"สัม":72699,"ä½ĵåŀĭ":72700,"Ġlongitud":72701,"书å±Ģ":72702,"åħĭéĩĮ":72703,"åĨľä¸ļåĨľæĿij":72704,"Ġfavors":72705,"}$.":72706,"said":72707,"ĠNormally":72708,"ĠSuzuki":72709,"_once":72710,"Ġinductive":72711,"ĠHb":72712,"大æłij":72713,"åºĦåŃIJ":72714,"]));Ċ":72715,"oliber":72716,"ĠMint":72717,"éķ¿å¤§äºĨ":72718,"Ġgrids":72719,"æĪ¿éĩĮ":72720,"Ġcerebell":72721,"=F":72722,"ĠPaste":72723,"ayah":72724,"Ġdepois":72725,"riding":72726,"rady":72727,"ĠسÙĦاÙħ":72728,"_points":72729,"Ġvastly":72730,"Ġdictate":72731,"ĠопÑĢеделиÑĤÑĮ":72732,"å²Ĥä¸įæĺ¯":72733,"Ġinvece":72734,"ĠSight":72735,"Thai":72736,"ĠNotification":72737,"ĠSolo":72738,"سباب":72739,"ĠConversions":72740,"Ġchuckled":72741,"ĠBolog":72742,"åĨĻ羣":72743,"κη":72744,"å°½æĹ©":72745,"={'":72746,"à¤ķà¥įत":72747,"æĵ¦æĭŃ":72748,"Ġwieku":72749,"liches":72750,"Ġlessen":72751,"Conc":72752,"æĺŁåħī":72753,"伺åĢĻ":72754,".ref":72755,"ĠFILE":72756,"cius":72757,"glut":72758,"æĨ§":72759,"ĠvÅ¡ak":72760,"Ġesk":72761,"æİ¨ä»ĭ":72762,"æķ°æį®åĪĨæŀIJ":72763,"ĠÑĤон":72764,"Ġкоман":72765,"Ġfrogs":72766,"Ġcohorts":72767,"Encoder":72768,"еÑģÑĤи":72769,"ÑĤнÑĭе":72770,"ä¸Ńå°Ĩ":72771,"ferably":72772,"åIJij举":72773,"Ġerhalten":72774,"Ġrepresenta":72775,"ĠChiefs":72776,"ÑĨионной":72777,"_Y":72778,"Ġwan":72779,"otrophic":72780,"ĠMaker":72781,"çĻ¾è´§":72782,"人ä¸İ人":72783,"纪å½ķçīĩ":72784,".default":72785,"æŃ©":72786,"assi":72787,"天çİĭ":72788,"ĠIsle":72789,"ä¹Łæĺ¯æľī":72790,"èĦ¸é¢Ĭ":72791,"Actual":72792,"ÑĢжа":72793,"ĠNab":72794,"äºĴéĢļ":72795,"ĠRatings":72796,"-er":72797,"ĠLemon":72798,"ĠSpell":72799,"\\infty":72800,"Ġepidemiology":72801,"åĩºåĬĽ":72802,"oused":72803,"è¡Įæ¥Ń":72804,"forma":72805,"Ġretin":72806,"Ġinfra":72807,"éļı身":72808,"å±ŀæĢ§çļĦ":72809,"Ġdeliveries":72810,"çݲçıij":72811,"ĠMANAG":72812,"_U":72813,"Ġresponsiveness":72814,"Ġinspector":72815,"Ġ];ĊĊ":72816,"Ġrenovation":72817,"Ġ{(":72818,"æ²īéĩįçļĦ":72819,"æľīæķο̧":72820,"Ġcorrespondent":72821,"åIJĮæĹ¶è¿ĺ":72822,"ĠBenefit":72823,"VELOP":72824,"oC":72825,"çī¹è´¨":72826,"æĨ¬":72827,".stringify":72828,"Rain":72829,"ĠPOP":72830,"iegel":72831,"Ġverge":72832,"給ä»ĸ":72833,"ĠEighty":72834,"ĠاÙĦØŃÙĬاÙĩ":72835,"Dynamic":72836,"rather":72837,"оÑĢож":72838,"ĠÚ©ÛĴ":72839,"ãĢįãĢĤĊĊ":72840,"è®ĵä½ł":72841,"bourg":72842,"عراض":72843,"ĠEksterne":72844,"ĠFract":72845,"å°ıçģ«":72846,"å°½äºĨ":72847,"å¿ħé¡»æľī":72848,"ĠApplicant":72849,"/log":72850,"Wa":72851,"_html":72852,"enig":72853,"redient":72854,"ocked":72855,"è®®é¢ĺ":72856,".Hash":72857,"è¤Ĵ":72858,"çļĦç͍æĪ·":72859,"ä¹Łç§°":72860,"ä½Ĩä¸įèĥ½":72861,"Ġbusca":72862,"าลัย":72863,"Ġdictionaries":72864,"Ġcheerful":72865,"Ġchac":72866,"вÑĪиÑħ":72867,"Ġassort":72868,"INST":72869,"ulte":72870,"ĠHubble":72871,"ĠProto":72872,"Ġmills":72873,"ĠProvided":72874,"_rec":72875,"æĥ³å¿µ":72876,"åıĺè´¨":72877,"æµģ产":72878,"转åŃIJ":72879,"Ġsuma":72880,"æIJŀå¾Ĺ":72881,"ispr":72882,"Ġanders":72883,"Ġqued":72884,"Ġsheath":72885,"ĠмÑĥÑĪ":72886,"çļĦäººæł¼":72887,"Ġcheckpoint":72888,"骨质":72889,"é¤IJé¦Ĩ":72890,"ĠÑħозÑıй":72891,"Ġmanipulating":72892,"ĠManit":72893,"cus":72894,"Ġworkspace":72895,"Ġorganizer":72896,"ĠоÑĢганиза":72897,"èĩªé©¾":72898,"çĤ¬":72899,"========================":72900,"Ġcorrobor":72901,"ratory":72902,"itre":72903,"ä¸Ńæłĩ":72904,"ÑĢак":72905,"çĸµ":72906,"åİĨæĹ¶":72907,"åĿļåĽº":72908,"çīĽé¡¿":72909,"ĠÐłÐ¾ÑģÑģийÑģкой":72910,"ĠwÅĤas":72911,"entries":72912,"åľ¨çľĭ":72913,"åĪĨéĴŁåIJİ":72914,"Ġmandated":72915,"alary":72916,"ĠvÉĻ":72917,"Ġмне":72918,"设å¤ĩåĴĮ":72919,"-regulation":72920,"åIJįçīĮ":72921,"樱æ¡ĥ":72922,"Ġspatially":72923,"代表æĢ§":72924,"ĠBritannica":72925,"kamp":72926,"賦":72927,"ÙĦÙħØ©":72928,"ĠУкÑĥпно":72929,"éĭª":72930,"åĩıéĢĢ":72931,"ש×Ļ×Ŀ":72932,"Ġconsonant":72933,"好æ¶Īæģ¯":72934,"è¿IJéĢģ":72935,"ĠWatts":72936,"Winter":72937,"ĠMiz":72938,"ĠECM":72939,"separ":72940,"失æİ§":72941,"ĠÙħÛĮاÙĨ":72942,"circle":72943,".ne":72944,"Pok":72945,"\\Delta":72946,"Ġrt":72947,"Ġobsolete":72948,"áĥľ":72949,"ĠXL":72950,"她çļĦæīĭ":72951,"(page":72952,"ĠdifÃŃ":72953,"æ¯Ķä»ĸ":72954,"ä»ĸä»¬ä¹Ł":72955,"oughton":72956,"æ´ģåĩĢ":72957,"ĠCounseling":72958,"Yesterday":72959,"Ġadtong":72960,"мон":72961,"ĠVerde":72962,"Ġì¤Ħ":72963,"oil":72964,"atham":72965,"ÙģØ§Øª":72966,".source":72967,"åĩĨå¤ĩäºĨ":72968,"غÙĨ":72969,"Ġdialysis":72970,"ĠMalaysian":72971,"æľ¬èĬĤ":72972,"Ġà¦¨à¦¿à¦ľ":72973,"åĽ½æľīèµĦ产":72974,"Ġgiorno":72975,"usahaan":72976,"sic":72977,"çļĦ第äºĮ":72978,"ĠHän":72979,"ĠÑģÑĤÑĢанÑĭ":72980,"@section":72981,"ibid":72982,"licts":72983,"ä¸ĵå±ŀ":72984,"æŃ¦å£«":72985,"à¸ģารà¸ĵà¹Į":72986,"Ġacidity":72987,"çļĦåıij":72988,"çļĦæľīåħ³":72989,"çļĦåĽ½éĻħ":72990,"Ġinformáció":72991,"ĠSophia":72992,"omrÃ¥":72993,"Ġmovimiento":72994,"à±įà°¨":72995,"Ġfestive":72996,"çļĦ游æĪı":72997,"ĠTay":72998,"ĠGym":72999,"å°±ä»İ":73000,"表åĨ³":73001,"æĹłæľº":73002,"äºĶ年级":73003,"ç»Ŀä¸į":73004,"顺çķħ":73005,"Ġmolti":73006,"Ġkolej":73007,"UDE":73008,"tube":73009,"Ġgere":73010,"ĠDixon":73011,"antz":73012,"Ġinterns":73013,"é¢Īæ¤İ":73014,"Ġtore":73015,"Ġencephal":73016,"Ġdurant":73017,"Ingredients":73018,"ĠMoy":73019,"ĠFold":73020,"æĻĵå¾Ĺ":73021,"Ġmatern":73022,"otechnol":73023,"èĢĮçİ°åľ¨":73024,"å°ijäºİ":73025,"Esta":73026,"Ġsurvivor":73027,"弩":73028,"åİŁåīĩ":73029,"rana":73030,"meth":73031,"ĠبÙĬت":73032,"Ġvarios":73033,"bio":73034,"Ġعبار":73035,"Season":73036,"Ġoat":73037,"ĠÙĦØ¥":73038,"äºīåIJµ":73039,"Ġspecifics":73040,"éĵ¶è¡Įä¸ļ":73041,"ĠPoems":73042,"Ġturbo":73043,"æĺ¯åħ¶":73044,"-store":73045,"ðĿijij":73046,"rypted":73047,"Ġcherche":73048,"æĴķè£Ĥ":73049,"Ġprocent":73050,"Ġunim":73051,"ĠдÑĢев":73052,"Ġprogrammers":73053,"Ġatyp":73054,"Ġroadmap":73055,"Ġpermutation":73056,"èIJ¬åħĥ":73057,"inux":73058,"Ġreleg":73059,"ĠMID":73060,"å°ı說":73061,"ĠоÑĪиб":73062,"åIJijä½ł":73063,"Ġmediate":73064,"ambigu":73065,"çĿ¡çĿĢäºĨ":73066,"FFECT":73067,"Operations":73068,"-result":73069,"Ġwanna":73070,"ÑĤнÑĭй":73071,"æĸ°å¨ĺ":73072,"ĠCookie":73073,"ĠAnthropology":73074,"ciences":73075,"ï¼ī=":73076,"çĭłæĬĵ":73077,"Ġà¹ĥหà¹ī":73078,"Ġcharcoal":73079,"лÑĮзÑı":73080,"ĠâĪ©":73081,"ãģĭãģ«":73082,"×ŀ×ĵ":73083,"Ġghosts":73084,"ĠAval":73085,"è¿ĽåĨĽ":73086,"Ġnegli":73087,"Seconds":73088,"å°įèijĹ":73089,"_loss":73090,"çŃīæķĪ":73091,"Ġrhs":73092,"Ram":73093,"åĩŃä»Ģä¹Ī":73094,"Ġwiele":73095,"Ġproducto":73096,"олÑĮно":73097,"-quarter":73098,"Ġbolts":73099,")T":73100,"å¤įä½į":73101,"ĠÕ¸ÖĢ":73102,"æĪijä»Ĭ天":73103,"éľĢè¦ģè¿Ľè¡Į":73104,"ĠÙĨدار":73105,"Ġসà¦Ļà§įà¦Ĺ":73106,"建ç«ĭä¸Ģ个":73107,"СÐļ":73108,"มาà¸ķ":73109,"rattutto":73110,"ĠاÙĦاعتداÙĦ":73111,"saurus":73112,"enton":73113,"owell":73114,"oplan":73115,"åĮĸèĤ¥":73116,"她èĩªå·±":73117,"ĠAless":73118,"worker":73119,"ĠREAL":73120,"Ġmediator":73121,"ĠElastic":73122,"Classes":73123,"èµŀåĬ©":73124,"ĠJosef":73125,"úa":73126,"èģĶç³»æĸ¹å¼ı":73127,"żej":73128,"ãĤŃãĥ£":73129,"Kal":73130,"vate":73131,"ĠTours":73132,"à¥įल":73133,"}}}{":73134,"ĠMaple":73135,"(un":73136,"reiche":73137,"ucceed":73138,"åIJĥåĸĿ":73139,"ाण":73140,"åħ¬æľīåζ":73141,"į":73142,"Ġalf":73143,"ĠLU":73144,"ä¸ĬåŃ¦æľŁ":73145,"ä¸ĩ个":73146,"ç§ģåĭŁ":73147,"Ġpériode":73148,"Ñģол":73149,"Ġclones":73150,"æ°ijçļĦ":73151,"á̾":73152,"竣çĦ¶æĺ¯":73153,"älle":73154,"åIJįé¢Ŀ":73155,"à¯ģà®±":73156,"èľ¡çĥĽ":73157,"åijĤ":73158,"Äįek":73159,"Ġréalis":73160,"ĠléÄį":73161,"-area":73162,"Ñĩении":73163,"ĠÙĤابÙĦ":73164,"ĠCalculus":73165,"Ġfuerza":73166,"Ġinaugural":73167,"uze":73168,"å¹³åĪĨ":73169,"Ġestekak":73170,"ÑĢиÑĺе":73171,"Ġgrandson":73172,"ĠUL":73173,"Ġprid":73174,"ianza":73175,"驯":73176,"ĠÐļом":73177,"ĠPediatrics":73178,"Civil":73179,"ĠMog":73180,"ä¸ļæĢģ":73181,"èĢĥåľº":73182,"רצ":73183,"å¥ĩæĢªçļĦ":73184,"Ġstitch":73185,"åľ¨äºº":73186,"æĹ¥è¶ĭ":73187,"æĺ¯å¤ļ":73188,"æĶ¶åī²":73189,"ðĿijł":73190,"交æĺĵçļĦ":73191,"ĠBrunswick":73192,"ĠBek":73193,"Ġdobr":73194,"Ġcontractions":73195,"Ġéén":73196,"Ġà¦Ĩমাদà§ĩর":73197,"ĠاÙĦرÙĪ":73198,"交æīĢ":73199,"ิส":73200,"èce":73201,"Ġcommenting":73202,"ĠWendy":73203,"ĠоÑĩеÑĢед":73204,"ubin":73205,"ái":73206,"åĽłåľ°":73207,"æ¶Ł":73208,"IDTH":73209,"(parent":73210,"Ġrejecting":73211,"ĠAurora":73212,"Completed":73213,"aisse":73214,"éĻĦçĿĢ":73215,"Ġfragmented":73216,"ĠAgile":73217,"ĠFrançais":73218,"Ġhypothalam":73219,"Ġvolunteering":73220,"Ġszczeg":73221,"pain":73222,"unched":73223,"oller":73224,"Ġbelts":73225,"aird":73226,"ł×Ĵ":73227,"è´µéĺ³":73228,"ĠìĿĺ미":73229,"'autres":73230,"ĠÑģвобод":73231,"agy":73232,"çŃIJ":73233,"Ġthemed":73234,"Ġanalogue":73235,"lius":73236,"Ġinventor":73237,"示èĮĥåĮº":73238,"ĠзадаÑĩ":73239,"Ġfountain":73240,"à¹ij":73241,"岡":73242,"Ïĥία":73243,"ẳ":73244,"ĠÑģегоднÑı":73245,"EARCH":73246,"å¹´äºĨ":73247,"Ġprenatal":73248,"curl":73249,"æĤ²åĵĢ":73250,"Ġresemblance":73251,"ĠRif":73252,"å±ĤéĿ¢çļĦ":73253,"ĠAccessibility":73254,"িতà§įর":73255,"Downloads":73256,"Street":73257,"analyse":73258,")P":73259,"ÑĩнÑĭм":73260,"erdings":73261,"Ġà¦Ńার":73262,"Ġì±ħ":73263,"ariamente":73264,"ä¸Ģ个éĹ®é¢ĺ":73265,"è§£èĦ±":73266,"Ġtranslator":73267,"în":73268,"Ġwilt":73269,"ä»ĸå®¶":73270,"Ġformación":73271,"è·¯æĺĵ":73272,"Ġinformations":73273,"æĨİ":73274,"æ©¡çļ®":73275,"æĸ°è¥¿åħ°":73276,"飽":73277,"ĠознаÑĩа":73278,"Ġdaerah":73279,"çĹĽå¿«":73280,"Ġpetals":73281,"æĬµæĮ¡":73282,"MOOCs":73283,"广æĴŃç͵è§Ĩ":73284,"cong":73285,"Ġimitation":73286,"Ġnovelty":73287,"ĠÐŁÑĢиÑģÑĤÑĥп":73288,"ĠCombine":73289,"Ġtranquil":73290,"ĠBecome":73291,"å±±ä¸ĭ":73292,"ÐłÐŀ":73293,"Ġreactors":73294,"Ġply":73295,"Ġstrap":73296,"ontrol":73297,"efit":73298,"argon":73299,"ĠÙĨس":73300,"Ġобозна":73301,"arÃŃa":73302,"usto":73303,"aremos":73304,"æµģéĢĿ":73305,"Ġinfancy":73306,"塾":73307,"моÑĤÑĢ":73308,"ĠNeurology":73309,"Ġhues":73310,"Ġanys":73311,"Ġabide":73312,"Ġlifts":73313,"Ġbrightly":73314,"ĠApproximately":73315,"ĠsarÃł":73316,"imoto":73317,"rax":73318,"ethoven":73319,"é£İæīĩ":73320,"è§īå¾Ĺå¾Ī":73321,"ClickListener":73322,"Ġসাম":73323,"ĠDOWN":73324,"äºĨä¸Ģä½į":73325,"çĨ¹":73326,"اءة":73327,"åĨįæĬĬ":73328,"åįĬæĻĮ":73329,"æĨ¤":73330,"Ġfreedoms":73331,"bx":73332,"æĹ¶å°±":73333,"дви":73334,"çļ®èĨļ":73335,"ÃĹĊĊ":73336,"âĸ¶":73337,"âĸĵ":73338,"ĠBaum":73339,"Ġinstrumentation":73340,"Ġperpetual":73341,"ĠPAN":73342,"ĠWien":73343,"Ġadecu":73344,"Ġriot":73345,"rero":73346,"Ġremnants":73347,"ĠProtect":73348,"Ġsociedade":73349,"临åºĬä¸Ĭ":73350,"ĠاÙĦØ·ÙģÙĦ":73351,"Ġpans":73352,"çļĦåı¤":73353,"çļĦåħĥç´ł":73354,"лÑıÑİÑĤ":73355,"Ġgoto":73356,"ĠEditors":73357,"ĠDenis":73358,"Ġreacting":73359,"ĠKerry":73360,"women":73361,"ĠTennis":73362,"ä¹ĭå¤ļ":73363,"åĮĸ管çIJĨ":73364,"Ġmarkings":73365,"ãĥ«ãģ®":73366,"Ġdiscriminate":73367,"åĪ»åº¦":73368,"ĠðŁĮ":73369,"ĠÐĿапÑĢимеÑĢ":73370,"Ġbreathed":73371,"gaben":73372,"kary":73373,"stituting":73374,"å°½æĥħ":73375,"ĠNotably":73376,"Ġdams":73377,"çŁ¿ä¸ļ":73378,"æĸ°åĨłçĹħæ¯Ĵ":73379,"ä¸ºå®ľ":73380,"Ġdistract":73381,"ç»ıèIJ¥çļĦ":73382,"кÑĥлÑĮ":73383,"åĬłå¤§å¯¹":73384,"æĪIJå½¢":73385,"rapie":73386,"鼶çĤ¹":73387,"é¤IJæ¡Į":73388,"Assessment":73389,"Ġaligning":73390,"èŁĴ":73391,"é¢łè¦Ĩ":73392,"Ġpamph":73393,"icke":73394,"置身":73395,"Ġsumber":73396,"ĠCNC":73397,"éĥ½åı¯":73398,"ĠRomanian":73399,"æĥ³è±¡çļĦ":73400,"ĠÙĩÙħÛĮÙĨ":73401,"Ġtroubleshooting":73402,"alach":73403,"Ġnotch":73404,"à¸Ńาà¸ģาร":73405,"Ġactivates":73406,"Ġterk":73407,"Ġessent":73408,"Ġbrainstorm":73409,"Ġrépond":73410,"ĠDegrees":73411,"ĠÃĵ":73412,"çģ«çĪĨ":73413,"Ġdivorced":73414,"-government":73415,"åħļç»Ħ书记":73416,"'clock":73417,"@{":73418,"ÃĪ":73419,"ĠкÑĢÑĭ":73420,"ç¡®åĪĩ":73421,"ĠØ´ÙħاÙĦ":73422,"çŁŃè§Ĩé¢ij":73423,"ĠDevelopments":73424,"Ġfurious":73425,"ujÄħce":73426,"èĦijåŃIJéĩĮ":73427,"à±įà°¤":73428,"Ġíĥľ":73429,"ãģ«éĸ¢ãģĻãĤĭ":73430,"Ġsare":73431,"),\\":73432,"åıªçŁ¥éģĵ":73433,"Ġsolute":73434,"Ġhanding":73435,"空æ´ŀ":73436,"ADO":73437,"Ġsplits":73438,"Strateg":73439,"Ġvielen":73440,"ĠExaminer":73441,"MK":73442,"Nat":73443,"[left":73444,"utex":73445,"ĠBess":73446,"omez":73447,"æĪij们ä¸įèĥ½":73448,"embang":73449,"volg":73450,"ĠGesund":73451,"à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī":73452,"红楼梦":73453,"genden":73454,"åѦåłĤ":73455,"æĹłäºĭ":73456,"Ġnosso":73457,"Ġelectronically":73458,"Ġlingering":73459,"ĠBrow":73460,"车åİ¢":73461,"applic":73462,"Ġsomeday":73463,"æIJIJ":73464,"rando":73465,"æī¹æ¬¡":73466,"åĪĺéĤ¦":73467,"Ġszko":73468,"اطÙĤ":73469,"Ġpessim":73470,"ĠHess":73471,"ä½łåıĪ":73472,"缸å°į":73473,"æ®´":73474,"опа":73475,"ĠListing":73476,"æ¸IJè¿Ľ":73477,"twitter":73478,"ĠRabbit":73479,"-functional":73480,"Ġlace":73481,"ért":73482,"éĻįè§£":73483,"æĬĹè®®":73484,"Ġcontexto":73485,"å¾Ģå¾Ģä¼ļ":73486,"è¿Ļæĸ¹éĿ¢çļĦ":73487,"Ġmodulated":73488,"åħ¬åı¸åĴĮ":73489,"inação":73490,"ĠHerb":73491,"Ġdissent":73492,"ança":73493,"Ġsworn":73494,"ç£ĭ":73495,"代表äºĨ":73496,"Ġà¦Ĩà¦Ľà§ĩ":73497,"Actually":73498,"Ġcommend":73499,"useppe":73500,"ASSWORD":73501,"Tre":73502,"æĸŁ":73503,"ä¸īç±»":73504,"ĠпÑĢием":73505,"éĢIJ漸":73506,"orch":73507,"æľīåĩł":73508,"reiben":73509,"Critical":73510,"YX":73511,"ĠExperiences":73512,"ĠвеÑģÑĮ":73513,"åĨ¶éĩij":73514,"ä½łä¸įèĥ½":73515,"é»İæĺİ":73516,"ðŁĮŁðŁĮŁ":73517,"=['":73518,"enance":73519,"çļĦåĬŁæķĪ":73520,"æĺİæĻº":73521,"Ġедин":73522,"AAAAAAAA":73523,"åħĥæ°Ķ":73524,"Annotation":73525,"éĺ¶æ¢¯":73526,"ìĦ¸ìļĶ":73527,"Ġunpublished":73528,")](":73529,"Ġfidelity":73530,"Ġبإ":73531,"ĠZap":73532,"é»Ħå¸Ŀ":73533,"àµįà´°":73534,"Ġmetastases":73535,"Ġpedagogy":73536,"-rank":73537,"zio":73538,"åħ¥çĿ¡":73539,"她è¦ģ":73540,"Ġsurgeries":73541,"åıijçĹħçİĩ":73542,"osas":73543,"åħŃ大":73544,"ĠNeutral":73545,"তার":73546,"ĠMagnus":73547,"Secondary":73548,"ĠÑģлÑĥÑĩаÑıÑħ":73549,"หมà¸Ķ":73550,"Ġniew":73551,"Ġdetachment":73552,"çĹħåı²":73553,"Ġpasture":73554,"Ġhesitated":73555,"}<":73556,"chr":73557,"regist":73558,"à¸ŀวà¸ģ":73559,"ĠاÙĦجز":73560,".\\)":73561,"ĠCec":73562,"身躯":73563,"ĠLeib":73564,"à¸Ķัà¸ĩ":73565,"æĢ¥è¯Ĭ":73566,"è§£åĨ³çļĦ":73567,"éĢıæĺİçļĦ":73568,"Ġcartridge":73569,"Ð¡Ð¡Ðł":73570,"å±±æŀĹ":73571,"borah":73572,"åıĥèĢĥ":73573,"Ġgermination":73574,".Arrays":73575,"è¿Ļå¹ħ":73576,"æ°ĵ":73577,"åħ¨å¿ĥ":73578,"èĢĥé¢ĺ":73579,"å¦ĩç§ij":73580,"Ġmigraine":73581,"ĠRandy":73582,"çĹ¢":73583,"à·Ħ":73584,"ĠANSW":73585,"ĠBrisbane":73586,".ar":73587,"©×Ķ":73588,"æ°´æ³µ":73589,"èħ«":73590,"æ®ĭå¿į":73591,"endregion":73592,"Ġlongtime":73593,"çŁ³å¢¨":73594,"ĠValle":73595,"Ġmurders":73596,"Ġznac":73597,"ĠVaugh":73598,"æĩ¼":73599,"åīªåĪĩ":73600,"/u":73601,"é¦Ļæ°´":73602,"èį¯ç²»çĸĹ":73603,"inally":73604,"ĠBates":73605,"Ġaliens":73606,"Ġpresupp":73607,"Ġgrabbing":73608,"ĠDahl":73609,"Ġdoivent":73610,"auh":73611,"Ġserait":73612,"Convers":73613,"Ġextravag":73614,"Ġdeterministic":73615,"opathic":73616,"isable":73617,"礦":73618,"adoop":73619,".es":73620,"speed":73621,"Ġicy":73622,"ĠFasc":73623,"ĠLiam":73624,"Ġamplit":73625,"Ġelites":73626,"ç»ĻçļĦ":73627,"Ġminimized":73628,"è¡ĽçĶŁ":73629,"vii":73630,"Ġpadd":73631,"æľīæĿ¡":73632,"ÃŃos":73633,"Ġprincipally":73634,"Ġmédia":73635,"Ġconocer":73636,"Ġsummoned":73637,")C":73638,"Ġappla":73639,"Å¡i":73640,"Typography":73641,"â̦..":73642,"à¹ģà¸Ķ":73643,"Ġeinige":73644,"Ġinformatie":73645,"Ġswoje":73646,"Ġatención":73647,"代è¨Ģ":73648,"羣èıĮ":73649,"Ġslider":73650,"ARDS":73651,"Ġlistings":73652,"åĮ»çĸĹåį«çĶŁ":73653,"ĠnumberOf":73654,"Ġأث":73655,"Ġfingert":73656,"(img":73657,"actors":73658,"å¹´åįİ":73659,"ĠMostly":73660,"ాన":73661,"Ġdisparity":73662,"ê´ij":73663,"ĠProsec":73664,"Ùĥار":73665,"å¾·å°Ķ":73666,"Ġpooled":73667,"Ġassigns":73668,"ανδÏģικÏĮ":73669,"பà¯į":73670,"ä¸ĢéĹ´":73671,"/hr":73672,"æĿ¾å¼Ľ":73673,"æļĹèĩª":73674,"æĺİç¡®è§Ħå®ļ":73675,"ÃŃtÄĽ":73676,"ĠBerger":73677,"çŃĶåºĶäºĨ":73678,"ĠDai":73679,"ä½ĵåĴĮ":73680,"è¾¾å°Ķ":73681,"çĶŁæ´»åĴĮ":73682,"åıįåºĶçļĦ":73683,"å§ijå§ij":73684,"éĺ¯":73685,"ĠклаÑģÑģи":73686,"Ġvesicles":73687,"ĠÑįнеÑĢгии":73688,"éĩįé»ŀ":73689,"æĢ¥äºİ":73690,"_part":73691,"Addr":73692,"(sizeof":73693,"eszcze":73694,"çļĦæĪIJ绩":73695,"ĠHLA":73696,"ĠSecrets":73697,"جÙĬÙĦ":73698,"ĠAmph":73699,"âĦĥï¼Į":73700,"Synonyms":73701,"Brian":73702,"æ¯İ":73703,"undert":73704,"å¨Ħ":73705,"Concept":73706,"æĻļæĬ¥":73707,"æģįæĥļ":73708,"pto":73709,"iret":73710,"culas":73711,"åIJįæł¡":73712,"è¯ĦåΤ":73713,"posta":73714,"ĠSemin":73715,"ĠCruise":73716,"ĠCoronavirus":73717,"ĠDollars":73718,"Ġremodeling":73719,"ĠEscherichia":73720,"Ġsuicidal":73721,"å¹¶æĬĬ":73722,"å²Ľå±¿":73723,"Ġdisappears":73724,"Ġprolific":73725,"ç¼ħç͏":73726,"male":73727,"бок":73728,"åĨħ容åĮħæĭ¬":73729,"éĢıäºĨ":73730,"Kar":73731,"Ġawhile":73732,"Ġwhipped":73733,"èĩªå·±åĴĮ":73734,"ĠArbor":73735,"Ġrozp":73736,"ĠвеÑĢÑħ":73737,"ĠÏĢαÏģα":73738,"Ġusability":73739,"ĠExpected":73740,"ÄĤ":73741,"é«ĺãģĦ":73742,"容è²Į":73743,"Ġplantations":73744,"éĤªæģ¶":73745,".â̦ĊĊ":73746,"ardia":73747,"ĠYin":73748,"deen":73749,"æŃ£æ°Ķ":73750,"slow":73751,"rebbero":73752,"facts":73753,"Ġlied":73754,"ä¸īèĢħ":73755,"骸":73756,"ä¸ĩè¾Ĩ":73757,"红åĪ©":73758,"à¸Īึà¸ĩ":73759,"Ġcatastrophe":73760,"Sleep":73761,"Ġkier":73762,"大åŁİå¸Ĥ":73763,"Ġprojecting":73764,"_cost":73765,"éļIJ约":73766,"åĬ±å¿Ĺ":73767,"à¸Ľà¸£à¸°à¹Ĥย":73768,"ĠGrat":73769,"ä¹ŁåIJĮæł·":73770,"Ġerro":73771,"å¼ķåĩº":73772,"åĢŁæŃ¤":73773,"Ġprincipals":73774,"opausal":73775,"å°Ĩ该":73776,"çļ®ä¸ĭ":73777,"é±¼çļĦ":73778,"ĠاÙĦبØŃر":73779,"declare":73780,"?\\":73781,"ä¸ī项":73782,"æĸ¯å¤§":73783,"INGTON":73784,"ì¶Ķ":73785,"odied":73786,"主åŃIJ":73787,"Ġemanc":73788,"æĽ´åĬłçļĦ":73789,"ë§Ŀ":73790,"ĠRoutes":73791,"èģĮèĥ½éĥ¨éŨ":73792,"hk":73793,"omination":73794,"ptides":73795,"åĬłå¼·":73796,"æ½ľèĥ½":73797,"æī«çłģ":73798,"ĠHEALTH":73799,"Ġpituitary":73800,"ĠBax":73801,"à¸Ĺัà¹Īว":73802,"ĠGli":73803,"Ġmez":73804,"ä½łå·²ç»ı":73805,"è¿ĺ说":73806,"离线":73807,"Ġconcave":73808,"éĽªå±±":73809,"ĠÑĤеÑĢа":73810,"Ġ×¢×":73811,"ĠVER":73812,"两ä¼ļ":73813,"Ġجا":73814,"ĠExecution":73815,"çĹĽèĭ¦çļĦ":73816,"çĭłçĭłçļĦ":73817,"gov":73818,"Ġsidewalk":73819,"Ġtaxonomy":73820,"ĠDerby":73821,"Ġconosc":73822,"ï¼ģï¼ģĊĊ":73823,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":73824,"elic":73825,"åľ°åĴĮ":73826,"æĶ¾çļĦ":73827,"Ġrevital":73828,"ĠнаÑĩала":73829,"éħµæ¯į":73830,"ĠPU":73831,"çİĭå®¶":73832,"ÏĮÏĦη":73833,"ãĤıãĤĮãĤĭ":73834,"åĢĨ":73835,"æīĵä»Ĺ":73836,"Ġcultivating":73837,"è³ĩæł¼":73838,"ĠOBJECT":73839,"Ġlumber":73840,"ĠEsk":73841,"metics":73842,"uestas":73843,"æ½ľä¼ı":73844,"Ġgossip":73845,"ĠWizard":73846,"Ġimpactful":73847,"åĨ·æ±Ĺ":73848,"âng":73849,"ÐŁÑĢе":73850,"ĠBusinesses":73851,"ĠSensing":73852,"hets":73853,"Ġreins":73854,"Ġenvy":73855,"æ¸ħé¦Ļ":73856,"ORN":73857,"Ġbusinessman":73858,"à¯ģà®Ł":73859,".ui":73860,"çļĦä¿ĿæĬ¤":73861,"obra":73862,"جاÙĩ":73863,"Arc":73864,"Ġможе":73865,"اØŃØ«":73866,"Ġbuildup":73867,"andung":73868,"plays":73869,"Ġshuff":73870,"ĠвоÑĤ":73871,"ittal":73872,"èĨł":73873,"åģľé¡¿":73874,"ĠÑĤаким":73875,"wx":73876,"Ľ×Ķ":73877,"ä¸ĢæīĢ":73878,"коле":73879,"chein":73880,"æĥ³èµ·æĿ¥":73881,"Ġخطر":73882,"æĭĸæĭī":73883,"ĠÑģлÑĥж":73884,"Ġmateri":73885,"ĠìĻĦ":73886,"\"=>":73887,"ĠFX":73888,"ä½İä»·":73889,"typeof":73890,"è¶ĬæĿ¥è¶Ĭ大":73891,"ãĤ³ãĥ³ãĥ":73892,"è£½ä½ľ":73893,"ĠÐŁÑĢиÑģÑĤÑĥпÑĻено":73894,"_format":73895,"fet":73896,"çļĦ她":73897,"ÑĪли":73898,"æľĽåIJij":73899,"纹çIJĨ":73900,"\\User":73901,"Ġдогов":73902,"Ġanimations":73903,"Ġfunctionalities":73904,"Ii":73905,"æĿ¥äºº":73906,"ĠChr":73907,"ĠShane":73908,"éĸĴ":73909,"={(":73910,"-Ass":73911,"Ġfonts":73912,"-ra":73913,"CK":73914,"]ãĢĤĊĊ":73915,"çĶŁåĩº":73916,"ÙĪØ±Ø´":73917,"Ġachievable":73918,"å±ĬæĹ¶":73919,"oof":73920,"èĥ½ç͍":73921,"è¡Įä¹ĭ":73922,"wee":73923,"æį®ç»Łè®¡":73924,"ĠعÙĦÛĮ":73925,"porate":73926,"Ġensl":73927,"æĺ¯åIJ§":73928,"æĺ¯åįģåĪĨ":73929,"å½İ":73930,"Ġcondens":73931,"ĠÙĤاÙĨÙĪÙĨ":73932,"ederbörd":73933,"Sand":73934,"]][":73935,"stelling":73936,"ä¸İä¼ģä¸ļ":73937,"ĠоказÑĭва":73938,"åĿļ飧":73939,"Ġsegreg":73940,"å²Ľä¸Ĭ":73941,"éĮ¯èª¤":73942,"Ġparticiple":73943,"à´ª":73944,"rö":73945,"Ġoblast":73946,"ØŃÙĬØ©":73947,"á»ķ":73948,"ĠпÑĢедÑģÑĤавлÑıеÑĤ":73949,"Alexander":73950,"ĠNorge":73951,"æīĵ磨":73952,"ĠLandes":73953,"Ġnev":73954,"ĠOPT":73955,"-server":73956,"uffix":73957,"Enjoy":73958,"ä¸Ŀ毫ä¸į":73959,"åįģäºĮ竳":73960,"-West":73961,"æ¡ĤèĬ±":73962,":',":73963,"bj":73964,"Ġcomún":73965,"æĸ°ä¸Ģè½®":73966,"ĠCompletion":73967,"eyn":73968,"Ġਹ":73969,"'+":73970,"ĠASE":73971,"ĠLut":73972,"Ġarranging":73973,"èģĶç³»çļĦ":73974,"Ġ׾×Ķ×Ļ×ķת":73975,"Ġparsley":73976,"Ġstenosis":73977,"_amount":73978,"æķĻä½ł":73979,"è§īæĤŁ":73980,"ĠÐľÑĭ":73981,"é¦Ĵ头":73982,"usic":73983,"è¿Ļ个åIJįåŃĹ":73984,"éĺ¿æł¹":73985,"Ġihnen":73986,"Ġতারিà¦ĸ":73987,"annt":73988,"æĺ¯ä»Ģ麼":73989,"жно":73990,"Ġ×ijש":73991,"à¹Ģà¸Ľà¸´à¸Ķ":73992,"âĹıĊĊ":73993,"[max":73994,"ĠBali":73995,"å¹´åĴĮ":73996,"èĢģçΏ":73997,"Ġmeticulously":73998,"Ġgrease":73999,"ĠScales":74000,"äºĭæĥħçļĦ":74001,"ĠÑģоÑģÑĤоÑıние":74002,"ĠNorte":74003,"ĉĉĠĠĠ":74004,"ĠÙģÛĮ":74005,"æij©å°Ķ":74006,"Ġguardians":74007,"/go":74008,"/Comment":74009,"Ye":74010,"igate":74011,"åı¯èĥ½å¯¼èĩ´":74012,"Ġlesbian":74013,"åĵĪä½Ľ":74014,"Ġcriticisms":74015,"çī¢è®°ä½¿åij½":74016,"ĠÙħردÙħ":74017,"tails":74018,"Ġtudo":74019,"ĠMuss":74020,"ĠInhib":74021,"æĿ¡çļĦè§Ħå®ļ":74022,"ĠدÙĦÛĮÙĦ":74023,"ĠDrum":74024,"ĠScriptures":74025,"çĹīæĮĽ":74026,"ĠCrop":74027,"åѰ":74028,"å°ijå¹´çļĦ":74029,"িà¦Ń":74030,"ISPR":74031,".Point":74032,"ĠpodrÃŃa":74033,"å¼±çĤ¹":74034,"ĠлиÑĨ":74035,"Ġplanners":74036,"Ġputative":74037,"apiro":74038,"cipitation":74039,"Ġkde":74040,"ulaire":74041,"áŀĵ":74042,"WORK":74043,"{[":74044,"Ġiç":74045,"åĿĤ":74046,"Ġসà§ĩ":74047,"ĠØ®ÙĦ":74048,"UGH":74049,"Ġhesitation":74050,"Ljava":74051,"è¦ģèµ°":74052,"Ġrak":74053,"Ġgrâce":74054,"à¤ķà¥ĭ":74055,"ĠФи":74056,"Ö¸Ö¼":74057,"ĠExpressions":74058,"ĠÐŀÑģнов":74059,"atitis":74060,"ĠGad":74061,"å¤Ħéķ¿":74062,"请æĤ¨":74063,"ĠPresence":74064,"éĢŁåº¦å¿«":74065,"Ġpolicing":74066,"Ignore":74067,"转è¿ĩ身":74068,"é¡«":74069,"Ġindifference":74070,"éķ·æľŁ":74071,"å®£ä¼łæķĻèĤ²":74072,"fass":74073,"ĠFiscal":74074,"Ġhera":74075,"ĠNiem":74076,"ä¼ļæĽ´":74077,"ĠZahl":74078,"è¾ĵäºĨ":74079,"缮åīį为æŃ¢":74080,"çķ¶åĪĿ":74081,"ĠинÑģÑĤиÑĤÑĥ":74082,"Ġíļ¨":74083,"Ġcrap":74084,"ĠUnve":74085,"æŀ¶ä¸Ĭ":74086,"ĠObserver":74087,"Ġnotwithstanding":74088,"ĠIni":74089,"áticos":74090,"åĬ¡å·¥":74091,"atoria":74092,"ĠWillis":74093,"Ġasymmetry":74094,"lord":74095,"æľīéĴĪ对":74096,"Ġprinters":74097,"shots":74098,"ĠRESP":74099,"Ġjov":74100,"é¢ĵ":74101,"Ġzde":74102,"Ġflashing":74103,"主é¢ĺæķĻèĤ²":74104,"pak":74105,"èĩªç«ĭ":74106,"äºĶ彩":74107,"JR":74108,"uding":74109,"ä½łéĥ½":74110,"åĨĻæ³ķ":74111,"Anti":74112,"Ġresentment":74113,"udder":74114,"Õ·":74115,"elim":74116,"ĠÂ¥":74117,"ukaan":74118,"Æ¡n":74119,"Ġantennas":74120,"×ķפף":74121,"ĠFerrari":74122,"åĪĩå¼Ģ":74123,"ĠRobotics":74124,"Ġtheorists":74125,"Ġseekers":74126,"Ġtasked":74127,"æīŃ头":74128,"Ġmonumental":74129,"ĠHole":74130,"æĪij被":74131,"åĪĨæµģ":74132,"æµ·ä¸Ń":74133,"ĠCSV":74134,"MenuItem":74135,"frequency":74136,"spects":74137,"ĠArrow":74138,"Ġpaso":74139,"infection":74140,"Professional":74141,"Ġgdzie":74142,"owatt":74143,"resist":74144,"ãĥļãĥ¼ãĤ¸":74145,"yet":74146,"teger":74147,"Ġinsomnia":74148,"Ġporosity":74149,"å®ģæĦ¿":74150,"Ġ×ij×Ļת":74151,"-black":74152,"Ġtraitement":74153,"Better":74154,"为ä¸Ģä½ĵ":74155,"ç§½":74156,"ipart":74157,"Ġabuses":74158,"çī¹åĮº":74159,"Ġpleasures":74160,"æĸ°æĿIJæĸĻ":74161,"çϽçĻľ":74162,"autre":74163,"édias":74164,"ĠCly":74165,"ä¸ĭåĽ¾":74166,"渾":74167,"ä¿¡èµĸ":74168,"Ġpsychosocial":74169,"ĠMobi":74170,"ç¥Ĥ":74171,"=\"\">Ċ":74172,"ĠProve":74173,"åĸª":74174,"åij½åIJį为":74175,"éħįä¸Ĭ":74176,"arbij":74177,"à¹Ģลà¹ĩà¸ģ":74178,"Clean":74179,"Applications":74180,"Agg":74181,"Ġtrough":74182,"ĠNun":74183,"å°±åľ°":74184,"Ġpreserves":74185,"Ġindividualized":74186,"à«įર":74187,"ĠRevelation":74188,"xtap":74189,"ĠYuk":74190,"çĤ¹åΰ":74191,"Ġimportancia":74192,"Ġstati":74193,"讲å¸Ī":74194,"设置äºĨ":74195,"ĠLaboratories":74196,"UU":74197,"chemy":74198,"×Ļ×£":74199,"ĠLeu":74200,"积水":74201,"ĠسÛĮستÙħ":74202,"Ġscuola":74203,"æĺ¯ä½ķ":74204,"রি":74205,"Ġpatio":74206,"åķĨæĪ·":74207,"á»ı":74208,"ĠGuides":74209,"ĠRemoval":74210,"ä¾įåį«":74211,"Õ©":74212,"æľĢ主è¦ģ":74213,"ĠConv":74214,"Philipp":74215,"æĢĴåIJ¼":74216,"music":74217,"åĴĮæĶ¿æ²»":74218,"Ġrespuesta":74219,"Ġimpending":74220,"è¶Ĭå°ı":74221,"ophobia":74222,"ĠмногиÑħ":74223,".As":74224,"entlich":74225,"åĽ½æĹĹ":74226,"èces":74227,"详è§ģ":74228,"æĺ¯ä¸įåı¯èĥ½":74229,"åħ±äº§åħļçļĦ":74230,"Ġtweets":74231,"caption":74232,"ĠsÄĥ":74233,"ĠNä":74234,"èĩŁ":74235,"å°ıç¨ĭåºı":74236,"æİĴæ³Ħ":74237,"æĥĬåı¹":74238,"ĠAbe":74239,"èĩªå¦Ĥ":74240,"Ġairflow":74241,"ĠMacbeth":74242,"åł¡åŀĴ":74243,"Ġgaseous":74244,"ĠYong":74245,"ä¸ĢçĤ¹éĥ½ä¸į":74246,"ĠÄijó":74247,"bigg":74248,"Ġmobilization":74249,"ĠíĥĢ":74250,"ousseau":74251,"ä¹łè¿ijå¹³æĸ°æĹ¶ä»£ä¸ŃåĽ½çī¹èī²ç¤¾ä¼ļ主ä¹īæĢĿæĥ³":74252,"Turkish":74253,"大å®Ĺ":74254,"ä¸ĵ项æķ´æ²»":74255,"à²Ĥದ":74256,"Ġlaquelle":74257,"Ġorderly":74258,"thening":74259,"Ġproblème":74260,"ĠSell":74261,"ĠWoj":74262,"ĠAnc":74263,"åĽĽä½į":74264,"ç®Ĺåĩº":74265,"æij¹":74266,"æĭ·è´Ŀ":74267,"Ġconcret":74268,"çĪ»":74269,"æŀģåĬĽ":74270,"ÏģοÏħ":74271,"Ġà¦ħনà§ģ":74272,"ĠProteins":74273,"Eu":74274,"ĠAo":74275,"绯":74276,"ĠÑģеÑĤ":74277,"bted":74278,"ĠÐĺÑģп":74279,"ĠÙĦÙĦت":74280,"ãĥ»ãĥ»ãĥ»":74281,"à¹ģà¸Ļว":74282,"integration":74283,"Ġherm":74284,"èħĭ":74285,"æĭīåĬ¨":74286,"ðŁļ":74287,"olls":74288,"ĠgetAll":74289,"æĬ¥åΰ":74290,"ĠXen":74291,"éĺ²èħIJ":74292,"Ġélect":74293,"Contrib":74294,"賺":74295,"åIJīå°Ķ":74296,"åŁºç¡Ģ设æĸ½å»ºè®¾":74297,"ĠÑģкоÑĢоÑģÑĤÑĮ":74298,"Ġnossa":74299,"Ġpropre":74300,"ecer":74301,"CPI":74302,"ulièrement":74303,"committee":74304,"Ġcampuses":74305,"ĠpÅĻÃŃpad":74306,"\"Oh":74307,"νÏī":74308,"ĠÐĵе":74309,"ĠакÑĤивно":74310,"ĠLancaster":74311,"-workers":74312,"jana":74313,"çļĦæľĢé«ĺ":74314,"лка":74315,"Ġ׾ש":74316,".degree":74317,"åĨįä¹Łæ²¡æľī":74318,"ând":74319,"ĠÑģÑĤаÑĤиÑģÑĤи":74320,"Ġdriveway":74321,"诧å¼Ĥ":74322,"ónica":74323,"åįģäºĮæľĪ":74324,"ĠÙħصر":74325,"Ġpequ":74326,"æĹłåģ¿":74327,"ÄĽst":74328,"unctional":74329,"userId":74330,"detail":74331,"Ġparasitic":74332,"ĠWolfgang":74333,"ĠпокÑĥ":74334,"ĠFlora":74335,"Ľ×ĸ":74336,"WG":74337,"äºŁ":74338,"Ġora":74339,"ä¹Łæĺ¯å¾Ī":74340,".'Ċ":74341,"Ġnég":74342,"legt":74343,"Ġ×ľ×ª":74344,"å¥ĩå¦Ļ":74345,"ĠGoodman":74346,"owler":74347,"平移":74348,"æİĪæ¥Ń":74349,"è´¢åĬ¡ç®¡çIJĨ":74350,"Ø·ÙĦÙĤ":74351,"ĠBiomedical":74352,"ĠAzerbaijan":74353,"Nic":74354,"è¿Ļåĩłå¹´":74355,"clic":74356,"жноÑģÑĤи":74357,"伤å¯Ĵ":74358,"æĦŁè§īèĩªå·±":74359,"Äĥng":74360,"çĶŁçĮª":74361,"Ġspre":74362,"é¢ĺ为":74363,"èIJ½åħ¥":74364,"ĠоÑĢиги":74365,"ĠMUST":74366,"ĠGou":74367,"enerated":74368,"STER":74369,"Ġspecializes":74370,"_first":74371,"æ»ijéĽª":74372,"ucci":74373,"mine":74374,"Ġwol":74375,"aday":74376,"Ġhandbook":74377,"大å¤ļæķ°äºº":74378,"ĠBolivia":74379,"çļĦåIJ§":74380,"ĠTWO":74381,"æĪijæľĥ":74382,"æĹłå¸¸":74383,"ãģıãĤĭ":74384,"ĠUseful":74385,"Õ¥Õ´":74386,"Ġsystolic":74387,"ëĥ":74388,"ĠÆ":74389,"igrant":74390,"åĽŀå®¶çļĦ":74391,"Ġsimplement":74392,"à¦ķল":74393,"ä½Ľå±±":74394,"ĠMatth":74395,"æ£Ģå¯Łæľºåħ³":74396,"ĠاطÙĦاعات":74397,"_th":74398,"Ġciel":74399,"Ġnama":74400,"æĪijå¿ĥ":74401,"azes":74402,"çĭĻ":74403,"è¿ľäºĨ":74404,"ĠPolym":74405,"DataSource":74406,"Ġپرد":74407,"Ġ×Ĺ×ĵ":74408,"ĠBST":74409,"Ġjeder":74410,"å¸ĥæĭī":74411,"çļĦåİ»":74412,"composition":74413,"èĭŀ":74414,"ãĢĭï¼ļâĢľ":74415,"tg":74416,"èĢģ天":74417,"ĠValueError":74418,"Ġcukup":74419,"Ġreel":74420,"unken":74421,"ĠKah":74422,"管çIJĨå±Ĥ":74423,"ĠÐŁÑĢ":74424,"Ġcuales":74425,"éĺŁåijĺ们":74426,"Ġaplik":74427,"ivol":74428,"åĶł":74429,"åī¯éĥ¨éķ¿":74430,"ูà¸Ļ":74431,"ĠHammer":74432,":]":74433,"Ġsund":74434,"çŁ¥è§ī":74435,"ä¸ĩä¸ĩ":74436,"æķħ宫":74437,"ÑģÑĤиÑĤÑĮ":74438,"Ġ×ľ×ª×":74439,"ĠاÙĦتÙĤ":74440,"åĮ¿åIJį":74441,"Texas":74442,"TX":74443,"Ġpů":74444,"اÙĦÙĤ":74445,"çŁŃ线":74446,"ĠباÙĦØ¥":74447,"itatea":74448,"Maria":74449,"çļĦè¯Ħä»·":74450,"emt":74451,"æĪij好":74452,"Ġmyc":74453,"ĠبÙħا":74454,"Ġfunnel":74455,"åĻľ":74456,"éĿŀéģĹ":74457,"åįĥåı¤":74458,"ĠAlready":74459,"å·¥ç¨ĭåѦéĻ¢":74460,"åī¯å¸Ĥéķ¿":74461,"ĠÙĪØ§ÙĦÙĨ":74462,"èµŀæī¬":74463,"ĠÑģло":74464,"attie":74465,"Ġdesignate":74466,"å¯ĨéĴ¥":74467,"èϽçĦ¶åľ¨":74468,"ç§ijæĬĢæĪIJæŀľ":74469,"Ġaltura":74470,"ར":74471,"Ġceramics":74472,"Obviously":74473,"iÅĤ":74474,"ĠðĿĴ":74475,"è®ļ":74476,"ĠÑģилÑĭ":74477,"ĠÑįлеменÑĤа":74478,"Ġпои":74479,"Ġprecursors":74480,"glise":74481,"ĠSurf":74482,"uddle":74483,"äººä¸ºæľ¬":74484,"Ġtion":74485,"ĠLAB":74486,"landers":74487,"çľ¼è§Ĵ":74488,"ucking":74489,".hash":74490,"Ġש׾×IJ":74491,"ÑĤÑĥÑĢÑĥ":74492,"æĬ¥åijĬä¸Ń":74493,"ÑĤивнÑĭÑħ":74494,"ниÑĨипа":74495,"íĶĮ":74496,"æĿĥåĬĽçļĦ":74497,"Ultimately":74498,"ç§ijåѦåıijå±ķè§Ĥ":74499,"ĠÄĩ":74500,"Ġdeity":74501,"ÙĪÙĬÙĥ":74502,"Ġhackers":74503,"ĠÑĢаÑģÑĤениÑı":74504,"æĪijä¸İ":74505,"对è§Ĵ":74506,"Ġsuburbs":74507,"ĠجسÙħ":74508,"æĮĩ导æĢĿæĥ³":74509,"Ġpolarized":74510,"Ġضد":74511,"ĠNaturally":74512,"åĮ»åĬ¡äººåijĺ":74513,"ÑĤого":74514,"主页":74515,"åĽºæľī":74516,"âĸij":74517,"Ġayuda":74518,"lesia":74519,"åıijå¸ĥæĹ¥æľŁ":74520,"ĠIhre":74521,"fighters":74522,"_api":74523,"ĠDON":74524,".Services":74525,"Chemical":74526,"ĠFot":74527,"Ġinterruption":74528,"кин":74529,"Worksheets":74530,"members":74531,"Ġcones":74532,"Ġاثر":74533,"åĪĨéļĶ":74534,"лакÑĤи":74535,"Ġillustrative":74536,"Ġquotid":74537,"åıijæĶ¹å§Ķ":74538,"zp":74539,"izziness":74540,"Ġprzyk":74541,"jut":74542,"ĠDrain":74543,"Ġnota":74544,"ĠStick":74545,"Ġ¬":74546,"Chief":74547,"Ġindebted":74548,"ĠÐĺÑģÑĤо":74549,"MH":74550,"daughter":74551,"æ¿ķ":74552,"ĠСШÐIJ":74553,"ียà¸ļ":74554,"ç»ķç»Ħ":74555,"Ġultrasonic":74556,"ént":74557,"ÙĪØŃ":74558,"ĠLands":74559,"Ġbenchmarks":74560,"'inter":74561,"ikai":74562,"ews":74563,"ĠAfrika":74564,"èĤīçľ¼":74565,"Ġpinpoint":74566,"Nevertheless":74567,"Kas":74568,"ĠCao":74569,"Ġwhichever":74570,"ptive":74571,"Ġspac":74572,"Ġsimulator":74573,"ĠDeborah":74574,"Ġbestimm":74575,"åľĨå¿ĥ":74576,"ĠEthn":74577,"ĠобоÑĢÑĥд":74578,"åĽ½å®¶æłĩåĩĨ":74579,"ĠStrange":74580,"ölker":74581,"è¾½å®ģçľģ":74582,"æĸ°åįİ社":74583,".twitter":74584,".exp":74585,"little":74586,"Ġbaj":74587,"ĠBalk":74588,"Ġdiber":74589,"Ġsixteenth":74590,">()":74591,"ÃŃculos":74592,"ĠÙħÙĦÙĬ":74593,"ARP":74594,"é»Ħèī²çļĦ":74595,"ĠLIKE":74596,"Ġসালà§ĩ":74597,"ĠZam":74598,"åĨįå°Ĩ":74599,"æ¿Ĵ":74600,"ĠÚ¯ÛĮ":74601,"ĠVisitors":74602,"ĠEgyptians":74603,"Ġsviluppo":74604,"é«ĺæ¡£":74605,"Ġmarketers":74606,"Ġconducts":74607,"ĠпÑĢоизводÑģÑĤва":74608,"ĠмеÑĢопÑĢиÑı":74609,"åĪ°ä½ł":74610,"ĠChung":74611,"å®ŀå¤Ħ":74612,"Ġdiscord":74613,"trzym":74614,"é»ĺé»ĺåľ°":74615,"rvats":74616,"ĠPretty":74617,"wagen":74618,"è¿ĺä¸įèĥ½":74619,"åħĪåİ»":74620,"Ġнаде":74621,"Ġdepiction":74622,"转账":74623,"ĠManuscript":74624,"Activities":74625,"ĠSommer":74626,"Ġpalabras":74627,"ĠCOURT":74628,"Cette":74629,"ĠBerm":74630,"ĠDru":74631,"æ²¹èıľ":74632,"better":74633,"Ġcomeback":74634,"ĠKick":74635,"交ç»ĩ":74636,"éĽĨä¸ŃçļĦ":74637,"Ġexecutes":74638,"Ġimpairments":74639,"Ġveggies":74640,"against":74641,"ẳng":74642,"åIJĮçIJĨ":74643,"iedades":74644,"åĽŀé¦ĸ":74645,"Ġconstipation":74646,"Ġmonol":74647,"ĠWilliamson":74648,"ãģ§ãģĻãģŃ":74649,"ä½ĵçݰåĩº":74650,"ãģķãĤīãģ«":74651,"ilor":74652,"ĠThin":74653,"åħīäºĨ":74654,"Ġhomogen":74655,"ĠBritt":74656,"çļĦç¥ŀæĥħ":74657,"ç®ĢçĽ´å°±æĺ¯":74658,"Ġbids":74659,"ĠWitch":74660,"ĠUCLA":74661,"Ġbuddy":74662,"áĥĺáĥľ":74663,"ĠDreams":74664,"æĭĽåķĨå¼ķèµĦ":74665,"Culture":74666,"Ġ****************************************************************":74667,"comput":74668,"éħįç͵":74669,"ĠJuni":74670,"Ġdoctrines":74671,"Ġdehydrogen":74672,"avat":74673,"éĥ½æ²Ĵ":74674,"(\"[":74675,"æĶ¶äºĨ":74676,"aucoma":74677,"_DATA":74678,"ĠLutheran":74679,"ĠNietzsche":74680,"-aff":74681,"Ġcontours":74682,"Ġcrear":74683,"áĥIJáĥł":74684,"Ġstereo":74685,"ÙĤÙĬÙĤØ©":74686,"ĠKrak":74687,"Ġhaber":74688,"æĢĢæĬ±":74689,"mooth":74690,"England":74691,"×Ļ׾×Ļ×Ŀ":74692,"åĴĨåĵ®":74693,".Key":74694,"çļĦ温度":74695,"ĠDIV":74696,"LOAT":74697,"ĠlÃŃnea":74698,"etra":74699,"æĺ¯ä»ĸ们":74700,"ĠOv":74701,"ä¸Ĭå²Ĺ":74702,"ĠInstructor":74703,"åĽ¾çĶ»":74704,"à¸Ĭà¸Ļิà¸Ķ":74705,"ĠгÑĢаждан":74706,"è¿ĩéĩı":74707,"å¿ĥäºĨ":74708,"ĠBeh":74709,"players":74710,"Ġmaison":74711,"ë§IJ":74712,"anch":74713,"ĠEigen":74714,"Ġtrader":74715,"Ġбол":74716,"éĻªçĿĢ":74717,"Ġnave":74718,"raum":74719,"ä¹Łæĺ¯åľ¨":74720,"Resolver":74721,"ĠCurve":74722,"éĿ¢ç§¯ä¸º":74723,"éĥ½ä¼ļæľī":74724,"ìŀIJìĿĺ":74725,"ிà®ķà¯įà®ķ":74726,"(it":74727,"ĠWerk":74728,"ignement":74729,"å¿ĥ室":74730,"æĥ³ä¸Ģæĥ³":74731,"/sub":74732,"Ġcalming":74733,"æľĢåIJİä¸Ģ次":74734,"åĺ´ä¸Ĭ":74735,"TPL":74736,"Ġbibliography":74737,"ĠHermann":74738,"ãĤĦãģĻãģĦ":74739,"Ġpä":74740,"çļĦæİªæĸ½":74741,"缸è¾ĥ":74742,"ÂłĠÂł":74743,"è¯Ńå¢ĥ":74744,"workers":74745,"ĠDoctors":74746,"Ġutilise":74747,"Ġদিন":74748,"èĬĻèĵī":74749,".swift":74750,"éĤ£èά":74751,"Ġchars":74752,"èĮ§":74753,"даÑĩа":74754,"ĠÐĴоз":74755,"HSV":74756,"Ġжидко":74757,"ĠMaharashtra":74758,"ĠÑĦилÑĮ":74759,"lá":74760,"Ġunaffected":74761,"åı¯ä¸º":74762,"çī©ä»¶":74763,"åıªè§īå¾Ĺ":74764,"ĠGrab":74765,"åĨ°åĨ°":74766,"ĠTrevor":74767,"Ġsoybean":74768,"_;Ċ":74769,"fielder":74770,"ĠBIG":74771,"ä½įå±ħ":74772,"æľĿèijĹ":74773,"æ²īéĻį":74774,"Ġtackles":74775,"Ġpermissible":74776,"å¦Ĥæŀľä»ĸ":74777,"-how":74778,"ĠмиÑĢ":74779,"æĪijçŃī":74780,"对åĩĨ":74781,"dead":74782,"æ¸ħæī«":74783,"ĠMacro":74784,"ĠGoldman":74785,"èµĮåįļ":74786,"ĠPainting":74787,"Ġadorned":74788,"Moving":74789,"hog":74790,"çļĦçĹĩçĬ¶":74791,"Ġprudent":74792,"ĠSusp":74793,"姥姥":74794,"以ä¸ĭåĩłä¸ªæĸ¹éĿ¢":74795,"Ġtedious":74796,"ĠTrop":74797,"ä¸Ģè´¯":74798,"ifie":74799,"вла":74800,"Ġreload":74801,"ĠJeremiah":74802,"Gas":74803,"ĠBJ":74804,"Ġstrides":74805,"ãĢĤãĢĤãĢĤãĢĤ":74806,"ĠDickens":74807,"以å¾ħ":74808,"Ġamusing":74809,"Ġserene":74810,"æŃ¤æ¬¡æ´»åĬ¨":74811,"FN":74812,"ĠMEN":74813,"ukun":74814,"ĠMarathon":74815,"ç§ģä¸ĭ":74816,"Ġlangue":74817,"zÄħt":74818,"pell":74819,"ĠEarn":74820,"èĢĮå¾Ĺ":74821,"ваний":74822,"客æłĪ":74823,"Ġburnout":74824,"Ġjuices":74825,"èĪŀåı°ä¸Ĭ":74826,"оÑĢÑĥ":74827,"Ġcompeted":74828,"èīºæľ¯åĵģ":74829,"çģŃ亡":74830,"(Long":74831,"-mentioned":74832,"Ġacom":74833,"Ġcontests":74834,"Ġcarga":74835,"uitable":74836,"similar":74837,"纲é¢Ĩ":74838,"丫鬣":74839,"Ġderecho":74840,"Iz":74841,"amino":74842,"Ġfilming":74843,"Ġpeninsula":74844,"ĠVictory":74845,"(app":74846,"onson":74847,"Ġwidened":74848,"ĠInvesting":74849,"à¸ģวà¹Īาà¸":74850,"æ¡ĪåŃIJ":74851,"skich":74852,"æ§ĭæĪIJ":74853,"Ġì¹´":74854,"Ġquarantine":74855,"Ġthrott":74856,"ulkan":74857,"Ġillicit":74858,"={`":74859,"ĠSTD":74860,"ายุ":74861,"驱éĢIJ":74862,"Ġoverlooking":74863,"hidupan":74864,"QB":74865,"pang":74866,"æ¸ħåģ¿":74867,"åıijå±ķè¶ĭåĬ¿":74868,"ĠPercy":74869,"ç´§åĩij":74870,"éĿ¢å¯¹éĿ¢":74871,"ĠSensors":74872,"(|":74873,")==":74874,"å½ĵäºĨ":74875,"便æ°ij":74876,"åľŁæľ¨":74877,".page":74878,"èĿł":74879,"-ever":74880,"Aqu":74881,"ultz":74882,"-Mar":74883,"itaria":74884,"æĻºèĥ½æīĭæľº":74885,"ĠObservation":74886,"Ġним":74887,"Ġexploiting":74888,"Ġbureaucracy":74889,"Cole":74890,"xsl":74891,"大åĶIJ":74892,"è¿ĻåĽŀ":74893,"Ġattachments":74894,"#{":74895,":layout":74896,"Ġgcd":74897,"Ġwhist":74898,"ĠClaus":74899,"Ġbrewing":74900,"IJת":74901,"({\\":74902,"ĠGore":74903,"à¤ı":74904,"ĠíĨł":74905,"Ġvoiced":74906,"çijļ":74907,"çº¸å¼ł":74908,"Ġosteoporosis":74909,"ĠRak":74910,"æ·±æĢĿ":74911,"æĹ©æľŁçļĦ":74912,"ĠвÑĭбоÑĢ":74913,"追éļı":74914,"糯米":74915,"Mutable":74916,"ĠÑģÑĢок":74917,"Ġsubtypes":74918,"ĠConven":74919,"çĦ¡æķ¸":74920,"-author":74921,"ĠABOUT":74922,"DEF":74923,"iram":74924,"tgn":74925,"ĠÑĢаза":74926,"Ñģад":74927,"éĺ¿éĩĮå·´å·´":74928,":H":74929,"chrom":74930,"äºĨä¸Ģéģĵ":74931,"æĺ¯ä¸Ģ種":74932,"ĠÑįÑĤÑĥ":74933,"ç§ĭåĨ¬":74934,"=false":74935,"ĠcDNA":74936,"ĠMadd":74937,"ä¸Ĭæīĭ":74938,"éĿ¢ä¸ĬçļĦ":74939,"heden":74940,"ĠPURPOSE":74941,"Ġcie":74942,"ий":74943,"åĹ·":74944,"Ġspindle":74945,"}ĊĊĊĊ":74946,"ponential":74947,"Ġgeared":74948,"Ġmagnets":74949,"åİĤéķ¿":74950,"æ±łå¡ĺ":74951,"Ġcardiomy":74952,"Ġvampire":74953,"ĠCrew":74954,"urz":74955,"为äºĨéģ¿åħį":74956,"husus":74957,"åĤ¬ä¿ĥ":74958,"åıĹ害èĢħ":74959,"-ret":74960,"\\.ĊĊ":74961,"ä¼łéĹ»":74962,"iscopal":74963,"оÑı":74964,"ä¸įèĩ³äºİ":74965,"ĠвклÑİ":74966,"Ġpolyp":74967,"ulsions":74968,"åľ¨è¿Ļæĸ¹éĿ¢":74969,"ĠоÑĢганизма":74970,"Ġzdrav":74971,"Ġenseñ":74972,"両":74973,"resources":74974,"æľīå¾ħ":74975,"æ¯Ķä½ł":74976,"äºĨä¸Ģèµ·":74977,"é¦ĸ缸":74978,".Type":74979,"渤":74980,"ĠÏĢληθ":74981,"Ġconnectors":74982,"grace":74983,"Ġmk":74984,"çļĦ模å¼ı":74985,"Ġquatro":74986,"ryan":74987,"×Ļ×Ļ×Ķ":74988,"åĬŁåĬ³":74989,"unners":74990,"诸ä½į":74991,"Ġisotopes":74992,"ĠTomas":74993,"oside":74994,"apar":74995,"ä¸ŃåĽ½ç»ıæµİ":74996,"Ġdépart":74997,"Ġmidpoint":74998,"-vers":74999,"Ġdó":75000,"Ġreyn":75001,"è°į":75002,"çī¹åľ°":75003,"ĠByron":75004,"åķĻæİĪ":75005,"ĠCleaning":75006,"[string":75007,"Ġkins":75008,"åĬ¨èį¡":75009,"лÑĮнÑĥÑİ":75010,"ĠAbel":75011,"å¦ĪçļĦ":75012,"iativa":75013,"Desktop":75014,"Ġdissociation":75015,"ĠMurder":75016,"Ġannouncements":75017,"ãģ¹ãģį":75018,"åIJīæŀĹçľģ":75019,"çļĦéĻIJåζ":75020,"Ġreplicated":75021,"Polish":75022,"Ġacrylic":75023,"å·²æľīçļĦ":75024,"æĽ´å¤ļçļĦ人":75025,"رÙĬÙĦ":75026,"SUM":75027,"immers":75028,"ĠнеÑģколÑĮ":75029,"ĠTakes":75030,"ĠVy":75031,"ĠÙĪÙħا":75032,"ĠDez":75033,"çªģå¦Ĥåħ¶":75034,"çħ¤æ°Ķ":75035,"Ġfruitful":75036,"iarism":75037,"Czech":75038,"Near":75039,"ÙĨÚ¯ÛĮ":75040,"à´²":75041,"Ġrespectable":75042,"_default":75043,"Ġcuring":75044,"ноп":75045,"å½Ĩ":75046,"пÑĢа":75047,"Ġausge":75048,"Ġavenue":75049,"ĠSé":75050,"Ġlocating":75051,"失æķĹ":75052,"åį°ç«ł":75053,"ĠYing":75054,"ĠBlut":75055,"ĠCompounds":75056,"Ġalbumin":75057,"ĠVariation":75058,"ĠداراÛĮ":75059,"ĠEmployer":75060,"Ġhomelessness":75061,"å½¢åĬ¿ä¸ĭ":75062,"ĠпоÑħ":75063,"'):Ċ":75064,"ĠMüller":75065,"ä¸ŃæŃ¢":75066,"被认为æĺ¯":75067,"éĿŀ线æĢ§":75068,"ĠColleges":75069,"Ġhabil":75070,"ázÃŃ":75071,"reira":75072,"alie":75073,"Ġlodge":75074,"ĠIEnumerable":75075,"Seven":75076,"èµŀæĪIJ":75077,"ç͵è§Ĩæľº":75078,"ĠEvaluating":75079,"轻轻çļĦ":75080,"ächst":75081,"ĠBeweg":75082,"éľĢè¦ģ注æĦıçļĦæĺ¯":75083,"Ġstaggering":75084,"ãĢĹ":75085,"å½¢ä½ĵ":75086,"æºIJçļĦ":75087,"aira":75088,"panies":75089,"-PCR":75090,"Ġrebuilding":75091,"CNN":75092,"ĠDenn":75093,"å®¶ä¼ģä¸ļ":75094,"åħįå¾Ĺ":75095,"è¨Ńç½®":75096,"Ġscrutin":75097,"Ġ×IJ×ķת×ķ":75098,"ĠÙħÙĨØ·ÙĤÙĩ":75099,"ĠMormon":75100,"Ġsuf":75101,"ä¸Ńæĸ¹":75102,"Ġintram":75103,"åºĶå°Ĩ":75104,"Ġë¸":75105,"è·¯åĨĽ":75106,"Ġplano":75107,"Ġpeeled":75108,"rán":75109,"Ġmoc":75110,"Ġhir":75111,"ĠLug":75112,"ĠGri":75113,"Ġsausage":75114,"Ġestates":75115,"æĴ²":75116,"mathscr":75117,"ä¸ĢçĤ¹ä¹Łä¸į":75118,"ĠΤο":75119,"Ġlän":75120,"åľ°ä¸ĬçļĦ":75121,"å°±æĺ¯è¿Ļ个":75122,"éłĥ":75123,"çļĦæĥħæĻ¯":75124,"ĠInglês":75125,"ongan":75126,"æī¿æİ¥":75127,"ä¹İä¹İ":75128,"Ġhorr":75129,"實é©Ĺ":75130,"Elizabeth":75131,"ĠUNIVERS":75132,"Ġanalysing":75133,"Ġillegally":75134,"}else":75135,"Ġbinder":75136,"éĥ½åºĶ该":75137,"åħ¶ä¸º":75138,"æĹ¥æ´»åĬ¨":75139,"Ġgrep":75140,"ENCY":75141,"หวัà¸Ķ":75142,"Ġlinguistics":75143,"åĩĿèģļåĬĽ":75144,"Łģ":75145,"Ġtá":75146,"Ġtrophy":75147,"iland":75148,"ä½Ł":75149,"å§Ŀ":75150,"åĥµç¡¬":75151,"顽强":75152,"velocity":75153,"ĠгÑĢи":75154,"cube":75155,"æľīä½ł":75156,"å¤ļ大çļĦ":75157,"headed":75158,"ĠBlockchain":75159,"ĠпеÑĢвÑĭй":75160,"Ġcog":75161,"ighted":75162,"weit":75163,"Ġâĩ":75164,"亲身":75165,"Ġsuperhero":75166,"åģľæ»ŀ":75167,"Ġخر":75168,"juven":75169,"ĠNordic":75170,"åĭĺå¯Ł":75171,"Git":75172,"泸":75173,"对åŃ©åŃIJ":75174,"å¼Ģå±Ģ":75175,"جÙĨ":75176,"è¦ģæ±ĤåĴĮ":75177,"ĠgroÃŁe":75178,"Ġenzymatic":75179,"編輯":75180,"èı©æıIJ":75181,"ĠParam":75182,"Ġiterate":75183,"Ġmurmured":75184,"Fish":75185,"lk":75186,"ĠPaolo":75187,"ãĤ¼":75188,"ਦ":75189,"Ġinspirational":75190,"ä¹Ĵä¹ĵçIJĥ":75191,"ĠIncluding":75192,"ĠResidential":75193,"ĠAuthent":75194,"ÃŃda":75195,"Ġsubmerged":75196,"ÏĦÏī":75197,"åĬŀçļĦ":75198,"емой":75199,"CLUD":75200,"oze":75201,"church":75202,"Ġhaunted":75203,"ãģijãģŁ":75204,"å¦ĸåħ½":75205,"iferous":75206,"ĠKyoto":75207,"ĠczÅĤowie":75208,"Ġchiam":75209,"indung":75210,"åħĥå¸ħ":75211,"ĠLeone":75212,"Receive":75213,"çµµ":75214,"Ġbarred":75215,"mmmm":75216,"åΏåķĨ":75217,"Scholar":75218,"Rose":75219,"ivert":75220,"Ġemergent":75221,"áĥĶáĥľ":75222,"åľ¨å½ĵæĹ¶":75223,"apr":75224,"suba":75225,"ä¼°è¨Ī":75226,"ĠWrest":75227,"Ġacronym":75228,"Ġboast":75229,"ilitating":75230,"ëłĩ":75231,"ãĤ¯ãĥª":75232,"Ġyouthful":75233,"Sym":75234,"už":75235,"头æĿ¡":75236,"Ġتک":75237,"zept":75238,"-present":75239,"-after":75240,"Ġdarauf":75241,"Multiply":75242,"+s":75243,"MX":75244,"ĠSiem":75245,"Ġjeszcze":75246,"éĥ½ç͍":75247,"âĢĶâĢĿĊĊ":75248,"ĠComun":75249,"untza":75250,"tin":75251,"窮":75252,"èĬ±èįī":75253,"éĢĻæīį":75254,"è¸Ĭ":75255,"phantom":75256,"ĠInvestments":75257,"ĠاÙĦÙģÙĦÙĥ":75258,".age":75259,"ä¹Łå°±ä¸į":75260,"çĿ¾":75261,"Ġflare":75262,"Ġestamos":75263,"æİĴ污":75264,"à¥įस":75265,"_items":75266,"Ġscop":75267,"Ġautour":75268,"æĭħè´Ł":75269,"Ġপà§įরথম":75270,"Organization":75271,"á»±c":75272,"(query":75273,"ÌĤ":75274,"åĮ®":75275,"èªķ":75276,".dto":75277,"ĠObesity":75278,"ĠHumidity":75279,"ĠConceptual":75280,"sent":75281,"Ġpiss":75282,"社ä¼ļä¸Ń":75283,"æĥ¯ä¾ĭ":75284,"çļĦæĹ¶éĹ´åĨħ":75285,"Ġwykorzyst":75286,"Ġbijvoorbeeld":75287,"Ġcontingency":75288,"Trend":75289,"ocortic":75290,"ubahan":75291,"Ġresolver":75292,"obox":75293,"缸æ¯Ķè¾ĥ":75294,"Õ¡Õ·":75295,"Ġeffortlessly":75296,"à§ĭà¦ľà¦¨":75297,"Ġlivro":75298,"ĠCYP":75299,"neal":75300,"Ġraced":75301,"æĤħ":75302,"åį°å°¼":75303,"Ġthinner":75304,"beda":75305,"éļ¨å¾Į":75306,"ĠVL":75307,"éĥ½æ¯Ķè¾ĥ":75308,"Ġflashed":75309,"æ¯ıç§į":75310,"Ġensino":75311,"ÙİÙĪ":75312,"Ġtrustees":75313,"Ġinterfering":75314,"Ġobtener":75315,"ĠGarn":75316,"éĿĴäºij":75317,"encers":75318,"ä¸įæĸŃåıijå±ķ":75319,"ĠMali":75320,"ĠDress":75321,"ĠFalk":75322,"æĥ®":75323,"åıĮèħ¿":75324,"Ġtouring":75325,"Ġколлек":75326,"Æ°á»Ľc":75327,"=/":75328,"å°¼åħĭ":75329,"ĠвÑģÑĤÑĢеÑĩа":75330,"ä¸įå°ıçļĦ":75331,"Ġunbiased":75332,"åĩºçı¾åľ¨":75333,"TRY":75334,"ãģ«ãģªãģ£ãģ¦":75335,"Ġfarewell":75336,"èĦijæµ·éĩĮ":75337,"ĠSHE":75338,"主æĿ¿":75339,"Ġempat":75340,"æľĢçα":75341,"Ġ\\(\\{":75342,"ĠEmmanuel":75343,"pour":75344,"isierung":75345,"çļĦè´¹ç͍":75346,"etings":75347,"Ġruth":75348,"shaw":75349,".Def":75350,"ĠÑģÑĤали":75351,"ücken":75352,"_op":75353,"asin":75354,"гал":75355,"Ġpropensity":75356,"Ġowl":75357,"人éģĵ":75358,"åѦçĶŁåŃ¦ä¹ł":75359,"ÑīаеÑĤ":75360,"注åĨĮä¼ļ计å¸Ī":75361,"èĬ³é¦Ļ":75362,"Åĵur":75363,"lakang":75364,"Ġamyloid":75365,"èİ«åIJįåħ¶å¦Ļ":75366,"vall":75367,"ĠLópez":75368,"club":75369,"ampal":75370,"ÑĤина":75371,"ogenes":75372,"ĠRede":75373,"execute":75374,"ĠÙĨسبت":75375,"Sr":75376,"jav":75377,"ä¹ĭé£İ":75378,"éĿ¢å®¹":75379,"Ġdeflection":75380,"неÑĢа":75381,":hover":75382,"ĠTehran":75383,"éĤ¸":75384,"-Americ":75385,"åł±å°İ":75386,"Ġjsem":75387,"vek":75388,"为人æ°ij":75389,"èĩªå¸¦":75390,"Ġregroup":75391,"Ġдок":75392,"æį¢ç®Ĺ":75393,"ç®Ģåįķåľ°":75394,"æŃ£ç¡®åľ°":75395,"ĠÄijưá»Ŀng":75396,"çłĤæµĨ":75397,"opathology":75398,"guez":75399,"è¿Ľè¡Įæ£ĢæŁ¥":75400,"oirs":75401,"éĽĩ主":75402,"deb":75403,"ç͵åİĤ":75404,"-Step":75405,"Ġdubbed":75406,"ankind":75407,"åĩĨæĹ¶":75408,"ĠUSC":75409,"ĠINR":75410,"-Saharan":75411,"åºĶç͍çļĦ":75412,"å°±ä¼ļ被":75413,"æ©Łæ¢°":75414,"èĺ¸":75415,"Ġdues":75416,"Ġenrol":75417,"ä½łçľŁçļĦ":75418,"å®¶åħ¬åı¸":75419,"äºij计ç®Ĺ":75420,"æı¡æīĭ":75421,"ĠвойнÑĭ":75422,"Ġparan":75423,"Ġestrat":75424,"oscale":75425,"ĠFrau":75426,"ĠBien":75427,"Ġcurry":75428,"Ġcharities":75429,"Ġساخت":75430,"ĠNottingham":75431,"-infected":75432,"è¾ľè´Ł":75433,"å¤ļä½į":75434,"Ġentender":75435,".AreEqual":75436,"ĠCafe":75437,"ĠReceived":75438,"社ä¼ļ责任":75439,"åĽ½æĥħ":75440,"ä¹ĭçİĭ":75441,"ixin":75442,"sonian":75443,"çĶļèĩ³è¿ĺ":75444,"éŃĶçİĭ":75445,"पà¥įर":75446,"ÑİÑīиÑħÑģÑı":75447,"-volume":75448,"ĠWirtschaft":75449,"åĨħèĦı":75450,"Coord":75451,"ĠKilogram":75452,"ĠjÄĻzy":75453,"Ċ":75874,"LEY":75875,"æ¶īæ¡Ī":75876,"ĠHelm":75877,"Ġinventions":75878,"試é¨ĵ":75879,"åľ¨åħ¬åı¸":75880,"Ġenvol":75881,"icho":75882,"erville":75883,"çĤ¹ä¸Ĭ":75884,"Ġiets":75885,"ç³ľ":75886,"è¿Ļæł·å°±":75887,"第äºĮå±Ĭ":75888,"atalytic":75889,"Ġwebpage":75890,"umsi":75891,".indexOf":75892,"experience":75893,"ãĤĤãģĹãĤĮ":75894,"ĠKop":75895,"éĥ½ä¸İ":75896,"æĬĹéľĩ":75897,"Ö¸×Ķ":75898,"à¸Ńะà¹Ħร":75899,"Ġbaff":75900,"Ġsehen":75901,"غط":75902,"Ġbloggers":75903,"ĠпоÑĩÑĤи":75904,"Ġhither":75905,"ĠTicket":75906,"å¤ĸåĮħ":75907,"ç»§ç͵åύ":75908,"ĠCookies":75909,"Descriptors":75910,"çļĦæ¯į亲":75911,"å΍":75912,"มืà¸Ńà¸Ļ":75913,"imentary":75914,"ĠAdvantage":75915,"ĠÐĹна":75916,"ĠINTEGER":75917,"Ġfiss":75918,"å¹´æľŁ":75919,"Ġamort":75920,"Ġmains":75921,"Ġboek":75922,"åĪĨæŀIJæ³ķ":75923,"ĠINST":75924,"ĠÐľÑĥ":75925,"åıªè¦ģæĺ¯":75926,"à¹Ģสริม":75927,"Ġdebugging":75928,"Å¿":75929,"è¦ģ使":75930,"æīĢåģļçļĦ":75931,"Ġmodific":75932,"åIJįè¨Ģ":75933,"awai":75934,"ĠìŀĪìĸ´":75935,"åįĥä¸ĩåĪ«":75936,"×Ļ×¢×Ķ":75937,":text":75938,"Train":75939,"ä¸įèµ°":75940,"ÃŃcios":75941,"Ġpoignant":75942,"пен":75943,"Ġà¦ħরà§įথ":75944,"Ġfiller":75945,"Ġpesquisa":75946,"Ġintensified":75947,"åľ¨ä¸įåIJĮçļĦ":75948,"ipada":75949,"ordinary":75950,"æľĪçIJĥ":75951,"(title":75952,"éģĹåĺ±":75953,"ĠFarmer":75954,"Ġkissing":75955,"esting":75956,"åı¯çĸij":75957,"åIJİå¤ĩ":75958,"Ġsponge":75959,"å¼ķåĬĽ":75960,"åķĨåĵģæĪ¿":75961,"Ġsucceeding":75962,"ĠвнÑĥÑĤÑĢи":75963,"çĶ»çĶ»":75964,"åįķä½įåĴĮ":75965,"æĽ²çº¿çļĦ":75966,"ãģĹãģ¦ãĤĤ":75967,"粪便":75968,"ç¤Ļ":75969,"ä¸ī缸":75970,"ĠConnor":75971,"åĩ¶æīĭ":75972,"å«ģç»Ļ":75973,"纪念é¦Ĩ":75974,"Ġscaffold":75975,"ä¸įæŃ£":75976,"rapped":75977,"Ġvolte":75978,"ä¹Łä¸įçŁ¥":75979,"OrDefault":75980,"Ġhemos":75981,"ĠUnderground":75982,"ÃŃna":75983,"Ġminutos":75984,"Ġglomer":75985,"-post":75986,"å¸Ĥåľºä»½é¢Ŀ":75987,"ĠPartage":75988,"ĠFishing":75989,"æ±¾":75990,"æľ¬æĺ¯":75991,"Ġelkaar":75992,"Italia":75993,"ĠSaúde":75994,"à¸Ĥà¸Ńà¸ĩà¸ģาร":75995,"ĠФедеÑĢаÑĨии":75996,"ĠSoy":75997,"Ġblonde":75998,"-btn":75999,"å¢ŀçĽĬ":76000,"-path":76001,"ĠÑĤоже":76002,"Ġlocales":76003,"гаÑĢ":76004,"ĠÑģобÑģÑĤвен":76005,"Ġhé":76006,"ĠпÑĢекÑĢа":76007,"ĠKelvin":76008,"ĠHassan":76009,"人å±ħ":76010,"太好":76011,"mA":76012,"Ġnik":76013,"ĠPizza":76014,"ĠBark":76015,"ä¸į失":76016,"ĠChal":76017,"è¿ĺæĺ¯ä¸Ģ":76018,"Ġnove":76019,"Ġعبر":76020,"cribes":76021,"ç®Ģè¿°":76022,"Modified":76023,"å°ıæĹ¶åIJİ":76024,"ĠPiper":76025,"ĠÑģÑĤановиÑĤÑģÑı":76026,"Ġmiesz":76027,"Ġго":76028,"èŀºä¸Ŀ":76029,"Processing":76030,"sers":76031,"ÃIJ":76032,"×ļ":76033,"çļĦæĪĺçķ¥":76034,"pees":76035,"çľĭçĹħ":76036,"-success":76037,"移交":76038,"ترÛĮ":76039,"cause":76040,"omány":76041,"ä¹Ĺ":76042,"æĮīéĶ®":76043,"Ġdém":76044,"ĠÑįкÑģпе":76045,"à§įলাহ":76046,"Ġgouvernement":76047,"aric":76048,"ĠJab":76049,"Ġequipo":76050,"ನà³įನà³ģ":76051,"builder":76052,"cra":76053,"ë¹":76054,"Ġvested":76055,"æľīèijĹ":76056,"ŀ×ĵ":76057,".Save":76058,"recated":76059,"ĠBulld":76060,"polar":76061,"ĠCY":76062,"æĢĿæ½®":76063,"Ġantico":76064,"allback":76065,"ä¹ĭ说":76066,"ethical":76067,"æ°ĶåĴĮ":76068,"Ġpreparedness":76069,"ÃŃtás":76070,"Ġtetap":76071,"Ġzdrow":76072,"etzung":76073,"EH":76074,"Ġthief":76075,"Ġkini":76076,"天åĨħ":76077,"Ġhorizons":76078,"Ġtint":76079,"å°Ħæīĭ":76080,"ĠRobb":76081,"Ġconocimiento":76082,"Authorization":76083,"kach":76084,"ĉC":76085,"åıijåĩºäºĨ":76086,"ä¸ī代":76087,"Ùĥثر":76088,"Ġtwitter":76089,"è¿ĻéĩĮéĿ¢":76090,"åįģäºĮæĿ¡":76091,"çĶĺèĤĥçľģ":76092,"-'":76093,"Ġcose":76094,"ä¸įåĬł":76095,"Ġagitation":76096,"æĹłå¿Į":76097,"_img":76098,"æ±Łå¸Ĥ":76099,"ITLE":76100,"ãĥ¬ãĤ¹":76101,"Anyone":76102,"忽çķ¥äºĨ":76103,"ä»ĸä¸įæĺ¯":76104,"èιåıª":76105,"Ġturf":76106,"Ġkdyž":76107,"ĠCarpenter":76108,"rne":76109,"Ġspores":76110,"éľĵ":76111,"çĶµåĽ¾":76112,"Ġdoubtful":76113,"欺è¯Ī":76114,"ĠBorough":76115,"äºĨä¸įèµ·":76116,"activate":76117,"åĪĨ段":76118,"主线":76119,"Ġautoc":76120,"Ġgeo":76121,"Ġsecund":76122,"ális":76123,"Relative":76124,"Esc":76125,"Ž":76126,"ģáĢ":76127,"Ġpours":76128,"Ġteg":76129,"Ġtransducer":76130,"Construct":76131,"Ġinspected":76132,"å¼ĢåıijèĢħ":76133,"Ġbelongings":76134,"ëĮĢë¡ľ":76135,"Ġinclu":76136,"ĠCovenant":76137,"isel":76138,"емÑĮ":76139,"='/":76140,"æĬ½æŁ¥":76141,"ozoic":76142,"æŁ¬":76143,"å·¥ä½ľè®¡åĪĴ":76144,"Ġindividuality":76145,"Ġrevolutionized":76146,"Ġpesticide":76147,"çļĦç³»ç»Ł":76148,"estim":76149,"çĶŁåīį":76150,"èĬ±æ¤Ĵ":76151,"׾×Ļת":76152,"িà¦ĵ":76153,"Ġmarca":76154,"ĠاÙĦØŃر":76155,"âĻ¥":76156,"Trade":76157,"ĠEuras":76158,"æľ¬æĽ¸":76159,"åħ¬åľĴ":76160,"篱":76161,"Ġpredic":76162,"ĠÑĢазÑĢе":76163,"é§IJ":76164,"vous":76165,"ĉde":76166,"ãģķãĤĮãģ¦ãģĦãģ¾ãģĻ":76167,"ĠFirefox":76168,"}e":76169,"Ġpaddle":76170,"åĨ¬å¥¥":76171,"ynthia":76172,"amation":76173,"é£İéĻ©ç®¡çIJĨ":76174,"Ġúnico":76175,"ĠMotivation":76176,"-xs":76177,"Ġpremiere":76178,"Ġcops":76179,"ĠTir":76180,"nev":76181,"ä¿Ĺè¯Ŀ说":76182,"æŃ¡è¿İ":76183,"èģ¯åIJĪ":76184,"Ġкомпон":76185,"ĠÏīÏĤ":76186,"Eric":76187,"{}Ċ":76188,"Ġincont":76189,"rains":76190,"ĠPathol":76191,"æĬĴæĥħ":76192,"ä¹Łè·ŁçĿĢ":76193,"é«ĺæ°´å¹³":76194,"社工":76195,"æł¼æŀĹ":76196,"ĠfamÃŃlia":76197,"ç»ĻäºĨæĪij":76198,"çļĦçī©è´¨":76199,"çĸ¸":76200,"Ġsunk":76201,"_AD":76202,"ĠAdmission":76203,"öld":76204,"çŁ³èĭ±":76205,"ĠManning":76206,"æĪªåĽ¾":76207,"ä¸įç͍æĭħå¿ĥ":76208,"Ġlivello":76209,"+h":76210,"ĠKod":76211,"ĠUncertain":76212,"æľªè§ģ":76213,"åij³ç²¾":76214,"extern":76215,"çϽçϽ":76216,"-elect":76217,"ĠкомпÑĮÑİ":76218,"ĠÑĢаÑģÑĩеÑĤа":76219,"$.Ċ":76220,"Tap":76221,"åij¼åij¼":76222,"jsce":76223,"Ġperfusion":76224,"professional":76225,"å¼Ģå¹ķå¼ı":76226,"Tot":76227,"Ġназна":76228,"ĠμM":76229,"åħ§çļĦ":76230,"Capacity":76231,"Ġíı¬íķ¨":76232,"Apart":76233,"ĉlist":76234,"ĠGale":76235,"æľ¬èįī":76236,"å¹³åĿ¦":76237,"æľįçļĦ":76238,"åĨ·æ·¡":76239,"çļĦ大éĩı":76240,"Ġtowels":76241,"esper":76242,"ä¼ļå±ķ":76243,"cardia":76244,"Ġintensely":76245,"Ġdiferencia":76246,"Pain":76247,"Ġcompressive":76248,"çī¹åĭĴ":76249,"iteration":76250,"à§ĩড":76251,"ĠJackie":76252,"ä¸įäºĨçļĦ":76253,"λλά":76254,"贪污":76255,"Ġ\"...":76256,"ĠRelation":76257,"Ġdigitally":76258,"åĪĽä½ľçļĦ":76259,"Ġlifestyles":76260,"Ġskeptic":76261,"贪婪":76262,"ĠÑĤÑĢÑĥд":76263,"Ġbustling":76264,"inous":76265,"ĠRough":76266,"orda":76267,"Ġ$(\"#":76268,"Going":76269,"Ġfirewall":76270,"ĠاÙĦربÙĬع":76271,"ä¸ŃæĹ¥":76272,"-inspired":76273,"Ġarrests":76274,"ipheral":76275,"Ġר×IJש":76276,"ében":76277,"Ġintelig":76278,"ferential":76279,"inky":76280,"Ġcompleteness":76281,"ĠJuvenile":76282,"çļĦåIJĪä½ľ":76283,"é«ĺæłĩåĩĨ":76284,"éĩijèī²çļĦ":76285,"iju":76286,"-trained":76287,"Ġcapitalize":76288,"ĠCircuits":76289,"san":76290,"inoa":76291,"Ġsexes":76292,"ç¶ĵæŃ·":76293,"abilitÃł":76294,"(let":76295,"_update":76296,"ĠRoles":76297,"Ġ구ìĦ±":76298,"ını":76299,"ầu":76300,"ĠпопÑĥла":76301,"Lew":76302,"为代表çļĦ":76303,"åıijèĬ½":76304,"cym":76305,"اÙĦج":76306,"æĢ»çĿ£":76307,"Terms":76308,"%D":76309,"ausend":76310,"ç¬ijèĦ¸":76311,"æľīä¸ĢæĿ¡":76312,"ĠìŀIJìĭł":76313,"|=":76314,"人就":76315,"ĠkN":76316,"Ġconta":76317,"祯":76318,"amping":76319,"allets":76320,"uvres":76321,"\\({}^{+}\\)":76322,"Ġdocumenting":76323,"误åĮº":76324,"Ġrhiz":76325,"æŀ¯çĩ¥":76326,"Ġpratique":76327,"Ġciclo":76328,"Ġlumen":76329,"åıĪæĬĬ":76330,"ĠYourself":76331,"Ġdownloading":76332,"Ġtierra":76333,"Ġseks":76334,"ĠSeventh":76335,"Ġfim":76336,"人ãģ¯":76337,"úng":76338,"åį¡è½¦":76339,"åĮ»çĸĹåĻ¨æ¢°":76340,"ĠاÙĦشخص":76341,"ettiin":76342,"ĠvÃło":76343,"enable":76344,"Ġyuan":76345,"CTV":76346,"ĠGeoffrey":76347,"Ġkró":76348,"缮çĿ¹":76349,"ĠElite":76350,"ĠTransit":76351,"ç½ij绾ä¸Ĭ":76352,"Ġê²Į":76353,"×Ļר×Ļ×Ŀ":76354,"Ġalumnos":76355,"virtual":76356,"â̰":76357,"对éĿ¢çļĦ":76358,"Ġnearer":76359,"å¥Ĺé¤IJ":76360,"æĶ¾å¿ĥåIJ§":76361,"zález":76362,"znych":76363,"Ġrealms":76364,"ATURE":76365,"é«Ķé©Ĺ":76366,"Ġsubstituting":76367,".concurrent":76368,"çĭ¼çĭĪ":76369,"Ġwhit":76370,"sofar":76371,"ühl":76372,"è¶³äºĨ":76373,"Ġmotivating":76374,"Ġimmensely":76375,"Wir":76376,"å¹´åĿĩ":76377,"ä¸ī峡":76378,"Ġvalore":76379,"Ġintensities":76380,"åĥµå°¸":76381,"á»ĭnh":76382,"æĺ¾å¾®éķľ":76383,"åı¯ç¬ij":76384,"жнÑĭÑħ":76385,"(source":76386,"æľŁéĹ´çļĦ":76387,"ï¹IJ":76388,"ÑĨенÑĤÑĢа":76389,"ĠJia":76390,"åı¯æİ§":76391,"iany":76392,"ãĢĤâĢĿãĢĬ":76393,"ĠContrast":76394,"ĠNurses":76395,"×ķפ×Ķ":76396,"ĠMobility":76397,"'r":76398,"NV":76399,"纶":76400,"Ġdevoid":76401,"ç»ıæµİçļĦåıijå±ķ":76402,"æĭĽæĶ¶":76403,"çī¹å¾ģçļĦ":76404,"ĠLisbon":76405,"acción":76406,"Ġrelativity":76407,"çϽç»Ĩèĥŀ":76408,"ãģĦãģ¾ãģĹãģŁ":76409,"ãģĮãģĤãĤĬ":76410,"设计ä¸İ":76411,"ä¹Łä¸įæľĥ":76412,"balances":76413,"ĠÙĦÙĦØŃ":76414,"ĠпÑĢоÑĨеÑģÑģе":76415,"éĢĻåħ©":76416,"Ġincision":76417,"غÙĦ":76418,"Ġtrainers":76419,"ĠMagnet":76420,"Ġmajestic":76421,"orientation":76422,"']ĊĊ":76423,"izzo":76424,"æĿ¥è¿ĻéĩĮ":76425,"ÑģкомÑĥ":76426,"USH":76427,"æĶ¿åºľéĥ¨éŨ":76428,"Ġà¦ķল":76429,"Ġpaternal":76430,"å®ļ为":76431,"áÅĻ":76432,"ä½Ĩä»į":76433,"éĩijåŃĹ":76434,"оÑĤпÑĥ":76435,"ãģĭãĤĭ":76436,"çķ¶ä¸Ń":76437,"Ġfolklore":76438,"缸çα":76439,"ç»ıæµİ建设":76440,"ĠInters":76441,"Ġplantas":76442,"Ġdissection":76443,"ĠJerome":76444,"ÙİÙĨÙĴ":76445,"Js":76446,"è¿´":76447,"website":76448,"Ġfamine":76449,"åħ¸èĮĥ":76450,"ĠÑĤам":76451,"Ġinstallment":76452,"Ġneutrality":76453,"ĠاÙĨتخ":76454,".Contains":76455,"ikawa":76456,"工人çļĦ":76457,"çħ²":76458,"schule":76459,"Ġfungsi":76460,"[label":76461,"Ġdamned":76462,"patrick":76463,"满满çļĦ":76464,"-cycle":76465,"Ġparsing":76466,"ä»ĸçļĦæīĭ":76467,"रà¥Ģ":76468,"å®īæİĴéĥ¨ç½²":76469,"ä¸ĵ项è¡ĮåĬ¨":76470,"Ġsoprattutto":76471,"ĠLös":76472,"Ġrisque":76473,"åĪĽæĸ°èĥ½åĬĽ":76474,"ĠìŬ룬":76475,"Ġtc":76476,"ĠJain":76477,"åĺĢåĴķ":76478,"Ġdici":76479,"Ġmoth":76480,"ouk":76481,"×Ļרת":76482,"Ġreconcile":76483,"ä¸īå±Ĥ":76484,"没æľī被":76485,"ĠÑĤомÑĥ":76486,"namen":76487,"ĠплоÑģко":76488,"×¨×Ľ×ª":76489,"ç¶ľåIJĪ":76490,"otec":76491,"å§Ĺ":76492,"Ġindifferent":76493,"èģĶ系人":76494,"ĠاÙĦجاÙħ":76495,"ĠобÑĬÑıÑģ":76496,"Ġdiaphragm":76497,"Ġaún":76498,"çļĦæķ°åŃĹ":76499,"èĩ³ä¸Ĭ":76500,"ادات":76501,"æĪIJåĬŁäºĨ":76502,"Ò»":76503,"alus":76504,"ĠتÙı":76505,"æĸŃå¼Ģ":76506,"Ġ×Ķ×ŀ":76507,"опÑĢи":76508,"IMO":76509,"cznych":76510,"Ġcalibrated":76511,"ĠBiodiversity":76512,"(pos":76513,"ĠDash":76514,"åľ¨å»º":76515,"Ġort":76516,"ï¼Īï¼īĊĊ":76517,"Ġmonoc":76518,"ĠعÙĤ":76519,"Ġdenies":76520,"åĢĭæľĪ":76521,"å°Ķçī¹":76522,"çļ®çļĦ":76523,"Ġ\")":76524,"ĠXOF":76525,"å¤įæĿĤæĢ§":76526,"ĠMembership":76527,"éĻº":76528,"tywn":76529,"ä»Ģä¹ĪæĦıæĢĿ":76530,"Ġbadan":76531,"çĥŁéĽ¾":76532,"樱èĬ±":76533,"osest":76534,"ĠNish":76535,"éĢŀ":76536,"çĽĺçĤ¹":76537,".build":76538,"ĠPROGRAM":76539,"YSIS":76540,"£p":76541,"ĠHaj":76542,"ÑĩноÑģÑĤÑĮ":76543,"åħ¬åħ³":76544,"огÑĥ":76545,"ä¼łçľŁ":76546,"ĠâĪĢ":76547,"åľ°æĸ¹çļĦ":76548,"Medium":76549,"Ġíݸ":76550,"Ġspins":76551,"è¥¿åŁŁ":76552,"å±Ģå±Ģéķ¿":76553,"ĠÑĪеÑģÑĤÑĮ":76554,"ĠPricing":76555,"akra":76556,"åıijæĺİçļĦ":76557,"æ·Ħ":76558,"æĿİä¸ĸ":76559,"Ġ×ŀ×ĸ":76560,"ĠتØŃÙĤÛĮ":76561,"à¸Ĭà¸Ļà¹Į":76562,"ĠÙĨÙĤØ´":76563,"Broad":76564,"jing":76565,"heten":76566,"Ġdeï¬ģ":76567,"ĠHSL":76568,"Ġpreocup":76569,"äºĮåĵ¥":76570,"çĻ½éĽª":76571,"ä¸Ĭä¸Ģç¯ĩ":76572,"ĠкваÑĢ":76573,"BACKGROUND":76574,"åıijè¡ĮçļĦ":76575,"Ġzug":76576,"Ġtrava":76577,"ÙĥاÙģ":76578,"Ġbooklet":76579,"å¼ĤæŃ¥":76580,"è§ĦåĪĻçļĦ":76581,"ĠLightning":76582,"Ġà¸Ħà¸Ļ":76583,"Ġtents":76584,"antar":76585,"æŀģå°ij":76586,"ĠCombining":76587,"lr":76588,"Ãij":76589,"elis":76590,"年以ä¸Ĭ":76591,"æ°´çħİ":76592,"è¿ĺæĺ¯ä¼ļ":76593,"ĠاÙĦØ£ÙĪÙĦÙī":76594,"ĠHos":76595,"é«ĺåľ°":76596,"Ġbedrooms":76597,"éĥ½æľīä¸Ģ个":76598,"èµ¶ä¸Ĭ":76599,"Ġsubstitutes":76600,"Conclusions":76601,"Legal":76602,"orget":76603,"ESC":76604,"Ġexperiencia":76605,"ĠEstimate":76606,"ç¹ģå¿Ļ":76607,"Ġaire":76608,"Ġ){Ċ":76609,"ĠErin":76610,"Ġnouv":76611,"}&\\":76612,"..................":76613,"orges":76614,".boot":76615,"Ġdisappro":76616,"Ġfortress":76617,"é̼è¿ij":76618,"網路":76619,"Ġthrombosis":76620,"绣é¢Ĩ":76621,"åĦ¡":76622,"Ġà¦ıà¦Łà¦¿":76623,"Ġborrower":76624,",W":76625,"ĠElections":76626,"Ġky":76627,"Club":76628,"ĠElijah":76629,"ÛĮدÙĨ":76630,"æĤ¬å´ĸ":76631,"Ġembarked":76632,"ĠDiploma":76633,"ĠFAC":76634,"iekt":76635,"ä½Ĩæĺ¯å¦Ĥæŀľ":76636,"Ġintercourse":76637,"ĠSeeds":76638,"ÏĥÏĥ":76639,".make":76640,"æŀ¶åŃIJ":76641,"ĠдоÑģÑĤÑĥп":76642,"èĤ¿èĥĢ":76643,"åݨå¸Ī":76644,"ĠLocated":76645,"Ġelicit":76646,"Ù¢":76647,"Ġmég":76648,"ä¸Ńåĩºçݰ":76649,"Ġcloning":76650,"åľ°æŃ¥":76651,"epoch":76652,"åĽ¾å½¢çļĦ":76653,"ĠتÙĩ":76654,"Ġseguridad":76655,"礼åĵģ":76656,"اØŃÛĮ":76657,"Ġgrocer":76658,"ç°¡çĽ´":76659,"èµĶåģ¿è´£ä»»":76660,"æİĴè¡Įæ¦ľ":76661,"Ġfaction":76662,"×ķ×Ķ×":76663,"Ġinitialized":76664,".stack":76665,"éĻªä½ł":76666,"Ãĭ":76667,"oum":76668,"Ġcatar":76669,"ĠVulner":76670,"çľĭä¸Ģçľĭ":76671,"ĠAngl":76672,"综åIJĪç´łè´¨":76673,"Privacy":76674,"Ġpadre":76675,"NZ":76676,"Ġconceive":76677,"åľ¨æĥ³":76678,"ĠпоÑıви":76679,"étés":76680,"Ġê°ľë°ľ":76681,"ĠRegularly":76682,"Ġdafür":76683,"ĠBamb":76684,"anska":76685,"åĽ½èIJ¥":76686,"umberland":76687,"缺氧":76688,"Ġmaupun":76689,"龸éģĵ":76690,"ĠкомплекÑģ":76691,"[pos":76692,"Ġaft":76693,"à¸Ļะ":76694,"Ġeigenen":76695,"æĪIJ交éĩı":76696,"ĠØŃÙĤÙĪÙĤ":76697,".ind":76698,"ĠDere":76699,"大åįĬ":76700,"×Ļ×§×":76701,"Ġtyph":76702,"导ç͵":76703,"Ġmilling":76704,"atiu":76705,"é̲äºĨ":76706,"Ġventral":76707,"ĠBrighton":76708,"ĠëIJľëĭ¤":76709,"onana":76710,"lles":76711,"äºĮåįĥ":76712,"ĠIrving":76713,"Ġclimax":76714,".{":76715,"Vers":76716,"æĸ°å¾ģç¨ĭ":76717,"ĠReset":76718,"ìĹ¼":76719,"Ġswell":76720,"Ġpsychotherapy":76721,"ĠDISC":76722,"Ġprerequisite":76723,"Ġnostalgia":76724,"Ġprocessus":76725,"argent":76726,"Äįka":76727,"ĠдоÑħод":76728,"Detailed":76729,"monton":76730,"Ġrecomend":76731,"ĠPARTIC":76732,"Mais":76733,"Ġdah":76734,"æĺ¯åIJĹ":76735,"Ġnaam":76736,"_nodes":76737,"Ġmengalami":76738,"Ġà¦¯à¦¾à§Ł":76739,"_event":76740,"Ġmohou":76741,"QUE":76742,"éļ¾çļĦ":76743,"(token":76744,"ĠReducing":76745,"ĠÑģоÑģÑĤоÑıниÑı":76746,"Ġwomb":76747,"Ġlounge":76748,"ĠPlane":76749,"Ġilust":76750,"ä¿¡ç͍è¯ģ":76751,"\\tau":76752,"Ġsummaries":76753,"éŃĶæľ¯":76754,"Others":76755,"ç´Ľç´Ľ":76756,"Dra":76757,"Rear":76758,"ovir":76759,"åŁ¹åħ»åѦçĶŁçļĦ":76760,"দà§įর":76761,"-plane":76762,"Ġczyli":76763,"棺æĿIJ":76764,"ĠPest":76765,"ĠRita":76766,"åĩºéĶĻ":76767,"åİŁçĤ¹":76768,"Forward":76769,"ìłij":76770,"Ġdétermin":76771,"åľĺéļĬ":76772,"?>Ċ":76773,"urved":76774,"åľ¨çĶŁäº§":76775,"ĠDispon":76776,"Priority":76777,"Ġcloak":76778,"ieb":76779,"æĹ¥åħī":76780,"èµĦæ·±":76781,"ydd":76782,"ç²¾ç¥ŀçĹħ":76783,"Ġlocker":76784,"Ġgrund":76785,".Image":76786,"KP":76787,"ĠHDL":76788,"å¿ĥçİĩ":76789,"ĠÑĢазÑĢа":76790,"à¸ľà¸´à¸Ķ":76791,"åĽŀåΰ家":76792,"Ġ×Ĺ×ijר":76793,"Ġживе":76794,"Ġreminders":76795,"-activated":76796,"mul":76797,"æľī空":76798,"å±±èį¯":76799,"Ġnovelist":76800,"ĠTurning":76801,"Ġaugmentation":76802,"ĠSis":76803,"åĴĮå¤ĸ":76804,"æľ¬çĹħ":76805,"æĬķèµĦ人":76806,"软骨":76807,"Ġlieutenant":76808,"ĠConnections":76809,"ĠHemisphere":76810,"Ġkedua":76811,"æ°ijèIJ¥ä¼ģä¸ļ":76812,"ĠAxis":76813,"ĠOUR":76814,"ĠKru":76815,"èĢģå¹²éĥ¨":76816,"iete":76817,"Ġapnea":76818,"å¿ĥçIJĨåĴ¨è¯¢":76819,"ĠWheeler":76820,"Ġstains":76821,"å¼Ģå°ģ":76822,"ĠQuite":76823,"ĠÕ¬":76824,"ĠFinish":76825,"è¡°åĩı":76826,":self":76827,"citation":76828,"npm":76829,"ĠимеÑĤÑĮ":76830,"Ġsheds":76831,"çݯå¢ĥ污æŁĵ":76832,"Ġhoje":76833,"å¹´åīįçļĦ":76834,"ĠwiÄĻcej":76835,"ана":76836,"çŃīäºĨ":76837,"å¸ĥèݱ":76838,"ĠÙĨÙĪØ´":76839,"ĠÐľÐ°Ðº":76840,"篮æĿ¿":76841,".DataFrame":76842,"pok":76843,"ĠMush":76844,"ĠÃį":76845,"â̦.ĊĊ":76846,"ĠExisting":76847,"çݯåį«":76848,"ลà¹Į":76849,"ĠPathology":76850,"Ġaerospace":76851,"Ġrodents":76852,"pole":76853,"æľ¬èģĮ":76854,"ä½ĵå¾ģ":76855,"æ·»åĬłåīĤ":76856,"Ġimpartial":76857,"Ñļима":76858,"Ġlượ":76859,"å¾Īæ¸ħæ¥ļ":76860,"åħļ课":76861,"å¤ľèī²":76862,"ánchez":76863,"å°±åľ¨è¿ĻæĹ¶":76864,"Ġsleeves":76865,"æĪĸå°ij":76866,"iqué":76867,"ĠLearners":76868,"ï¼ŁãĢįĊ":76869,"elier":76870,"ÑħаÑĢ":76871,"below":76872,"为åѦçĶŁ":76873,"Citations":76874,"çļĦè·¯ä¸Ĭ":76875,"Ġvenge":76876,"Ġdanced":76877,"ĠìĦ¸ê³Ħ":76878,"grown":76879,"Ê¿":76880,"ĠStraw":76881,"денÑĤ":76882,"Ġcoraz":76883,"ç«ĭ项":76884,"æľįèį¯":76885,"çij¤":76886,"ĠModelling":76887,"](../":76888,"ĠоÑģновним":76889,"³³³³³³³³³³³³³³³³":76890,"ĠSomalia":76891,"polit":76892,"iners":76893,"ĠNPR":76894,"Ġrasa":76895,"ĠKC":76896,"éģĵæķĻ":76897,"Ġscam":76898,"约æľī":76899,"çIJĨæĥ³ä¿¡å¿µ":76900,"Ġseb":76901,"ä»ĸæĽ¾":76902,"ĠYuta":76903,"ĠUni":76904,"리를":76905,"Ä«n":76906,"ä¸Ĭåįĩåΰ":76907,"Ġvertebra":76908,"fw":76909,"fax":76910,"lx":76911,"æĹ¶æīĢ":76912,"weis":76913,"æİ¥è§¸":76914,"Ġremission":76915,"elser":76916,"彩票":76917,"Ġførste":76918,"VIII":76919,"ïľ":76920,"Ġaustral":76921,"ĠGink":76922,"Ġparab":76923,"ãĢįï¼Ī":76924,"缴æİ¥çļĦ":76925,"Truth":76926,"Ġuniformity":76927,"é»ijé¾Ļæ±Łçľģ":76928,"ᱣá±":76929,"eat":76930,"ĠNamed":76931,"åĩºéĿ¢":76932,"Ġdownside":76933,"Ġvuel":76934,"ĠFighting":76935,"å¡Ĺ":76936,"iasi":76937,"æľ±åħĥ":76938,"Ġflooring":76939,"validator":76940,"Ġintrigued":76941,".not":76942,"Yo":76943,"æĥ¬":76944,"æĮīä½ı":76945,"ietic":76946,"表示为":76947,"markets":76948,"ĠинÑģÑĤÑĢÑĥ":76949,"ĠInspection":76950,"Collins":76951,"Ġkale":76952,"çĹĬ":76953,"rictions":76954,"èµĦæºIJåĴĮ":76955,"Ġuniforms":76956,"Ġcontradictions":76957,"éģİç¨ĭä¸Ń":76958,"Ġwi":76959,"Ġgeno":76960,"åij¼åı«":76961,"epen":76962,"Ġsurreal":76963,"æľīä»Ģ麼":76964,"æĪijåģļ":76965,"对è§Ĩ":76966,"çī¹éķ¿":76967,"æµģéĢŁ":76968,"textarea":76969,"Ġconverges":76970,"èĥĨåŃIJ":76971,"Ġpitches":76972,"à§İস":76973,"Î¥":76974,"Ġmound":76975,"Ġstrutt":76976,"è¿IJè¡ĮçļĦ":76977,"Ġξα":76978,"ĠExhibition":76979,"pring":76980,"Ġtheaters":76981,"anuts":76982,"Ġjoyful":76983,"اÙĪØª":76984,"çĸ¯äºĨ":76985,"ĠdifÃŃcil":76986,"¢×Ķ":76987,"è§Ħéģ¿":76988,"ĠBlo":76989,"ç¼ĸåī§":76990,"Ġbedtime":76991,"乳头":76992,"ç¥ŀç»ıåħĥ":76993,"æĭĴç»ĿäºĨ":76994,"Ġinformációk":76995,"ĠLists":76996,"ког":76997,"ارات":76998,"ä¼ĺç¾İçļĦ":76999,"æĺ¯ä¸Ģéĥ¨":77000,"ĠÙĤاعدÙĩ":77001,"-report":77002,"ÑĪаеÑĤÑģÑı":77003,"æ¼Ĥæµ®":77004,"Ġmultimédias":77005,"Ġspleen":77006,"ä¹Ŀå·ŀ":77007,"Ġviolates":77008,"CMYK":77009,"áģĭ":77010,"ĠëĶĶ":77011,".row":77012,"Ġpreached":77013,"Ġworkings":77014,"Ġkonnte":77015,"ĠInher":77016,"çϼåĩº":77017,"åıĤä¸İåΰ":77018,"Ġorientations":77019,"Ġdeploying":77020,"ĠDimension":77021,"ĠEnhancing":77022,"Mesh":77023,"önt":77024,"اÛĮر":77025,"Ġà¦ľà¦¾à¦¤":77026,"èģªæĺİçļĦ":77027,"ĠCITY":77028,"ÑĩнÑĥÑİ":77029,"åħ¨å¤©":77030,"responding":77031,"åįĸå®¶":77032,"Peer":77033,"éģķãģĦ":77034,"ĠTruman":77035,"ç¨İè´¹":77036,"æľīå¤ļ大":77037,"Ġaspirin":77038,"äºĨçľ¼":77039,"åı¤ç±į":77040,"æ´ŀå¯Ł":77041,"Ġchromatin":77042,"Ġlaptops":77043,"å¯Ŀ室":77044,"-going":77045,"ĠSAF":77046,"ĠMär":77047,"ä¹Łæ¯Ķ":77048,"æŀľæłij":77049,"åĩĿè¡Ģ":77050,"ĠبعدÙĩ":77051,"Ġíķ¨ê»ĺ":77052,"çļĦå®ŀæĸ½":77053,"essori":77054,"Ġdisso":77055,"..........":77056,"Ġxxx":77057,"ĠChristie":77058,"olon":77059,"vectors":77060,"Ġoranges":77061,"wof":77062,"ä¸Ģèάéĥ½æĺ¯":77063,"-component":77064,"Ġtactic":77065,"Ġattentive":77066,"Ġcleansing":77067,"Ġmúlt":77068,"dv":77069,"Ġcobalt":77070,"ĠPreston":77071,"Orden":77072,"å¦ĤæŃ¤çļĦ":77073,"иÑģÑĭ":77074,"ç¥Ŀä½ł":77075,"ĠÑģлÑĥÑĩай":77076,"ĠBosnia":77077,"mui":77078,"ução":77079,"åĵĪåĪ©":77080,"ÙĬÙijØ©":77081,".amazon":77082,"lampi":77083,"{Y":77084,"isty":77085,"ĠEas":77086,"éĢĻä¹Ī":77087,"-lim":77088,".dll":77089,"åŃķæľŁ":77090,"寶寶":77091,"à«ģàªĤ":77092,"TPS":77093,"ÅĤych":77094,"Ġhalo":77095,"Ġdumped":77096,"imetry":77097,"迸":77098,"ä¸ĩæĪ·":77099,"çŁ³çļĦ":77100,"commission":77101,"Ġvoork":77102,"uição":77103,"Columns":77104,"Ġì²Ń":77105,"Friends":77106,"Ġhamb":77107,"åı¯åıĺ":77108,"Ġsofter":77109,"ĠSimmons":77110,"Phylum":77111,"ĠEtim":77112,"ĠShelley":77113,"à¹Ģà¸ķà¸Ńรà¹Į":77114,"giv":77115,"åĨ½":77116,"次åºı":77117,"çłĶä¿®":77118,"Ġadvising":77119,"Ġbroccoli":77120,"à¨Ĥ":77121,"\\nu":77122,"ÑĢовой":77123,"Someone":77124,"çĶŁéķ¿åıijèĤ²":77125,"etu":77126,"ĠCork":77127,"Ġbead":77128,"apoda":77129,"ccc":77130,"ä»»çͱ":77131,"ç²¾åŃIJ":77132,"Ġimmature":77133,".total":77134,"ĠConsent":77135,"Ġfj":77136,"å°ıåŃ¦æł¡":77137,"价款":77138,"ãĢijï¼Į":77139,"ĠBarber":77140,"Ġwelcomes":77141,"RequestMapping":77142,"æĻĭ级":77143,"Ġקר":77144,"à¹Ģà¸īà¸ŀาะ":77145,"大ä¸Ģ":77146,"нин":77147,"ÏĦεÏĤ":77148,"Ġবà§įযবহার":77149,"\"-":77150,"ä¸Ń举":77151,"æ¶Ŀ":77152,"ĠдеÑĢ":77153,"åĽĽäºĶ":77154,"ĠEncyclop":77155,"Ġfireworks":77156,"Äģt":77157,"elligence":77158,"Ġmicrop":77159,"ĠÄijiá»ĩn":77160,"ophyta":77161,"ĠHypothesis":77162,"รà¹Īาà¸ĩà¸ģาย":77163,"ĠReb":77164,"ecost":77165,"å¾´":77166,"å°ıé±¼":77167,"ยม":77168,"é¡¹çĽ®ç®¡çIJĨ":77169,"é¢Ŀå¤ĸçļĦ":77170,"-hop":77171,"ĠاÙĦØ´Ùħس":77172,"Ġkomunik":77173,"çαå°Ķ":77174,"Ġbrokers":77175,"å½Ĵè¿ĺ":77176,"éĴ¢æĿ¿":77177,"organized":77178,"'alt":77179,"ĠUM":77180,"Ġ/ĊĊ":77181,"çĹħçĹĩ":77182,"è¯»éŁ³":77183,"Ġstehen":77184,"Unix":77185,"ĠPreservation":77186,"Ġmoderne":77187,"ĠCounc":77188,"大å¥ĸ":77189,"æ´»äºĨ":77190,"æĶ¾æĺł":77191,"ĠÑĤоп":77192,".grid":77193,"è¸ıä¸Ĭ":77194,"orce":77195,"åĨħæł¸":77196,"Ġspectators":77197,"اضÙĬ":77198,"Ġ...,":77199,"adrat":77200,"åύä¸Ń":77201,"ç»¿åľ°":77202,"Ġש×Ŀ":77203,"Ġulcers":77204,"ĠÑģколÑĮко":77205,"ĠBarton":77206,"ĠÑģоÑģÑĤоиÑĤ":77207,"asional":77208,"fern":77209,"åĿ·":77210,"åĽŀåij³":77211,"äºĨä¸Ģä»¶":77212,"Ġarticulation":77213,"à§įযà§ĩর":77214,"ĠMetals":77215,"æ··åIJĪçī©":77216,"Ġtentative":77217,"ĠпоÑĤен":77218,"Ġsignify":77219,"åģ¥èĦ¾":77220,"ä»ĻåŃIJ":77221,"ĠëĮĢíķ´":77222,"ĠKanpo":77223,"ĠÑĥÑĢавнение":77224,"Ws":77225,"omn":77226,"ĠTend":77227,"åΰ家":77228,"наÑĤа":77229,"æĢ»åĴĮ":77230,"-treatment":77231,"Entre":77232,"ĠFritz":77233,"Ġspäter":77234,"ãĢĤâĢĿ(":77235,"Ġpresses":77236,"ĠÙĥÙĩ":77237,"éļĶçĿĢ":77238,"ĠÑģин":77239,"Ġassassination":77240,"leh":77241,"agara":77242,"illage":77243,"çϽè¡Ģ":77244,"สà¹Į":77245,"æ²¹çĶ»":77246,"ĠBoost":77247,"Neil":77248,"ãģ§ãģįãģªãģĦ":77249,"Ġpigments":77250,"为èĩªå·±çļĦ":77251,"Ġ-(":77252,"ĠSeal":77253,"éĿŀçī©è´¨":77254,"Ġë°ķ":77255,"ĠاÙĦخط":77256,"Ġjsme":77257,"Ġbattling":77258,"Ġmundane":77259,"ério":77260,"åĬ¨æijĩ":77261,"è§ģ她":77262,"overflow":77263,"ĠоÑĤмеÑĤ":77264,"é¢ijé¢ij":77265,"à¹Ĥà¸Ľà¸£":77266,"éĴ¢ä¸Ŀ":77267,"IFICATION":77268,"Ġεξ":77269,"..................................................................":77270,"à¦ŀà§įà¦ļ":77271,"çļĦ马":77272,"Ġcloves":77273,"æĢ¥éĢŁ":77274,"Ġredsh":77275,"à¹ĩà¸Ī":77276,"æĥ³è±¡åĬĽ":77277,"Ġjavascript":77278,"ĠباشÙĨد":77279,"à¸ģิà¸Īà¸ģรรม":77280,"Ġnouvelles":77281,"().":77457,"Ky":77458,"Ġbeasts":77459,"**.ĊĊ":77460,"æĮĩæİ§":77461,"åIJĥçĿĢ":77462,"Ġzeigt":77463,"ĠConfidence":77464,"Ġphospholip":77465,"åħ¬å¸ĥçļĦ":77466,"ĠKosovo":77467,"_the":77468,"âĢĥ":77469,"Ġatmos":77470,"Ġmarc":77471,"}}(\\":77472,"ĠCrom":77473,"çĶŁåŃIJ":77474,"çĦ¶åľ°":77475,"°.":77476,"ä½Ĩæĺ¯åį´":77477,"βα":77478,"ĠGeorgian":77479,"ὴν":77480,"ulants":77481,"ĠâŁ":77482,"交éģĵ":77483,"è§ģåΰäºĨ":77484,"Ġpope":77485,"Ġdiversi":77486,"Ġfurry":77487,"Ġwod":77488,"ĠEy":77489,"controllers":77490,"é£ŀèι":77491,"رÙĬÙħ":77492,"-olds":77493,"{W":77494,"Ġkj":77495,"大ä¸ĵ":77496,"åĴĮå®īåħ¨":77497,"ĠReformation":77498,"èĤī身":77499,"ĠмалÑĭ":77500,"Ġdej":77501,"umu":77502,"things":77503,"ĠskÅĤad":77504,"åħ«æĸ¹":77505,"بدأ":77506,"Ġ=-":77507,"Ġmucus":77508,"Ġasymptotic":77509,"Ġanchored":77510,"Ġmanier":77511,"Ġattr":77512,"äºĨä¸Ģéĺµ":77513,"Lean":77514,"-leading":77515,"ãĥģãĥ£":77516,"æĸ°ä¸ŃåĽ½æĪIJç«ĭ":77517,"ayaan":77518,"ĠاÙĦتارÙĬØ®":77519,"Ġmaladie":77520,"Ġনির":77521,"nk":77522,"assemb":77523,"Ġstartled":77524,"iums":77525,"Ġsalient":77526,"ربÛĮ":77527,"æ¹¾åĮº":77528,"ĠMey":77529,"Ġentail":77530,"Ġдем":77531,"(\"{":77532,"REEN":77533,"波浪":77534,"ä½ľåĵģä¸Ń":77535,"Ġflown":77536,".sqrt":77537,"ĠعاÙĦÙħ":77538,"ĠHarriet":77539,"-reaching":77540,"Ġmesma":77541,"Ġnods":77542,"Ġživ":77543,"Ġnarrowly":77544,"Ġintertwined":77545,"Ġzest":77546,"Ġconsortium":77547,"ĠتÙĨاÙĪÙĦ":77548,"AMES":77549,"ĠChess":77550,"æ³ķå¾ĭ责任":77551,"Ġmiglior":77552,"neas":77553,"ç¼ĸæİĴ":77554,"设置çļĦ":77555,"èµ¶å¿Ļ":77556,"Ġbraking":77557,"ĠÕ¥Õ¶":77558,"dle":77559,"enten":77560,"èĢĮ使":77561,"Äħt":77562,"_db":77563,"Ġidiot":77564,"ÙĴÙĦ":77565,"æľĢ好æĺ¯":77566,"wendung":77567,".Assert":77568,"ĠCret":77569,"ĠMAG":77570,"ayas":77571,"å¦ĤæľŁ":77572,"Ġblanc":77573,"ociate":77574,"Ġviewpoints":77575,"ĠдеÑĢев":77576,"Ġunpack":77577,"Ġremun":77578,"ĠðŁĩ":77579,"èķ©":77580,"æľĢ大éĻIJåº¦åľ°":77581,"-rest":77582,"enity":77583,"ä¸įèµ·æĿ¥":77584,"ĠlastName":77585,"ĠØ¥ÙĦÙĬ":77586,"à©ģ":77587,"å®Ŀå®ĿçļĦ":77588,"Ġúltimos":77589,"Corn":77590,"Ġquod":77591,"åĩºæ°´":77592,"åħ¨çıŃ":77593,"عاÙħ":77594,"æĪ´ä¸Ĭ":77595,"-hearted":77596,"ĠназÑĭваеÑĤÑģÑı":77597,"LIN":77598,"ç®į":77599,"æİ¨ç¿»":77600,"èĥĥåı£":77601,"гÑĥÑĢа":77602,"Ġwandered":77603,"ालà¥ĩ":77604,"Ġfishermen":77605,"Australian":77606,"-ID":77607,"Cer":77608,"atzen":77609,"ĠStones":77610,"åįķåIJij":77611,"æķĻåѦ缮æłĩ":77612,"ológica":77613,"ĠMozart":77614,"(end":77615,"Hier":77616,"nesty":77617,"Ñģен":77618,"ï¼Łï¼ģâĢĿĊĊ":77619,"æ·±éĤĥ":77620,"ytics":77621,"Sharp":77622,"ĠADC":77623,"ÏĢοι":77624,"ĠAux":77625,"Ġcomplements":77626,"ÙĬاÙĩ":77627,"å¦ĸæĢª":77628,"Ġfrances":77629,"æĺŁæľŁåħŃ":77630,"สิà¸Ĺà¸ĺิ":77631,"éĵħç¬Ķ":77632,"conde":77633,"ĠCentimeters":77634,".strip":77635,"è§Ĥå¯Łåΰ":77636,"Ġsophistication":77637,"Ġcomorbid":77638,"åĪĨé¡ŀ":77639,"举æ±ī":77640,"ÑĽÐµ":77641,".scss":77642,"olg":77643,"Ġexogenous":77644,"Ġclaws":77645,"azol":77646,"racia":77647,"Independent":77648,"éĹ²ç½®":77649,"Ġdragons":77650,"Ġunrealistic":77651,"ĠProfessionals":77652,"Night":77653,"Ġheuristic":77654,"åĪĨ红":77655,"å½ĵæĻļ":77656,"æĤ¶":77657,"Ġì§ĢìĹŃ":77658,"ặt":77659,"ìŁģ":77660,"ĠãĢijĊĊ":77661,"orre":77662,"Ġseptic":77663,"Ġindiv":77664,"derabad":77665,"Reason":77666,"brevi":77667,"ĠЧе":77668,"Ġalgumas":77669,"è¾²æ¥Ń":77670,"ĠITS":77671,"髦":77672,"å®ŀå®ŀåľ¨":77673,"å¾®ç¬ijçĿĢ":77674,"Tips":77675,"sce":77676,"ä¸įæĸ¹ä¾¿":77677,"ä½łæ²¡æľī":77678,"নি":77679,"人åijĺåľ¨":77680,"Ġtroop":77681,"çĽ¯èijĹ":77682,"Ġযদ":77683,".username":77684,"ëĬĶëĭ¤":77685,"ĠSpringfield":77686,"ĠKlaus":77687,"ĠManufacturers":77688,"ĠнепоÑĤпÑĥ":77689,"Ġشرکت":77690,"è¿Ļå°ıåŃIJ":77691,"Ġblanks":77692,"Ġchores":77693,"gaard":77694,"ä»ĺåĩºçļĦ":77695,"Ġinformasi":77696,"åĸĬçĿĢ":77697,"ĠдиамеÑĤ":77698,"ĠFault":77699,"ç¾¹":77700,"Ġradiant":77701,"ĠPerkins":77702,"Ġprav":77703,"Ġθα":77704,"à¹Ģà¸Ĭืà¹īà¸Ń":77705,"ĠDietary":77706,"寵":77707,"Ġparticulier":77708,"ĠÙģÙĪ":77709,"ĠÙĨÙĪØ±":77710,"ĠWillie":77711,"ÑĤивнÑĭе":77712,"æĶ¿åįıå§Ķåijĺ":77713,"ailles":77714,"ĠElla":77715,"ĠGong":77716,"Ġquais":77717,"ĠпоÑıвлÑı":77718,"Ġplanta":77719,"ĠWM":77720,"Ġkonse":77721,"ĠGamma":77722,"Ġshaken":77723,"Ġdelim":77724,"medicine":77725,"Ġorigen":77726,"æĺ¾ç¤ºäºĨ":77727,"ĠDevon":77728,"é£İæł¼çļĦ":77729,"можно":77730,"ĠGabri":77731,"ĠÑĤеÑĢÑĢиÑĤоÑĢии":77732,"Jam":77733,"tok":77734,"ĠSAN":77735,"ĠCoordinate":77736,"åĤĢåĦ¡":77737,"áĨ":77738,"imetric":77739,"è§ij":77740,"Ġappelle":77741,"ä¸ŃçļĦä½ľç͍":77742,"ĠпоÑģÑĤ":77743,"Ġoriginates":77744,"幾天":77745,"ìĨ¡":77746,"æĹłçº¿ç͵":77747,"çĦļçĥ§":77748,"ĠPang":77749,"ç͍ä»Ģä¹Ī":77750,"ĠXavier":77751,"маÑı":77752,"à¸Ħรู":77753,"ĠдомаÑĪ":77754,"ĠÒ»":77755,"ĠCalled":77756,"หà¹Į":77757,"umerator":77758,"ĠMartÃŃ":77759,"Ġcoastline":77760,"à³Ĩಯ":77761,"Ġwatt":77762,"ä¸ĢçŃī":77763,"å®īåİ¿":77764,"-final":77765,"ì§ĢëĬĶ":77766,"ç»ıåħ¸çļĦ":77767,"Ġreagents":77768,"fixed":77769,"ĠViolet":77770,"第åįģåħŃ":77771,"generate":77772,"Observer":77773,"ĠWindsor":77774,"æįħ":77775,"Inventory":77776,"æĤĸ":77777,"Ġliste":77778,"Ġnumerically":77779,"蹤":77780,"ĠMaintaining":77781,"Ġexcessively":77782,"hely":77783,"åĴĮçłĶç©¶":77784,"Ġplanner":77785,"ä¹Łèĥ½å¤Ł":77786,"è¿Ľä¿®":77787,"Ġ'\"":77788,"ĠRever":77789,"äv":77790,"lexia":77791,"Ġakhir":77792,"Ġqualité":77793,"=r":77794,"ĠRaff":77795,"Ġ\"))Ċ":77796,"Ġpolish":77797,"ieties":77798,"Ġwonderfully":77799,"Ġdryer":77800,"approach":77801,"ÑĢеменно":77802,"åĿļå®ļä¸įç§»":77803,"Ġindefinite":77804,"DX":77805,"Ġonboard":77806,"åĩºåĬ¨":77807,"æºIJæĢ§":77808,"åij¨åħŃ":77809,"ĠاÙĦسرعÙĩ":77810,"ĠCONDITIONS":77811,"å°±è§īå¾Ĺ":77812,"è°´":77813,"اÙĦÙĬد":77814,"éĢģä½ł":77815,"Ġvelmi":77816,"Ġdiffus":77817,"Springer":77818,"tanleria":77819,"iline":77820,"ĠLIFE":77821,"ĠMinim":77822,"ä¸įåı¯æĪĸ缺":77823,"ĠKenny":77824,"à¹Ģà¸Ĥา":77825,"Ġcultivars":77826,"ĠKNOW":77827,"Ġapós":77828,"}}}\\":77829,"Ġpiez":77830,"åĪĽéĢłåĬĽ":77831,"ĠCSR":77832,"ĠMLB":77833,"ĠسرعÙĩ":77834,"Ġuplift":77835,"flutter":77836,"å¿«æ¨Ĥ":77837,"Ġaprendizaje":77838,".cloud":77839,"]):Ċ":77840,"mak":77841,"æ³¨çĽ®":77842,"æ²§æ¡ij":77843,"Ġmasc":77844,"ĠInk":77845,"comments":77846,"æłijä¸ĭ":77847,"Ġtutors":77848,"-kind":77849,"Constraint":77850,"ĠAO":77851,"igms":77852,"ĠгÑĥ":77853,"é¹Ĭ":77854,"Ġmastered":77855,"è®¤çľŁåŃ¦ä¹ł":77856,"åįģä¸Ģ竳":77857,"Ġbetrayed":77858,"Ġzituen":77859,"GEN":77860,"并举":77861,"ĠاÙĦÙħدار":77862,"Ġpretending":77863,"ĠHomeschool":77864,"Hindi":77865,"Qt":77866,"æĥŃ":77867,"Ġzir":77868,"åıĪå¼Ģå§ĭ":77869,"کز":77870,"ارض":77871,"å·¥ç¨ĭéĩı":77872,"çļĦäºĭåĦ¿":77873,"ĠBookmarks":77874,"×ķ׾×Ļ":77875,"Ġdilute":77876,"Ġadvisers":77877,"å®°çĽ¸":77878,"éŁ§å¸¦":77879,"Ġparalysis":77880,"Ġaggressively":77881,"imil":77882,"åľ°æ¯¯":77883,"主干":77884,"Ġextrac":77885,"åĨľä½ľçī©":77886,"æ±Łæ²³":77887,"ĠدرÛĮ":77888,"å¡«è¡¥":77889,"çĤ«èĢĢ":77890,"Impact":77891,"erView":77892,"ĠTx":77893,"tochrome":77894,"ĠRecording":77895,"æĪijåį´":77896,"æľĢéĢĤåIJĪ":77897,"Ġexplica":77898,"çļĦä¸Ģæł·":77899,"×ij×Ļ×Ŀ":77900,"Ġearns":77901,"åħ¨æĹ¥åζ":77902,"ä¸Ģå±Ĭ":77903,"Ġbelle":77904,"Ġloin":77905,"ĠMercy":77906,"çľĭåIJijäºĨ":77907,"ĠECG":77908,"ç»Łæ²»èĢħ":77909,"Nam":77910,"æĻĹ":77911,"缴è¨Ģ":77912,"éĢĻå°±æĺ¯":77913,"amental":77914,"Ġglaciers":77915,"hw":77916,"ĠBonds":77917,"ĠGert":77918,"æŃ»äºº":77919,"å¿§èĻij":77920,"Ġkonts":77921,")...":77922,"ZY":77923,"ÊĮ":77924,"ortune":77925,"æľĢç¾İçļĦ":77926,"ĠEnterprises":77927,"ĠWhitney":77928,"ĠREPORT":77929,"lod":77930,"Ġvere":77931,"compar":77932,"åįıåĬĽ":77933,"Ġyoungsters":77934,"æĶ¿åºľåĴĮ":77935,"ĠDecisions":77936,"atok":77937,"Ġcorso":77938,"Ġfollower":77939,"ĠCumm":77940,"ĠLicht":77941,"ortality":77942,"Ġshipment":77943,"idente":77944,"-from":77945,"Ġcrashing":77946,"ĠСк":77947,"æ³¢æ¾ľ":77948,"iotensin":77949,"çļĦåĨħæ¶µ":77950,"ĠAbdullah":77951,"Ġbipart":77952,"ĠاصÙĦÛĮ":77953,"ĠTus":77954,"ĠHume":77955,"erma":77956,"å¹¶çͱ":77957,"æķĻåŃ¦è®¾è®¡":77958,"Ġблиз":77959,"кономÑģки":77960,"\\neq":77961,"Ġسپ":77962,"产åĵģåĴĮ":77963,"æľ¨å¤´":77964,"âŦ":77965,"à¹ģละà¸ģาร":77966,"ué":77967,"ĊĠĊ":77968,"ĠEf":77969,".,ĊĊ":77970,"Ġprescriptions":77971,"ĠÑģпÑĢа":77972,"Ġpositives":77973,"ĠGroÃŁ":77974,"Ny":77975,"çļĦ产çĶŁ":77976,"ĠRelevant":77977,"\\)_":77978,"ä¿¡æģ¯åĴĮ":77979,"Ġìłij":77980,"Worker":77981,"Ġtoen":77982,"ĠRender":77983,"å°ı鼨":77984,"ĠExtrem":77985,"ç¶²ç«Ļ":77986,"advantages":77987,"æłĩæĿĨ":77988,"ĠOrb":77989,"incare":77990,"ĠBev":77991,"ãĤ¸ãĤ§":77992,"Ġmasked":77993,"ĠÙĦجرÙħ":77994,"Ġfringe":77995,"ĠDrosophila":77996,"Ġindist":77997,"Ġcolonists":77998,"åħijçݰ":77999,"æİ¡ç͍":78000,"ĠNatalie":78001,"Åģ":78002,"Ġjaren":78003,"ĠUma":78004,"æİ°":78005,"Ġskate":78006,"æŃ¥æŃ¥":78007,"ĠPrevalence":78008,"Ġforgiven":78009,",...ĊĊ":78010,"jc":78011,"éĿ¢ç©į":78012,"ĠQuote":78013,"araan":78014,"æįŁçĽĬ":78015,"æ©Łåύ":78016,"OBJECT":78017,"人家çļĦ":78018,"Ġhaw":78019,"ĠاÙĦتش":78020,"िम":78021,"鸡èĤī":78022,"ĠкÑĢай":78023,"ĠÑģекÑĥн":78024,"probably":78025,"ĠÑĺÑĥнÑĥ":78026,"QM":78027,"سات":78028,"äºĨä¸Ģå¥Ĺ":78029,"模çī¹":78030,"-dess":78031,"Ġsocialization":78032,"ĠкаÑĤего":78033,"IVER":78034,".Label":78035,"Ġnosotros":78036,"Ġbiomarker":78037,"ä¸Ģè¾Ī":78038,"äºĮ代":78039,"äºĨä¸Ģ段":78040,"ημ":78041,"å°¿éģĵ":78042,"ä¸Ģåħ±æľī":78043,"erva":78044,"ç¢İäºĨ":78045,"ĠSublunar":78046,"GMT":78047,"vee":78048,"ĠVintage":78049,"æ³ķå®Ŀ":78050,"اÙĨا":78051,"ĠÔµ":78052,"street":78053,"/sec":78054,"Around":78055,"[_":78056,"ÙĶ":78057,"åı¯èĥ½åľ¨":78058,"åİĭæł¹":78059,"Ġevento":78060,"Ġâ̦,":78061,"Ġoccupying":78062,"主ä½ĵ责任":78063,"Ġ×ĸ×IJת":78064,"éĨ«çĻĤ":78065,"ĠBroadcasting":78066,"\\gamma":78067,"fait":78068,"money":78069,"Ġpardon":78070,"Ġforestry":78071,"ÈĽie":78072,"ĠCarmen":78073,"wir":78074,"Ġmanganese":78075,"ĠGear":78076,"Ġrer":78077,"ĠProposal":78078,"azor":78079,"æľįåĬ¡äºİ":78080,"ĠImmediately":78081,"Ġgymn":78082,"æĪIJåĵ¡":78083,"æīĢ为":78084,"ĠتÙĪØ³":78085,"(sys":78086,"ĠINV":78087,"Ġaltijd":78088,"å®īå¸Ĥ":78089,"ĠÏĦÏį":78090,"åĢŁçĿĢ":78091,"个æľĪçļĦ":78092,"æīĢè¿°çļĦ":78093,"ĠколиÑĩеÑģÑĤва":78094,"aldehyde":78095,"Ġindefinitely":78096,"Josh":78097,"\\Component":78098,"ĠDoyle":78099,"ĠÙĪØ§":78100,"Ġmodernity":78101,"äºĮåįģåħ«":78102,"Ġmesmer":78103,"Urls":78104,"ĠVoIP":78105,"ë²Īíĺ¸":78106,"ãģĦãģĨ":78107,"ĠÑĦÑĥн":78108,"éļľå®³":78109,"表çݰå¾Ĺ":78110,"æĸ½å·¥çİ°åľº":78111,"被害人":78112,"¥å¹¸":78113,"Ġdiver":78114,"ĠLer":78115,"лоб":78116,"ĠStyles":78117,"ä½łåĸľæ¬¢":78118,"éĩ῏©":78119,"ĠTRANS":78120,"änger":78121,"Ġunreliable":78122,"éĿĻéĿĻåľ°":78123,"ĠSalam":78124,"riere":78125,"åĬ¨äºº":78126,"ujemy":78127,"åļ£å¼ł":78128,"HIP":78129,"Temperature":78130,"Ġperplex":78131,"ĠUrb":78132,"ĠKilograms":78133,"ç·©ç·©":78134,"dbo":78135,"Ġcomenz":78136,"Ġatrib":78137,"rouse":78138,"Ġropes":78139,"马çļĦ":78140,"Ġgreedy":78141,"ĠиндивидÑĥалÑĮ":78142,"ixels":78143,"ĠAssoc":78144,"etted":78145,"为äºĨä¿Ŀè¯ģ":78146,"Ġnuovo":78147,"à¸ķรà¸ĩ":78148,"નà«ĩ":78149,"/blog":78150,"åĩĿç»ĵ":78151,"ĠLenin":78152,"Ġpuff":78153,"chap":78154,"ä¸įè¿ľå¤Ħ":78155,"èµ°è¿ĩåİ»":78156,"Ġrecursion":78157,"Ġtsunami":78158,"Ġweniger":78159,"ĠHernandez":78160,"ĠاÙħر":78161,"ĠWhilst":78162,"è¡ĢèĤī":78163,"é£Łçī©çļĦ":78164,"å®ıä¼Ł":78165,"å£ĩ":78166,"Ġscissors":78167,"ç¼ī":78168,"ĠEngel":78169,"Ïĥαν":78170,"ĠÙĩÙĦ":78171,"èIJĮèĬ½":78172,"Ġsourcing":78173,"'}":78174,"imester":78175,"éħįåζ":78176,"çł´æįŁ":78177,".Fore":78178,"Figures":78179,"handlung":78180,"ĠARM":78181,"åŁİéķĩåĮĸ":78182,"ĠгодÑĭ":78183,"á̏áĢ":78184,"åİĭåĬĽçļĦ":78185,"ë¡ľìļ´":78186,"knowledge":78187,"Ġreperc":78188,"ëĪ":78189,"ĠFinger":78190,"为æĮĩ导":78191,"å®ļå±ħ":78192,"èε":78193,"æ·©":78194,"ä¿Ŀ湿":78195,"å¿«æŃ¥":78196,"ä¼ģä¸ļæĸĩåĮĸ":78197,"ĠPerth":78198,"æ±īåŃIJ":78199,"åĩ¹éĻ·":78200,"Ġnib":78201,"Ġconferred":78202,"ĠBN":78203,"人éĢł":78204,"Ġslate":78205,"ĠVisualFractions":78206,"gray":78207,"ża":78208,"ĠMultimedia":78209,"ãģĬãĤĪ":78210,"å½ĵçĿĢ":78211,"çļĦä¸Ģåı¥è¯Ŀ":78212,"é§Ľ":78213,"Ġtratamiento":78214,".controller":78215,"Ġtyrosine":78216,"ĠминÑĥÑĤ":78217,"ĠÙħجÙħÙĪØ¹Ùĩ":78218,"reten":78219,"Ġsings":78220,"Ġinvestigative":78221,"ãĥĭãĥ¥":78222,"(<":78223,"Ġdared":78224,"Ġthá»ĥ":78225,"Ġleuc":78226,"ÙģÙĪ":78227,"ä¾Ľæ±Ĥ":78228,"Ġsemif":78229,"Ġtemas":78230,"修补":78231,"ĠEducación":78232,"ĠQuestionnaire":78233,"ç§īæī¿":78234,"Ġdeutschen":78235,"ertas":78236,"æĹ¥è¯Ń":78237,"ĠÑļ":78238,"åĢįçļĦ":78239,"imbing":78240,"å£ģåŀĴ":78241,"å®īè£ħåľ¨":78242,"á¿ĨÏĤ":78243,"为大":78244,"åīĥ":78245,"Ġtriang":78246,"ابÙĤ":78247,"é½Ĵ":78248,"æĢĢä¸Ń":78249,"èµĦæľ¬çļĦ":78250,"總æĺ¯":78251,"Ġlichaam":78252,"วิà¸Īัย":78253,"Ġducks":78254,"Ġanh":78255,"管å§Ķä¼ļ":78256,"Ġelectrom":78257,"ĠпоÑĤом":78258,"Ġzáklad":78259,"Ġcélulas":78260,"æľ¯åīį":78261,"Ġcertaines":78262,"ĠActing":78263,"ൽ":78264,"ĠÑĤоÑĤ":78265,"Ġphenomenal":78266,"Ġcumpl":78267,"åĴĮå¤ĦçIJĨ":78268,"++]":78269,"ĠChecks":78270,"Ġinternationale":78271,"ĠSampling":78272,"Ġpublik":78273,"購買":78274,"ĠAlgeria":78275,"ĉname":78276,"Ġlute":78277,"çŃł":78278,"Ġعب":78279,"+t":78280,"Ġcannon":78281,"éĤ£æ¨£":78282,"ède":78283,"Ġembodies":78284,"Ġ×ķ×Ĵ":78285,"Ġpagan":78286,"çε士":78287,"_as":78288,"copyright":78289,"Ġdá":78290,"lean":78291,"åĴĮç»Ħç»ĩ":78292,"Ġintolerance":78293,"åīįä¸ĸ":78294,"ĠÙĪÙħع":78295,"isman":78296,"Ġwaits":78297,"Ġarid":78298,"縫":78299,"ä¸ĸ纪åĪĿ":78300,"ĠTGF":78301,"Ġvyt":78302,"åı¯æĥ³":78303,"她已ç»ı":78304,"Ġxen":78305,"ának":78306,"klich":78307,"ĠBuildings":78308,"Azure":78309,"ĠвопÑĢоÑģÑĭ":78310,"åĽ½ä¹ĭ":78311,"à¹Ģà¸Ĥียà¸Ļ":78312,"æ°´åŁŁ":78313,"ä»Ģä¹Īéĥ½ä¸į":78314,"à¸ķà¹ī":78315,"åı¦è¡Į":78316,"Leon":78317,"ĠCatch":78318,"vero":78319,"ä½łèªª":78320,"ĠнаÑĪей":78321,"çļĦä¸Ģå¹ķ":78322,"Ġexcuses":78323,"æīİæł¹":78324,"æĺĨä»ij":78325,"Ġcalmly":78326,"-European":78327,"-cur":78328,"/react":78329,"mad":78330,"åħ¨éķ¿":78331,"æĦı象":78332,"å¤ĩ注":78333,"Ġsuperst":78334,"ĠMetab":78335,"Decision":78336,"ĠNegot":78337,"Ġthighs":78338,"ç͵èĥ½":78339,"æ¸ħäºĨ":78340,"è¡Ģ红":78341,"หà¸į":78342,"èĻļæĹł":78343,"ĠAddressing":78344,"Ġknit":78345,"ç¼ĺåĪĨ":78346,"Historical":78347,"ĠDaisy":78348,"thanks":78349,"æ°´è§£":78350,"ĠBeruf":78351,"ĠкÑĥлÑĮÑĤÑĥÑĢ":78352,"methyl":78353,"rÄĻ":78354,"}.Ċ":78355,"两éĿ¢":78356,"èĢģæĺ¯":78357,"Ġfunctionally":78358,"ĠMech":78359,"ĠPeriodic":78360,"Ġprzedstaw":78361,"ĠLuxembourg":78362,"uation":78363,"ĠBits":78364,"isex":78365,"ERENCE":78366,"æ³¢æĸ¯":78367,"ĠìĥĿìĦ±":78368,"Ġglacier":78369,"OY":78370,"Ġdiscourses":78371,"ĠPacket":78372,"ĠCarbohyd":78373,"ĠTU":78374,"ĠBRA":78375,"åĴĮæĹ¶éĹ´":78376,"ĠJest":78377,"dew":78378,"ë¡Ģ":78379,"èµĦæĸĻçļĦ":78380,"à®°à¯įà®ķ":78381,"ĠÄĮesk":78382,"æĨ§æĨ¬":78383,"Royal":78384,"åľ°åIJį":78385,"æķĻåĬ¡":78386,"è¦ıåīĩ":78387,"ĠпÑĢодÑĥкÑĨии":78388,"çļĦçĶŁ":78389,"Ġtrên":78390,"-fin":78391,"_case":78392,"Ĺש×ij":78393,"ĠÅ¡kol":78394,"Ġpredecessors":78395,"ä¸Ńéĸĵ":78396,"Ġ\\{":78397,"解説":78398,"é£ŀå¿«":78399,"Ġpolymeric":78400,"Ġenhancements":78401,"ாய":78402,"Ġrejects":78403,"ĠмеÑĤоди":78404,"Film":78405,"Ġinstituted":78406,"uncher":78407,"رÙĬÙĥا":78408,"urized":78409,"å¾Ĺä¸Ģ":78410,"èĭĩ":78411,"_source":78412,"èĤ¾åĬŁèĥ½":78413,"-Verlag":78414,"Specific":78415,"]$":78416,"Ġhade":78417,"æī¾ä½ł":78418,"ĠVerify":78419,"Ġdiferente":78420,"ä»ĸè§īå¾Ĺ":78421,"ĠKE":78422,"éĥ½å±ŀäºİ":78423,"ä¸ĩ亿":78424,"å¾·åįİ":78425,"ĠÐŁÑĥ":78426,"è¿Ľä¸ĢæŃ¥æıIJåįĩ":78427,"Ġselenium":78428,"èį¡èį¡":78429,"ÛĮØ´Ùĩ":78430,"伪è£ħ":78431,"èħ¦è¢ĭ":78432,"ĠCandidates":78433,"Ġlek":78434,"åı¯éĢī":78435,"Ġworkflows":78436,"åĨľçī§":78437,"è´¢åĬĽ":78438,"ĠDepth":78439,"ĠخدÙħ":78440,"æī©åħħ":78441,"éĺĪå̼":78442,"Ġshakes":78443,"Ġcrater":78444,"强è°ĥäºĨ":78445,"×ŀ×Ļ×Ŀ":78446,"Binomial":78447,"uche":78448,"ç³§":78449,"æİĴ骨":78450,"æŃ»æŃ»":78451,"åĸĿéģĵ":78452,"ĠArabian":78453,"Ġextraordin":78454,"Ġatividades":78455,"Ġsitt":78456,"æĺ¾çĦ¶æĺ¯":78457,"Ġannotated":78458,"ĠìĦ¤ëªħ":78459,"Ġforge":78460,"Ġ#[":78461,"åĬŀ好":78462,"Ġ×Ķ×ĸ×Ķ":78463,"æ¤įæłij":78464,"×ķצ×IJ":78465,"æĬĹæĹ¥æĪĺäºī":78466,"Wie":78467,"vp":78468,"agli":78469,"人æķĻçīĪ":78470,"ĠQB":78471,"زا":78472,"Greek":78473,"Ġdst":78474,"Ġdese":78475,"宫廷":78476,"IRS":78477,"-indust":78478,"ĠÑĤÑĭÑģ":78479,"å°ıèĬ±":78480,"Ġ×ŀ×ĺ":78481,"placeholder":78482,"å®Īåį«":78483,"OTH":78484,"çijª":78485,"ĠSTEP":78486,"ç³ĸæŀľ":78487,".debug":78488,"åħ³èĬĤçĤİ":78489,"Ġpresumption":78490,"Ġlingkungan":78491,"å¤ļæĥ³":78492,"æ¯Ķä¸Ĭå¹´":78493,"Ġedo":78494,"é£İéĻ©çļĦ":78495,"Ġsedent":78496,"ĠклаÑģÑģа":78497,"ĠVernon":78498,"ä¸İéĿŀ":78499,"Ġcommunion":78500,"Ġchlorophyll":78501,"Tables":78502,"Ġws":78503,"ifiques":78504,"ologne":78505,"ĠAdemás":78506,"à¸Īัà¸ģ":78507,".setState":78508,"æŃ»äº¡çļĦ":78509,"Ġobstructive":78510,"}.ĊĊ":78511,"Ġdisconnect":78512,"æµ·æ£ł":78513,"çĬ¶çļĦ":78514,"责任人":78515,"aktion":78516,"è¿Łè¿Ł":78517,"ĠSidney":78518,"ĠpÅĻes":78519,"çªģåıijäºĭä»¶":78520,"éŁŃèıľ":78521,"æĺ¯å¤©":78522,"ĠRW":78523,"æ¶²æĻ¶":78524,"çĮ®è¡Ģ":78525,"Ġúltima":78526,"ihat":78527,"rotic":78528,"ĠDram":78529,"ĠComedy":78530,"å¤įæĹ¦":78531,"Ġgiov":78532,"Guide":78533,"'ag":78534,"/ad":78535,"漩":78536,"å¤įåı¤":78537,"à¹ģม":78538,"åħĴç«¥":78539,"çѹèµĦ":78540,"odzi":78541,"Include":78542,"(\"-":78543,"Ġelett":78544,"éĿĻç͵":78545,"æĢĿæĥ³ä¸Ĭ":78546,"ĠÐŀна":78547,"Ġadmitting":78548,"辨è¯ģ":78549,"jarah":78550,"ä¹ŁåĽłæŃ¤":78551,"achie":78552,"æ°´æ±ł":78553,"ALK":78554,"ĠапÑĢе":78555,"Ġsectional":78556,"Ġwaard":78557,"Ġepidemiological":78558,"ä¸Ģåĩ»":78559,"sharp":78560,"éĥ½æĺ¯çͱ":78561,"ĠпÑĢиме":78562,"åįĸæĸ¹":78563,"IRA":78564,"å·§åIJĪ":78565,"帶ä¾Ĩ":78566,"ĠHawk":78567,"è±IJå¯Į":78568,"æĶ¹æĪIJ":78569,"ĠбÑİ":78570,"Ġkomon":78571,"ĠFIL":78572,"cliffe":78573,"代表æĢ§çļĦ":78574,"Ġminimizes":78575,"è½ī身":78576,"çĽ¸ä¼¼çļĦ":78577,"Ġprzypadku":78578,"vell":78579,"åıĪå¦Ĥä½ķ":78580,"à´ķ":78581,")=>":78582,"Ġunsatisf":78583,"Ġtradicional":78584,"à§Ĥরà§įণ":78585,"å¼Ĭ端":78586,"ĉbool":78587,"ĠCEST":78588,"ĠVij":78589,"æ°ijçľ¾":78590,"Ġпам":78591,"ĠInfectious":78592,"Ġinglés":78593,"Jane":78594,"ĠSaving":78595,"åľ°é»Ħ":78596,"èĢĮ没æľī":78597,"æķĻå®ĺ":78598,"计çĶŁ":78599,"è¿IJåĬ¿":78600,"åīªè¾ij":78601,"(TreeNode":78602,"èł»":78603,"Construction":78604,"ĠÑĪколÑĭ":78605,"-Star":78606,"Ġcommits":78607,"਼":78608,"æĿĤèįī":78609,"žÃŃvá":78610,"çļĦåĬ¨åĬĽ":78611,"ĠBenn":78612,"лÑıÑħ":78613,"éħ¸çĽIJ":78614,"ĠÐĺва":78615,"ãģĪãģŁ":78616,"ĠShirley":78617,"Cra":78618,"ĠKatz":78619,"زاÙħ":78620,"ĠEditing":78621,"ਹ":78622,"Ġlecturer":78623,"æ»ĭåħ»":78624,"Ġà¦¸à¦®à§Ł":78625,"ĠFus":78626,"ç»´å°Ķ":78627,"ابد":78628,"åĪºåı²":78629,"Ġ×ij×Ļ×ķתר":78630,"å®ļä¹īçļĦ":78631,"Ġmandates":78632,"æĶ¾å¤§åύ":78633,"vf":78634,"çľĭå®Ī":78635,"ĠMayer":78636,"Ġbloodstream":78637,"Trump":78638,"ĠExtract":78639,"Ġbetrayal":78640,"bots":78641,"kot":78642,"Ġpensions":78643,"ä¸įåħ·å¤ĩ":78644,"æĿ¥å®ĮæĪIJ":78645,"ordre":78646,"å°ıé»ij":78647,"她æīį":78648,"æĺ¯ä¸Ģ座":78649,"encoded":78650,"ĠInterval":78651,"åĬ£åĬ¿":78652,"Ġremediation":78653,"ĠMuller":78654,"wg":78655,"ĉĉĠ":78656,"ieli":78657,"ç²¾ç¾İ":78658,"æĶ¯è¡Į":78659,"Ġtalags":78660,"çļĦ主è¦ģåİŁåĽł":78661,"Ġmotivational":78662,"Ġmundial":78663,"orgen":78664,"ï¼£":78665,"ĠSnyder":78666,"è¯įçļĦ":78667,"ĠConfigure":78668,"ä¸ĵåĪ©æĿĥ":78669,"ĠbÄĻdÄħ":78670,"åĴĮåŃ¦ä¹ł":78671,"åĽ½èµĦ":78672,"Ġtee":78673,"Ġtwisting":78674,"niu":78675,"Ġê²ĥìľ¼ë¡ľ":78676,"ĠTalking":78677,"bear":78678,"ĠCyp":78679,"说起æĿ¥":78680,"racuse":78681,"æľĽè¿ľ":78682,"éłĹ":78683,"ç´§äºĨ":78684,"Ġestaba":78685,"Ġpasado":78686,"ĠìĿ´íķ´":78687,"ä¸ĭä¸ĢåĪ»":78688,"ีà¹īย":78689,"æĺŁæľŁäºĶ":78690,"Ġcursed":78691,"å¤īåĮĸ":78692,"dies":78693,"åľ¨æĪij们çļĦ":78694,"éĤ£æ¬¡":78695,"æľªæĪIJå¹´":78696,"Ġ!Ċ":78697,"Units":78698,"ç¯ĩå¹ħ":78699,".Base":78700,"æ·±åħ¥çļĦ":78701,"ĠMahm":78702,"Promise":78703,"agher":78704,"رخ":78705,"æĹ¥æ¸IJ":78706,"å±ķå¼ĢäºĨ":78707,"Ġlongue":78708,"æ¶²ä¸Ń":78709,"ظÙĬÙħ":78710,"ĠÚ¯ÛĮر":78711,"å¯ĨåĪĩ缸åħ³":78712,"ĠпÑĢогÑĢаммÑĭ":78713,"Jones":78714,"Ġreinst":78715,"Ġuninter":78716,"çŃīè¿Ľè¡Į":78717,"åĬłæĮģ":78718,"æģº":78719,"Ġcheating":78720,"ĠÙĤÙĦ":78721,"å°ıæľĭåıĭ们":78722,"[Hentet":78723,"_if":78724,"ĠBai":78725,"å¤ĸ伤":78726,"Ġmatriz":78727,"âĪĢ":78728,"emaakt":78729,"Ġtanah":78730,"apsing":78731,"Ġبرگ":78732,"Ġaffordability":78733,"almaz":78734,"icl":78735,"人人éĥ½":78736,"priv":78737,"å±ķéĸĭ":78738,"Ġíķ©":78739,"Ġmisunderstand":78740,":I":78741,"åľ¨ç¬¬ä¸Ģ":78742,"åĬłçĤ¹":78743,"åIJĦæĿij":78744,"é¢Ħè¨Ģ":78745,"Ġbaptized":78746,"prés":78747,"åŁİå¸Ĥ建设":78748,"èģļåIJĪçī©":78749,"éīĦ":78750,"ĠлÑİби":78751,"Ġoutweigh":78752,"ä»ĸ表示":78753,"å¤ļæľī":78754,"azen":78755,"Ġsoftened":78756,"ková":78757,"ĠíħĮ":78758,"sid":78759,"Ġdär":78760,"ĠLIN":78761,"Ġاغ":78762,"ovina":78763,"èĩªå·±æīĢ":78764,"rani":78765,"Ġmemoria":78766,"ä¸ĩå¤ļ":78767,"Ġgrounding":78768,"Ġstrengthens":78769,"Ġinspires":78770,"大å°ıå§IJ":78771,"states":78772,"Ġemph":78773,"iha":78774,"ÙĨدا":78775,"Ġtenderness":78776,"ateness":78777,"人éĻħåħ³ç³»":78778,"ĠPlymouth":78779,"Ġtalagsaon":78780,"Runtime":78781,"æīĭ游":78782,"æµģ泪":78783,"ÑĤел":78784,"æĶ¾æ£Ħ":78785,"社ä¼ļåĮĸ":78786,"ĠPerception":78787,"ĠØ´Ú©ÙĦ":78788,"Ġmůž":78789,"Ġcôté":78790,"Ġluk":78791,"Ġperish":78792,"ãģ®ãģł":78793,"boa":78794,"urse":78795,"å¹´ãģ«":78796,"ĠUnified":78797,"Ġcostitu":78798,"èĭ¦æģ¼":78799,"Ġdroits":78800,"Ġignores":78801,"Ġrationality":78802,"ĠÙĪÙĩذا":78803,"ĠÑĦÑĥнкÑĨиÑı":78804,"Ġsidlakan":78805,"ĠRahmen":78806,"Ġseawater":78807,"-rated":78808,";a":78809,"Ġfury":78810,"Ġcommonplace":78811,"ÑĩиÑģли":78812,"ĠCirculation":78813,"aeus":78814,"⣨":78815,"ä¼ļå°Ĩ":78816,"aryl":78817,"ĠÑģвеÑĢ":78818,"å¸Ĥä¸Ńå¿ĥ":78819,"ãĢĭâĢľ":78820,"Ġmonoxide":78821,"CHA":78822,"Си":78823,"ĠBias":78824,"ÚĺÙĪÙĩ":78825,"Ġ×Ļ׼×ķ׾":78826,"ĠÑĢоÑĴено":78827,"/image":78828,"hya":78829,"Ġmansion":78830,"Ġhyperbolic":78831,"Ġà´µ":78832,"Ġhurdles":78833,"ĠCyr":78834,"èİ·èĥľ":78835,"ĠìĿ´ë¦Ħ":78836,"-response":78837,"ĠвоÑģпа":78838,"Ale":78839,"FH":78840,"]];Ċ":78841,"ĉj":78842,"Ġvont":78843,"ĠØŁ":78844,"-gradient":78845,"Ġsweating":78846,"Ġmuitas":78847,"Ġpentru":78848,"Ġважно":78849,"ĠHeide":78850,"å¤ĸåĬł":78851,"Ġcarotid":78852,"âĪ©":78853,"çĥŃçĥĪçļĦ":78854,"ä¹Łæľī人":78855,"ĠÑģмÑĭÑģ":78856,"ä¸¤çľ¼":78857,"-series":78858,"ä½İä½į":78859,"红èĬ±":78860,"диÑĤÑĮ":78861,"ĠPoster":78862,"à¹Ģà¸Ĭืà¹Īà¸Ń":78863,"Tow":78864,"гÑĸ":78865,"讲äºĨ":78866,"æĶ»æīĵ":78867,"Ġpursuits":78868,"Ġnobility":78869,"))ĊĊĊ":78870,"Ġsecolo":78871,"Ġcanals":78872,"ĠDesktop":78873,"å½ķç͍":78874,"åī§åľº":78875,"Ġphenotypic":78876,"checkbox":78877,"Feed":78878,"è°¦èĻļ":78879,"Evaluation":78880,":P":78881,"lbs":78882,"Ġthì":78883,"ĠRide":78884,"太å®Ĺ":78885,"Ġhumming":78886,"ènes":78887,"markt":78888,"çıįè´µçļĦ":78889,"μαÏĦοÏĤ":78890,"élior":78891,"Ġtravellers":78892,"å®´ä¼ļ":78893,"{(}\\":78894,"ĠvÃŃce":78895,"ĠPCA":78896,"سط":78897,".First":78898,"ĠпÑĢеобÑĢаз":78899,"-blind":78900,"ĠCarmichael":78901,"ĠÑĢелиги":78902,"Ġinsc":78903,"ctl":78904,"umbo":78905,"åľ¨è·¯ä¸Ĭ":78906,"骷":78907,"çĽijå§Ķ":78908,"å°½åħ¨åĬĽ":78909,"Ġfacebook":78910,"åįģä¸ĢæľĪ":78911,"æ£ķèī²":78912,"kJ":78913,"ĠWaves":78914,"бÑĢÑĮ":78915,"çģ¯çģ«":78916,"ĠTimer":78917,"Ġaffidavit":78918,"}u":78919,"Ġcreek":78920,"声ä¸Ń":78921,"ลาย":78922,"ĠUSS":78923,"ĠSmooth":78924,"EPT":78925,"asmus":78926,"Ġdiscret":78927,"ãģ«ãģ¨":78928,"voll":78929,"Ψ":78930,"åĮ£":78931,"âĪĺ":78932,"ĠFrontier":78933,"çµĮæ¸Ī":78934,"Ġtighter":78935,"onate":78936,"åľ¨åĽ½å®¶":78937,"èĢĥéĩı":78938,"æ·±å±Ĥ":78939,"æľ¨è´¨":78940,"/users":78941,"è¿ĺä¸įçŁ¥éģĵ":78942,"Ġamel":78943,"åħ¨éĿ¢æİ¨è¿Ľ":78944,"Ġtête":78945,"çļĦæĹ¶åĪ»":78946,"Ġrins":78947,"åĴĮç²¾ç¥ŀ":78948,"åºĶ交":78949,"è¢ĭåŃIJ":78950,"Latin":78951,"ifl":78952,"ä½łå¿ħé¡»":78953,"Ġretour":78954,"çļĦä¸Ģå®¶":78955,"æ½į":78956,"æĬ½æIJIJ":78957,"Ġbombard":78958,"äºĶå®ĺ":78959,"Ġokre":78960,"ç»Łè®¡åѦ":78961,"Ġdeserted":78962,"owanych":78963,"äºĨæĮĩ":78964,"Ġimprint":78965,"åħ¨éķĩ":78966,"[node":78967,"Ġhic":78968,"ä¸įäºĨè§£":78969,"éĹĨ":78970,"Ġcliffs":78971,"ç©¿çļĦ":78972,"Ġsecreted":78973,"Ġtambé":78974,"à´¤àµįà´¤":78975,"+D":78976,"Ġdessa":78977,"çļĦè¯Ńæ°Ķ":78978,"ĠBram":78979,"Ġhasht":78980,"ä½Ĩå®ŀéĻħä¸Ĭ":78981,"ĠEngels":78982,"Ġbiolog":78983,"Ġsax":78984,"å¿ĥéĩĮçļĦ":78985,"åºĶä¸İ":78986,"åĨįä¸ī":78987,"ÑĤим":78988,"ĠOrdin":78989,"ĠRaum":78990,"WARE":78991,"mour":78992,"çļĦè¡ĮåĬ¨":78993,"Ġasphalt":78994,"Ġinstru":78995,"æĶ¾çĿĢ":78996,"ĠRepública":78997,"_split":78998,"å¸ĮæľĽèĥ½å¤Ł":78999,"Ġmelodies":79000,"ä¸į太好":79001,"ŀצ×IJ":79002,"nova":79003,"hemer":79004,"åŃĹå½¢":79005,"ĠدÙĦ":79006,"Compat":79007,"åıijæĮ¥ä½ľç͍":79008,"åºĶæĢ¥é¢Ħæ¡Ī":79009,"crum":79010,"Ġreclaim":79011,"Ġseab":79012,"Ġréfé":79013,"åħ³å¿ĥçļĦ":79014,"ähl":79015,"ä¾Ĩçľĭ":79016,"ĠPlanck":79017,"Ġgeben":79018,"èµ·é£ŀ":79019,"Ġcalcular":79020,"Ġreferee":79021,"æĭ¿åΰäºĨ":79022,"èĤīç±»":79023,"Ġαá½IJ":79024,"硬å¸ģ":79025,".Run":79026,"æĭĸåĬ¨":79027,"ĠStafford":79028,"ĠPokemon":79029,"/Al":79030,"Ô±":79031,"ç͍è¯Ń":79032,"ĠCanberra":79033,"çĿ¡åīį":79034,"Acts":79035,"è¡Ģ液循çݯ":79036,"åĸ°":79037,"_words":79038,"æŁ¥çľĭäºĨ":79039,"apture":79040,"ISP":79041,"æĹħéģĬ":79042,"Ġwraps":79043,"éo":79044,"å¼ĢæĮĸ":79045,"æ¨Ł":79046,"Ġglare":79047,"èŀįåĮĸ":79048,"Ġmassacre":79049,"ĠKingston":79050,"ç¼łç»ķ":79051,"æĶ¥":79052,"èĩªçŁ¥":79053,"å¾Ĺ失":79054,"Ġfinan":79055,"ä¸įæĺ¯è¯´":79056,"éĢĴç»Ļ":79057,"ãĤıãģij":79058,"FY":79059,"Ġgracious":79060,"缼ä¸ĸ":79061,"æij¸æij¸":79062,"ubbing":79063,"çµ±è¨Ī":79064,"ĠNumerous":79065,"ÙĨتاج":79066,"Ġcaterpill":79067,"asch":79068,"å°±è¿ij":79069,"æĹłé¡»":79070,"书åĮħ":79071,"åįĥçĵ¦":79072,"OTA":79073,"Ġescort":79074,"çݰå®ŀä¸Ń":79075,"ิà¸ļัà¸ķิ":79076,"åIJŃ":79077,"rompt":79078,"对åIJ§":79079,"罪åIJį":79080,"åĪĬçĻ»":79081,"ä¸į对åĬ²":79082,"[f":79083,"åıijæĬĸ":79084,"Ġappellate":79085,"以ä¸ĭåĩłçĤ¹":79086,"âij¥":79087,"ĠUNIX":79088,"ĠMessenger":79089,"FDA":79090,"åĩºä¸į":79091,"Ġcheat":79092,"Ġ×ķ×ij":79093,"ãĤ¸ãĥ£":79094,"=S":79095,"пом":79096,"表çݰçļĦ":79097,"ĠAffordable":79098,"odea":79099,"׾×ij":79100,"ä¿®çħī":79101,"Ġreceptive":79102,"\"Is":79103,"iab":79104,"Ġquarts":79105,"Ġsubstring":79106,"Ġheartfelt":79107,"äºĮåıīæłij":79108,"ĠTun":79109,"among":79110,"éĩľ":79111,"æľ¬æĢ§":79112,"湿çĥŃ":79113,"×¢×ķת":79114,"Ġbakter":79115,"owiÄħ":79116,"ĠâĢ»":79117,"对æĪij说":79118,"ĠZip":79119,"Ġelective":79120,"åħ«å¤§":79121,"Ġsoundtrack":79122,"Ġhybrids":79123,"Ġmadre":79124,"ĠPhillip":79125,"Ġconceded":79126,"Ġcorpse":79127,"hay":79128,"Ġপà§ģর":79129,"ĠDayton":79130,"æ³īå·ŀ":79131,"Ġëĭ¤ìĸij":79132,"溢åĩº":79133,"Constraints":79134,"Ġmédico":79135,"ĠÑĢиÑģÑĥн":79136,"Ġliaison":79137,"ĠResilience":79138,"ĠWalmart":79139,"åı·ç§°":79140,"Manufact":79141,"åĽ½åĨħçļĦ":79142,"ĠУкÑĢа":79143,"æįķèİ·":79144,"æĦ§çĸļ":79145,"Silver":79146,"quiv":79147,"okal":79148,"ĠProz":79149,"ETF":79150,"omycin":79151,"éķ·èĢģ":79152,"(color":79153,"fed":79154,"è¦ģ好":79155,"Ġstrata":79156,"Ġrealt":79157,"ä¸ĥçϾ":79158,"âī¡":79159,"oules":79160,"ĠCunningham":79161,"нож":79162,"ĠZeb":79163,"åįİä¸Ń":79164,"è¿Ļæĺ¯ä¸ª":79165,"Ġcapacitors":79166,"ÙħاÙĭ":79167,"è¦ĭéģİ":79168,".Font":79169,"å¥ĭåıij":79170,"ÑijÑĢ":79171,"ĠÙħتÙĨ":79172,"ĠProducer":79173,"çļĦ樣åŃIJ":79174,"è¿Ļåı¯":79175,"ĠQuotes":79176,"à¸ŀà¸Ń":79177,"æĺ±":79178,"è°ĥåīĤ":79179,"Ġbootstrap":79180,"PQ":79181,"lion":79182,"çļĦåĮºåŁŁ":79183,"è¦ģ让":79184,"è£ħåħ¥":79185,"Ġphenyl":79186,"ä¸į带":79187,"Ġexits":79188,"ĠأبÙĪ":79189,"-Me":79190,"èĢĺ":79191,"Ġchia":79192,"ertos":79193,"åħīæłĩ":79194,"ĠÙħÙĨذ":79195,".Ab":79196,"æµĵåİļçļĦ":79197,"Ġoxidized":79198,"Ġzorg":79199,"é£ŁçĽIJ":79200,"æī¾ä¸Ģ个":79201,"çĦ¶åIJİæĬĬ":79202,"Nu":79203,"ĠTrem":79204,"åľ¨ä¸ĬéĿ¢":79205,"éĢļåijĬ":79206,"ä½Ĩæľī":79207,"еÑĢÑĤ":79208,"æĸĹå¿Ĺ":79209,"Ġmembres":79210,"ç¼Ķ约":79211,"ĠHospitals":79212,"Ġunderlined":79213,"áĢ·":79214,"arlow":79215,"_dim":79216,"çĶŁåij½åĬĽ":79217,"Ġsmoothing":79218,"ĠArabidopsis":79219,"solution":79220,"Ġoutlining":79221,"æıIJé«ĺåΰ":79222,"鲨":79223,"罪æģ¶":79224,"Ġphonetic":79225,"Ġurea":79226,"åıijåŀĭ":79227,"uali":79228,"éĤ£æ®µ":79229,"Ġposing":79230,"Struct":79231,"è¯Ĺåı¥":79232,"Registry":79233,"ibilidade":79234,"ĠPVC":79235,"ibit":79236,"Ġaccents":79237,"æŃ£å¤Ħäºİ":79238,"ç¦ıçī¹":79239,"åĢĴåľ¨åľ°":79240,"urgence":79241,"ocht":79242,"ç»ı常ä¼ļ":79243,"inherit":79244,"Wik":79245,"Ġ*.":79246,"éĤ£åıĮ":79247,"axi":79248,"Ġvolleyball":79249,"Ġenamel":79250,"åłĤåłĤ":79251,"Ġcommunicates":79252,"Ġvelocidad":79253,"-dark":79254,"Ġfronts":79255,"ĠStarbucks":79256,"åįģä¸ĢæĿ¡":79257,"è·Łè¿Ľ":79258,"河边":79259,"ĠÑģÑĤÑĢок":79260,"ĠEmbassy":79261,"Ġhippocampus":79262,"Ui":79263,"inem":79264,"ubation":79265,"Ġpositivity":79266,"-hidden":79267,"Ġmemorize":79268,"Ġtoddlers":79269,"ĠOsw":79270,"ಬ":79271,"è¿ŀåIJĮ":79272,"éĢĤç͍çļĦ":79273,"室温":79274,"levance":79275,"_parent":79276,"è¦ıåĬĥ":79277,"ãĥĹãĥª":79278,"ãģ«å¯¾ãģĹãģ¦":79279,"emarks":79280,"Ġarbe":79281,"åĮĹæŀģ":79282,"Ġconvict":79283,".nih":79284,"çģĮæľ¨":79285,"缸çŃīçļĦ":79286,"Ġpoziom":79287,"flage":79288,"å±±åĿ¡":79289,"å¢ŀæ®ĸ":79290,"ĠÙĬص":79291,"æŃİ":79292,"èĬĤæ°Ķ":79293,"ĠCasino":79294,"Ġsteadfast":79295,"ĠرسÙĪÙĦ":79296,"Ġsoutheastern":79297,"Fetch":79298,"ĠCement":79299,"ĠPension":79300,"ĠFG":79301,"Ġguild":79302,"å®ĿèĹı":79303,"logram":79304,"haven":79305,"Ġsinks":79306,"ä¸Ģè¯ķ":79307,"ĠBytes":79308,"æĺ¥å¤©çļĦ":79309,"æĢ¥äºĨ":79310,"Ġpetty":79311,"ĠоÑĤноÑĪениÑı":79312,"Ġarsenic":79313,"stim":79314,"Ġstroll":79315,"quares":79316,"å¹¶å°Ĩåħ¶":79317,"ursions":79318,"æī¹å¤į":79319,"ĠTracy":79320,"ĠRubin":79321,"electronic":79322,"Ġforts":79323,"Projects":79324,"ĠBeethoven":79325,"ç¿Į":79326,"}{*":79327,"Ġexploits":79328,"微微ä¸Ģ":79329,"æ£Ģå¯Łå®ĺ":79330,"}A":79331,"Ġhinter":79332,"ä½ļ":79333,"ĠPW":79334,"å·¥ä½ľæĹ¥":79335,"æł¡å¤ĸ":79336,"ĠÑĥÑĩен":79337,"sku":79338,"Со":79339,"à¹Ģà¸Ķิà¸Ļ":79340,"(*)":79341,"ĠAndersen":79342,"-api":79343,"︰":79344,"Ġrecycle":79345,"åŁºæľ¬åİŁåĪĻ":79346,"Ġহতà§ĩ":79347,"Ġfruct":79348,"æĺ¯åŁºäºİ":79349,"Ġevit":79350,"ambo":79351,"顺åĬ¿":79352,"rabble":79353,"æĥ³åΰè¿ĻéĩĮ":79354,"GRect":79355,"Ġenlightenment":79356,"تÙı":79357,"è°Ľ":79358,"Ġpatag":79359,"Ġplaywright":79360,"àµģà´Ĥ":79361,"irá":79362,"Ġdislik":79363,"é¢ĦåIJİ":79364,"Ġsuffice":79365,"Ġettä":79366,"Ġê·ľ":79367,"Ġeukaryotic":79368,"-string":79369,"]])":79370,"Ġunanswered":79371,"×ķ×ij×ĵ":79372,"емÑĭй":79373,"ABS":79374,"subsection":79375,"Discussion":79376,"ĠKazakhstan":79377,"-add":79378,"cé":79379,"alta":79380,"ĠÑģÑĢазÑĥ":79381,"Ġtransnational":79382,"Ġincrements":79383,"Ġbastante":79384,"ĠتارÛĮØ®":79385,"-position":79386,"elp":79387,"ĠKathy":79388,"ä¹Łä»İ":79389,"ĠAsper":79390,"å¸Ĥåľºä»·æł¼":79391,";\"><":79392,"ĠËĨ":79393,"Ġretiring":79394,"Ġмон":79395,"_category":79396,"æľ¬çļĦ":79397,"åįķåįķ":79398,"Italy":79399,"模樣":79400,"åIJ¬éĹ»":79401,"Ġauthorize":79402,"ĠEffectiveness":79403,"lauf":79404,"chas":79405,"-toggle":79406,"å¾·æĭī":79407,"structured":79408,"ĠABCD":79409,"ç¾İæľ¯é¦Ĩ":79410,"Ġefekt":79411,"Jen":79412,"elope":79413,"è¿Ļä¼ļåĦ¿":79414,"æĹ¶éĻIJ":79415,"Ġintrus":79416,"çIJĨçļĦ":79417,"Analy":79418,"Ġdispersal":79419,"cÄħ":79420,"ĠWB":79421,"ä¹ŁæĮº":79422,"æĹłä»İ":79423,"Ġâŀ":79424,"ãģ®ãģĬ":79425,"-stre":79426,"æīŃ磩":79427,"Ġданной":79428,"Ġenfrent":79429,"Ġstrawberry":79430,"cartes":79431,"ĠPatriots":79432,"jury":79433,"()`":79434,"社ä¼ļå®ŀè·µ":79435,"é»ĦåľŁ":79436,"-SA":79437,"ĠMagist":79438,"Ġdoping":79439,"Ġmulai":79440,"bund":79441,"é£ŁæĮĩ":79442,"æ²¹èħ»":79443,"å®ĹéŨ":79444,"à¦Ĥশ":79445,"Ġescola":79446,"å¹»çģ¯çīĩ":79447,"设为":79448,"Ġмед":79449,"驾é©Ń":79450,"HashMap":79451,"Ġplacenta":79452,"bys":79453,"Ġlords":79454,"ĠSessions":79455,"ĠDinner":79456,"Ġjars":79457,"ĠKoz":79458,"æľĢå¿«çļĦ":79459,"-domain":79460,"åĽłä¸ºè¿Ļ个":79461,"客æĪ¶":79462,"Ġmicrostructure":79463,"rotate":79464,"Ġmau":79465,"ĠкоммÑĥ":79466,"å°±ç®ĹäºĨ":79467,"sfc":79468,"ĠÙħجÙħÙĪØ¹Ø©":79469,"vio":79470,"ä¸įéķ¿":79471,"uret":79472,"ĠJPL":79473,"ستÛĮ":79474,"éĩĩ访æĹ¶":79475,"CAS":79476,"Ġonemoc":79477,"Ġkemb":79478,"éĥ½å·²":79479,"Anth":79480,"综述":79481,"Slot":79482,"ĠScotia":79483,"çķ°å¸¸":79484,"District":79485,"Ġtừ":79486,"æķ£åıijçĿĢ":79487,".randint":79488,"Ġconjecture":79489,"(other":79490,"urin":79491,"Ġintangible":79492,"åζæĪIJçļĦ":79493,"Ġcaramel":79494,"Ġgovernors":79495,"éĥ½æĺ¯æľī":79496,"è¯ļæĦı":79497,"Ġdisciplined":79498,"é£ĺé£ĺ":79499,"ĠÑĤепло":79500,"Ġcomprendre":79501,"Ġcontagious":79502,"Ġteil":79503,"次ä¼ļè®®":79504,"è¿Ļç§įçݰ象":79505,"Ġpourrait":79506,"Ġurbanization":79507,"ĠClayton":79508,"}))":79509,"igator":79510,"ä¸ĢæĹ©":79511,"Ġdoomed":79512,"غÙĬ":79513,"ijnen":79514,"}/\\":79515,"æĭ¨æ¬¾":79516,"è¯ģ人":79517,"çĶŁäº§æĢ»å̼":79518,"çĴŁ":79519,"Ġczyn":79520,"ĠParticle":79521,"滿足":79522,"'{":79523,"ĠBür":79524,"éĥ½è§īå¾Ĺ":79525,"psin":79526,"Ġenthal":79527,"æĺ¯åIJ¦ç¬¦åIJĪ":79528,"ĠEnsuring":79529,"é«ĺäºĨ":79530,"vency":79531,"ĠÐļÑĢа":79532,"ленной":79533,"æĭŁåIJĪ":79534,"è½´çļĦ":79535,"nymi":79536,"æĬijéĥģçĹĩ":79537,"schema":79538,"resp":79539,"_{-":79540,"飬":79541,"åĮĹä¸Ĭ":79542,"è¿Ļä¹Īå¿«":79543,"রà§įশ":79544,"ĠVikings":79545,"¤×Ļ×Ŀ":79546,"Ġasi":79547,"éĢļè¯Ŀ":79548,"Ġtransporter":79549,"åģľäºĨ":79550,"å°¼å°Ķ":79551,"åŃĶéĽĢ":79552,"Ġfuera":79553,"ä¹³èħºçĻĮ":79554,"Ġassez":79555,"Ġarbitrarily":79556,"å°ıå··":79557,"è°ĥéħį":79558,"å¤§å®¶åľ¨":79559,"_top":79560,"åľ°ä¸ĭæ°´":79561,"çļĦåħ´è¶£":79562,"禦":79563,"Ġprogramas":79564,"Ġlimite":79565,"-pound":79566,"(base":79567,"åijĬè¯ī她":79568,"Ġতবà§ĩ":79569,"èĮħåı°":79570,"åı¯æĮī":79571,"æĶ¶èµ·":79572,"çĬ¶åħĥ":79573,"Ġeinz":79574,"ÙĦÙĬات":79575,"ษà¸IJ":79576,"Ġ×ij×ŀ×§":79577,"Ġhobbies":79578,"ä¸Ģè§Ī":79579,"ãĢģãĢģ":79580,"ĠJian":79581,"ĠKerr":79582,"Ġfinanced":79583,"ĠÐŀÑĢ":79584,"ÙħراÙĩ":79585,"/wp":79586,"Ġverschiedenen":79587,"Ġflashes":79588,"æ°ijæĦı":79589,"æĤ¯":79590,"sko":79591,"Ġinformações":79592,"ĠÄijá»ĥ":79593,"Ġà®ħவ":79594,"<>(":79595,"antib":79596,"ĠStokes":79597,"æľįåĬ¡å¹³åı°":79598,"ضÙħ":79599,"-stim":79600,"骨æŀ¶":79601,"Ġкаждой":79602,"æľīåħ´è¶£":79603,"代åı·":79604,"åIJĦæĸ¹éĿ¢çļĦ":79605,"èĬ±æł·":79606,"ĠPeck":79607,"ÏĮγ":79608,"koa":79609,"èĥ¶åĽĬ":79610,"Ġdiversion":79611,"Ġ민":79612,"ĠKathleen":79613,"_ad":79614,"ptus":79615,"esez":79616,"Ġthermometer":79617,"UMBER":79618,"Ġplainly":79619,"éĽĻæīĭ":79620,"ĠRapids":79621,"ĠPresbyterian":79622,"\"Well":79623,"ivorous":79624,"ĠMoor":79625,"riam":79626,"社ä¼ļåıijå±ķ":79627,"ottest":79628,".local":79629,"Ġilmu":79630,"Intent":79631,"éĺ»åĩ»":79632,"Ġsenators":79633,"Ġocclusion":79634,"Ġpembelajaran":79635,"Made":79636,"ç»Ļå®ĥ":79637,"Ġplaneta":79638,"ĠÑģÑĤÑĢан":79639,"webkit":79640,"ĠTECHN":79641,")//":79642,"Ġtaux":79643,"Ġnemat":79644,"ä»ĸæĮĩåĩº":79645,"Ġunderstandings":79646,"ÅĽcia":79647,"Ġimplanted":79648,"Ġyen":79649,"estar":79650,"大é»Ħ":79651,"èĬĤåģĩæĹ¥":79652,"éĻIJæľŁ":79653,"ophosph":79654,"Strings":79655,"å¤ľçļĦ":79656,"ĠкоÑĤоÑĢÑĥÑİ":79657,"-virtual":79658,"ĠMozamb":79659,"-One":79660,"ĠWahl":79661,"ĠLIB":79662,"ä¸Ń人":79663,".gz":79664,"Ġcabo":79665,"capital":79666,"ĠCornwall":79667,"Ġfluxes":79668,"culoskeletal":79669,"ĠпиÑĤаниÑı":79670,"-ness":79671,"RV":79672,"Ġern":79673,"éĥ¨éĥ¨éķ¿":79674,"èĤ¡åĪ©":79675,"宣称":79676,"Ġalters":79677,"ä¸ĭä¸Ģç¯ĩ":79678,"好çľĭçļĦ":79679,"tas":79680,"åĵģ質":79681,"eração":79682,"èĸ°":79683,"adoras":79684,"èµŀåı¹":79685,"Jackson":79686,"OUS":79687,"Ġnautical":79688,"Ġgeld":79689,"Ġ*,":79690,"æķ´é«Ķ":79691,"Ġdirectives":79692,"è¡Į为人":79693,"ĠдиагноÑģÑĤи":79694,"ł×ķ×¢":79695,"leurs":79696,"ä¸ĭè°ĥ":79697,"è¿ĺ以为":79698,"æŀľåŃIJ":79699,"ĠShu":79700,"æĭīä½ı":79701,"rafts":79702,"ĠDiscipline":79703,"çªĹå¸ĺ":79704,"Ġpronunci":79705,"ĠниÑĺе":79706,"èĩªè§īåľ°":79707,"Ġê¸Ģ":79708,"ĠWish":79709,"-select":79710,"ĠEverybody":79711,"Ġcytos":79712,"Middleware":79713,"Lecture":79714,"ä¸İæĪij们":79715,"æĥ³æĬĬ":79716,"external":79717,"Ġbenar":79718,"áŀĦ":79719,"Ġjuxtap":79720,"ĠPapua":79721,"Ġmengenai":79722,"esley":79723,"åĩºæ±Ĺ":79724,"Ġdiagon":79725,"Ġbacterium":79726,"æĴ¤ç¦»":79727,"reibung":79728,"ultatua":79729,"ĠTheories":79730,"ä¸ĭä¸Ģ次":79731,"APS":79732,"Ġwebinar":79733,"angelo":79734,"Ġgamers":79735,"Ġkontsultatua":79736,"ä¸Ģ审":79737,"ä¸į代表":79738,"æ±¶":79739,"Ġalkali":79740,"à¸Ńยà¹Īาà¸ĩà¹Ħร":79741,"Ġмолод":79742,"åĩºçĶŁçļĦ":79743,"encephal":79744,"×ķ×ķ×Ķ":79745,"ĠSev":79746,"наÑĢ":79747,"Ġblueprint":79748,"Ġminimally":79749,"åĪĽä¸ļèĢħ":79750,"Ġrectangles":79751,"Ġà¸ŀระ":79752,"对åħ¶è¿Ľè¡Į":79753,"ĠStraight":79754,"ĠOmar":79755,"ĠToast":79756,"ä¸įæĸŃå®ĮåĸĦ":79757,"å¤ļå°ij人":79758,"è¨ĺéĮĦ":79759,"Ġmarching":79760,"Ġcarc":79761,"çģŃäºĨ":79762,"ĠAutomated":79763,"Ġsucked":79764,"çĤ¹äº®":79765,"Ġbiotechnology":79766,"æķĻåѦæĸ¹æ³ķ":79767,"ĠогÑĢаниÑĩеÑļима":79768,"大èĴľ":79769,"ä¿Ŀå§Ĩ":79770,"èĥľä»»":79771,"åģıè§ģ":79772,"------------------------------------------------------------------------":79773,"Ġtopping":79774,"ÏĦηÏĤ":79775,"è¶Ĭéĩİ":79776,"Noiz":79777,"}y":79778,"Ġtarde":79779,"ĠIris":79780,"uala":79781,"ãģĨãģ¡":79782,"éŃĶåĬĽ":79783,"견":79784,"æ©ŁéĹľ":79785,"/ac":79786,"/uploads":79787,"mil":79788,"zos":79789,"Ġأرب":79790,"صابة":79791,"Ġdiagnost":79792,"çģĮ注":79793,"Ġchampionships":79794,"çİĭå°ı":79795,"Spain":79796,"Ġsociological":79797,"ÐĵÐŀ":79798,"หà¸Ļà¹īา":79799,"-condition":79800,"ĠSail":79801,"ĠFamiliar":79802,"好æĦŁ":79803,"engage":79804,"Ġsimp":79805,"à¥įà¤ľ":79806,"Ġannum":79807,"æ®ĸæ°ijåľ°":79808,"ĠпÑĢедпÑĢиÑıÑĤиÑı":79809,"ĠBEL":79810,"à§ĩহ":79811,"éĽĨä½ĵç»ıæµİ":79812,"à¸Ħรัà¸ļ":79813,"ĠPrincip":79814,"érieure":79815,"ĠEthiopian":79816,"BBC":79817,"\\quad":79818,"Ġdemean":79819,"åIJĥä¸į":79820,"į¼":79821,"éress":79822,"Ġgoose":79823,"Ġgrated":79824,"æŃ¦æŀĹ":79825,"ç»§èĢĮ":79826,"smanship":79827,"ä¸įåıĺçļĦ":79828,"ĠFleming":79829,"oblastoma":79830,"(col":79831,"enal":79832,"Ġkasar":79833,"ipro":79834,"éĥ½æ¯Ķ":79835,"å®ŀåĬĽçļĦ":79836,"Ñĩай":79837,"ุษ":79838,"Õ«Õ¯":79839,"Ġflavorful":79840,"Ġreplica":79841,"è¶´åľ¨":79842,"\\usepackage":79843,"uins":79844,"è¿Ļçķª":79845,"мб":79846,"æĶ¿å§Ķ":79847,"åĬŁè¯¾":79848,"Ġprotested":79849,"racket":79850,"ĠвеÑīеÑģÑĤв":79851,"Ġà´ķ":79852,"ãĥ¡ãĥ³ãĥĪ":79853,"ĠвоÑģÑģÑĤанов":79854,"Ġflagship":79855,"'][":79856,"æ°ĶçIJĥ":79857,"during":79858,"ĠPuzzle":79859,"被è§Ĩ为":79860,"ĠBeast":79861,"Ġensuing":79862,"igraphic":79863,"Ġjealousy":79864,"å®¶åįıä¼ļ":79865,"åıĹ人":79866,"è¯Ħæ¯Ķ":79867,"ÑĩаÑĤÑĮ":79868,"楼å¸Ĥ":79869,"åĪĽéĢłåĩº":79870,"ĠRicardo":79871,"Ġempirically":79872,"Ġà¦ķথা":79873,"EPS":79874,"趨":79875,"Ġchoses":79876,"овÑĭе":79877,"à´¸":79878,"ĠFamous":79879,"éļħ":79880,"eseorang":79881,"à¥ĩश":79882,"ĠDetective":79883,"моÑĤÑĢеÑĤÑĮ":79884,"éĬ·åĶ®":79885,"(all":79886,"Moh":79887,"Ġapo":79888,"/dist":79889,"ĠGOOD":79890,"Ġornamental":79891,"åΰåĵªéĩĮ":79892,"Ġziek":79893,"ĠArcher":79894,"ĠAssy":79895,"ä»»åĬ¡æĺ¯":79896,"æĬ½çĥŁ":79897,"æĸ°éĹ»ç½ij":79898,"pag":79899,"Ġnós":79900,"Ġerano":79901,"Ġfluent":79902,"TextField":79903,"社ä¼ļ主ä¹īå¸Ĥåľºç»ıæµİ":79904,"ུ":79905,"Ġnombreuses":79906,"Ġì°½":79907,"-ent":79908,"-che":79909,"天èī²":79910,"æŃ£ä¸Ń":79911,"æĽ¾ä»»":79912,"çļĦ大åĬĽ":79913,"Ġrotations":79914,"ĠPentagon":79915,"коÑģÑĤÑĮ":79916,"à¹Ģà¸Ļิà¸Ļ":79917,"ĠFalcon":79918,"åı£å¾Ħ":79919,"æķijäºĨ":79920,"ĠÑĦоÑĢме":79921,"ÑĨионнÑĭе":79922,"Ġreaff":79923,"ä¸ĢåŃ£åº¦":79924,"ĠDSM":79925,"angements":79926,"Ġadverb":79927,"Ġparticipatory":79928,"Ġsegmented":79929,"Ġpenetrating":79930,".Update":79931,"**)":79932,"åIJĮæĢ§":79933,"éĢļ车":79934,"ä½Ĩè¦ģ":79935,"äºĶæĺ¯":79936,"Ġpostpartum":79937,"Introdu":79938,"LET":79939,"Ġfilaments":79940,"æł¹æľ¬å°±ä¸į":79941,"ĠFuller":79942,"åĴĮè´¨éĩı":79943,"辫":79944,"issan":79945,"ĠÙħÙĪØ§ÙĦÙĬد":79946,"ĠCochrane":79947,"ĠCardiac":79948,"ĠTrustees":79949,"ĠRajas":79950,"(sc":79951,".me":79952,"owment":79953,"ç¥ŀæĿ¥":79954,"ĠScal":79955,"μÏĨ":79956,"usercontent":79957,"Ġdakong":79958,"LOC":79959,"[@":79960,"malloc":79961,"Ġbằng":79962,"ä¸į强":79963,"ĠVB":79964,"оге":79965,"ĠEnable":79966,"baik":79967,"é»ĥéĩij":79968,"Ġмногие":79969,"ĠspoÅĤecz":79970,"-responsive":79971,"Ġatrophy":79972,"Ġlevy":79973,"çĥŁçģ«":79974,"Ġhormon":79975,"ç»ı纪人":79976,"Ġmouvement":79977,"Ġbegging":79978,"åIJĮä»ģ":79979,"Ġemblem":79980,"ĠSpaces":79981,"ãģ¨ãģĭ":79982,"Ġnewsletters":79983,"Ġанглий":79984,"rill":79985,"ä¸Ĭè·¯":79986,"ä¹ĭäºĮ":79987,"羣æľī":79988,"ĠAllergy":79989,"Ġpods":79990,".Event":79991,"Ġbreaths":79992,"æģ¢å¤įæŃ£å¸¸":79993,"ĠлекаÑĢ":79994,"饿äºĨ":79995,"Ġ길":79996,"à¸ķวà¹Į":79997,"-standard":79998,"ĠThou":79999,"èµ°è¿ĽäºĨ":80000,"unnable":80001,"ä¹ĺ车":80002,"Ġrebuilt":80003,"या":80004,"Ġlantern":80005,"qing":80006,"etet":80007,"Ġreusable":80008,"æ²»çĸĹçļĦ":80009,"æ´ĽæĿī":80010,"ĠÚ©ÙĨÛĮÙħ":80011,"Ġskiing":80012,"\"--":80013,"Ġanarch":80014,"ĠDex":80015,"ÙĪØ±Øª":80016,"Unless":80017,"è§£åĨ³çļĦéĹ®é¢ĺ":80018,"unnan":80019,"ĠNCERT":80020,"estyle":80021,"åĴĮåºĶç͍":80022,"assed":80023,"inders":80024,"ĠProposed":80025,"æĦŁè§¦":80026,"Ġdevise":80027,"Ġà¦ķà§ĭ":80028,"Supplementary":80029,"ĠLiberation":80030,"饼干":80031,"arriage":80032,"ĠmV":80033,"Ġkehidupan":80034,"ivalence":80035,".fill":80036,"ĠbackgroundColor":80037,"交éĢļå·¥åħ·":80038,"ãĤıãĤĬ":80039,"á̽áĢ":80040,"åĩºäºĭ":80041,"ilee":80042,"ĠConcentration":80043,"énergie":80044,"기ìĹIJ":80045,"रà¥įव":80046,"Ġważ":80047,"ĠSupervisor":80048,"åı¯è°ĵæĺ¯":80049,"ÕŃ":80050,"Ġmango":80051,"ĠVish":80052,".Current":80053,"×ŀ×ķ":80054,"ĠHCC":80055,"äºĶç§į":80056,"ĠPhar":80057,"Closed":80058,"ženÃŃ":80059,"éĻįåΰ":80060,"Ġconceptions":80061,"æľºæ¢°åĮĸ":80062,"JK":80063,"IJ×ķת":80064,"ä½łæ²¡":80065,"西æ±ī":80066,"Ġrestless":80067,"è¿ŀ线":80068,"æĥĬå¥ĩ":80069,"ÑĢанениÑı":80070,"åĭ¤åĬ³":80071,"ä»ķäºĭ":80072,"maps":80073,"widget":80074,"׳×ķ":80075,"åıĸçļĦ":80076,"ÑĤива":80077,"è´§çī©çļĦ":80078,"Santa":80079,"åĪĨæ¯į":80080,"éĥ¨ä»½":80081,"æĸĻéħĴ":80082,"ÏĥÏī":80083,"Ġknockout":80084,"ĠاÙĦÙħجتÙħع":80085,"Ġgobierno":80086,"ĠCoh":80087,"æĹłè¯¯":80088,"åΩ害":80089,"-div":80090,"çϾ家":80091,"èϽæľī":80092,"ĠдеÑģÑı":80093,"ĠíĺĦìŀ¬":80094,"Ġreap":80095,"å°±å¦Ĥ":80096,"æľ¬èµĽåŃ£":80097,"è¿Ļ个è¿ĩç¨ĭ":80098,"ĠPerforming":80099,"ĠAlexandra":80100,"ĠاÙĦزاÙĪÙĬÙĩ":80101,"é¾IJ":80102,"Ġarchived":80103,"Ġcasinos":80104,"èħ°æ¤İ":80105,"datetime":80106,"Ġconsolidate":80107,"Ġlle":80108,"storms":80109,"ĠFü":80110,"æĶ¶åħ»":80111,"ĠСан":80112,"æ°¸ä¸į":80113,"è®¤çľŁåľ°":80114,"Canadian":80115,"ником":80116,"ĠPrompt":80117,"ĠMesopot":80118,"Ġsynthesize":80119,"Ġsedimentary":80120,"nod":80121,"Ġevolves":80122,"åħ¥èģĮ":80123,"Ġdeforestation":80124,"ktf":80125,"Ġingin":80126,"碳水":80127,"ç͵åĬ¨æ±½è½¦":80128,"Ġunserer":80129,"Ġforn":80130,"Ġstature":80131,"åĴİ":80132,"Ġskulle":80133,"åħ±èµ¢":80134,"Ø·ÙĬÙĨ":80135,"é£ŀè·ĥ":80136,"Ġingestion":80137,"ĠSymfony":80138,"Ġayant":80139,"áĢĶáĢºáĢ":80140,"-tallet":80141,"Sie":80142,"ntown":80143,"åħ³éŨ":80144,"éĩĮè¾¹":80145,"ä¿®è¾ŀ":80146,"èµĽéģĵ":80147,"Ġkinadul":80148,"Ġdictated":80149,"Ġnuevas":80150,"ĠlỼ":80151,"ĠMega":80152,"ĠUEFA":80153,"æĬĢæľ¯ä¸İ":80154,"ĠRecipes":80155,"æ¼Ĩé»ij":80156,".per":80157,"ĠAST":80158,"Ġstent":80159,"ĠfirstName":80160,"centos":80161,"æĢĿæĶ¿":80162,"ä¸Ģ次次":80163,"اعت":80164,"Ġstarvation":80165,"ĠвозвÑĢа":80166,"ãģŀãĤĮ":80167,"Offic":80168,"ibu":80169,"åIJij社ä¼ļ":80170,"anking":80171,"Ġsummed":80172,"Ġutama":80173,"å°±ä¼ļæľī":80174,"zerw":80175,"ĠJudges":80176,"ĠMesa":80177,"为æŃ£":80178,"åĢĶ":80179,"åIJįå®¶":80180,"â̦â̦ãĢįĊĊ":80181,"uiten":80182,"à§Łà¦¾à¦°":80183,"celain":80184,"ĠавгÑĥ":80185,"ĠBildung":80186,"Ġreluctance":80187,"Cou":80188,"ĠHick":80189,"/mod":80190,"ĠGuill":80191,"ĠØ£ÙĨÙĩا":80192,"åĸ·å°Ħ":80193,"Ġpropagate":80194,"sense":80195,"Ġphe":80196,"æµģæĺŁ":80197,"åħ¨ä½ĵåħļåijĺ":80198,"åįģä¸ī竳":80199,"Ġsparkling":80200,"rk":80201,"ĠãĦ":80202,"æĢ»èĥ½":80203,"è°ĥä¾ĥ":80204,".Next":80205,"ĠCardinals":80206,"ĠLouisville":80207,"å±Īæľį":80208,"Ġoats":80209,"Ġrèg":80210,"ividade":80211,"å¢ŀéķ·":80212,"çu":80213,"ĠBooth":80214,"etable":80215,"olus":80216,"å¤Ħ以":80217,"çĭŀ":80218,"åĮĹå¹³":80219,"Amb":80220,"approximately":80221,"ĠÑģамÑĭÑħ":80222,"ĠÑģÑĥÑīеÑģÑĤвÑĥеÑĤ":80223,".Start":80224,">`":80225,"æĺ¯æĹłæ³ķ":80226,"Ġminut":80227,"ĠLeicester":80228,"èĽ¤":80229,"è·³åĬ¨":80230,"åıĮæĸ¹çļĦ":80231,"ĠEmpirical":80232,"Ġrepairing":80233,"ovábbi":80234,"Ġwinters":80235,"icer":80236,"缧":80237,"åįģéĩĮ":80238,"Ġdistillation":80239,"Ġwording":80240,"çŁ³æ¦´":80241,"μÏĮÏĤ":80242,"ãĤģãģŁ":80243,"Ġdaripada":80244,"à¹Ħมà¹Īมี":80245,"dasarkan":80246,"Bh":80247,"leben":80248,"ofi":80249,"é¦ĻçĥŁ":80250,"å¢Ļä½ĵ":80251,"ĠPCs":80252,"ีà¹Īยà¸ĩ":80253,"ĠBattalion":80254,"Ġcorticoster":80255,"Wenn":80256,"讥":80257,"ĠStuttgart":80258,"ĠPsychiatric":80259,"Ġseluruh":80260,"éĩįåIJ¯":80261,"annotation":80262,"ĠباÙĦا":80263,"ç¾Ĭæ¯Ľ":80264,"digital":80265,"=models":80266,"ĊĊĊĊĊ":80267,"Ġitandi":80268,"ĠAdolf":80269,"ಣ":80270,"çļĦ人æĿ¥è¯´":80271,"hael":80272,"Ġà¦ıস":80273,"ĠmiÄĻdzy":80274,"ĠMadagascar":80275,"æĪijå¾Ĺ":80276,"Ġmodality":80277,"è§£å¼Ģ":80278,"attention":80279,"èѦæĪĴ":80280,"Ù¾ÛĮ":80281,"à¦¾à¦Ľà§ĩ":80282,"çı¾å¯¦":80283,"ĠTuc":80284,"ĠPens":80285,"Ġwaterproof":80286,"å¼łæī¬":80287,"Ġpotency":80288,"大家åı¯ä»¥":80289,"Ġconcomitant":80290,"¢×ª":80291,"æµģè¡Ģ":80292,"æĭīåĬĽ":80293,"æ¯įåŃIJ":80294,"Ġκά":80295,"ĠKimber":80296,"ĠPompe":80297,"Ġstaircase":80298,"Ġ×Ķס×":80299,"ðŁĻ":80300,"office":80301,"æĥĬåij¼":80302,"¤×¡":80303,"满足äºĨ":80304,"vÄĽt":80305,"ĠSco":80306,"Ġattackers":80307,"Ġà°Ĺ":80308,"Ġfibrous":80309,"})\\),":80310,"Ġpodcasts":80311,"æľ±åħĥçĴĭ":80312,"ëģ":80313,"Ġdato":80314,"ĠSCC":80315,"Ġalph":80316,"人æīĭ":80317,"ä¹Łéļıä¹ĭ":80318,"eway":80319,"Ġê¶Į":80320,"Ġcoma":80321,"åİ¿åŁŁ":80322,"Ġgilay":80323,"SerializeField":80324,".Command":80325,"_root":80326,"iala":80327,"å°ı说çļĦ":80328,"çŃīåĬŁèĥ½":80329,"æĪĸç͍":80330,"ãģĮå¿ħè¦ģ":80331,"(right":80332,"boss":80333,"áš":80334,"posable":80335,"å±ŀåľ°":80336,"çŃĶæĩī":80337,"åĪĨæŀIJå¸Ī":80338,"Ġpermutations":80339,"Ġsvé":80340,"pure":80341,"é»Ħçĸ¸":80342,"å¸ĥæĸ¯":80343,"çķĻçĿĢ":80344,"Orders":80345,"elingen":80346,"Ġantiviral":80347,"Northern":80348,"Ġosp":80349,"ĠAngles":80350,"ãĤ¤ãĥ³ãĥ":80351,"ãģĿãĤĮãģŀãĤĮ":80352,"Ġmilitia":80353,"ĠUruguay":80354,"ĠTig":80355,"illor":80356,"Ġjong":80357,"ĠChurches":80358,"Ġshortcut":80359,"åĢĴæķ°":80360,"Ġintellectuals":80361,"Ġluar":80362,"Ġshielding":80363,"Ġholog":80364,"estra":80365,"Ġожи":80366,"头çļ®":80367,"ç»Ļä»ĺ":80368,"ĠеÑģÑĤе":80369,"Ġà¦ħধ":80370,"夹æĿĤ":80371,"ĠVaccine":80372,"\".\"":80373,"repository":80374,"ĠMitch":80375,"é¤ħ":80376,"arence":80377,"è¿Ŀ竳":80378,"åıĤä¸İäºĨ":80379,"ĠMarty":80380,"ĠSnake":80381,"ĠвоздÑĥÑħа":80382,".connect":80383,"Ġoor":80384,"olazione":80385,"åľ¨çݰ代":80386,"Ġ\":":80387,"عÙĬ":80388,"ĠÙħÙĪØ³":80389,"Ġabandonment":80390,"ĠCrypto":80391,"ĠRouge":80392,"-haired":80393,"åĮ»ç§ij大åѦ":80394,".####":80395,"åı¯ä»ĸ":80396,"Ġfiner":80397,"ListItem":80398,"ĠÙĥØ«ÙĬر":80399,"/dL":80400,"θή":80401,"æĦĪåıij":80402,"å¤ļåĬŁèĥ½":80403,"å®ŀæĥł":80404,"è½®æµģ":80405,"å¼¹åĩºçļĦ":80406,"}=(":80407,"ĠStevenson":80408,"BH":80409,"ĠTensor":80410,"è¦ģè¿Ľè¡Į":80411,"å±±åºĦ":80412,"åŁ¹é¤Ĭ":80413,"ÐłÐ¸Ñģ":80414,"Ġطب":80415,"×ķש×Ķ":80416,"Browser":80417,"rein":80418,"atrice":80419,"ĠMp":80420,"转弯":80421,"Ġdownto":80422,"ĠRolle":80423,"Ġhợp":80424,"侥幸":80425,"Ġmayo":80426,"Ġdette":80427,"è¡ĢèĦĤ":80428,"æ²³æµģåŁŁ":80429,"填空é¢ĺ":80430,"AO":80431,"åĩºåĩ»":80432,"Outlet":80433,"éĽķåĥı":80434,"ĠEspañ":80435,"ZH":80436,"}A":81939,"Ġricon":81940,"æĪIJäºĨä¸Ģ":81941,"é©ħ":81942,"ä¸ĭä¸Ģ代":81943,"Ġtouchdowns":81944,"Ġfrem":81945,"å¹´éĩij":81946,"ĠStella":81947,"èĦħ":81948,"çĹħçģ¶":81949,"åĽĬèĤ¿":81950,"ĠاÙĦÙĦغة":81951,"Directions":81952,"[e":81953,"ĠSword":81954,"Ġ=.":81955,"大衣":81956,"è£ĺ":81957,"å°±æĺ¯æĬĬ":81958,"女婿":81959,"Asian":81960,"ĠÙĩدÙģ":81961,"æĢİä¹Īçľĭ":81962,"ĠGlac":81963,"Ġpodle":81964,"ôte":81965,"òng":81966,"ä¼Ĭæĭīåħĭ":81967,"Ġìłľê³µ":81968,"ĠпокÑĢÑĭ":81969,"ĠAerospace":81970,"cluster":81971,"èµ°åĩºæĿ¥":81972,"âĢĶâĢĶâĢĿ":81973,"楼å±Ĥ":81974,"Ġaggregated":81975,"ä¾ĿæĹ§æĺ¯":81976,"Ġ모ëijIJ":81977,"Ġhuv":81978,"Ġvzd":81979,"ilyn":81980,"代表人":81981,"Ġcirculated":81982,"Ġdusty":81983,"!\")Ċ":81984,"zier":81985,"åľ¨åĽ¾":81986,"大æŃ¥":81987,"天çļĩ":81988,"峪":81989,"patients":81990,"Ġpleading":81991,"æľ´ç´ł":81992,"Ġrepentance":81993,"åľ¨ä½łçļĦ":81994,"åĸļ":81995,"ĠZum":81996,"Ġgras":81997,"ãģªãģŁ":81998,"éĸĭå¿ĥ":81999,"Ġtriglycer":82000,"MathStep":82001,"ĠÙħÛĮÚ©ÙĨÙĨد":82002,"à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļà¹Į":82003,"SESSION":82004,"åıĮèĦļ":82005,"Ġsekitar":82006,"Ġbuckets":82007,"ä»İ严治åħļ":82008,"dream":82009,"ĠTrim":82010,"ĠDefining":82011,"ziak":82012,"wives":82013,"Ġsx":82014,"èĩªå·±å¯¹":82015,"اÙĦÙģ":82016,"çļĦ大éĥ¨åĪĨ":82017,"ĠÐľÐµÑĤ":82018,"GridView":82019,"Fixture":82020,"æ¯Ķäºļ迪":82021,"FG":82022,"kN":82023,"zhen":82024,"Ùł":82025,"åľĥ":82026,"ä¸ĭåıij":82027,"Ġmatière":82028,"ebug":82029,"Ġloaf":82030,"ĠPayne":82031,"ĠNapier":82032,"à¸Īัà¸Ķà¸ģาร":82033,"Ġmoyen":82034,"ĉsuper":82035,"Ġ../":82036,"ĠWings":82037,"æķ´å½¢":82038,"Ġspeeding":82039,"Ġdissimilar":82040,"μαν":82041,"ĠWWII":82042,"Ġgeopolitical":82043,"ĠбиблиоÑĤе":82044,"IUM":82045,"vote":82046,"ença":82047,"ĠMia":82048,"emerg":82049,"ĠBene":82050,"å°±æĺ¯ä½ł":82051,"ľ×ļ":82052,"ĠخصÙĪØµ":82053,"ĠÑģÑĤал":82054,"Ġontology":82055,"ĠCrossRef":82056,"ĠRoof":82057,"ĠkoÅĦ":82058,"/an":82059,":+":82060,"Ġtm":82061,"åĴĮèĢģ":82062,"allenges":82063,"ipolar":82064,"ä»İæĪij":82065,"Ġgrowers":82066,"STOR":82067,"缸åħ³è´Łè´£äºº":82068,"Ġburger":82069,"Ġpeacefully":82070,"æĶ¾åľ¨äºĨ":82071,"ĠTelephone":82072,"Ġpreschoolers":82073,"Buk":82074,"Ġprescribing":82075,"ìĿij":82076,"ĠBereich":82077,"_rows":82078,"ĠSDS":82079,"Ġalarms":82080,"orel":82081,"à¸ļรร":82082,"ä¸įä»ħèĥ½":82083,"ĠFounder":82084,"åı¬å¼ĢçļĦ":82085,"Ġreconcil":82086,"Ġdunay":82087,"æķ°ãģ®":82088,"odeficiency":82089,"Ġeasing":82090,"大家åºŃ":82091,"Ġcounselors":82092,"'ent":82093,"åľ¨åIJĮ":82094,"åĵģç±»":82095,"Strip":82096,"ÏĦί":82097,"Ġgele":82098,"ĠSwing":82099,"çĿ¡çĿĢ":82100,"ĠMLA":82101,"åŁºç¡ĢçļĦ":82102,"秸ç§Ĩ":82103,"Jay":82104,"ubu":82105,"å°ıçĭĹ":82106,"ierno":82107,"Ġsuggestive":82108,"Above":82109,"Ġglutamate":82110,"Cómo":82111,"lost":82112,"chars":82113,"æľīåIJįçļĦ":82114,"ĠTheo":82115,"éĹ®è´£":82116,"ãģ§ãģĤ":82117,"ĠGoddess":82118,"ĠÐļÑĢоме":82119,"繪":82120,"æ½ľæ°´":82121,"ĠÑĩаÑīе":82122,"ätze":82123,"onan":82124,"ĠBold":82125,"ĠkPa":82126,"è¦ģéĢļè¿ĩ":82127,"Ġélé":82128,"ĠнелÑĮзÑı":82129,"Neither":82130,"污æŁĵéĺ²æ²»":82131,"æ°¸è¿ľä¸įä¼ļ":82132,"ĠвлаÑģÑĤи":82133,"ĠHeroes":82134,"ĠWikisource":82135,"serve":82136,"åĪ¶åº¦æĶ¹éĿ©":82137,"à¥Ĥन":82138,"ĠczÄĻÅĽci":82139,"Ġà¸ľ":82140,"çı©":82141,"ãģ«è¡Į":82142,"ployed":82143,"Ġrecorder":82144,"Ġdroplet":82145,"ĠJonas":82146,"हà¥ĩ":82147,"à§ģরà§ģ":82148,"Ġwartime":82149,"[right":82150,"Ġwickets":82151,"Ġinscribed":82152,"ĠLucky":82153,"åIJİèĥĮ":82154,"ï¼Łï¼ģĊĊ":82155,"å°Ĩä»İ":82156,"åĪĻä¼ļ":82157,"夫æĸ¯åŁº":82158,"Env":82159,"-written":82160,"cou":82161,"çļĦç͵影":82162,"leader":82163,"Ġmodulate":82164,"ĠLean":82165,"Ú¯ÙĪÙĨÙĩ":82166,"令æĪij":82167,"販":82168,"èĤ©è´Ł":82169,"Ġdiplomat":82170,"æµıè§Ī次æķ°":82171,"Jobs":82172,"ĠYao":82173,"Ġmechanically":82174,"ĠAutonomous":82175,".Autowired":82176,"Ġatypical":82177,"gat":82178,"arying":82179,"Ġrecruits":82180,"乡亲":82181,"Ġnormalize":82182,"å£ģçĶ»":82183,"顺åĪ©è¿Ľè¡Į":82184,"ĠPlacement":82185,"Norwegian":82186,"Ġlance":82187,"Ġgö":82188,"ä¸ĭèIJ½":82189,"اÙĨزÙĬاØŃ":82190,"mini":82191,"Ġilluminate":82192,"Ġbitterness":82193,"Ġsponsorship":82194,"িষà§įà¦ł":82195,"Ġbs":82196,"ĠFond":82197,"Ġobraz":82198,"aleigh":82199,"ACP":82200,"éĻįä»·":82201,"(total":82202,"Ġnovice":82203,"éĢĴåĩı":82204,"ĠконкÑĥÑĢ":82205,"æİ©æĬ¤":82206,"ç¥ŀç§ĺçļĦ":82207,"hdad":82208,"mV":82209,"ĠSikh":82210,"好å¿ĥ":82211,"å·¥ä½ľæĢ»ç»ĵ":82212,"ĠShows":82213,"åıĺå¾ĹæĽ´":82214,"=B":82215,"åĴĮçĿ¦":82216,"ĠHawkins":82217,"Ġmaks":82218,"olulu":82219,"ä¸īäºĶ":82220,"Ġدرصد":82221,"追寻":82222,".facebook":82223,"Ġtekn":82224,"æ·±æ²ī":82225,"Ġcambios":82226,"çľ¯çľ¯":82227,"Ġenvisioned":82228,"Ġtad":82229,"Ġevoked":82230,"INO":82231,"ìĬ¹":82232,".nextToken":82233,"ĠDEVELOP":82234,"ìĿ´ëĿ¼ê³ł":82235,"ĠBET":82236,"ÑīÑij":82237,"éĩĩåıĸæİªæĸ½":82238,"Ġsyndromes":82239,"Ġkec":82240,"ysql":82241,"çļĦä¸ĢåIJį":82242,"æī¾åĩĨ":82243,"åĪĿä¸Ģ":82244,"å·ŀåĮº":82245,"Ġsnel":82246,"дÑĥÑĤ":82247,"æĸ½å·¥åįķä½į":82248,"ĠBologna":82249,"Cop":82250,"ivar":82251,"outed":82252,"å°±å¦ĤåIJĮ":82253,"æĹ¥èIJ½":82254,"ç»ıæµİæįŁå¤±":82255,"ICEF":82256,"æ²¹æ¼Ĩ":82257,"Ġeinzel":82258,"åIJīä»ĸ":82259,"Ġgospod":82260,"CAN":82261,"aturity":82262,"çĺ«çĹ":82263,"Ġź":82264,"à¦ļà§įà¦ļ":82265,"âķIJâķIJâķIJâķIJ":82266,"ponde":82267,"ĠConsumers":82268,"çļĦèĤ©èĨĢ":82269,"yset":82270,"æįį":82271,"ĠدÙĪÙħ":82272,"Ġhumanities":82273,"ä¹°å®¶":82274,"â̲(":82275,"çľīéłŃ":82276,"Ġmasking":82277,"ÑĴÑĥ":82278,"letons":82279,"åıijçĶŁè¿ĩ":82280,"зад":82281,"Ġfortunes":82282,"ĠLN":82283,"кÑĤа":82284,"Ġdecipher":82285,"è´¨æĬ¼":82286,"åĥıæĪij":82287,"æµĭéĩıçļĦ":82288,"ĠConscious":82289,"ർ":82290,"Ġkinahabogang":82291,"Ġcourageous":82292,"hc":82293,"Ġdès":82294,"ĠToul":82295,"ĠVä":82296,"èIJ¤":82297,"å¢ŀ产":82298,"é¡¹çĽ®ä¸Ń":82299,"Ġbitcoin":82300,"ĠRanch":82301,"ĠBuffer":82302,"ocellular":82303,"书ä¸Ĭ":82304,"å¤įä»ĩ":82305,"}}^{\\":82306,")_{":82307,"Ġanion":82308,"εÏĢ":82309,"анÑĤи":82310,"ĠwystÄĻp":82311,"æĺ¯è¯´":82312,"好åIJİ":82313,"éĤ£éĩĮçļĦ":82314,"æ¡Ģ":82315,"ними":82316,"Ġreviewer":82317,"ãĤ¢ãĥ¡ãĥª":82318,"Ġcapsules":82319,"á¸į":82320,")e":82321,"Lists":82322,"_day":82323,"writers":82324,"ĠRiemann":82325,"åĴĮå¼ł":82326,"Ġspit":82327,"çī¹è®¸":82328,"书é¦Ĩ":82329,"æ±ī代":82330,"ĠEvolutionary":82331,"Ġunittest":82332,"红æŀ£":82333,"æĹ©èµ·":82334,"宣èªĵ":82335,"ĠWorkplace":82336,"ĠMultic":82337,"ĠDaniels":82338,"Ġsupremacy":82339,"igar":82340,"beans":82341,"ĠCanvas":82342,"Ġসাহ":82343,"åħ¬å¸ĥäºĨ":82344,"Cd":82345,"ĠSoup":82346,"utsch":82347,"ĠCove":82348,"Ġ\\$":82349,"×Ļ×ŀ×Ķ":82350,"ç¾¤å²Ľ":82351,"Ġ×ij׳×Ļ":82352,"ĠÑıзÑĭка":82353,"Ġcensorship":82354,"ĠVolunteers":82355,"atório":82356,"çļĦè¶ħ":82357,"ruch":82358,"Ġflowed":82359,"第åįģä¸ī":82360,"ĠاÙĦاÙĨزÙĬاØŃ":82361,"Ġbaskets":82362,"jung":82363,"Ġavez":82364,"æĽĨ":82365,"Ġexpire":82366,"Ġsubunits":82367,"Ġrunway":82368,"æķĻèĤ²åİħ":82369,"лÑıеÑĤ":82370,"æ¶ĪåĮĸéģĵ":82371,"binary":82372,"иÑģп":82373,"iaux":82374,"Ġquan":82375,"æľªæľī":82376,"å®Įåħ¨ä¸į":82377,"ĠDiary":82378,"Ġà¦ıমন":82379,"(length":82380,"çĺ«çĹª":82381,"Guest":82382,"Ġditch":82383,"-million":82384,"Ġнеиз":82385,"Ġرشد":82386,"ÅĽwiad":82387,"ĠEstad":82388,"Ġcamel":82389,"ĠSUV":82390,"ĠManitoba":82391,"Ġà¹ģลà¹īว":82392,"Ġà·Ģ":82393,"riak":82394,"油价":82395,"讲课":82396,"лÑĥб":82397,"empel":82398,"GFloat":82399,"Ġorbitals":82400,"Indonesian":82401,"ĠTür":82402,"ĠTovábbi":82403,"ä¹ĭ主":82404,"ĠpeÅĤ":82405,"Ġdecorate":82406,"+q":82407,"оÑĨи":82408,"æīĵåΰ":82409,"Ġidentifiers":82410,"ĠизгоÑĤов":82411,"ãģ£ãģ¦ãģĦãģŁ":82412,"对è¯Ŀæ¡Ĩä¸Ń":82413,"ĠоÑĤноÑģиÑĤелÑĮно":82414,"à§įà¦ŀান":82415,"BMI":82416,"ĠIsolation":82417,"à¹Ģวลา":82418,"ĠIhr":82419,"è¿ĺå¾Ī":82420,"请示":82421,"Ġintegrals":82422,"ĠLPS":82423,"åı¯åIJ¦":82424,"Ġexpelled":82425,"åĩĢå̼":82426,"Ġzeal":82427,"Ġastronomers":82428,"Ġwhiskey":82429,"Ġoverdose":82430,"Ġmama":82431,"åıij声":82432,"ĠConcent":82433,"温水":82434,"第äºĮæĿ¡":82435,"american":82436,"_context":82437,"`)Ċ":82438,"econom":82439,"adge":82440,"ĠPose":82441,"ainan":82442,"èĩ³æŀģ":82443,"Ġideologies":82444,"ĠплаÑģÑĤи":82445,"Ġhangs":82446,"Ġureth":82447,"Ġreckless":82448,"éĿ¢åĽ¢":82449,"ĠмиÑĢе":82450,"سÙħÙī":82451,"Ġbuffalo":82452,"Ġharbour":82453,"alat":82454,"etin":82455,"ĠMere":82456,"åľ¨æľĢ":82457,"æĪij覺å¾Ĺ":82458,"é«ĺæĢ§èĥ½":82459,"çĤ¹ä»Ģä¹Ī":82460,"Ġtying":82461,"ר×Ļת":82462,"Ġniño":82463,"å½»åºķçļĦ":82464,"Ġpalliative":82465,"æĢħ":82466,"çłĶåΤ":82467,"ĠReprint":82468,"TU":82469,"lst":82470,"Ġ________________________________":82471,"ĠводÑĥ":82472,"Ġdipped":82473,"å¦Ĥæŀľä½łçļĦ":82474,"-mass":82475,"ToList":82476,"ä¸ĸçķĮä¸Ń":82477,"æķ£äºĨ":82478,"Ġprogressing":82479,"æ·¡å®ļ":82480,"Ġcupc":82481,"Ġbaggage":82482,"ĠSear":82483,"ĠTense":82484,"表åįķ":82485,"à§įধ":82486,"Ġskins":82487,".dir":82488,"à¯įà®®":82489,"ÙĪÛĮÛĮ":82490,"Ġshrinking":82491,"ãĤ¢ãĥ¡ãĥªãĤ«":82492,"Ġئ":82493,"Ġpoorest":82494,"-informed":82495,"ĠProductivity":82496,"Ġfigurative":82497,"]\"":82498,"ĠABA":82499,"ä¸Ńä¸ĵ":82500,"æĹ¶ä¼ļ":82501,"廿":82502,"ucional":82503,"Ġfactores":82504,".lower":82505,"丽èİİ":82506,"лекÑĤÑĢи":82507,"Ġmetaphysical":82508,"ĠJesús":82509,"Ġunintended":82510,"/file":82511,"à¦¾à¦ł":82512,"ëĭ´":82513,"ĠÐŁÐµ":82514,"çĹĽçļĦ":82515,"çĪĨ竹":82516,"Ġeyebrow":82517,"çļĦéĿ¢ç§¯":82518,"ĠReef":82519,"æķĻåħ»":82520,"ĠÑĦев":82521,"年代çļĦ":82522,"nonatomic":82523,"éªļæī°":82524,"Ġinterm":82525,"azionale":82526,"åį´ä¹Ł":82527,"èĥ¡åIJĮ":82528,"Ġridges":82529,"ĠDalton":82530,"Ġczasie":82531,"+N":82532,"NATIONAL":82533,"Ġinser":82534,"Ġspecializing":82535,"è§ĦåĪĴåĴĮ":82536,"Experimental":82537,"ĠعÙħÙĦÙĬØ©":82538,"Ġcomunicación":82539,"urbed":82540,"Ġchromium":82541,"&E":82542,"/un":82543,"iO":82544,"çļĦåIJĦ":82545,"æľ¬çİĭ":82546,"ĠSchu":82547,"Ġstoryline":82548,"-structured":82549,"èģ½èªª":82550,"ĠëĺIJíķľ":82551,"ĠSauce":82552,"Ġì¶ľëł¥":82553,"ัà¸ĩà¸ģฤษ":82554,"ä½ľæģ¯":82555,"Ġetiology":82556,"Ġconfisc":82557,"æıIJä¾Ľä¸Ģ个":82558,"è¯ģåĪ¸äº¤æĺĵæīĢ":82559,"Ġtertentu":82560,"é£ŀç¿Ķ":82561,"å¯Į士":82562,".Back":82563,"Ġfingertips":82564,"Ġuv":82565,"æĪij们家":82566,"Ġtotals":82567,"ĠâĪª":82568,"çĶŁåij½åij¨æľŁ":82569,"ĠìĿ¼ë³¸":82570,"Brad":82571,"ZO":82572,"aturing":82573,"çļĦéĩı":82574,"äºĭä¾ĭ":82575,"åħ¥åºĵ":82576,"ĠSchuster":82577,"ÄĽr":82578,"Õ¥ÖĢÕ«":82579,"éģ©ç͍":82580,"Ġê³¼ìłķ":82581,"Miller":82582,"_msg":82583,"Ġfü":82584,"ĠÑĥÑģÑĤойÑĩи":82585,"Ġgenau":82586,"_null":82587,"ĠTimeline":82588,"ĠкиÑģлоÑĤ":82589,"annes":82590,"å¸Īå¼Ł":82591,"åħ¬åı¸æ³ķ":82592,"Ġcommentators":82593,"第åįģåħ«":82594,"奴婢":82595,"oglobulin":82596,"Ġ.....":82597,"à¦ĺ":82598,"èµĥ":82599,"ä¸Ĭä¸ĭæĸĩ":82600,"龸çİĭ":82601,"ĠвоÑģемÑĮ":82602,"-help":82603,"\\rho":82604,"iin":82605,"Ġsyl":82606,"adura":82607,"Ġcommunism":82608,"ĠMedien":82609,"åİ¿åħ¬å®īå±Ģ":82610,"æŁIJä¸Ģ个":82611,"ĠпиÑīе":82612,"rases":82613,"ĉfloat":82614,"ĠEig":82615,"Ġthereon":82616,"æĬĬå®ĥ们":82617,"Ġsalads":82618,"æĹ¥æľ¬ãģ®":82619,"Ġresistors":82620,"Smallest":82621,"å¤įå·¥å¤į产":82622,"{|":82623,"aliation":82624,"ameth":82625,"äºĽè®¸":82626,"(\"\");Ċ":82627,"åķĨæ¥Ń":82628,"Ġconcord":82629,"ĠParse":82630,"nÃŃk":82631,"ĠNumerology":82632,"æ«ĥ":82633,"fried":82634,"便èĥ½":82635,"缮åīįå·²":82636,"以ä¸ĭãģ®":82637,"паÑĢ":82638,"ĠSundays":82639,"宾è¯Ń":82640,"Virgin":82641,"Ġslogan":82642,"ĠGenre":82643,"oji":82644,"ĠCLE":82645,"èĩªå·²":82646,"èģ°":82647,"ĠÎĪ":82648,"æľĪ饼":82649,"æ°Ķåİĭ":82650,"Ġbelum":82651,"管çIJĨä½ĵåζ":82652,"èĬĴæŀľ":82653,"åįģä¸īæĿ¡":82654,"Ġenriching":82655,"(de":82656,"[T":82657,"ppel":82658,"ĠKons":82659,"é£İçŃĿ":82660,"è¢ĸåŃIJ":82661,"ĠBedford":82662,"Ġlaut":82663,"ä½Ĩåħ¶å®ŀ":82664,"ÑĤал":82665,"ÅĤÄĻ":82666,"Ġbiologically":82667,"ĠÙħÛĮÚº":82668,"שר":82669,"Paths":82670,"lug":82671,"åīįå¤ķ":82672,"Ġflakes":82673,"ĠLeah":82674,"æĺ¯åIJ¦èĥ½":82675,"Ġfooter":82676,"২০০":82677,"ĠGustav":82678,"bringing":82679,"Infl":82680,"太ä¹ħ":82681,"æĸĩåĮĸ产ä¸ļ":82682,"ammers":82683,"à¥ģà¤":82684,"ĠPassage":82685,"ĠлеÑĩение":82686,"Ġà¦ļল":82687,"ĠмаÑĤеÑĢиал":82688,"Ġìĺģíĸ¥":82689,"ĠبØŃØ«":82690,"Ġacquaintance":82691,"ĠSidd":82692,"Ġvinden":82693,"ÛĮÙĦÛĮ":82694,"ergies":82695,"èIJ¥åĪ©":82696,"Ġaccession":82697,"ByName":82698,"顺æīĭ":82699,"æIJŀå®ļ":82700,"Ġδὲ":82701,"ÙĪØ£":82702,"å±±ä¸Ń":82703,"ĠGould":82704,"æį¨":82705,"Ġskut":82706,"åŁİåĨħ":82707,"åĪĿè¡·":82708,"Ġspiritually":82709,"èµĦæĸĻæĿ¥æºIJ":82710,"ĠStrept":82711,"omaterials":82712,"ĠRost":82713,"è¿ĻéŨ":82714,"èĩªæķij":82715,"èĢĮ导èĩ´":82716,"æĸĩæŃ¦":82717,"URA":82718,"æ±ĩéĽĨ":82719,"ĠFeeling":82720,"ĠMetrics":82721,"Perfect":82722,"Ġdrifted":82723,")âĢĵ":82724,"Ġmalloc":82725,"รà¹īà¸Ńย":82726,"éĢĤåºĶçļĦ":82727,"ĠReliability":82728,"Ġrichest":82729,"ĠпÑĢоÑĨедÑĥ":82730,"dorf":82731,"Ġศ":82732,"æľ¬éĻ¢":82733,"ickle":82734,"Ġsimbol":82735,"è¿IJä¼ļ":82736,"ĠColour":82737,"éĢĢäºĨ":82738,"functional":82739,"åIJĮå¿Ĺ们":82740,"Ġgalvan":82741,"ĠBenson":82742,"ĠÑĥдов":82743,"Ġnonfiction":82744,"ĠÙħÛĮزاÙĨ":82745,"ĠLiouville":82746,"Ġdeparting":82747,"Ġurgently":82748,"моÑģ":82749,"åħįéϤ":82750,"Ġpowdered":82751,"еÑĤелÑĮ":82752,"Ġwenig":82753,"æĿ¥ä¿¡":82754,"è§ĤæľĽ":82755,"æī¿è¿IJ":82756,"è¿ħçĮĽ":82757,"Industry":82758,"ĠBlessed":82759,"ĠÙĪØµÙĦØ©":82760,"Possible":82761,"ĠLithuania":82762,"Nan":82763,"éĢĤéĩıçļĦ":82764,"èĨĪ":82765,"Ġberl":82766,"è§ĦèĮĥçļĦ":82767,"æī¾åΰä¸Ģ个":82768,"ĠLimitations":82769,"Ġmemorandum":82770,";[":82771,"Ide":82772,"_port":82773,"ä¸ĵåζ":82774,"涨价":82775,"ĠкаÑĩеÑģÑĤва":82776,"jp":82777,"ĠHMS":82778,"åľ¨æīĭ":82779,"çłĶç©¶åıijçݰ":82780,"ãģªãĤĬ":82781,".payload":82782,"EventArgs":82783,"ĠÙħØŃد":82784,"ĠAccountability":82785,"ãģ®ãģ§ãģĻãģĮ":82786,"responsible":82787,"Girl":82788,"ä¸Ģçĵ¶":82789,"ayana":82790,"illian":82791,"åĩºå¤´":82792,"ä¸Ģå®ļç¨ĭ度ä¸Ĭ":82793,"ĠSensitivity":82794,"æĩ·çĸij":82795,"åĴĮçĶŁäº§":82796,"çĽĬæ°Ķ":82797,"Ġunfavorable":82798,"è¸ıåħ¥":82799,"Ġimmunosupp":82800,"Ġbanc":82801,"åıijæ³Ħ":82802,"ohol":82803,"Ġchoix":82804,"ĠGuided":82805,"Ã¥k":82806,"Ġë²Ħ":82807,"Ġkry":82808,"çŁ¥å·±":82809,"à¹Ģà¸ĺ":82810,"ĠÐļогда":82811,"اذا":82812,"rÃŃguez":82813,".aut":82814,"Ġsplic":82815,"éĿ¢çĽ¸":82816,"å¤įæķ°":82817,"æĺĵæĩĤ":82818,"对äºİä¸Ģ个":82819,"Ġppt":82820,"æľºåĴĮ":82821,"Ġfils":82822,"-config":82823,"ืà¹Īà¸Ńà¸Ļ":82824,"Donnell":82825,"ложениÑı":82826,"IFIED":82827,"MON":82828,"dg":82829,"æĹ¶åı¯":82830,"åıªæīĭ":82831,"eco":82832,"Ġminors":82833,"ä¾Ľåħ»":82834,"è®®äºĭ":82835,"ç»´äºļ":82836,"ç±³å°Ķ":82837,"Ġpropria":82838,"Browse":82839,"зеÑĢ":82840,"æĺİ確":82841,"Ġspeculate":82842,"ĠAugustus":82843,"Ġreassuring":82844,"Electronic":82845,"åĿİåĿ·":82846,"nad":82847,"åĩºäºĨä¸Ģ个":82848,"huis":82849,"ÑĤелÑıм":82850,"ĠCoordination":82851,"ç©©å®ļ":82852,"Ġflattened":82853,"-State":82854,"å°Ĩ为":82855,"å°±æĺ¯ä¸ª":82856,"Ġautistic":82857,"çļĦä¸Ģçīĩ":82858,"é´»":82859,"Ġbekannt":82860,"'},Ċ":82861,"Difference":82862,"çļĦæĶ¶åħ¥":82863,"Ġforaging":82864,"ellan":82865,"Ġix":82866,"æĢİæ¨£":82867,"à¸Ĺยà¹Į":82868,"à§ĩতà§įর":82869,"Ġà®İன":82870,"å·¥ä½ľä»»åĬ¡":82871,"Ġpolitiques":82872,"opters":82873,"ãģĹãģ¦ãģ¿":82874,"Logged":82875,"iazza":82876,"Ġadept":82877,"红èĸ¯":82878,"hoz":82879,"éĻįåİĭ":82880,"patcher":82881,"Ġлиней":82882,"ĠÑıзÑĭк":82883,"аÑħаÑĢ":82884,"Ġinhaled":82885,"çļĦæĿ±è¥¿":82886,"Ġjas":82887,"ĠZug":82888,"ä»·ä½į":82889,"ä¼ģäºĭä¸ļåįķä½į":82890,"ĠØ´Ú©":82891,"_match":82892,"Ġmodernization":82893,"æĺ¾ç¤ºå±ı":82894,"ĠChandler":82895,"é»ĦèĬª":82896,"Ġmason":82897,"Ġvive":82898,"é«ĺåĪĨ":82899,"ĠIndoor":82900,"Reports":82901,"è¿Ļç§įäºĭæĥħ":82902,"ãģıãĤĬ":82903,"ÙıÙĪØ§":82904,"Ġallegiance":82905,"Wiki":82906,"Ġode":82907,"Ġlij":82908,"å½ĵä¸Ģ个":82909,"åı¯ä»¥èĢĥèĻij":82910,"èĪªæ¯į":82911,"ียà¸Ķ":82912,"Ġjuin":82913,"æµ¦ä¸ľ":82914,"åīĸéĿ¢":82915,"Poor":82916,"URCE":82917,"å³¥":82918,"ä¿ĿæĬ¤çļĦ":82919,"Newton":82920,"ĠSemester":82921,"Ġcucumber":82922,"ĠtÃŃch":82923,"ĠRUB":82924,"resión":82925,"endre":82926,"身å¾Į":82927,"à¥įह":82928,"åı«ä»Ģä¹Ī":82929,"Ġ-*-Ċ":82930,"(ll":82931,"rude":82932,"anonymous":82933,"ĠRocket":82934,"æŀŃ":82935,"mina":82936,"ัà¸ķร":82937,"ĠÙĩÙħراÙĩ":82938,"æķ°æį®ç±»åŀĭ":82939,"(default":82940,"wissenschaft":82941,"Ġbaz":82942,"mares":82943,"ARGET":82944,"ä½Ļåľ°":82945,"ĠCompact":82946,"åľĨå¼§":82947,"æĹģçļĦ":82948,"×ķ×ijר":82949,"ĠTecn":82950,"ÊĶ":82951,"Ġfum":82952,"åѦ好":82953,"èϧ":82954,"å®Ŀçī©":82955,"omeric":82956,"Ġlungo":82957,"-Level":82958,"itution":82959,"ä¸įè¦ĭ":82960,"çĤ¯":82961,"issä":82962,"èĢĥä¸Ĭ":82963,"ä½İäºĨ":82964,"ĠGlory":82965,"Ġethos":82966,"TextBox":82967,"ĠSiO":82968,"第åįģä¸ĥ":82969,"å¾Ģåīįèµ°":82970,"Ġfibroblasts":82971,"ĠAvery":82972,"âĢľâ̦â̦":82973,"ĠNim":82974,"Ġpreterm":82975,"Ġzonder":82976,"Ġguerr":82977,"æĬĵèİ·":82978,"matched":82979,"Ġaanv":82980,"Ġâľħ":82981,"JI":82982,"åı¯åı¯":82983,"Ġunde":82984,"Ġtransports":82985,"áĥIJáĥ¡":82986,"大æ±ī":82987,"åģļ强":82988,"ç»´å¥ĩ":82989,"à¸Ħà¸ĩ":82990,"à¹ĥà¸ķà¹ī":82991,"ĠRepro":82992,"Ġlogarithmic":82993,"ĠÑĪÑĤо":82994,"giore":82995,"าวิà¸Ĺย":82996,"Ġhau":82997,"iceps":82998,"åı¯ä»¥åIJij":82999,"æĿijéķ¿":83000,"ç»Ħç»ĩå®ŀæĸ½":83001,"ĠWorlds":83002,"zeni":83003,"Ġstressors":83004,"åŁºéĩij管çIJĨ":83005,"young":83006,"ĠпÑĢакÑĤиÑĩеÑģки":83007,")}{\\":83008,"以达åΰ":83009,"cheid":83010,"çϽåıij":83011,"ĠUSER":83012,"Ġtwor":83013,"ĠобÑĢазÑĥ":83014,".Application":83015,"(br":83016,"çļĦç̧":83017,"elius":83018,"ãĢĤ>>":83019,"Ġreunion":83020,"ĠAFP":83021,"Ġvaleurs":83022,"ĠоÑīÑĥ":83023,"æŃ£å¼¦":83024,"Ġadvises":83025,"ĠاÙĦتÙģ":83026,"å¿įçĿĢ":83027,"Ġorthodox":83028,"Ġsolves":83029,"-borne":83030,"Ġfrü":83031,"ุล":83032,"Ġplatelets":83033,"fuer":83034,"atism":83035,"éĢĻ話":83036,"åĨĽæ°ij":83037,"ĠBeam":83038,"Ġvoed":83039,"Ġafric":83040,"çļ±çº¹":83041,"ĠAdaptation":83042,"ĠMek":83043,"å¼łå¤§":83044,"Ġbedding":83045,"ĠElectoral":83046,"Ġselves":83047,"Ġatherosclerosis":83048,"ä¸Ģ转":83049,"åĬłæģ¯":83050,"Ġraff":83051,"°ï¼Į":83052,"åħħæ²Ľ":83053,".Has":83054,"ĠÎijÏģÏĩ":83055,"Ġfd":83056,"Ġбога":83057,"ĠSchro":83058,"Ġradios":83059,"ÙĬÙħÙĥÙĨ":83060,"à¹īาห":83061,"ä¸Ģåģļ":83062,"æ·¡çĦ¶":83063,"àµįà´¯":83064,"æĩī該æĺ¯":83065,"Ġprofesional":83066,"inander":83067,"Ġספר":83068,"æ¸ħèĦĨ":83069,"Ġpawn":83070,"skie":83071,"è¡Įä¸ļåįıä¼ļ":83072,"ĠPlaintiffs":83073,"à¹Ģลืà¸Ńà¸Ķ":83074,"/Second":83075,"Ġtabel":83076,"ä¸īåħĥ":83077,"çݰéĩijæµģ":83078,"çļĦæĬ¥åijĬ":83079,"ĠPixel":83080,"ĠEph":83081,"æĸĩåѸ":83082,"æŀĹæľ¨":83083,"Ġleftover":83084,"κB":83085,"(numbers":83086,"追赶":83087,"ĠاÙĦأخ":83088,"ĠÐķго":83089,"Å«n":83090,"iconductors":83091,"人称":83092,"Ġsufic":83093,"åĴĮæķĻèĤ²":83094,"å®ŀç͍çļĦ":83095,"irchen":83096,"ĠSozial":83097,"ðĿijŁ":83098,"é½IJå¿ĥ":83099,"Neuro":83100,"'ass":83101,"ĠNora":83102,"åħī度":83103,"ç½ijæ°ij":83104,"Ġà¦Ńাল":83105,":T":83106,"Flu":83107,"ĠFans":83108,".....ĊĊ":83109,"Ġdiscontinued":83110,"Ġpartisan":83111,"ampuan":83112,"::$":83113,"ä¿®ä»Ļ":83114,"Ïĥί":83115,"Ġunsur":83116,"Confirm":83117,"-valued":83118,"Ġpinned":83119,"åľ¨æİ¥åıĹ":83120,"è¿Ľåľº":83121,"Ġdiastolic":83122,"numero":83123,"ãĤ·ãĥ¥":83124,"Ġchond":83125,"ĠвÑĭбÑĢа":83126,"Ġtrimmed":83127,"ĠÃŃnd":83128,"angka":83129,"ä»ĸä¸Ģ缴":83130,"å°ıé¼ł":83131,"Ġamalg":83132,"==ĊĊ":83133,"æµ·è¾¹":83134,"Ġconfessed":83135,"èģĶç»ĵ":83136,"ĠاÙĦÙħÙģ":83137,"-invasive":83138,"ĠBoom":83139,"åĮĸåѦåıįåºĶ":83140,"ĠSavannah":83141,"Ġsagt":83142,"ĠzostaÅĤ":83143,"Ġroar":83144,"æĥ³è¯´":83145,"ĠXCT":83146,"æĢ¥çĿĢ":83147,"诺è´Ŀå°Ķ":83148,"à½Ħ":83149,"Ġaffiliates":83150,"ĠÑĥгол":83151,"educated":83152,"Ġpueblo":83153,"Ġexcretion":83154,"æĿİå°ı":83155,"è¿Ļç§įäºĭ":83156,"EFT":83157,"æĦŁæŁĵèĢħ":83158,"Ġquadril":83159,"Ġmujer":83160,"ĠÏĢÏģÏī":83161,"ĠмикÑĢо":83162,"KF":83163,"áln":83164,"Ġ))":83165,"Ġbatches":83166,"åĩºåıijçĤ¹":83167,"ĠاÙĦÙħسÙĦÙħ":83168,"zal":83169,"çļĦ女åĦ¿":83170,"ĠMIS":83171,"æĶ¶çº³":83172,"strateg":83173,"å¸Įå°Ķ":83174,"Ġscriptures":83175,"ç«¶çĪŃ":83176,"Ġ'*'":83177,"arella":83178,"Ġpartecip":83179,"ç»Ļèį¯":83180,"ĠZimm":83181,"лии":83182,"å¸Īå¾·":83183,"ĠForg":83184,"äºĨä¸Ģä¸Ŀ":83185,"Ġlimp":83186,"ĠâĨĵ":83187,"drive":83188,"Ġpadres":83189,"åįģä¸ĥ竳":83190,"çī¢åĽºæłijç«ĭ":83191,"Ġbumps":83192,"åľ¨å·¥ä½ľä¸Ń":83193,"à¹īาร":83194,"Ġworldly":83195,"èĥ½åĬĽå¼º":83196,"ÐĴÑģе":83197,"åĽŀçŃĶéģĵ":83198,"Ġmixes":83199,"ĠTrinidad":83200,"Ġï¼Ŀ":83201,"Ġunden":83202,"ĠQt":83203,"åij¨ä¸ī":83204,"Ġsummation":83205,"ĠCurry":83206,"ĠØŃدÙĪØ¯":83207,"ĠDestroy":83208,"Ġks":83209,"çŃīä»·":83210,"è§Ħå¾ĭçļĦ":83211,"Ġdendritic":83212,"Ò³":83213,"Ġhati":83214,"lias":83215,"Ġmagnification":83216,"Ġimagining":83217,"Ġgiá":83218,"åĦĦåħĥ":83219,"environ":83220,"åįĹéĢļ":83221,"ypse":83222,"Ġseeming":83223,"ĠExplained":83224,"ĠWeekend":83225,"ĠпеÑĢвого":83226,"Important":83227,"isés":83228,"=\"../":83229,"èĬĤæ°´":83230,"è¿ŀ带":83231,"ĠPrz":83232,"货款":83233,"ä»ĺåĩºäºĨ":83234,"çĽĺçļĦ":83235,"লà§įপ":83236,"ĠMillions":83237,"кови":83238,"Ġëħ¼":83239,"Lorem":83240,"ä¸ļçķĮ":83241,"ĠImag":83242,"ĠpÅĻip":83243,"HQ":83244,"demo":83245,"人æĹı":83246,"Ñĩном":83247,"Ġfirstly":83248,"öss":83249,"LLE":83250,"Ġweighting":83251,"Ġį":83252,"oraly":83253,"辨认":83254,"ĠRFID":83255,";}":83256,"ĠTina":83257,"ĠTaste":83258,"ĠMild":83259,"大åĵŃ":83260,"ï¼ļ[":83261,"Ġapprend":83262,"è¿ĺåŃĺåľ¨":83263,"æĹłå°½":83264,"æµĭç»ĺ":83265,"λÏį":83266,"æ··èĽĭ":83267,"ĠÐĽÑİ":83268,"èª¿æŁ»":83269,"ĠATT":83270,"Ġbolster":83271,"ĠاÙĦثاÙĨÙĬ":83272,"ĠEndocrinol":83273,"ĠTrophy":83274,"ĠJUST":83275,"æĦŁæĥ³":83276,"Ġberat":83277,"æľ«å°¾":83278,"åĴ¬çĿĢ":83279,"Ġoutsourcing":83280,"tant":83281,"ĠMih":83282,"æĶĺ":83283,"éĢłåı¥":83284,"åij¨åĽĽ":83285,"Ġcopolymer":83286,"Descriptor":83287,"Ek":83288,"raiser":83289,"Ġheures":83290,"ового":83291,"Ġvarias":83292,"éľĢè¦ģ对":83293,"risis":83294,"ĠCLI":83295,"hundrede":83296,"ä¸įæİī":83297,"两çϾ":83298,"æİ¨åIJij":83299,"Äįné":83300,"Ġsymbolizes":83301,"Ġweakening":83302,".order":83303,"_button":83304,"Ġbh":83305,"èµ·åĬ¨":83306,"Ġimpacto":83307,"ĠEVs":83308,"머":83309,"Salt":83310,"dump":83311,"unen":83312,"ĠRousseau":83313,"ĠHomo":83314,"ä½İä¼°":83315,"}{(":83316,"äºĴæį¢":83317,"é¹ĥ":83318,"ĠSilk":83319,"Ġstratified":83320,"ittel":83321,"Ġgenerals":83322,"Ġdevastated":83323,"Ġanz":83324,"Ġkhusus":83325,"æĺ¯ä¸įåı¯èĥ½çļĦ":83326,"Considering":83327,"Ġìĵ°":83328,"伸å±ķ":83329,"Ïĩή":83330,"èĥ¸èĨĽ":83331,"çϽçĻľé£İ":83332,"depth":83333,"åİĨå¹´":83334,"Ġsquamous":83335,"äºīåħĪ":83336,"åŁİå¸ĤåĮĸ":83337,"VG":83338,"Ġsinter":83339,"ãĢĤï¼ī":83340,"å®¶éŨ":83341,"iffany":83342,"OTS":83343,"Ġsexy":83344,"Ġپزش":83345,"Ġfashionable":83346,"_VERSION":83347,"Ġconhecimento":83348,"Ġverwendet":83349,"缸éĢļ":83350,"-------":83351,"å¾Īåĥı":83352,"åij¨æĺĵ":83353,"å¸ĮæľĽå¯¹":83354,"ĠELECT":83355,"Ġà¦¹à§Łà§ĩ":83356,"моÑĤÑĢим":83357,"[...":83358,"Ġmc":83359,"choline":83360,"ĠProspect":83361,"ìĹIJëıĦ":83362,"å¸ĮæľĽéĢļè¿ĩ":83363,"lenÃŃ":83364,"ĠáĥĻ":83365,"combe":83366,"ulling":83367,"åĽłä¸ºæľī":83368,"ĠÙħÙĪØ§Ø±Ø¯":83369,"åѤåĦ¿":83370,"ĠëĤł":83371,"總統":83372,"ifikasi":83373,"è¿ijæĿ¥":83374,"Äģs":83375,"å±ĭåŃIJéĩĮ":83376,"ÑĬл":83377,"Ġtidy":83378,"Survey":83379,"ĠContinuing":83380,"ĠZambia":83381,"ĠStad":83382,"Ġ')":83383,"umba":83384,"Ġflavon":83385,"ĠRuiz":83386,"ĠRudolf":83387,"Ġgezond":83388,"ĠInverse":83389,"ãģĦãĤį":83390,"ĠReviewed":83391,"æ°ijæĹıåĽ¢ç»ĵ":83392,"Ġllegar":83393,"ĠAnglican":83394,"Eg":83395,"ĠLadies":83396,"Ġcompt":83397,"intes":83398,"ĠرÙĬ":83399,"Ġsilhou":83400,"åįĪåIJİ":83401,"æ§Ł":83402,"宽æķŀ":83403,"ë§ī":83404,"æĭ¨æīĵ":83405,"ÙħجرÙĩ":83406,"ĠÒ»ÓĻм":83407,"-La":83408,"Ġfaktor":83409,"Ġrepar":83410,"Ġantiquity":83411,"اÛĮØ·":83412,"çĦ¶åIJİåıĪ":83413,"ëĤł":83414,"Ġcrispy":83415,"ëķĮ":83416,"æİ¨åĭķ":83417,"Ġviscous":83418,"ĠImmune":83419,"ĠESG":83420,"Ġexacerbated":83421,"ĠPou":83422,"å¹¶ä¸įçŁ¥éģĵ":83423,"াফ":83424,"iamond":83425,"ĠпÑĢоÑĨ":83426,"èİ«åIJįçļĦ":83427,"è¿Ķ乡":83428,"Ġfunciones":83429,"Ġchatting":83430,"ĠSMEs":83431,"为导åIJij":83432,"ethods":83433,"Ġhomme":83434,"×Ļשר×IJ׾":83435,"Ġpopulação":83436,"Brazil":83437,"jat":83438,"ĠPST":83439,"ĠHolder":83440,"Ġziem":83441,"åıªç͍":83442,"æĭ¿åĩºä¸Ģ":83443,"_main":83444,"volent":83445,"Ġomit":83446,"Ġalerg":83447,"Ġheed":83448,"Ġblond":83449,"åįģå¤ļ":83450,"ranking":83451,"Ġmenopause":83452,"à¶½":83453,"Ġquadr":83454,"éĢıæĺİ度":83455,"Ġannealing":83456,"Χ":83457,"åŃIJåľ¨":83458,"ethane":83459,"Ġindign":83460,"æıIJè´¨":83461,"Ġattire":83462,"åĨįèĢħ":83463,"Ġvisceral":83464,"åĪĿä¸ī":83465,"ç§ijæĬĢè¿ĽæŃ¥":83466,"øn":83467,"ä¸ĸçºªæľ«":83468,"Batch":83469,"ÅĮÄĨ":83470,"orange":83471,"Ġperts":83472,"Ġsideways":83473,"Clock":83474,"Logo":83475,"éĢĤåºĶæĢ§":83476,"Ġfleeing":83477,"Ġprecipitate":83478,"åĽłåľ°åĪ¶å®ľ":83479,")Skip":83480,"åĩºåİĤ":83481,"phrase":83482,"ĠداÙĬرÙĩ":83483,"ĠاÙĦشعاعÙĬÙĩ":83484,"ä¸įåĭķ":83485,"è¾Ĺ":83486,"ĠÙĤطع":83487,"ائÙĤ":83488,"ĠIrene":83489,"Ġdescriptor":83490,"Ġvagu":83491,"ãĥĹãĥŃãĤ°ãĥ©":83492,".math":83493,"cÃŃ":83494,"Ġrepos":83495,"æ°°":83496,"itez":83497,"اÙĦÙĩ":83498,"-soluble":83499,"Ġmencion":83500,"Ġprecisa":83501,"åĶIJè¯Ĺ":83502,"å§ĵæ°ı":83503,"Ġcontrole":83504,"ĠвÑĭполнÑı":83505,"Ġê¸Ī":83506,"keepers":83507,"Ġoverseeing":83508,"Fresh":83509,"ëĨ":83510,"Ġwhims":83511,"Ġchefs":83512,"Ġà¤Ľ":83513,"anao":83514,"河西":83515,"åĿIJä¸ĭæĿ¥":83516,"Ġprotease":83517,"æĸĩä»¶åIJį":83518,"éĹªèĢĢ":83519,"ÓĻн":83520,"Ġklass":83521,"ĠسÙĨÚ¯":83522,"×ķ×ŀ×Ļ":83523,"Ġtester":83524,"Ġvant":83525,"åºĶå±Ĭ":83526,"Ġconvergent":83527,"ĠUR":83528,"клоп":83529,"psum":83530,"çİ°åľ¨æĪij们":83531,"ĠAnnals":83532,"éĢĥçĶŁ":83533,"ĠìĹŃìĤ¬":83534,"Ġkondisi":83535,"lant":83536,"ÃĬ":83537,"åĴĮä¸ĢäºĽ":83538,"æıIJçĿĢ":83539,"annie":83540,"车祸":83541,"Ġgrooves":83542,"Ġstratification":83543,"ĠìŀijìĦ±":83544,"ĠCVD":83545,"广ç͵":83546,"ĠëıĮ":83547,"[len":83548,"askell":83549,"ĠDesigned":83550,"stituto":83551,"CODE":83552,"æ·¡æ°´":83553,"ÑĻе":83554,"ÙĥتÙĪØ±":83555,"Ġinpatient":83556,"estination":83557,"以身":83558,"Ġagr":83559,"ÙİÙĥ":83560,"Ġnationals":83561,"ĠCreativity":83562,"夹è§Ĵ":83563,"_child":83564,"zg":83565,"ĠMünchen":83566,"acock":83567,"ogt":83568,"asca":83569,"ĠOutstanding":83570,"éĤ®ç¥¨":83571,"åĬ²åĦ¿":83572,"ĠاÙĦربÙĬعÙī":83573,"à¸ĩà¹Īาย":83574,"Ġreduz":83575,"оÑģÑĢед":83576,"ĠÙ¾ÚĺÙĪÙĩ":83577,"ä¹Łåı¯ä»¥æĺ¯":83578,"æķ¸éĩı":83579,"ĠGrandma":83580,"åĤ³ä¾Ĩ":83581,"ëIJĺìĹĪ":83582,"å¿ħä¸įåı¯å°ijçļĦ":83583,"ãĤĴãģĻãĤĭ":83584,"çĭ¬å®¶":83585,"Ġgrasping":83586,"æ°ijäºĭè¯ī讼":83587,"Ġrejoice":83588,"Ġstrangely":83589,"ĠMOV":83590,"æľ¬å¸Ĥ":83591,"ĠLeist":83592,"åĽłä¸ºæĺ¯":83593,"éĢĥèĦ±":83594,"çѹåĪĴ":83595,"ĠBangalore":83596,"ĠìĿ¼ë°ĺ":83597,"åħ¶çī¹å¾ģåľ¨äºİ":83598,"bok":83599,"Ġquoting":83600,"éĢļæ°Ķ":83601,"å°±æĺ¯äºĨ":83602,"失衡":83603,"ĠDrivers":83604,"çĿ«æ¯Ľ":83605,"+R":83606,"ĠtÃŃm":83607,"ÑĢÑİ":83608,"opat":83609,"大åĪĩ":83610,"াৰ":83611,"Ġparsed":83612,"Ġsmugg":83613,"anken":83614,"ĠQuarters":83615,"ĠCoat":83616,"çĶļèĩ³åľ¨":83617,"_numbers":83618,"åħ¨åĽ½åIJĦåľ°":83619,"æĮijè¡ħ":83620,"Ġmuitos":83621,"Ġambiental":83622,"омеÑĤÑĢи":83623,"Ġwürde":83624,"Jason":83625,"ĠdÄĽt":83626,"éĥ½æľīäºĽ":83627,"oye":83628,"Ġoppressed":83629,"ituary":83630,"ĠСÑĤ":83631,"Ġtorment":83632,"æĺ¾èijĹçļĦ":83633,"対çŃĸ":83634,"Ġphysicist":83635,"Ġsulphur":83636,"ĠHY":83637,"ĠLNG":83638,"Ġshrine":83639,"没éĤ£ä¹Ī":83640,"Ġprovoke":83641,"Ġdecks":83642,"åģıä½İ":83643,"Refresh":83644,"ĠÑģооÑĤвеÑĤÑģÑĤвÑĥÑİÑīи":83645,"Ġsecrecy":83646,"ĠÖĦ":83647,"eson":83648,"å¼ĢæºIJ":83649,"ishly":83650,"çł¾":83651,"Ġglacial":83652,"ĠScr":83653,"åĩıåİĭ":83654,"новника":83655,"ĠHawks":83656,"ëIJĺìĹĪëĭ¤":83657,"Ļà§įà¦ķ":83658,"-guided":83659,"ĠHuntington":83660,"Ġmalfunction":83661,"-ear":83662,".Code":83663,"ذر":83664,"ĠApproved":83665,"ĠاÙĦØ«ÙĤ":83666,"Ġunderscore":83667,"Ġ(+)":83668,"ĠAnalyzing":83669,"\\delta":83670,"cov":83671,"é¢Ħè§Ī":83672,"coles":83673,"åĮ»çĸĹæľįåĬ¡":83674,"Ġonclick":83675,"æĪIJè´¥":83676,"ĠاÙĦاÙĤتص":83677,"Ġpurpos":83678,"Ġinvoluntary":83679,"æī§åĭ¤":83680,"ĠÕ·":83681,"é¢ĿçļĦ":83682,"è±Ĩçĵ£":83683,"Ġprevailed":83684,"ä¸ĭä¸Ģç§Ĵ":83685,"Ġmisunderstood":83686,"æĸ¯å¤§æŀĹ":83687,"}={":83688,"ĠðĿ":83689,"è¿ĩçĿĢ":83690,"è¨Ŀ":83691,"ĠIDs":83692,"ĠاÙĦÙĨس":83693,"ĠTHC":83694,"McC":83695,"Missing":83696,"Ġpellets":83697,"Ġteoria":83698,"æīĢåıĹ":83699,"主æīĵ":83700,"Ġagony":83701,"Ġعرض":83702,"Produ":83703,"两个åŃĹ":83704,"ĠÑĤакого":83705,"ziaÅĤa":83706,"Ġrobe":83707,"ophysics":83708,"èĩªçĦ¶çģ¾å®³":83709,"ÑĨионного":83710,"測試":83711,"Ġcanoe":83712,"åľ°æ®µ":83713,"åħļ代ä¼ļ":83714,"Ġpatiently":83715,"ĠLiability":83716,"-Rel":83717,"ĠBurma":83718,"ĠвÑģей":83719,"è°£è¨Ģ":83720,"áī":83721,"åIJĦè¡Į":83722,"ĠHarlem":83723,"æ´ĭèij±":83724,"ĠGDPR":83725,"管线":83726,"ossing":83727,"软弱":83728,"Ġoblique":83729,"MU":83730,"ĠMerr":83731,"quake":83732,"ĠTherapeutic":83733,"ával":83734,"米":83735,"éļıé£İ":83736,"Ġlatin":83737,"absorb":83738,"umont":83739,"izk":83740,"ऽ":83741,"缴æİ¥å°Ĩ":83742,"æĢªä¸įå¾Ĺ":83743,"Ġ미êµŃ":83744,"ĠRandall":83745,"Ġexhilar":83746,"Cards":83747,"aution":83748,"Ġechter":83749,"Ġ{},Ċ":83750,"æĪIJæīį":83751,"é«ĺ涨":83752,"åıĺ大":83753,"íļį":83754,"ĠPhilosophical":83755,"èĻIJå¾ħ":83756,"waters":83757,"ĉget":83758,"ä¸Ĭè¿Ľè¡Į":83759,"Ġspoj":83760,"ĠRecon":83761,"Ġformulae":83762,"Ġsubscriptions":83763,"åįĹä¸ĭ":83764,"ĠBelief":83765,"à¹Ģà¸ģà¹ĩà¸ļ":83766,"Ġdisparate":83767,"ĠSubstance":83768,"Ġש×Ķ×ķ×IJ":83769,"Wilson":83770,"æĹłå°½çļĦ":83771,"arguments":83772,"èµ°ç§ģ":83773,"SSL":83774,"ĠRESEARCH":83775,"éĢļäºĨ":83776,"ลำ":83777,"çģ«äºĨ":83778,"Ġsalty":83779,"Ġدربار":83780,"ĠÑĢезÑĥлÑĮÑĤаÑĤ":83781,"Ġвозможно":83782,"etik":83783,"èIJ½äºĨ":83784,"è¶³å¤ł":83785,"éķ¿åº¦ä¸º":83786,"/man":83787,"×ķש×IJ":83788,"Ġservicios":83789,"ç»´åŁĥ":83790,"ĠPolsce":83791,"état":83792,"Ġvirtu":83793,"æĪIJåijĺåĽ½":83794,"_FAIL":83795,"Anderson":83796,"æ³¢ç½Ĺ":83797,"ிவ":83798,"Ġrép":83799,"çļĦæľĢ好":83800,"_graph":83801,"åīĬåĩı":83802,"æľĢæĹ©çļĦ":83803,"ĠCBSE":83804,"}.\\]":83805,"ãĢĤ)":83806,"otas":83807,"äºİå¿ĥ":83808,"çľĭæľĽ":83809,"cheon":83810,"Ġdissatisfaction":83811,"wirk":83812,"ĠBarker":83813,"éĵł":83814,"è»Į":83815,"éĩijèŀįå¸Ĥåľº":83816,"Ġwoodland":83817,"ĠHebrews":83818,"rily":83819,"Ġkhi":83820,"Ġupfront":83821,"Ġफ":83822,"大家ä¸Ģèµ·":83823,"èĭ¥ä¸į":83824,"Ġmorals":83825,"åı³ä¸Ĭ":83826,"æķĻåŃ¦è´¨éĩı":83827,"éĩİåħ½":83828,"Ukrainian":83829,"ĠBenchmark":83830,"rips":83831,"åĨ·èĹı":83832,"_frame":83833,"ĠPortrait":83834,"çįµ":83835,"她们çļĦ":83836,"à¸ģลัà¸ļ":83837,"elden":83838,"ĠGeg":83839,"被æī§è¡Į":83840,"åĨĽéĺĢ":83841,"åıijçĶŁåIJİ":83842,"Evidence":83843,"developed":83844,"è¯ħ":83845,"ä¼ģä¸ļç»ıèIJ¥":83846,"é¢ĦçķĻ":83847,"ĠинÑĤеÑĢе":83848,"ĠпÑĢомÑĭÑĪ":83849,"اÙĦÙħÙĬÙĦاد":83850,"roma":83851,"Ġoverhaul":83852,"ниÑĨе":83853,"-dollar":83854,"ĠCoaching":83855,"ç¨ĭåºıåijĺ":83856,"ĠMillimeters":83857,"çļĦå¿ĥæĢĿ":83858,"à¥ĥष":83859,"fors":83860,"çŃīå¼ı":83861,"ç²¾èĩ´çļĦ":83862,"üb":83863,"æķĻèĤ²åٹè®Ń":83864,"ÙİÙī":83865,"å®Ĺ主":83866,"Ġwidening":83867,"ĠCOLOR":83868,"Ġperten":83869,"تش":83870,"ĠTrich":83871,"Ġbehaves":83872,"-hard":83873,"Ġfactions":83874,"Endpoint":83875,"è´Ī":83876,"Ġbrethren":83877,"extends":83878,"Ġviolently":83879,"າ":83880,"Ġpráctica":83881,"ç»Ļ人ä¸Ģç§į":83882,"ĠSpotify":83883,"Tar":83884,"Ġaisle":83885,"Ġdifferentially":83886,"åįĩ温":83887,"ĠÙħÙĨاسب":83888,"ĠConsistent":83889,".login":83890,"Ġscratching":83891,"ĠгÑĢÑĥн":83892,"ĠParticipant":83893,"Ġfak":83894,"ç͍æĦı":83895,"erno":83896,"导读":83897,"æ¯ıæ¯ı":83898,"Ġcaptivated":83899,"èĪªè¿IJ":83900,"-Free":83901,"ĠLegends":83902,"ählt":83903,"æĸ°åĨłèĤºçĤİçĸ«æĥħ":83904,"ĠSergeant":83905,"windows":83906,"ĠCain":83907,"å¹´å°ij":83908,"该æĸ¹æ³ķ":83909,"ç»Ŀä¸įä¼ļ":83910,"Ġpanjang":83911,"èĥĨåĽĬ":83912,"ĠFORM":83913,"'}Ċ":83914,"çĶŁéķ¿çļĦ":83915,".COM":83916,"ç¨İéĩij":83917,"phthal":83918,"Ġdemost":83919,"ĠкаÑģа":83920,"Ġreferrals":83921,"_local":83922,"à½ĵ":83923,"ÐľÐµ":83924,"ãĤ³ãĥ³":83925,"Kat":83926,"eas":83927,"Ġnc":83928,"ãĢĤ...ĊĊ":83929,"ĠPris":83930,"plash":83931,"Ġsozial":83932,"ijks":83933,"åĬ©åѦ":83934,"covering":83935,"ÙĦÙĬس":83936,"ç¼ĿåIJĪ":83937,"ĠAuburn":83938,"ãĢģãĢIJ":83939,"ĠConsequences":83940,"èĢĥãģĪãĤĭ":83941,"æłĩåĩĨåĴĮ":83942,"-covered":83943,"tiny":83944,"amatan":83945,"ĠFris":83946,"车éŨ":83947,"å©ļåIJİ":83948,"×ijר×Ļ×Ŀ":83949,"ĠFragen":83950,"大家éĥ½çŁ¥éģĵ":83951,"ĠMongolia":83952,".Al":83953,"çĥ½":83954,"Ġbrim":83955,"ï¼Į\"":83956,"Ġfamously":83957,"åŃĺåħ¥":83958,"åĦ¿ç«¥çļĦ":83959,":<":83960,"ĠPip":83961,"ĠHouses":83962,"ÙĦغ":83963,"Ġteh":83964,"ÃŃdu":83965,"Ġsmirk":83966,"é»ĦçļĦ":83967,"æł¹æį®èĩªå·±çļĦ":83968,"Ġtaxonomic":83969,"Ġpremiers":83970,"ãĥ©ãĥ³ãĤ¹":83971,"Ġpelvis":83972,"Ġclaro":83973,"-small":83974,"{":84645,"秦å§ĭçļĩ":84646,"Ġmigratory":84647,"Ġunterst":84648,"Ġvaguely":84649,"+âĢĿ":84650,"ĠFail":84651,"Ġinterstitial":84652,"Ġswamp":84653,"ĠGetty":84654,"Ġpouco":84655,"Ġniveles":84656,"BST":84657,"Ton":84658,"ĉA":84659,"Ġhikes":84660,"ĠFavorite":84661,"æĪijåıªèĥ½":84662,"æ´»åĮĸ":84663,"-self":84664,"Ġantiqu":84665,"ì§Ģ를":84666,"认è¯ĨäºĨ":84667,"utilisation":84668,"亨åĪ©":84669,"å°±æĺ¯æĪij们":84670,"avez":84671,"ĠSpani":84672,"ĠParagu":84673,"ĠMassive":84674,"หà¸Ļัà¸ģ":84675,"ĠMensch":84676,"Ġtenses":84677,"iede":84678,"æ·±åİļçļĦ":84679,"ĠاÙĦÙĨجÙħ":84680,"Ġfosse":84681,"Ġdisbelief":84682,"社群":84683,"åķĨè®®":84684,"ĠMein":84685,"åħ³éĶ®æĹ¶åĪ»":84686,"çĶµè·¯ä¸Ń":84687,"æ·®åįĹ":84688,"ĠElias":84689,"ĠCitizenship":84690,"-types":84691,"Bat":84692,"Pear":84693,"æĺ¯ç¾İåĽ½":84694,"ĠWWE":84695,"å¹¶èĤ©":84696,"ä¸įèĥ½å¤Ł":84697,"Ġcommunicative":84698,"ravings":84699,"ĠABSTRACT":84700,"ĠCMOS":84701,"é쮿Į¡":84702,"Ġembraces":84703,"滤波åύ":84704,">';Ċ":84705,"ĠOrion":84706,"Ġcoursework":84707,"UMENT":84708,"uencia":84709,"çļĦæŃ»":84710,"åѦåĪĨ":84711,"à¹Ģà¸ľ":84712,"æ½ľèīĩ":84713,"Ġeins":84714,"Ġlö":84715,"Ġkort":84716,"éĩijéϵ":84717,"èģĶéĢļ":84718,"ожеÑĤ":84719,"宾客":84720,"Ġinversely":84721,"cape":84722,"çħ½":84723,"ç²¾çĽĬ":84724,"ĠAntio":84725,"Ġballots":84726,"à¸Ńà¸ģà¸Īาà¸ģ":84727,"æĶĢåįĩ":84728,"Ġunresolved":84729,"want":84730,"å°ıæīĭ":84731,"Ġendblock":84732,"çĭ¬åħ·":84733,"讨好":84734,"à«ĭàª":84735,"Ġnombres":84736,"Ġenslaved":84737,"ĠCater":84738,"าà¸ŀ":84739,"ĠëĴ":84740,"å̼å®Ī":84741,"å¢ŀ设":84742,"Ġhomologous":84743,"sztaÅĤ":84744,"çĸ²å̦":84745,"ä½ıæĪ¿åħ¬ç§¯éĩij":84746,"Ġrealizado":84747,"hteet":84748,"Ġamused":84749,"ĠSouthampton":84750,"éĻĨåľ°":84751,"è¯Ħ论åĮº":84752,"pressure":84753,"สัà¸ĩà¸Ħม":84754,"çļĦéĹ®éģĵ":84755,"èĥ½åģļåΰ":84756,"åĽĽåįĥ":84757,"æĢ»æĺ¯åľ¨":84758,"ĠLeigh":84759,"à¸ķำ":84760,"ĠActivation":84761,"Ġsustent":84762,"èµ¢äºĨ":84763,"Ġ기ìĪł":84764,"ĠEntrepreneurship":84765,"Ġundeniable":84766,"/MS":84767,"ĠDup":84768,"梦éĩĮ":84769,"ĠVertex":84770,"èĻļæŀĦ":84771,"æĮģç»ŃæĹ¶éĹ´":84772,"Ġgrassroots":84773,"Ġgrup":84774,"Ġintimidating":84775,"onis":84776,"人以":84777,"人éĢī":84778,"cloth":84779,"ĠHowe":84780,"æĢ»åħ¬åı¸":84781,"ĠGoldberg":84782,"Ġниж":84783,"ĠWORLD":84784,"Ġconspicuous":84785,"ä¸Ģæĥ³åΰ":84786,"ĠBayer":84787,"ĠWow":84788,"Ġverifying":84789,"æĢ¥ä¿ĥ":84790,"اسخ":84791,"Ġsyntactic":84792,"Ġpagina":84793,"Ġshowcased":84794,"oan":84795,"olle":84796,"她没æľī":84797,"ä¸¤å¼ł":84798,"ä¸ŃåĽ½ç§ijåѦéĻ¢":84799,"çİĩè¾¾":84800,"Ġà¦ķà§ĩন":84801,"juk":84802,"ĠSUM":84803,"ĠAmend":84804,"åįĥç§ĭ":84805,"ĠضÙħÙĨ":84806,"ĠPrairie":84807,"Ġболезни":84808,"Ġসà¦Ļà§įà¦Ĺà§ĩ":84809,"ĠJAMA":84810,"Ġunsc":84811,"Ġdetain":84812,"Ġexperiential":84813,"ಹ":84814,"ĠEdmonton":84815,"ĠInterventions":84816,"LAST":84817,"Ġruim":84818,")/((-":84819,"arán":84820,"ĠRPM":84821,"ä¸Ĭ空":84822,"åķĨåŁİ":84823,"éļ¾çľĭ":84824,"Ġbois":84825,"Ġdivent":84826,"éĢĤéħį":84827,"_description":84828,"×Ļפ×ķ׾":84829,"ĠشخصÙĬÙĩ":84830,"VIP":84831,"Ġcords":84832,"Ġrevert":84833,"Ġcurt":84834,"married":84835,"ĠмаÑĤÑĢи":84836,"Ġfirmware":84837,"Setup":84838,"忧伤":84839,"对çħ§ç»Ħ":84840,"??ĊĊ":84841,"Ġregión":84842,"ç»ĵå®ŀ":84843,"opharmac":84844,"habi":84845,"Ġë¶Ģë¶Ħ":84846,"Southern":84847,"Ġ'[":84848,"-brain":84849,"å®ĥæīĢ":84850,"ĠBrands":84851,"Nel":84852,"Ġrejuven":84853,"ollah":84854,"Ġoverexpression":84855,"çĨŁçļĦ":84856,"Ġvacancy":84857,"Helpers":84858,"Ġsakit":84859,"istische":84860,"åĮĸåIJĪ":84861,"éĩijå¸ģ":84862,"ĠGuitar":84863,"ĠEquivalent":84864,"Ġfeminism":84865,"åĦªç§Ģ":84866,"Ġpharmacokin":84867,"ĠTunisia":84868,"Kini":84869,"çļĦåIJ«éĩı":84870,"óź":84871,"çº¢æŁ¿":84872,"åIJ¸è¡Ģ":84873,"ĠGABA":84874,"Ġchassis":84875,"urname":84876,"çĤ¹å¿ĥ":84877,"æĺİåªļ":84878,"Chair":84879,"ä¼ļè®®çͱ":84880,"ĠEphes":84881,"å±łæĿĢ":84882,"rizzle":84883,"ãĢĭï¼ļ":84884,"åĵ¥ä¼¦":84885,"Ġrevolutions":84886,"å®ĩæĸĩ":84887,"å¹³è¡ĮåĽĽè¾¹å½¢":84888,"Ġà¸Īึà¸ĩ":84889,"Ġchiral":84890,"plots":84891,"assuming":84892,"éģĵåħī":84893,"exports":84894,"常éĩı":84895,"Ġbuena":84896,"åı¤è¯Ĺ":84897,"Ġweld":84898,"recipe":84899,"è¨Īçķ«":84900,"Ġaccelerator":84901,"å¿ĥçģµçļĦ":84902,"å°±åıªæľī":84903,"ĠAfro":84904,"ারà§įথ":84905,"ĠSignature":84906,"ĠDickinson":84907,"à¸Ľà¸ıิà¸ļัà¸ķิ":84908,"opper":84909,"political":84910,"ä¹ĭåŁİ":84911,"åºĶ纳ç¨İ":84912,"opsida":84913,"Ġà°¦":84914,"EXP":84915,"éĩĮéĿ¢æľī":84916,"Ġchiefs":84917,"ধান":84918,"кладÑĭ":84919,"ĠINSERT":84920,".word":84921,"ĠSánchez":84922,"Ġimporting":84923,"flight":84924,"Ġsymphony":84925,"çļĦäºĭ项":84926,"Redirect":84927,"åįģä¹Ŀ竳":84928,"ä¸ĭæłĩ":84929,"зон":84930,"coord":84931,"æ´Ĺ礼":84932,"Ġë§ŀ":84933,"locked":84934,"Õ¸ÖĤÕ½":84935,"Ġâĸ¼":84936,"Ġtheo":84937,"åĪĨæĭħ":84938,"Ġoutra":84939,"Ġinterés":84940,"åĬłåĵ¥":84941,"éĹ®ä½ł":84942,"ategori":84943,"å·¥ç¨ĭæĬĢæľ¯":84944,"à¸Ĺาà¸ĩà¸ģาร":84945,"Ġpilgrimage":84946,"Ġamelior":84947,"ĠNolan":84948,"Ġhail":84949,"Ġاک":84950,"æīĵåĬ¨":84951,"åıijå±ķä¸Ń":84952,"ĠColony":84953,"ipple":84954,"认å®ļ为":84955,"hera":84956,"Ġunderline":84957,"åij¨äºĮ":84958,"åºĶå½ĵæĮīçħ§":84959,"Ġquotations":84960,"ä¸įè¯Ń":84961,"åľ¨éĢīæĭ©":84962,"Ġshrug":84963,"讲åΰ":84964,"lickr":84965,"çļĦä»»ä½ķ":84966,"ä¸Ģåį·":84967,"å¦Ĥä¸Ĭ":84968,"æĹłç¼ĺ":84969,"éĢīä¿®":84970,"çĨµ":84971,"梯形":84972,"Ġ기본":84973,"Ġsécurité":84974,"uddin":84975,"Ġhides":84976,"ĠBRO":84977,"ĠLowe":84978,"Ġheirs":84979,"Ġ\\(|":84980,"羣åĪĩ":84981,"åıĸäºĨ":84982,"åij¨æľŁçļĦ":84983,"eredith":84984,"è´Łæľī":84985,"ÙİÙĤ":84986,"ĠOliveira":84987,"ĠAppalach":84988,"é¾ĻéŨ":84989,"Ġrevived":84990,"ĠAlternatives":84991,"ĠConcern":84992,"Ġlobbying":84993,"ilog":84994,"izu":84995,"ĠChloe":84996,"á»ī":84997,"ï½ŀĊĊ":84998,"OURNAL":84999,"ĠrealtÃł":85000,"png":85001,"åı¯ä»¥çļĦ":85002,"ixes":85003,"ĠÑĢаÑģÑĤв":85004,"Ġtreacher":85005,"è¸IJ":85006,"åIJĮåѦçļĦ":85007,"å¥Ķèµ´":85008,"Ġvertebral":85009,"ĠпÑĤи":85010,"产çļĦ":85011,"åIJĥ飯":85012,"æijĨåĬ¨":85013,"ÑģÑĤвеннаÑı":85014,"çļĦé«ĺ级":85015,"å·¡åĽŀ":85016,"ĠÑģеÑĢÑĮ":85017,"-eye":85018,"-Unis":85019,"Cancer":85020,"YE":85021,"ĠMets":85022,"oretic":85023,"å±ī":85024,"Ġprise":85025,"åİĨæĿ¥":85026,"çĶµè·¯çļĦ":85027,"=\"#\"":85028,"Ġpharmacies":85029,"=M":85030,"没éĴ±":85031,"æ°´ä½ĵ":85032,"æĹłå¿ĥ":85033,"-faced":85034,"ĠÙĬر":85035,"BOOL":85036,"િàª":85037,"Ġprincipe":85038,"æľī声":85039,"å»Ł":85040,"-menu":85041,"åIJĥäºı":85042,"à¸ķล":85043,"建设åįķä½į":85044,"éĢĢåĽŀ":85045,"ĠRemed":85046,"ĠSPSS":85047,"æĿŃå·ŀå¸Ĥ":85048,"Ġadversary":85049,"âł":85050,"çļĦä½ł":85051,"igheid":85052,"-selling":85053,"å¦Ĥæŀľæĥ³":85054,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":85055,"Ġrevive":85056,"ĠAnniversary":85057,"åĽºå®ļåľ¨":85058,"Ġwearable":85059,"Ġtécnica":85060,"æĺ¯ä½łçļĦ":85061,"ĠDix":85062,"Ġenn":85063,"çĶŁåĬ¨çļĦ":85064,"æīĭè¡ĵ":85065,"éĩij丹":85066,".âĢĿ[":85067,"aison":85068,"æĦ¿æĻ¯":85069,"kira":85070,"åĩ¡äºº":85071,"交æĺĵæĹ¥":85072,"ipy":85073,"Ġemitter":85074,"æľĪæľ«":85075,"æĢ»è¦ģ":85076,"Ġslap":85077,"çļĦæĺ¯ä¸Ģ个":85078,"ĠDarkness":85079,"èªĵè¨Ģ":85080,"ç쵿ķı度":85081,".EntityFrameworkCore":85082,"anze":85083,"Ġrites":85084,"天ç¥ŀ":85085,"ĠÙĪØ¥ÙĨ":85086,"Ġnuisance":85087,"ר×IJ×Ķ":85088,"å¿ij":85089,"ĠJF":85090,"Ġdesem":85091,"å½ĵä¸ŃçļĦ":85092,"isses":85093,"زب":85094,"Ġjeu":85095,"礼æĭľ":85096,"æĢ»ç»ĵäºĨ":85097,".op":85098,"ným":85099,"笳":85100,"obot":85101,"ungtor":85102,"强æĤį":85103,"ĠgroÃŁ":85104,"æIJį失":85105,"ĠHELP":85106,"Ġpää":85107,"ï¼ĮâĢĿĊĊ":85108,"ĠICD":85109,"åħ·æľīèī¯å¥½çļĦ":85110,".Time":85111,"å͝çĭ¬":85112,"Collabor":85113,"(View":85114,"dong":85115,"年为":85116,"rita":85117,"Ġpropulsion":85118,"åĿıçļĦ":85119,"ĠHorizontal":85120,"ĠHoover":85121,"Traditional":85122,"Ġsaucepan":85123,"Ġï¬ģrst":85124,"formerly":85125,"Ġlangsung":85126,"guan":85127,"ĠGG":85128,"åįķæį®":85129,"ĠÑĩлен":85130,"åį¡çļĦ":85131,"Ġqualidade":85132,"帮åĬ©ä»ĸ们":85133,"fonts":85134,"Ġ......":85135,"Ġgerman":85136,"ĠIngen":85137,"Ġ¯":85138,"ĠMarines":85139,"éĢıéķľ":85140,"Ġassertions":85141,"ĠминÑĥ":85142,"ĠConcert":85143,"ĠмаÑĤеÑĢиалов":85144,"-access":85145,"elay":85146,"å¯¹ä½łçļĦ":85147,"ĠStake":85148,"交çķĮ":85149,"Ġseconda":85150,"ĠاÙĦÙħÙĦÙĥ":85151,".match":85152,"ĠodreÄij":85153,"Ġdosing":85154,"ĠJoão":85155,"Ġneuroscience":85156,"Ġshamp":85157,"稷":85158,"ponder":85159,"绵绵":85160,"éĽĩåijĺ":85161,"Ġintrigue":85162,"ĠGalileo":85163,"ä¸įåΰä½į":85164,"apiens":85165,"ĠLucia":85166,"Ġkarakter":85167,"Ġordinarily":85168,"надле":85169,"Ġmendapatkan":85170,"aggreg":85171,"åѦçłĶç©¶":85172,"-su":85173,"-dem":85174,"Into":85175,"ĠPORT":85176,"åľ¨æķĻåѦ":85177,"Ġà°ļ":85178,"Ġovat":85179,"ützen":85180,"Ġapostles":85181,"|x":85182,"Ġhym":85183,"ĠTact":85184,"ä¸ĢåĽ½":85185,"Ġlandfill":85186,"å¥ĩçī¹":85187,"ĠMontessori":85188,"éĽĻæĸ¹":85189,"atamente":85190,"Ġsoaring":85191,"ĠCalder":85192,"ä¹ħä¹ĭ":85193,"ĠMonkey":85194,"Ġtougher":85195,"'art":85196,"Ġtém":85197,"Ġhottest":85198,"اÙĪÙĨ":85199,"éĢłæŀĹ":85200,"glio":85201,"åħ¼ä»»":85202,"Ġdefeating":85203,"è¾ĸåĮºåĨħ":85204,"Ġbureaucratic":85205,"ĠÙĨÙ쨳Ùĩ":85206,"dade":85207,"Outside":85208,"Ll":85209,"ä»Ģä¹Ī人":85210,"Conversion":85211,"Ġসময়":85212,"Ġinconsist":85213,"Alternative":85214,"esthetics":85215,"Ġprogrammable":85216,"åı°è¯į":85217,"Ġallowable":85218,"Ġsinful":85219,"ĠHyde":85220,"Ġseptembre":85221,"rinsic":85222,"Ġgaug":85223,"åŃļ":85224,"acios":85225,"åħ¥åľº":85226,"Ġdrunken":85227,"behavior":85228,"Ġdaher":85229,"å°Ķå¤ļ":85230,"Ġmotto":85231,"Ġdisappearing":85232,"æĤłéĹ²":85233,"aussian":85234,"ĠاداÙħÙĩ":85235,"Pixel":85236,"_inter":85237,"ĠFreed":85238,"ĠLeng":85239,"çIJ¶":85240,"è¿ĽåŁİ":85241,"æĬĬæĪij们":85242,"é£İä¸Ń":85243,"åıĤéĺħ":85244,"ä¹Łä¸įåĨį":85245,"Ġclosures":85246,"Ġscrambled":85247,"ĠHodg":85248,"_ne":85249,"Ġops":85250,"ä¼ļè°Ī":85251,"Thom":85252,"ÏĦÏīν":85253,"λλο":85254,"ĠBelieve":85255,"Ġbaths":85256,"éĪ´":85257,"æĪijåΰ":85258,"æ°ijå¿ĥ":85259,"åħ»åĪĨ":85260,"计åĪĴåĴĮ":85261,"Ġnarration":85262,"ĉlet":85263,"ç͍好":85264,"åij¨åĪĬ":85265,"éĢĢç¨İ":85266,"å°ļæľī":85267,"çĸı导":85268,"èĬĿåĬłåĵ¥":85269,"ĠìĦłíĥĿ":85270,"她äºĨ":85271,"ä¾Ľè´§":85272,"å¯Į豪":85273,"Ġhomage":85274,"Ġgrandeur":85275,"éĺ»æĸŃ":85276,"ĠÅŀ":85277,"æľ¬æĿ¥å°±":85278,"âĢĶâĢĶâĢĶ.":85279,"\"So":85280,"_const":85281,"iada":85282,"ä¸ĢçĶŁçļĦ":85283,"cccc":85284,"èĢĮå¼Ĥ":85285,"åı¯ä»¥è¿Ľè¡Į":85286,"éħĭ":85287,"Ġpartnered":85288,"个æľĪåĨħ":85289,"è´¢åĬ¡æĬ¥è¡¨":85290,"Ġà¦Ńাষ":85291,"ç¬ijçĿĢ说éģĵ":85292,"æķıæĦٿ̧":85293,"ĠпаÑĢалле":85294,"anski":85295,"Ġaccret":85296,"ĠоÑĩи":85297,"èĤ²åĦ¿":85298,"à¸Ķà¹Į":85299,"(func":85300,"Ġbrightest":85301,"çĽĪä½Ļ":85302,"ĠHunger":85303,"ĠCategoryTreeLabel":85304,"Ġlt":85305,"ĠSECTION":85306,"ĠIber":85307,"arend":85308,"ä¹ŁåįģåĪĨ":85309,"åIJijæĪij们":85310,"已达":85311,"ĠÙħÙĨابع":85312,"Ġabscess":85313,"æŁĶæĢ§":85314,"éĤ®ç¼ĸ":85315,"éĢĻä»¶äºĭ":85316,"Ġtreasury":85317,"ĠзамеÑĤ":85318,"Ġreasoned":85319,"Ġkelu":85320,"æķ·è¡į":85321,"Ñıвление":85322,"%)Ċ":85323,")--":85324,"ĠGom":85325,"èĥ½ä¸İ":85326,"åĮĹ京æĹ¶éĹ´":85327,"(\\,":85328,")D":85329,"+p":85330,"Ġà¸Īาà¸ģ":85331,"ä¹ĭä¹ħ":85332,"Ġeminent":85333,"æĪĸå°Ĩ":85334,"ĠклÑİ":85335,"åıĭ人":85336,"à¸Ĭัà¹īà¸Ļ":85337,"âĦĸ":85338,"ĠDELETE":85339,"Ġcondemnation":85340,"Ġamplitudes":85341,"uniform":85342,"orexia":85343,"å¿IJ":85344,"åIJĮæ¡Į":85345,"-project":85346,"Ġfluctuation":85347,"Ġunconstitutional":85348,"Ġmathematician":85349,"Ġwob":85350,"ideal":85351,"byt":85352,"Ġterap":85353,"Ġpolitik":85354,"Trim":85355,"Ġопла":85356,"éĥijéĩį":85357,"Ġwetland":85358,"-web":85359,"repo":85360,"åı¯èĥ½æľĥ":85361,"LEFT":85362,"ĠTechnician":85363,"ĠÐĽÑĥ":85364,"Ġconservatives":85365,"Ġاساس":85366,"ĠPec":85367,"ä¸ĬåĬł":85368,"ICY":85369,"å°įæīĭ":85370,"çļĦé«ĺä½İ":85371,"强åζæĢ§":85372,"Ġbenzene":85373,"ivu":85374,"ĠChern":85375,"acted":85376,"ĠKafka":85377,"åIJİåį«":85378,"Ġmats":85379,"äºijçļĦ":85380,"immel":85381,"大æ¦Ĥçİĩ":85382,"åζæľį":85383,"ĠÙĪØ§Ø³Øª":85384,"ĠAudience":85385,"ĠðĿIJµ":85386,"æĿıä»ģ":85387,"çijķçĸµ":85388,"óż":85389,"å¤Ħ女":85390,"Ġমার":85391,"çĽijçĿ£ç®¡çIJĨå±Ģ":85392,"Forg":85393,"主è¦ģ以":85394,"两个人çļĦ":85395,"çŃĶæ¡Ī为":85396,"åĽŀçŃĶ说":85397,"æ¶īåıĬçļĦ":85398,"æĭĸçĿĢ":85399,"åĴ³åĴ³":85400,"ä¹ĭéĸĵçļĦ":85401,"à¹ģà¸ģà¹ī":85402,"влека":85403,"Ori":85404,"ĉcount":85405,"aney":85406,"Ġperic":85407,"Ġdisrespect":85408,"Ġsubspace":85409,"-ev":85410,"æķij人":85411,"Ġcasually":85412,"Ġàªħ":85413,"Ġcoworkers":85414,"ĠMug":85415,"ĠDashboard":85416,"Ġheck":85417,"Ġrigu":85418,"åı¯çľŁ":85419,"Ġregião":85420,"ĠпеÑģ":85421,"ĠìĨIJ":85422,"-rounded":85423,"ĠBike":85424,"éĹ®é¢ĺæĹ¶":85425,"é¢Ĩçķ¥":85426,"çϾæĹ¥":85427,"ĠEpstein":85428,".githubusercontent":85429,"Ġsurfactant":85430,"'Brien":85431,"вÑĪие":85432,"Ġresponsibly":85433,"ä¿ĿæĬ¤åĴĮ":85434,"ĠповÑĤоÑĢ":85435,"èī°å·¨":85436,"iopathic":85437,"Ġktórym":85438,"RATION":85439,"inx":85440,"çĶŁäº§æĪIJæľ¬":85441,"è§ĦèĮĥæĢ§":85442,"Ġpiping":85443,"digit":85444,"çĥĺå¹²":85445,"å¿ĥæĦ¿":85446,"argas":85447,"à¸ķà¸ģ":85448,"åĿĩå̼":85449,"æĭįçļĦ":85450,"ĠSmoking":85451,"æ»´å®ļ":85452,"é¾Ļ头ä¼ģä¸ļ":85453,"нÑĨиклоп":85454,"(sp":85455,"Gab":85456,"ä¼ļ说":85457,"å°ıèħ¿":85458,"çĸĻ":85459,"Ġsomme":85460,"maximum":85461,"寺éĻ¢":85462,"Ġmourn":85463,"Ġawakening":85464,"arez":85465,"Ġfirsthand":85466,"çİ©äºĨ":85467,"ĠCardiol":85468,"缴æĴŃéĹ´":85469,"гоÑĢод":85470,"-fluid":85471,"Ġimposition":85472,"Ġchildbirth":85473,"Ġstructurally":85474,"ĠAllies":85475,"èĭ±å°º":85476,"-wrapper":85477,"éĸĭæĶ¾":85478,"!!Ċ":85479,"第åįģä¹Ŀ":85480,"Ġcryptography":85481,"æĬijåζåīĤ":85482,"ĠгÑĢадÑĥ":85483,"ĠArgentine":85484,"Ġrecessive":85485,"ĠشراÛĮØ·":85486,"Ġfibrillation":85487,"Lady":85488,"ĠFever":85489,"nehm":85490,"ä¿Ŀæ´ģ":85491,"åıĹéĻIJ":85492,"ufe":85493,"ä¸ĸçķĮéĩĮ":85494,"åŃĻæĤŁç©º":85495,"/year":85496,"okka":85497,"Ġtemperatur":85498,"Äģd":85499,"Ġimmuno":85500,"åįģä¹Ŀå±Ĭ":85501,"-earth":85502,"ä¸įæĸĻ":85503,"Ġacción":85504,"èIJ½åIJİçļĦ":85505,"ropract":85506,"å᡿ĭī":85507,"åģ¥åº·æĪIJéķ¿":85508,"æĭ¥æľīä¸Ģ":85509,"ĠVoices":85510,"ĠCeleb":85511,"Ġsilicone":85512,"katan":85513,"Ġeut":85514,"å¤ĸåħ¬":85515,"ĠAdoption":85516,"éģİçļĦ":85517,"ĠRivera":85518,"ä¸Ĭä¸Ģå±Ĥ":85519,"Ġcheapest":85520,"ç´«å¤ĸ线":85521,"ĠÃītats":85522,"Ġlässt":85523,"!:":85524,"cpp":85525,"ĠEarnings":85526,"大çϽ":85527,"еннаÑı":85528,"Ġendings":85529,"Ġparasit":85530,"ĠPanthers":85531,"Ġboron":85532,">\\)":85533,"aré":85534,"Ġtableau":85535,"ĠاÙĦÙĨÙ쨳":85536,"ĠReflect":85537,".There":85538,"?>":85539,"ĠKost":85540,"Ġlongo":85541,"éĨ¬":85542,"人åijĺåĴĮ":85543,"æ²īçĿĢ":85544,"ï¼ģâĢĿâĢľ":85545,"ĠاستاÙĨ":85546,"uyên":85547,"èĿĻ":85548,"Ġà®ķà¯Ĭ":85549,"ĠPendidikan":85550,"Eight":85551,"zuk":85552,"Ġgoalk":85553,"ä¸īè½®":85554,"Ġservings":85555,"ĠرÙĪØ§ÙĨ":85556,"Ġà¦ķà§įর":85557,"ĠRecruitment":85558,"ĠBrush":85559,"Ġëĭ´":85560,"çĵ¦æĸ¯":85561,"ĠNEED":85562,"æŀķ头":85563,"Ġabbiamo":85564,"Ġhukum":85565,"åľ¨ä¸Ģ次":85566,"å¹³æĪIJ":85567,"åĬ³ç´¯":85568,"ترÙĪÙĨ":85569,"ĠCardiff":85570,"-=":85571,"Safety":85572,"æīĵåħ¥":85573,"Ġauthorised":85574,"à¹ĩà¸ĩ":85575,"Ġpuberty":85576,"dzi":85577,"ĠLun":85578,"Ġjaws":85579,"好ç¬ij":85580,"èĥ¥":85581,"Ġcharger":85582,"åIJ¬è§ī":85583,"Ġshortening":85584,"Shader":85585,"æ²Ļçī¹":85586,"æĨ©":85587,"Ġenfant":85588,"Ġconjugation":85589,"ìķĺëĭ¤":85590,"Ġkör":85591,"è¾¹æ¡Ĩ":85592,"ĠгÑĢе":85593,"Ġterrace":85594,"IPP":85595,"ĠÙĤØ·":85596,"âĸĴ":85597,"çĿ¡ä¸įçĿĢ":85598,"ĠUnternehmen":85599,"-fer":85600,"ĠRental":85601,"ç¾İéĩij":85602,"ĠSovere":85603,"Geometry":85604,"ĠобÑīеÑģÑĤва":85605,"ĠSinai":85606,"ĠMalt":85607,"åIJĪæ³ķçļĦ":85608,"Ġdijo":85609,"å¼łå°ı":85610,"ç³»ç»ŁæĢ§":85611,"å¾Įãģ®":85612,"niÄĻ":85613,"çĺ©":85614,"à©Ī":85615,"JsonProperty":85616,"Africa":85617,"ĠSadly":85618,"Ġgiorni":85619,"roly":85620,"ĠAED":85621,"ĠMX":85622,"åĴĮè¡Į为":85623,"Ġtrainees":85624,"æĹłå¼Ĥ":85625,"èĤīä½ĵ":85626,"ĠWalton":85627,"Ġnaturaleza":85628,"Ġlupus":85629,"=l":85630,"Michel":85631,"ĠNes":85632,"ogas":85633,"Ġchu":85634,"Ark":85635,"åĮħæĭ¬äºĨ":85636,"å¿ħçĦ¶ä¼ļ":85637,"Ġundersc":85638,"िया":85639,"éĿŀçī©è´¨æĸĩåĮĸéģĹ产":85640,"หà¸įิà¸ĩ":85641,":R":85642,"Ġpopping":85643,"åıĭåĸĦ":85644,"Ġgasped":85645,"çķ¶å¹´":85646,"ĠSunshine":85647,"woods":85648,"arbonate":85649,"ĠâĹİ":85650,"ĠDeadline":85651,"olism":85652,"quire":85653,"ilea":85654,"Ġformação":85655,"ITDA":85656,"ικÏİν":85657,".pyplot":85658,"âĨĵâĨĵ":85659,"çļĦéĶĻ误":85660,"Ġhardships":85661,"ĠGone":85662,"Ġshoved":85663,"ä»ĸåı¯ä»¥":85664,"åĪĨæŀIJä¸İ":85665,")\\]ĊĊ":85666,"Firstly":85667,"-components":85668,"èĪªç©ºåħ¬åı¸":85669,"-ru":85670,"-plan":85671,"ulación":85672,"ĠFriendly":85673,"èĥ½åĬ¨":85674,"Ñģког":85675,"çͷ士":85676,"ĠFlint":85677,"Ġshipments":85678,"VIR":85679,"ĠBraz":85680,"è¦ģç´§":85681,"åIJĪä¹İ":85682,"æĥħè¶£":85683,"ä¼ĺéĢī":85684,".mark":85685,"个人æīĢå¾Ĺç¨İ":85686,"Ġautomobiles":85687,"æĮijçľī":85688,"çŁ¿çī©è´¨":85689,"ativi":85690,"Ġmicrons":85691,"Ġintersections":85692,"轨éģĵ交éĢļ":85693,"alink":85694,"ä»ĸä¹Łæĺ¯":85695,"irez":85696,"çݰä»Ĭ":85697,"ĠÑģенÑĤ":85698,"è¿Ļä¹Īä¹ħ":85699,"Ġtranscends":85700,".);":85701,"dater":85702,"getting":85703,"Ġchildcare":85704,"干货":85705,"िà¤ı":85706,"CY":85707,"_keys":85708,"ĠBaj":85709,"æľīæĹ¶éĹ´":85710,"thorne":85711,"ocating":85712,"Ġploraly":85713,"å½ĵå®¶":85714,"常å·ŀ":85715,"Ġidé":85716,"èıľåĵģ":85717,"Ġsorte":85718,"Ġcinematic":85719,"ĠμεÏĦα":85720,"大éĺª":85721,"å®ī康":85722,"åij¨èº«":85723,"สà¸ļ":85724,"索尼":85725,"ĠÑģвоими":85726,"érience":85727,"ĉcontinue":85728,"ä¹ĭåĬŁ":85729,"Ġmodelled":85730,"ĠWebs":85731,"ĠзаклÑİÑĩа":85732,"ç»ĪçĶŁ":85733,"Ġtrumpet":85734,"Ġtides":85735,"вÑĪий":85736,"â̦)":85737,"æĹ©å¹´":85738,"Ġgeothermal":85739,"ĠNecess":85740,"!âĢĻĊĊ":85741,"æ³Ĺ":85742,"å·²ç»ıå¾Ī":85743,"ĠCharity":85744,"Ġhatten":85745,"Ġíķ©ëĭĪëĭ¤":85746,"嬴":85747,"ĠоÑĢганизм":85748,"éĢĿä¸ĸ":85749,"ĠмаленÑĮ":85750,"é쏿Ĭŀ":85751,"ï¼įï¼įï¼įï¼į":85752,"Aust":85753,"Ġstitches":85754,"Ġonge":85755,"emes":85756,"ĠÙĬÙĨا":85757,"ðĿijĵ":85758,"ĠCastell":85759,"Ġpiel":85760,"Ġzost":85761,"æĪ¿ä¸ľ":85762,"дел":85763,"ĠÑħи":85764,"ÑĤивно":85765,"{Doxy":85766,"ĠMash":85767,"é¢ĺåºĵ":85768,"Ġattest":85769,"åħ±ç͍":85770,"ĠtemplateUrl":85771,"Ġibid":85772,"Ġnuevos":85773,"ĠиммÑĥ":85774,"DV":85775,"ĠMimi":85776,"Ġ\"{":85777,"æĢ§è³ª":85778,"Ġprovoked":85779,"Ġbuku":85780,"æł¼æł¼":85781,"红éħĴ":85782,"ä½Ľæ³ķ":85783,"ĠÏĥÏĦα":85784,"Ġpounding":85785,"-":86586,"ĠRIS":86587,"主è¯Ń":86588,"åĪ©å°¿":86589,"ciente":86590,"Ġhijos":86591,"ĠParticularly":86592,":,":86593,">[":86594,"Qi":86595,"ĠCBC":86596,"ноз":86597,"é«ĺçĤ¹":86598,"转è¿ĩ头":86599,"æ¯įæł¡":86600,"ginas":86601,"åΤæĸŃé¢ĺ":86602,"ĠPlayStation":86603,"ĠReflections":86604,"Ġhayop":86605,"kx":86606,"Ġbucks":86607,"Ġbeck":86608,"ä¸įåIJĮç¨ĭ度çļĦ":86609,"County":86610,"ĠвозможноÑģÑĤи":86611,"Ġpuppies":86612,"csv":86613,"lut":86614,"ĠtÅĤ":86615,"Ġpami":86616,"Ġdrip":86617,"راÙĤ":86618,"Protein":86619,"afar":86620,"Ġlogos":86621,"åıĮèĩĤ":86622,"ĠÄijá»ĭnh":86623,"ÙĦاة":86624,"ĠChemicals":86625,"Ġkurang":86626,"Late":86627,"ĠLans":86628,"Ġmecan":86629,"ĠYEAR":86630,"åĨħéĺģ":86631,"Ġgoodwill":86632,"Ġconfines":86633,"Ġdestru":86634,"Ġfilmmakers":86635,"Ġbleak":86636,"对å¤ĸè´¸æĺĵ":86637,"_API":86638,"whole":86639,"ĠмаÑģÑģа":86640,"Ġμια":86641,"ãģĬãĤĪãģ³":86642,"Luc":86643,"tools":86644,"ĠSofia":86645,"è¦ĥ":86646,"ç©¿æĪ´":86647,"å¼Ģå±ķçļĦ":86648,"çĿ£å¯Ł":86649,"никами":86650,"Ġshields":86651,"ĠاÙĦدÙĪÙĦØ©":86652,"routine":86653,"ĠTracing":86654,"ĠPunk":86655,"æŃ¦éģĵ":86656,"ĠØ®ÙĪÙĨ":86657,"ï½į":86658,"éī´èµı":86659,"対象":86660,"ĠЯн":86661,"িষà§įà¦Ł":86662,"imu":86663,"Ġendorse":86664,"}\\)\\(\\":86665,"åŃĶéļĻ":86666,"ĠاÙĦÙĤÙĦب":86667,"بÙĦغ":86668,"======Ċ":86669,"ĠпÑĢиводиÑĤ":86670,"dain":86671,"meters":86672,"åį«è§Ĩ":86673,"èĪįå¾Ĺ":86674,"ĠUndergraduate":86675,"ĠاسÙĦاÙħÛĮ":86676,"Wo":86677,"è¿Ļè¾ĪåŃIJ":86678,"éĩĮ头":86679,"æĹłæķħ":86680,"åħļå·¥å§Ķ":86681,"ĠBlanc":86682,"ĠCarrie":86683,"Ġsieve":86684,"ç¨įæľī":86685,"Ġbranched":86686,"ëĿ½":86687,"oitation":86688,"å¾Ĺåħ¶":86689,"èµ°å¾Ĺ":86690,"æĢĿç»´æĸ¹å¼ı":86691,"æĭĨåį¸":86692,"èIJĮèIJĮ":86693,"ĠSistema":86694,"ĠEukary":86695,"ĠзÑĥб":86696,"à§ĩপ":86697,"steady":86698,"ĠEdith":86699,"ĠMonark":86700,"Ġtrousers":86701,"ĠдÑĢÑĥга":86702,"-reviewed":86703,"nienia":86704,"ĠBret":86705,"ĠDFS":86706,"ĠRegg":86707,"Ġallowances":86708,"çĩķåŃIJ":86709,"究竣æĺ¯":86710,".ly":86711,"表çϽ":86712,"表åĵ¥":86713,"пон":86714,"Ġinvade":86715,"ÙĪÙĦÙĪ":86716,"-Aug":86717,"Ġgestational":86718,"ãģĿãĤĮãģ¯":86719,"Ġতারা":86720,"ĠSurveillance":86721,"aeda":86722,"ĠCaleb":86723,"عادة":86724,"æķ´æµģ":86725,"Ġinsulated":86726,"转èĢĮ":86727,"ĠNeal":86728,"äºļåİĨ":86729,"/files":86730,"ĠTRAN":86731,"ĠТакже":86732,"Cookie":86733,"kam":86734,"{'":86735,"Ġï¼īĊ":86736,"çļĦ缴æİ¥":86737,"Ġranc":86738,"é»Ħå±±":86739,"èĵ¦":86740,"Columb":86741,".jupiter":86742,"étude":86743,"å¹ķåIJİ":86744,"Ġਨ":86745,"ĠThankfully":86746,"ĠBaghdad":86747,"å°ıåĵ¥":86748,"usses":86749,"ATUS":86750,"à§ĩà¦Ĺ":86751,"fecture":86752,"Ġballoons":86753,"ترÙĥ":86754,"Ġlure":86755,"è¿ĺç»Ļ":86756,"æĽ´éľĢè¦ģ":86757,"åı°è´¦":86758,"czes":86759,"ĠSyracuse":86760,"Ġ×Ķ×ŀ×§":86761,"Ġpsoriasis":86762,"Sv":86763,"нованиÑı":86764,"åĴĮæľĭåıĭ":86765,"éĿ¢æĹł":86766,"Ġinterv":86767,"æį»":86768,"Ġseront":86769,"çľģåĨħ":86770,"çζçļĩ":86771,"Ġà°¤":86772,"åķĨä¸ļ模å¼ı":86773,"cited":86774,"åıijèĩª":86775,"Ġprogramma":86776,"åħļç»ĦæĪIJåijĺ":86777,"-element":86778,"Avg":86779,"çļĦæīĵ":86780,"ĠвÑĢед":86781,"ÑĪком":86782,"è¯ĨåŃĹ":86783,"Ġsenso":86784,"avorites":86785,"=P":86786,"Kin":86787,"éĩįä»»":86788,"Ġblan":86789,"олог":86790,"å¢ŀåĩı":86791,"èī¯ä¹ħ":86792,"æ¹ĸæ°´":86793,"Ġordained":86794,"àŃģ":86795,"มาà¸Ī":86796,"ãĥĸãĥŃ":86797,"Ġaliqu":86798,"ä¸Ĭè°ĥ":86799,"æĹ¶é«¦":86800,"оÑĢоÑĤ":86801,"ĠSprache":86802,"æŀģæĺĵ":86803,"çľĭåΰä»ĸ":86804,"ĠIntegral":86805,"ĠYahweh":86806,"Ġsquirrels":86807,"åıĺå°ı":86808,"åIJĦå®¶":86809,"èŀįæ´½":86810,"eax":86811,"anglement":86812,"Ġcovert":86813,"-ground":86814,"輸åħ¥":86815,"èĿĻèĿł":86816,"ä¹ĭè¡Į":86817,"表å¾ģ":86818,"ä¸ĩ亿åħĥ":86819,"logue":86820,"ĠاÙĦÙĨظاÙħ":86821,".createElement":86822,"Ġvt":86823,"Ġheraus":86824,"Ġanticoag":86825,"Fri":86826,"ĠOman":86827,"天é¹ħ":86828,"åĸ³":86829,"å¸Īéķ¿":86830,"åĸľçαçļĦ":86831,"èµĦæľ¬å®¶":86832,"à§ĩনà§įà¦Ł":86833,"æİ¥è§¦åΰ":86834,"æ·»åĬłåΰ":86835,"Ġconfronting":86836,"Ġdormant":86837,"Ġà¸Ķัà¸ĩ":86838,"ĠBeverly":86839,"èĮīèİī":86840,")ãĢĭ":86841,"lld":86842,"ĠSib":86843,"ĠCody":86844,"artist":86845,"sof":86846,"身çĿĢ":86847,"åģļ为":86848,"å°ijåIJĥ":86849,"æłĩè¯Ń":86850,"Reverse":86851,"Soon":86852,"ĠDesigns":86853,"åĮĸåѦåĵģ":86854,"çIJĨæīĢå½ĵçĦ¶":86855,"ĠPulse":86856,"æĺ¯æĸ°":86857,"platin":86858,"ĠÙħض":86859,"åĵģä½į":86860,"ÑĤай":86861,"宽带":86862,"ë¶Ī":86863,"Ġundertook":86864,"ĠTonight":86865,"å´Ńæĸ°çļĦ":86866,"éķ¿å¤§çļĦ":86867,"shake":86868,"Ġvoce":86869,"åIJĮæ¯Ķä¸ĭéĻį":86870,"fuel":86871,"çļĦ缮çļĦæĺ¯":86872,"ĠGat":86873,"æľĢåŁºæľ¬çļĦ":86874,"两å§Ķ":86875,"æµ·äºĭ":86876,"eroon":86877,"åįļä¼ļ":86878,"ĠاÙĦأخرÙī":86879,"PMID":86880,"Ġdarling":86881,"Ġgigantic":86882,"Ġtowering":86883,"Ġauthored":86884,"Ġunanimously":86885,"ç´łè´¨æķĻèĤ²":86886,"ĠпÑĥÑĤем":86887,"ĠBahrain":86888,"ç´§ç´§åľ°":86889,"éĥ½å¯¹":86890,"Contains":86891,"ĠÑĢазно":86892,"ระยะ":86893,"éĺ´èĻļ":86894,"ĠExecute":86895,"Ġì¶Ķê°Ģ":86896,"BACK":86897,"ĠNouns":86898,"oviet":86899,"ksam":86900,"çħ§æĸĻ":86901,"Ġchois":86902,"ĠAugusta":86903,"Ġsinh":86904,"åĺīåħ´":86905,"æħĪæĤ²":86906,"åĬĿ说":86907,"aston":86908,"æ¹§":86909,"æĽ¾åĽ½":86910,"ĠкоÑĺи":86911,"éĤ®ç͵":86912,"èIJ¨æĸ¯":86913,"confidence":86914,"Ġ문ìŀIJ":86915,"ÙĨاÙħج":86916,"Ġоднако":86917,"zés":86918,"大ä¼Ļ":86919,"Ġenigmatic":86920,"åĽłä¸ºè¿Ļ":86921,"éĶĻè¿ĩäºĨ":86922,"Ġunfinished":86923,"ÑĽÐ¸":86924,"ĠмножеÑģÑĤво":86925,"ĠGENERAL":86926,"ĠMANAGEMENT":86927,"Ġrecited":86928,"ä¹Łæĺ¯éĿŀ常":86929,"æĮªå¨ģ":86930,"计æķ°åύ":86931,"ĠNavigating":86932,"'ac":86933,"omar":86934,"getahui":86935,"åķ¶":86936,"-fire":86937,"ÑĪими":86938,"Psalm":86939,"×ŀ×Ļ":86940,"Ġsnippet":86941,"nict":86942,"}|\\":86943,"Ġdnia":86944,"æĿ¥åİĨ":86945,"Ġprez":86946,"ĠFlav":86947,"éĤĦæľĥ":86948,"ÑģолÑİÑĤ":86949,"JT":86950,"QP":86951,"Ġdrowning":86952,"ĠRedis":86953,"Ġknights":86954,"Ġprak":86955,"Ġmanuals":86956,"-unit":86957,"Pic":86958,"ologiques":86959,"ĠAbbas":86960,"Ġassesses":86961,"ามารà¸ĸ":86962,"ãĤĪãģĦ":86963,"æĮºå¥½çļĦ":86964,"ĠImportantly":86965,"çļĦå¢ŀåĬł":86966,"ĠOfic":86967,"Ġjue":86968,"ç͍å¤Ħ":86969,"ĠاÛĮÙħ":86970,"åīįè¿°":86971,"Ġ`ĊĊ":86972,"ĠÐļо":86973,"罪è¡Į":86974,"Bei":86975,"ikhail":86976,"ĠздоÑĢовÑĮÑı":86977,";**":86978,"Ġdever":86979,"ĠLTD":86980,"让èĩªå·±çļĦ":86981,"Ġlayouts":86982,"Deleted":86983,"ĠGallon":86984,"Greater":86985,"ĠаппаÑĢа":86986,"Divid":86987,"äºīæī§":86988,"篡":86989,"åı³éĶ®":86990,"ĠSimult":86991,"çļ±çĿĢ":86992,"ØŃÙĬØŃ":86993,"Ġenfermedades":86994,"åıĸå̼èĮĥåĽ´":86995,"Ġestructura":86996,"Nb":86997,"Ġaorta":86998,"ĠKyr":86999,"ucchini":87000,"ãģĤãģªãģŁ":87001,"åı¦ä¸Ģè¾¹":87002,"é³Ħ":87003,"ê·ł":87004,"ĠKY":87005,"Ġscala":87006,"å¹¶æıIJåĩº":87007,"ĠDeleg":87008,"ðĿijIJ":87009,"ĠконÑĨенÑĤÑĢа":87010,"éijij":87011,"dropdown":87012,"[num":87013,"Ġclasp":87014,"ä¹ĭä¹ī":87015,"ç¥Ł":87016,"åıĺæĢ§":87017,"ä½Ĩæĺ¯æĪij们":87018,"UBLE":87019,"Bird":87020,"éĥ½åºĶ":87021,"訳":87022,"ç»ĵæŀľæĺ¾ç¤º":87023,"ä¸įæĸŃå¢ŀ强":87024,"erdem":87025,"åĽ´ç»ķçĿĢ":87026,"氢氧åĮĸ":87027,"สิà¸ļ":87028,"Ġгид":87029,"Ġdreadful":87030,"Vertical":87031,"诲":87032,"Ġenquiry":87033,"ä¹ĭç͍":87034,"ĠYards":87035,"Ġcoy":87036,"اÙħÙĬÙĨ":87037,"ç¨ĭåºıä¸Ń":87038,"structural":87039,"å¹´ä»£æľ«":87040,"éªijè¡Į":87041,"Operating":87042,"Ġintervening":87043,"IGHTS":87044,"LOR":87045,"Ġpinn":87046,"ĠпиÑģÑĮ":87047,"Ġacceso":87048,"Ġparler":87049,"Ġpetits":87050,"Visibility":87051,"Ġkembali":87052,"viii":87053,"ä¸įåħī":87054,"ä½łçľŁ":87055,"afia":87056,"夫çļĦ":87057,"ĠOuter":87058,".\\,":87059,"ĠновÑĭÑħ":87060,"ocentric":87061,"qua":87062,"ĠWrit":87063,"Ġindig":87064,"æĶ¹è£ħ":87065,"_two":87066,"(src":87067,"ĠØŃÙĪ":87068,"ç»ıè¿ĩäºĨ":87069,"Ġsedang":87070,"Mol":87071,"دÙĪ":87072,"æĸĩå¸Ŀ":87073,"ĠвÑħод":87074,"äºĶ人":87075,"ĠMeer":87076,"ĠرÙħ":87077,"åįģäºĶæĿ¡":87078,"ĠCivic":87079,"ĠSTUDY":87080,"Ġanonymity":87081,"Ġlượng":87082,"(position":87083,"=T":87084,"å°±åΰ":87085,"å°ıå®¶ä¼Ļ":87086,"inschaft":87087,"ä½Ĩæĺ¯çͱäºİ":87088,"æĹĭåį³":87089,"è¿ŁæĹ©":87090,"×ķ×IJר":87091,"acqua":87092,"乡ä¸ĭ":87093,"ðĿijĿ":87094,"éĵģçŁ¿":87095,"Ġpasar":87096,"ĠQuesto":87097,"Ġotten":87098,"Ġexceedingly":87099,"ASCADE":87100,"Ġproblèmes":87101,"Vitamin":87102,"ĠÐłÑĥÑģ":87103,"âĹĭâĹĭ":87104,"ĠØŃاÙĦØ©":87105,"Ġcuff":87106,"Ġslash":87107,"çħ§æł·":87108,"ĠCenturies":87109,"огÑĢад":87110,"Ġagonist":87111,"Ġitinerary":87112,"ĠIEL":87113,"Ġatual":87114,"é«ĺé£İéĻ©":87115,"-local":87116,"Ġabsolut":87117,"اÙĤات":87118,":**:":87119,"Ġbardziej":87120,"oron":87121,"ĠBAL":87122,"ritte":87123,"Ġpea":87124,"éĢļåħ³":87125,"éĢ£çºĮ":87126,"POL":87127,"Щ":87128,"Ġsuk":87129,"ä¸īäºļ":87130,"Ġsemin":87131,"Regardless":87132,"à¸Ľà¸£à¸°à¸¡à¸²à¸ĵ":87133,"DIS":87134,"entie":87135,"coins":87136,"åı¤å¸ĮèħĬ":87137,"稻èįī":87138,"ĠLevine":87139,"ĠYugoslavia":87140,"ĠRFC":87141,"forum":87142,"åºľçļĦ":87143,"Ġembro":87144,"ĠJournalism":87145,"à©Ĥ":87146,"ĠPRODUCT":87147,"ĠparseInt":87148,"åĢŁæ¬¾äºº":87149,"![](":87150,".Format":87151,"ä¹ĭä¸į":87152,"-tw":87153,"ä½ıæĪ·":87154,"Ġlima":87155,"ÄĽÅĻ":87156,"åĿı人":87157,"ÑĢовки":87158,"crumb":87159,"Ġgerade":87160,"Ġstereotyp":87161,"Ġíķ´ëĭ¹":87162,"Ġegin":87163,"Ġstu":87164,"åħ¬å¼Ģåıij":87165,"×Ļש×":87166,"гон":87167,"æĶ¾å®½":87168,"Ġavian":87169,"举åĿ¡":87170,"ási":87171,"Ġpourquoi":87172,"ĠHSV":87173,"ĠnÄĽkol":87174,"kcji":87175,"Ġcrawling":87176,"Ġ׼×IJשר":87177,"etten":87178,"æľºæ²¹":87179,"ĠبÙIJ":87180,"书信":87181,"è¿Ļç§į人":87182,"Ġparticipates":87183,"Ġanimales":87184,"connecting":87185,"æIJŀç¬ij":87186,"æģ¶æĢ§èĤ¿çĺ¤":87187,"Ġverschiedene":87188,"ruff":87189,"æĬĢæ³ķ":87190,"ronomy":87191,"ÄĻtr":87192,"ĠScopus":87193,"-wheel":87194,"çļĩ室":87195,"Golden":87196,"Snow":87197,"çµ¶":87198,"Ġsemakin":87199,"_mult":87200,"驾车":87201,"çĭ¬ç«ĭæĢ§":87202,"ä¸¥æł¼éģµå®Ī":87203,"OHN":87204,"Ġtingkat":87205,"Ġìĸ´ëĸ¤":87206,"ĠÑģокÑĢа":87207,"ãĢĤĊĊĊ":87208,"تÙĥ":87209,"åľºé¦Ĩ":87210,"ücks":87211,"çļĦä¸Ģæĸ¹":87212,"åºķéĿ¢":87213,"è¿Ļæĺ¯æĪij们":87214,"ר×ij×¢":87215,".store":87216,"ĠâĬĨ":87217,"ĠWirk":87218,"ĠLOS":87219,"Ġintimately":87220,"æľĢèĥ½":87221,"åĪĻ以":87222,"Ġmidfielder":87223,"Ġselalu":87224,"ĠDetermining":87225,"charged":87226,"Ġpaving":87227,"太ç¥ĸ":87228,"åIJĥäºĨä¸Ģ":87229,"çŁ³åύ":87230,"ĠNeon":87231,"Ġcontainment":87232,"Ġfermented":87233,"ĠEmpower":87234,"моÑĤÑĢÑı":87235,")t":87236,"Ġinund":87237,"Ġbefind":87238,"åĽ¤":87239,"ĠGilles":87240,"ĠOnd":87241,"ä»ĸ以":87242,"请注æĦı":87243,"ĠMehr":87244,"ãģĭãģ®":87245,"зиÑı":87246,"ãĢĤ(ãĢĬ":87247,"ÙĪÛĮت":87248,"ajÄħcych":87249,"Ġà´ħ":87250,"Ġmildly":87251,"ĠBeginners":87252,"ĠSTATES":87253,"Ġusando":87254,"Ġcompañ":87255,"Ġতà§Ī":87256,"åį«çĶŁåģ¥åº·":87257,"Locale":87258,"è°´è´£":87259,"åıĸèĥľ":87260,"ĠзÑĢениÑı":87261,"ĠÑĢазлиÑĩнÑĭе":87262,"ĠHai":87263,"æĺ¥å¤ı":87264,"ÏĨα":87265,"smart":87266,"StatusCode":87267,"缸æ¯Ķä¹ĭä¸ĭ":87268,"YYYY":87269,"GROUP":87270,"Ġalmonds":87271,"çļĦçζæ¯į":87272,"è¿ĩéķ¿":87273,"æĢ»åĨ³èµĽ":87274,"æľªè¢«":87275,"åı¦ä¸Ģæĸ¹":87276,"缸å½ĵçļĦ":87277,"Ġpartnering":87278,"ĠTribe":87279,"ĠETF":87280,"Nous":87281,"VAR":87282,"è¥¿çº¢æŁ¿":87283,"Quad":87284,"IPE":87285,"éģįäºĨ":87286,"çĽĽå®´":87287,"Ġthreaded":87288,"Ġdeterminado":87289,"-intercept":87290,"ðŁĵį":87291,"à§ĩà¦Ľà¦¿à¦²à§ĩন":87292,"Ġdestroys":87293,"VF":87294,"_active":87295,"wash":87296,"ä¸ĢæĪĺ":87297,"äºĶåij³":87298,"åIJ«ç³Ĭ":87299,"æĨIJ":87300,"ä¹Łä¼ļæľī":87301,"Ġgranules":87302,"Pray":87303,"Ġiniz":87304,"åĴĮåľ¨":87305,"ä½łéĤ£":87306,"èĢģéĹĨ":87307,"κÏģα":87308,"Ġglyph":87309,"arvard":87310,"ÙĪÙģÙĬ":87311,"épend":87312,"Ġresusc":87313,"æł·æĿ¿":87314,"桨":87315,"Ġsmug":87316,"åıĸæļĸ":87317,"èĬ±åĦ¿":87318,"Ġprojectile":87319,"Ġ׼ף":87320,"Ġcoward":87321,"ĠBASIS":87322,"è¦ģåΰ":87323,"(\"../":87324,"Ġتب":87325,"APE":87326,"çĶ³è¯·ä¹¦":87327,"ĠTimber":87328,"Ġprincipale":87329,"airobi":87330,"Ġunflagged":87331,"ĠSwim":87332,"Ġtranslational":87333,"ä¹Įé²ģ":87334,"Ġcarte":87335,"详ç»Ĩä»ĭç»į":87336,"Ġsabwag":87337,"opedic":87338,"CX":87339,"ä¸Ĭæĸ°":87340,"æĮĩäºĨæĮĩ":87341,"认æ¸ħ":87342,"ä¸ĸåŃIJ":87343,"Ġstabilizing":87344,"ĠоÑģобен":87345,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĊ":87346,"Ġwards":87347,"Ġmuz":87348,"Ġlids":87349,"ĠDK":87350,"ĠLoch":87351,"Ġrov":87352,"à¹Ģà¸Ńà¸ģ":87353,"æķ´å¥Ĺ":87354,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":87355,"è°Īæģĭçα":87356,"Philosoph":87357,"ÙijÙı":87358,"Ġdisclosures":87359,"Ġparadigms":87360,"Ġbarr":87361,"راÙĨ":87362,"Ġselv":87363,"_SET":87364,"çİĦæŃ¦":87365,"ĠERROR":87366,"rae":87367,"山峰":87368,"Ġwaterfall":87369,"ĠвÑĭзÑĭва":87370,"沿岸":87371,"multiple":87372,"ĠìľĦì¹ĺ":87373,"---------------+":87374,"Ġkernels":87375,"Cool":87376,"Ġsinners":87377,"åĴĮåѦçĶŁ":87378,"å°±æĺ¯ä»ĸ":87379,"keen":87380,"请åģĩ":87381,"å¢ŀåĬłçļĦ":87382,"eksi":87383,"ĠCapac":87384,"æ¯ķä¸ļ论æĸĩ":87385,"_exp":87386,"Ġ}.":87387,"ixe":87388,"Ġresearches":87389,"积èĵĦ":87390,"é¢Ħåζ":87391,"-grid":87392,"MLA":87393,"processed":87394,"ändern":87395,"ĠоÑĢганов":87396,"ĠStrauss":87397,"ĠÑĤÑĢеÑĤÑĮ":87398,"èIJ½åΰå®ŀå¤Ħ":87399,"issage":87400,"ĠسÙģ":87401,"ĠTwilight":87402,"åĽ°éļ¾åĴĮ":87403,"主èIJ¥ä¸ļåĬ¡":87404,"ampere":87405,"ç͍çļĦæĺ¯":87406,"Ġпен":87407,"ĠResident":87408,"ĠCommissioners":87409,"اجة":87410,"triangle":87411,"Nombre":87412,"=N":87413,"Poll":87414,"åIJİåįĬ":87415,"两éģĵ":87416,"ushima":87417,"âĿĹ":87418,"hz":87419,"rored":87420,"Ġchilling":87421,"Ġpepp":87422,"å·²åĽŀçŃĶ":87423,"Ġservir":87424,"ä¸įæĺ¯æĪij":87425,"ĠDescartes":87426,"Ġapproximations":87427,"ĠSunny":87428,"lemagne":87429,"Ġdoubted":87430,"ifiz":87431,"Ġsuperintendent":87432,"ä¸įä¼ļåĨį":87433,"绿èĮ¶":87434,"ÉĻËĪ":87435,"èIJ½å®ŀåΰ":87436,"Ġclockwise":87437,"Topics":87438,"erland":87439,"çļĦåĪ¶ä½ľ":87440,"ĠLLP":87441,"çIJĨåıij":87442,"æľºè½¦":87443,"Ġtesto":87444,"ัà¹Īว":87445,"Ġworldview":87446,"ĠпоÑĢаж":87447,"ĠGoethe":87448,"ôi":87449,"ordeaux":87450,"ĠSas":87451,"Ġbegged":87452,"æľ¬åįķä½į":87453,"æĸĩéĿ©":87454,"ساÙĨÛĮ":87455,"ĠpoÄį":87456,"ĠShall":87457,"èĸĦèį·":87458,"åıĤæķ°çļĦ":87459,"Ġalcune":87460,"ĠFutures":87461,"bars":87462,"fers":87463,"Ľ×ķת":87464,"Ġ×Ķ×¢×":87465,"³à¯į":87466,"Ġاجر":87467,"Ġnoir":87468,"Ġinsider":87469,"ãģ²ãģ¨":87470,"ĠMIL":87471,"æľ¬å®ŀç͍æĸ°åŀĭ":87472,"ált":87473,"çĤ¹æ»´":87474,"ĠÑģеÑĤи":87475,"ieważ":87476,"ĠмеÑĢе":87477,"liw":87478,"ĠMarqu":87479,"éĢģå¾Ģ":87480,"ĠSerie":87481,"ä»·å̼åĴĮ":87482,"æĪIJåijĺçļĦ":87483,"éĢĻåĢĭæĻĤåĢĻ":87484,"çŁ¥ä¹İ":87485,"Ġconstitutive":87486,"rystals":87487,"itosan":87488,"ĠSquadron":87489,"itars":87490,"Ġleth":87491,"ffiti":87492,"Ġdismin":87493,"ĠPhonics":87494,"åĽºæī§":87495,"å®ĥ们æĺ¯":87496,"ĠAdvocate":87497,"ä¸Ģå®ļè¦ģ注æĦı":87498,"ĠEduardo":87499,"Ġdrowned":87500,"两èĢħçļĦ":87501,"ضة":87502,"رÙĪØ³":87503,"Ġkonstru":87504,"èĵĦçĶµæ±ł":87505,"ĺף":87506,"åIJĪæ³ķæĢ§":87507,"Ġintrins":87508,"uele":87509,"Ġintuit":87510,"cian":87511,"å¹³çļĦ":87512,"Ġinsistence":87513,"æł¹éĥ¨":87514,"Annotations":87515,"Ġseasoning":87516,"Ġcreditor":87517,"IRED":87518,"िश":87519,"-shot":87520,"çµĦåIJĪ":87521,"åįļ士åѦä½į":87522,"ĠëłĪ":87523,"jär":87524,"Ġtinnitus":87525,"vertices":87526,"åģĩåĨĴ":87527,"Ġrecommending":87528,"建çŃijçļĦ":87529,"ĠGreenwood":87530,"Ġvisionary":87531,"frontal":87532,"нÑİÑİ":87533,"(en":87534,"Ġnghi":87535,"çĶŁãģį":87536,"Ġinfinit":87537,"è£ħè½½":87538,"ذÛĮر":87539,"-production":87540,"èģĮä¸ļæĬĢèĥ½":87541,"ãģķãĤĮãģ¦":87542,"огÑĢаÑĦии":87543,"ĠознаÑĩаеÑĤ":87544,"vole":87545,"å¼Ģåľº":87546,"ridine":87547,"Ġзако":87548,"ĠÑĤой":87549,"Ġstandardization":87550,"ç§ģãģ¯":87551,"Ġniece":87552,"Ġrevolutionize":87553,"éģĭç͍":87554,"ĠоблаÑģÑĤÑĮ":87555,"Ġzpůsob":87556,"ĠHitch":87557,"æĮĩæı®":87558,"ä¼łåΰ":87559,"ä»ĸ们认为":87560,"Ġdomic":87561,"åĩĮäºij":87562,"ждение":87563,"ĠDewey":87564,"Ġодинаков":87565,"Ġunatt":87566,"Ġ{-":87567,"Ġdoom":87568,"åħ¬å·®":87569,"Ġreplacements":87570,"æ´ĽæĿī磶":87571,"çļĦ女åŃIJ":87572,"Ġcling":87573,"å¼Ģåĩº":87574,"Ġsuburb":87575,"çĭ¬ä¸ĢæĹł":87576,"Ġawal":87577,"Ġanalyzer":87578,"Ġpygame":87579,"ĠSeparation":87580,"æ¦ľåįķ":87581,"Ġbiasanya":87582,"ĠFernández":87583,"淹没":87584,"akume":87585,"Ġquelli":87586,"æĢ§å¥½":87587,"ĠÑĤÑĢÑĥб":87588,"Players":87589,"/config":87590,"ĠKeb":87591,"åľ°åĬ¿":87592,"θÎŃ":87593,"ানি":87594,"è³ĩçĶ¢":87595,"åľ°ä½įçļĦ":87596,"ĠSupported":87597,"omie":87598,"esség":87599,"ĠNass":87600,"对æķ°":87601,"inkan":87602,"åıĪåı¯ä»¥":87603,"çķĮ线":87604,"Ġunfores":87605,"connections":87606,"ĠاÙĨÙĪØ§Ø¹":87607,"èħ¹èĥĢ":87608,"ÙĪÛĮس":87609,"etheus":87610,"ĠÙĩÛĮÚĨ":87611,"ĠоÑĩеÑĢедÑĮ":87612,"çļĦ第ä¸Ģ个":87613,"etri":87614,"Ġjon":87615,"Ġcontrat":87616,"Ġdismay":87617,"çݰ身":87618,"ä¿¡å¾Ĵ":87619,"hetamine":87620,"ÐķÐĿ":87621,"ĠHexapoda":87622,"ĠContracts":87623,"Ġelucidate":87624,"Zo":87625,"Ġona":87626,"Ġmeisten":87627,"Ãłnh":87628,"uellement":87629,"æ¤ħä¸Ĭ":87630,"ĠAren":87631,"oters":87632,"ĠMMP":87633,"Ġacetic":87634,"ĠпÑĢинадле":87635,"Ġcutter":87636,"伯çε":87637,"å¼·èĢħ":87638,"Ġowes":87639,"Ġrok":87640,"ificação":87641,"ĠØŃتÛĮ":87642,"éĬ³":87643,"ĠعÙĦÙĪÙħ":87644,"Sav":87645,"[tex":87646,"jl":87647,"忱":87648,"缸èģļ":87649,"çIJĨ论åŃ¦ä¹ł":87650,"Ġpropio":87651,"æ´Ľä¼Ĭ":87652,"dou":87653,"éĥ½æĺ¯ä»¥":87654,"ETY":87655,"è¿ĺæľī人":87656,"æıĴ座":87657,"Ġmurderer":87658,"Ġпопа":87659,"ĠVersailles":87660,"Cells":87661,"Ġthwart":87662,"ĠuÄį":87663,"æĿĤçī©":87664,"ĠMitglied":87665,"ANGU":87666,"çļĦçαæĥħ":87667,"ostÄĻp":87668,"rité":87669,"éĺ²åį«":87670,"éħįæĸĻ":87671,"-counter":87672,"ä»ħä¾ĽåıĤèĢĥ":87673,"+f":87674,"Ġsth":87675,"åľ¨åĨľæĿij":87676,"ssch":87677,"Ïģια":87678,"缴æİ¥æĬĬ":87679,"伸缩":87680,"-score":87681,"âĢĿ(ãĢĬ":87682,"Ġlobes":87683,"াধà§įযম":87684,"\")ĊĊĊ":87685,")p":87686,".center":87687,"çļĦåľ°åĮº":87688,"ĠReleased":87689,"Ðļон":87690,".Con":87691,"Gray":87692,"mens":87693,"Ġ{$":87694,"éķ°":87695,"à¸ģิà¸Ļ":87696,"Ġ==>":87697,"Ġcontrario":87698,"ĠìķĦëĭĪëĿ¼":87699,"аÑħаÑĢÑħой":87700,"ĠToll":87701,"对ä¸Ģ个":87702,".Select":87703,"ä½Ĩæĺ¯å¥¹":87704,")=-":87705,"(bool":87706,"Ġlandscaping":87707,"ÑģÑĤвима":87708,".exit":87709,"=[]Ċ":87710,".Empty":87711,"Ġживело":87712,"endorf":87713,"年产":87714,"çļĦ人æĺ¯":87715,"å½±åĵįçĿĢ":87716,"COP":87717,"ĠSummar":87718,"Coin":87719,"稿件":87720,"zug":87721,"()),Ċ":87722,"æİ¥ä¸ĭä¾Ĩ":87723,"\\);":87724,"ophil":87725,"认è¯ĨåĴĮ":87726,"丰å¯ĮäºĨ":87727,"Ġinventive":87728,"åħļåĴĮåĽ½å®¶":87729,"'ob":87730,"åľ¨çĶŁæ´»ä¸Ń":87731,"Ġprecon":87732,"ificantly":87733,"Includes":87734,"atured":87735,"manent":87736,"ĠCRISPR":87737,"Ġkönnte":87738,"ĠukÅĤad":87739,"Ġinadvertently":87740,"(',":87741,"Vin":87742,"ÄĢ":87743,"Ġstessa":87744,"creto":87745,"æĺ¾èĢĮæĺĵ":87746,"Ġизде":87747,"Ġorganizz":87748,"atonin":87749,"ĠAdolescent":87750,".identifier":87751,"Bol":87752,"Ġspacer":87753,"Ġblender":87754,"è£ħåį¸":87755,"à¹ĩม":87756,"ĠاÙĦأساس":87757,"Ġjednot":87758,"使èĩªå·±":87759,"ä¾ĽçĥŃ":87760,"åįİçļĦ":87761,"Ġresponders":87762,"ĠMilky":87763,"Ġà¹Ģà¸Ķà¹ĩà¸ģ":87764,"ç¾İæĦŁ":87765,"æĮīè¦ģæ±Ĥ":87766,"Ġefficace":87767,"mmHg":87768,",''":87769,"ĠارائÙĩ":87770,"åĬłæ²¹ç«Ļ":87771,"BRA":87772,"Ġpave":87773,"Ġdizziness":87774,"ĠPike":87775,"iennes":87776,"ENA":87777,"鸥":87778,"}}-":87779,"Ġpendulum":87780,"ĠPicasso":87781,"Ġanglès":87782,"Ġcoagulation":87783,"Ġartificially":87784,"Ġgroceries":87785,"DY":87786,"бÑĢан":87787,"æķ°æį®ç»ĵæŀĦ":87788,"mmm":87789,"Records":87790,"iesiÄħt":87791,">>ĊĊ":87792,"Ġslick":87793,"ediatric":87794,"æ²½":87795,"çIJµ":87796,"è¿ĩä¸Ģ个":87797,"éĩĮåİ»":87798,"æ¯ıå°ıé¢ĺ":87799,"æĸŃè·¯":87800,"æİĴåľ¨":87801,"æĸ¹æ³ķæĿ¥":87802,"åĬŁèĥ½éļľç¢į":87803,"ĠMoreno":87804,"ä¹Łæľīä¸ĢäºĽ":87805,"躯ä½ĵ":87806,"à¦ıà¦ĩ":87807,"Ġastronauts":87808,"Race":87809,"äºĨçĦ¶":87810,"appiness":87811,".Color":87812,"Ġinventories":87813,"Ġétudes":87814,"ĠSegmentation":87815,"ä¸įçͱèĩªä¸»":87816,"ĠLEDs":87817,"Ġreiterated":87818,"Ġпедагоги":87819,"ĠJules":87820,"éģĵåıĭ":87821,"èįŁ":87822,"ÏħνÏĦ":87823,"éħĿ":87824,"绣绣":87825,"Whit":87826,"CHAR":87827,"Ġ×ł×ª":87828,"ĠkV":87829,"Ġdistancia":87830,"Ġgrabs":87831,"Ġdonné":87832,"Profit":87833,"Ġprimero":87834,"ská":87835,"æĶ¿åºľå¯¹":87836,"ĠÐĴлади":87837,"å²ģ以ä¸Ĭ":87838,"Ġadmirable":87839,"ÅĻÃŃklad":87840,"training":87841,"gte":87842,"running":87843,"icom":87844,"ĠTRI":87845,"pline":87846,"Ġabre":87847,"Ġlax":87848,"å¥½ä¸ľè¥¿":87849,"ä¸īåįģå¹´":87850,"çĵ·åύ":87851,"Ġì²ľ":87852,"åīįæ®µæĹ¶éĹ´":87853,"ssh":87854,"计æıIJ":87855,"åºıå¹ķ":87856,"ĠàªĨ":87857,"ĠFemin":87858,"ĠArchaeological":87859,"Ġomin":87860,"Ġdrilled":87861,"ĠPolski":87862,"æĶ¿æ²»å±Ģ":87863,"à½ĺ":87864,"Ġelaborated":87865,"çī²çķľ":87866,"ĠÑģÑħем":87867,"Choosing":87868,"Zm":87869,"ĠRPG":87870,"æİ¥åĬĽ":87871,"éĺ²å¤ĩ":87872,"สาย":87873,"ĠکاÙħ":87874,".Tab":87875,"Ġepigenetic":87876,"ĠÙħÙĦÙģ":87877,"å¾Īæĺ¾çĦ¶":87878,"Ġблок":87879,"Ġbookmark":87880,"羣çļĦ好":87881,"رÙĬÙĩ":87882,"slides":87883,"åįģä¸īäºĶ":87884,"åįłæį®äºĨ":87885,"å°ĭæī¾":87886,"Ġreduct":87887,"ä¹ĭæľ¬":87888,"Ġrestrained":87889,"Ġдело":87890,"æįŁå¤±çļĦ":87891,"Ġশà§įর":87892,"Ġadipis":87893,"Ġeased":87894,"ĠBuzz":87895,"åħ¨æĿij":87896,"æģĨ":87897,"problems":87898,"æīĵ交éģĵ":87899,"æ±ŁåĮĹ":87900,"iati":87901,"ĠPowered":87902,"ĠWilde":87903,"à¥ĭà¤Ĺ":87904,"ĠдиÑĦ":87905,"bnb":87906,"ĠCombination":87907,"erase":87908,"ĠBé":87909,"placing":87910,"Ġherds":87911,"Ġcommute":87912,"å¾Īéĩįè¦ģçļĦ":87913,"×ķ×IJ×":87914,"æĬķå°Ħ":87915,"èĴ¿":87916,"ĠPaÃŃs":87917,"Ġconstrucción":87918,"ĠÏĮÏĦι":87919,"Syntax":87920,"Ġhype":87921,"ĠÑģейÑĩаÑģ":87922,"Ġamely":87923,"ahuan":87924,"ãģĻãĤĮãģ°":87925,"çľģçļĦ":87926,"è¿Ļé¦ĸæŃĮ":87927,"\"},":87928,"ĠTata":87929,"ĠFI":87930,"ĠWyd":87931,"ieck":87932,"åĴĮå¤ļ":87933,"Ġshone":87934,"ç»ĻåĪ«äºº":87935,"rische":87936,"-coll":87937,"ãĥ¼ãĤº":87938,"謹":87939,"åıĺå¾Ĺè¶ĬæĿ¥è¶Ĭ":87940,"ĠHelping":87941,"ĠpolÃŃtico":87942,"Ġelongation":87943,"Ñķ":87944,"çļĦéĿ¢åīį":87945,"Ġdean":87946,"Ġ´":87947,"æĹłä¸º":87948,"æĶ¹åĬ¨":87949,"Ġtemos":87950,"EFL":87951,"ĠNumerade":87952,"Ġcranial":87953,"Meg":87954,"Ġids":87955,"ä¸Ńèİ·å¾Ĺ":87956,"Ùħبر":87957,"...)":87958,"afen":87959,"Ġ׾פ×Ļ":87960,"éĢĤåIJĪèĩªå·±çļĦ":87961,"Ġsouthwestern":87962,"æī¿æĭħ责任":87963,"ĠبازÛĮ":87964,"Nutrition":87965,"ĠHague":87966,"okus":87967,"æ²īåIJŁ":87968,"Ġingenu":87969,"Ġpromoters":87970,"çªģçł´äºĨ":87971,"nich":87972,"Ġapprox":87973,"Ġcrecimiento":87974,"åħ±çĶŁ":87975,"Ġpostwar":87976,"ĠÑĦоÑĤо":87977,"æĮĤäºĨ":87978,"'":89657,"_service":89658,"ĉstruct":89659,"ĠÑģбоÑĢ":89660,"åİŁèijĹ":89661,"ת×Ļ":89662,"å©ļ纱":89663,"éĢŁåº¦åĴĮ":89664,"{(}":89665,"à§Ĥরà§įব":89666,"ĠاÙĦبØŃØ«":89667,"(private":89668,"Ġneoliber":89669,"大æĥĬ":89670,"ĠVAR":89671,"Ġinteresse":89672,"Ġcoales":89673,"Ġmedically":89674,"Ġstrives":89675,"åºķæ°Ķ":89676,"çıŃä¼ļ":89677,"Ġfactoring":89678,"àµĢ":89679,"Ġweathering":89680,"Ġ×§×ij":89681,"Ġreversing":89682,"niz":89683,"ĠClem":89684,"Ġprolet":89685,"ĠHIS":89686,"ocuments":89687,"Ġsapp":89688,"Pros":89689,"rafted":89690,"ĠVerification":89691,"Ġhypnot":89692,"å·¥ä¸ļåĴĮ":89693,"æ¶Īå¤±åľ¨":89694,"islav":89695,"_O":89696,"ĠLAS":89697,"Ġphil":89698,"åŁºçŁ³":89699,"Ġsmashed":89700,"çłĶ究室":89701,"å¾·åĽ½çļĦ":89702,"åı³ä¸ĭ":89703,"èĪªæµ·":89704,"Ġsands":89705,"ì°°":89706,"walks":89707,"occupied":89708,"Ġmikro":89709,"ĠLähteet":89710,"Diet":89711,"ulif":89712,"åĴĮéĺ¿":89713,"èIJ¦":89714,"-sal":89715,"éĽĨå¸Ĥ":89716,"Ġoppressive":89717,".dis":89718,"ä¹Ŀé¾Ļ":89719,"æ£ĢæŁ¥åĴĮ":89720,"æĸ¹åIJijçĽĺ":89721,"ç¨Ģçĸı":89722,"æIJľç´¢å¼ķæĵİ":89723,"boldmath":89724,"ĠLepid":89725,"æĺ¯åĪ©ç͍":89726,"ĠDatuak":89727,"Ġì±Ħ":89728,"éĩįè¿Ķ":89729,"Ġcarbs":89730,"Ġdistributor":89731,"æķ¬æĦı":89732,"ç»Ŀ对å̼":89733,"çĸı忽":89734,"Ġrozd":89735,"çķħéĶĢ":89736,"æĮ¡ä½ı":89737,"-enabled":89738,"Ġattenuated":89739,"ĠBacteria":89740,"ĠJT":89741,"å½Į":89742,"ĠIncent":89743,"ç³¾":89744,"æīįå¼Ģå§ĭ":89745,"è¿ĻäºĽè¯Ŀ":89746,"è¿ŀæİ¥çļĦ":89747,"Ġespéc":89748,"Ġlactose":89749,"Improved":89750,"Bool":89751,"Ġð":89752,"éħ£":89753,"èĭ±ä¿Ĭ":89754,"Ġfullest":89755,"å¿ħè¦ģæĢ§":89756,"ĠAlexa":89757,"Ġrozw":89758,"ĠudziaÅĤ":89759,"Ġrifles":89760,"Maker":89761,"adav":89762,"ogli":89763,"åıĬãģ³":89764,"ÏĢά":89765,"ĠSOFT":89766,"Ġnecesidad":89767,"melon":89768,"缴åįĩæľº":89769,"Ġsublime":89770,"fatt":89771,"inom":89772,"Ġstaan":89773,"å·¥ä½ľå²Ĺä½į":89774,"ogno":89775,"åħ«å¹´çº§":89776,"æĮ¥åıij":89777,"Ġmolded":89778,"(`${":89779,"lel":89780,"rake":89781,"è¿ĻèĤ¡":89782,"yma":89783,"çĥŃæIJľ":89784,"ÙĴتÙİ":89785,"éĤ»éĩĮ":89786,"ĠSomerset":89787,"ì½":89788,"recomm":89789,"itzen":89790,"ä¸įéľĢ":89791,"Ġirresist":89792,"ĠMerlin":89793,"çļĦæĸ°åŀĭ":89794,"ährung":89795,"è°İè¨Ģ":89796,"ĉq":89797,"abouts":89798,"Ġregimens":89799,"ĠScha":89800,"ĠEssentially":89801,"ÑĨиÑıÑħ":89802,"ĠLjava":89803,"åīįè¨Ģ":89804,"æĿ¡çº¹":89805,"论çĤ¹":89806,"第äºĮå¹´":89807,"ĠExplor":89808,"失败äºĨ":89809,"×ķצ×Ķ":89810,"ĠпÑĢоÑĤивоп":89811,".Order":89812,";s":89813,"Dave":89814,"Rx":89815,"endes":89816,"å¼Ĥçī©":89817,"çļ®å¸¦":89818,"ĠBenz":89819,"ĠSuperman":89820,"UCK":89821,"èĬ¬èĬ³":89822,"Gross":89823,"Ġtending":89824,"Ġauss":89825,"以满足":89826,"对åIJĦ":89827,"æĢ»äº§å̼":89828,"éĿŀ常大":89829,"Checked":89830,"ĠASSERT":89831,"gj":89832,"renn":89833,"жем":89834,"èij©":89835,"æĻĤãģ«":89836,"Ġdedicate":89837,"áŀĺ":89838,"ĠìĿ´ë¯¸":89839,"Ġdoped":89840,"nasium":89841,"æļ§æĺ§":89842,"çIJ¥":89843,"管åĨħ":89844,"帮åĬ©åѦçĶŁ":89845,"éĢĴ交":89846,"褥":89847,"ĠÙħØ´Ú©":89848,"PATH":89849,"çļĦæľ¨":89850,"Ġrecreate":89851,"äºĨä»ĸ们":89852,"æĦıæĥ³ä¸įåΰ":89853,"ĠArlington":89854,"ä¿®éģĵ":89855,"Ġauditing":89856,"èĤ¥çļĤ":89857,"Ġθε":89858,"åķĨåĬ¡åį°ä¹¦é¦Ĩ":89859,"horse":89860,"ĠокÑĤÑı":89861,"Kindergarten":89862,"ServletRequest":89863,"\"):Ċ":89864,"Fortunately":89865,"Ġridd":89866,"ĠChor":89867,"ungtod":89868,"ĠÐĵÐŀ":89869,"Ġburner":89870,"Ġadjuvant":89871,"×Ļקר":89872,"Ġregenerative":89873,"ĠMärz":89874,"åĩºåĵģ":89875,"æĸ¹åľĨ":89876,"å·²æĪIJ":89877,"åIJįèĥľ":89878,"REAM":89879,"ãĥĥãĥī":89880,"Ġneuropathy":89881,"ĠSergio":89882,"\\Omega":89883,"Ġاشار":89884,"åIJİæĦŁ":89885,"éĥ½ä¸º":89886,"ä½įåĪĹ":89887,"å¼łè´´":89888,"ĠÑĪколе":89889,"Ġáĥł":89890,"ĠìĤ¬ìĿ´":89891,"Ġdisproportionately":89892,"åĩ¦çIJĨ":89893,"ĠEmbedded":89894,"Gest":89895,"enching":89896,"ĠBW":89897,"åħī亮":89898,"åĪĻéľĢè¦ģ":89899,"à¸Ħà¹Ĥà¸Ļ":89900,"ĠرئÙĬس":89901,"Ġqi":89902,"ĠBurger":89903,"Ġcereals":89904,"ĠLuca":89905,"æīĭç»Ńè´¹":89906,"-described":89907,"ografic":89908,"Ġnanotubes":89909,"-connected":89910,"ÉĴ":89911,"ombs":89912,"ĠRanger":89913,"ĠEQ":89914,"å°±åıªèĥ½":89915,"对åı£":89916,"ahami":89917,"Ġstrlen":89918,"Ķ×Ĵ":89919,"å°½èģĮ":89920,"åħ¨éĿ¢èIJ½å®ŀ":89921,"ĠUntersuch":89922,"ĠNickel":89923,"ĠÑĢезÑĥлÑĮÑĤаÑĤÑĭ":89924,"æĪĺåľºä¸Ĭ":89925,"ĠÄijá»Ļng":89926,"BRE":89927,"Ġfurl":89928,"ĠGus":89929,"çĶŁæł¹":89930,"ä¸ĭåľº":89931,"å¤ļäºİ":89932,"åĮ»ç͍":89933,"ophilus":89934,"æķ¬èĢģ":89935,"æľīçĤ¹åĦ¿":89936,"Ġtrademarks":89937,"_modules":89938,"ĠScores":89939,"ĠCAGR":89940,"coni":89941,"åĪĨäºĨ":89942,"好èĩªå·±çļĦ":89943,"trigger":89944,"asadpang":89945,"Ġcomputes":89946,"åıĬæĻĤ":89947,"éĶĦ":89948,"è·¯çģ¯":89949,"ĠSpir":89950,"Ġsuperim":89951,"ĠMaÃŁ":89952,"Ġkabungtor":89953,"Ġplagued":89954,"ĠEVERY":89955,"kowski":89956,"大æĪIJ":89957,"ãĤĤãģĨ":89958,"ĠEstonia":89959,"Ġдели":89960,"Alternatively":89961,"Ġapprehend":89962,"mong":89963,"pir":89964,"Ġoncology":89965,"-bi":89966,"æĿĥè¡¡":89967,"Ġsuccumb":89968,"Ġunanimous":89969,"Ġkabungtoran":89970,"ÃŃk":89971,"缸éĢ¢":89972,"æ´»å¾Ĺ":89973,"ĠHighland":89974,"æ°ıæĹı":89975,"Ġfavoured":89976,"amilton":89977,"æ¸ĬæºIJ":89978,"Ġredshifts":89979,"opping":89980,"çļĦæī§è¡Į":89981,"äºĭåıĺ":89982,"ighbour":89983,"à¸Ńà¹Īาà¸Ļ":89984,"texttt":89985,"äºĶ代":89986,"Ġизме":89987,"ä¸Ģä¸ĭåIJ§":89988,"Ġdéb":89989,"OMO":89990,"krieg":89991,"ĠBd":89992,"çĶŁäº§èµĦæĸĻ":89993,"helpers":89994,"ĠFeatured":89995,"illusion":89996,"æĻ¤":89997,"-py":89998,"Ġfilmmaker":89999,"ä¼¼ä¹İåľ¨":90000,"à·ļ":90001,"让æĪij们ä¸Ģèµ·":90002,"ĠÔ²":90003,"Ġconveyor":90004,"ĠغذاÛĮÛĮ":90005,"iceless":90006,"least":90007,"Ġench":90008,"å¾ģåľ°":90009,"Ġlabyr":90010,"åŃĻ女":90011,"Ġthermodynamics":90012,"Ġmengandung":90013,"ĠProviders":90014,"ĠStaphylococcus":90015,"ĠIELTS":90016,"Ġcatech":90017,"ä¸įèĢĥèĻij":90018,"ç»Ĩèĩ´çļĦ":90019,"å·´å°Ķ":90020,"Ġaudible":90021,"пиÑĤÑĮ":90022,"Kenn":90023,"Ġrelocated":90024,"两åı£":90025,"ĠÑĥÑĢок":90026,"康德":90027,"çģµçٳ":90028,"Ġ».ĊĊ":90029,"å±Ĭä¸ī":90030,"ä¸į对称":90031,"ĠRossi":90032,"bereich":90033,"ĠÑĢеализаÑĨии":90034,"Ġtectonic":90035,"peÅĤ":90036,"Ġsmoot":90037,"Ġéd":90038,"Ġém":90039,"èĤīçļĦ":90040,"è·³åĩº":90041,"ĠÙħجرÙĩ":90042,"Ø®ÙĦاÙĤ":90043,"ĠBIOS":90044,"ĠMickey":90045,"kid":90046,"ĠMarm":90047,"Ġplunge":90048,"é¦ĸæŃĮ":90049,"Ġpaar":90050,"à¥įà¤ŀ":90051,"Ġcutaneous":90052,"åĩĨå¤ĩ好äºĨ":90053,"feedback":90054,"ণà§įড":90055,"åįļ士çĶŁ":90056,"Ġgangs":90057,"Ġжелез":90058,"ĠPSA":90059,"platz":90060,"ä¸Ĭ个":90061,"ĠChiang":90062,"Ġforwarding":90063,"ãĥ©ãĥ¼":90064,"-auth":90065,"èħIJçĥĤ":90066,"ĠExtraction":90067,"ĠConnected":90068,"ĠFrei":90069,"Career":90070,"Ġgadgets":90071,"çľ©æĻķ":90072,"¤×Ķ":90073,"ĠKü":90074,"强度çļĦ":90075,"åĿļ强çļĦ":90076,"Ġà´®":90077,"/provider":90078,"ingles":90079,"è¦ģä¿ĿæĮģ":90080,"Ġprimordial":90081,"äºĮåįģä¸ĥ":90082,"çģ¾åĮº":90083,"Ġentitlement":90084,"ĠLens":90085,"Ġcharacterizing":90086,"缺å¸Ń":90087,"ï½¥":90088,"ĠPetr":90089,"åĽŀå®¶äºĨ":90090,"Ġprincipais":90091,"-team":90092,"ĠCommitment":90093,")}\\)":90094,"åĽ½åºĵ":90095,"Ġetapa":90096,"izzard":90097,"èªŀæ°£":90098,"Ġescalation":90099,"Ġplutôt":90100,"Ġfict":90101,"ĠIngg":90102,"ĠMarse":90103,"aturally":90104,"Ġmisinformation":90105,"ĠSalz":90106,"ERTY":90107,"icolor":90108,"Ġfleeting":90109,"ιαÏĥÏĦ":90110,"ĠíĮIJ":90111,")ãĢĬ":90112,"ĠEbola":90113,"ĠFrid":90114,"ä½łå¿«":90115,"ç´IJ":90116,"æĦŁäºº":90117,"åĬŁåĬĽ":90118,"æĪ¿è´·":90119,"Õ¡ÖĦ":90120,"ìĸµ":90121,"ĠÑģилÑĥ":90122,"Ġnodules":90123,"罢工":90124,"Ġspoil":90125,"bef":90126,"Ġbesser":90127,"roff":90128,"asten":90129,"åĩºä¸ĸ":90130,"formations":90131,"iteur":90132,"æĻĤåĪ»":90133,"éĥ½æľīçĿĢ":90134,"ä¿ĿéĻ©è´¹":90135,"ĠMagdal":90136,"æĸĩæĺİåŁİå¸Ĥ":90137,"æŀļ举":90138,"Ry":90139,"ĠBars":90140,"缾":90141,"ovou":90142,"ického":90143,"Ġsentir":90144,"Ġжел":90145,"çªģåĩºéĹ®é¢ĺ":90146,"ĠÑĤÑĢебованиÑı":90147,"ĠاÙĦÙĤرÙĨ":90148,"qp":90149,"Ġgazed":90150,"Ġsubcutaneous":90151,"ridged":90152,"äºĴ为":90153,"Ġcompletamente":90154,"ĠDEV":90155,"ĠVenture":90156,"ĠPereira":90157,"ÃŃpio":90158,"ĠSü":90159,"ĠMata":90160,"åĴĮåIJĦ":90161,"ustering":90162,"礴":90163,"Ġraining":90164,"ĠZinc":90165,"çľ¼è§ģ":90166,"lista":90167,"Ġκο":90168,"OI":90169,"ĠPCT":90170,"èĩªè¨Ģ":90171,"ç¥ŀæĺİ":90172,"ONY":90173,"ĠAngola":90174,"ÐĴо":90175,"(lst":90176,"èĪĪ奮":90177,"ĠHeidegger":90178,"Ġcirrhosis":90179,"Ġpernah":90180,"æł¼éĽ·":90181,"}},Ċ":90182,"IPC":90183,"身边çļĦ人":90184,"ĠDoesn":90185,"белÑĮ":90186,"Ġbloed":90187,"estershire":90188,"}{*}{":90189,"Ġunavoidable":90190,"Letters":90191,"æł¼åŃIJ":90192,"Orth":90193,"Cycle":90194,"croft":90195,"ãĤ·ãĤ¹ãĥĨãĥł":90196,"ç͍å®ĥ":90197,"ATEG":90198,"å°±ä¼ļåĩºçݰ":90199,"严éĩįå½±åĵį":90200,"Ġanthropogenic":90201,"nodes":90202,"Ġdeserts":90203,"çī¹å¤§":90204,"Ġesfuer":90205,"æĹ¶éĹ´æĺ¯":90206,"离éĢĢä¼ij":90207,"ĠScher":90208,"Ġлоги":90209,"åį«åģ¥":90210,"鸡汤":90211,"Ġmegabits":90212,"åįģä¸ĥæĿ¡":90213,"è´¬å̼":90214,"Ġpalabra":90215,"èħİ":90216,"ä½İè¿·":90217,"Ġtypename":90218,"ĠEmotion":90219,"èĮ¶æĿ¯":90220,"ĠHilfe":90221,"çļĦåIJĦ项":90222,"ä¹Łå¾Ī好":90223,"管åŃIJ":90224,"享åıĹåΰ":90225,"ĠBalancing":90226,"æŃ¦æ±īå¸Ĥ":90227,"ĠÙĪØ¬Ùĩ":90228,"ĠRNAs":90229,"Ġstipulated":90230,"+A":90231,"_head":90232,"ĠWak":90233,"é«ĺåĵģè´¨":90234,"éĥ¨ä¸ĭ":90235,"Ġcoff":90236,"-Te":90237,"Signal":90238,"ĠHomeland":90239,"/https":90240,"ĠWhis":90241,".nlm":90242,"éĻªæĪij":90243,"ĠPassive":90244,"Ġdodat":90245,"Ġpancakes":90246,"Ġvengeance":90247,"Ġdeformed":90248,"Ġascent":90249,"ichter":90250,"ç²½åŃIJ":90251,"éĵ°":90252,"Ġcelles":90253,"åĿĩä»·":90254,"ĠMatte":90255,"Ġchromosomal":90256,"ĠEggs":90257,"Ġunderestimate":90258,"Ġtú":90259,"Ġforage":90260,"geometry":90261,"éķ¿åīij":90262,"åĮħçļĦ":90263,"κά":90264,"icycle":90265,"åı«ä½ł":90266,"åįĸäºĨ":90267,"'''ĊĊ":90268,"ĠPeny":90269,"Ġgrasped":90270,"ãĤµãĤ¤ãĥĪ":90271,"ĠBett":90272,"æĹ¶è®¸":90273,"Ġparted":90274,"ĠÙĪØºÙĬر":90275,"ijs":90276,"Ã¥rd":90277,".Display":90278,"社åĮºå±ħæ°ij":90279,"Ġминима":90280,"opportun":90281,"Ġpearl":90282,"ĠPioneer":90283,"辦åħ¬å®¤":90284,"Ġmelancholy":90285,"?**ĊĊ":90286,"ĉarr":90287,"ĠDess":90288,"ĠVand":90289,"è¿ĽæĿ¥çļĦ":90290,"æĪ·åŀĭ":90291,"ĠAccred":90292,"parametric":90293,"à¥Ģà¤Ĥ":90294,"主é¢ĺæ´»åĬ¨":90295,"泡泡":90296,"åľ°çIJĨä½įç½®":90297,"ĠEuph":90298,"Ġwes":90299,"Ġalat":90300,"ĠOc":90301,"éĥ½ç»Ļ":90302,"åĿį":90303,"æī¿èªį":90304,"å°į象":90305,"ðĿijĩ":90306,"ленноÑģÑĤи":90307,"Ġcolonialism":90308,"æ©ĺåŃIJ":90309,"ĠìłĢìŀ¥":90310,"ĠDividing":90311,"çļĦä¾Ŀæį®":90312,"ĠSper":90313,"ĠRSA":90314,"ĠHeld":90315,"ĠHUM":90316,"天åij½":90317,"×ķש×":90318,"å·¥ä½ľéĩı":90319,"ANTS":90320,"AMD":90321,"-yil":90322,"Ġasymmet":90323,"olson":90324,"Ġgt":90325,"ä¸įåħ·æľī":90326,"ĠheiÃŁ":90327,"ĠKass":90328,"ĠKats":90329,"creative":90330,"Ġmaintenant":90331,"Ġâ΍":90332,"iyembre":90333,"(http":90334,"eys":90335,"rän":90336,"essä":90337,"-fed":90338,"Ġarmour":90339,"åħ®åħ®":90340,"NYSE":90341,"åijIJåĸĬ":90342,"Ġmaternity":90343,"ukunft":90344,"Lik":90345,"nite":90346,"çļĦ被":90347,"æģ¯æģ¯":90348,"Ġcustomizable":90349,"帮她":90350,"è½´ä¸Ĭ":90351,"èļĮ":90352,"ÃŃstico":90353,"Ġarrogant":90354,"Inflater":90355,"Ġpéd":90356,"igon":90357,"以åĮĹ":90358,"Ġsais":90359,"ĠHeating":90360,"导æķ°":90361,"zaam":90362,">Ċ":91695,"ĠUNDER":91696,"èĦ±é¢ĸèĢĮåĩº":91697,"Rather":91698,"}using":91699,"Ġclima":91700,"ĠVue":91701,"Ġfunzione":91702,"Ġproté":91703,"Ġissuer":91704,"ĠRetrie":91705,"ĠMerchant":91706,"Ġfatalities":91707,"Ġeind":91708,"ä½ľæ¡Ī":91709,"çĿij":91710,"èĢģåħĪçĶŁ":91711,"åŁŁç½ij":91712,"è³Ī":91713,"æĿ¾åĬ¨":91714,"æĿIJæĸĻåĴĮ":91715,"ĠÙĪØªØ³":91716,"Ġmuncul":91717,"-IV":91718,"cum":91719,"ī´":91720,"webs":91721,"пÑĢави":91722,"é͵":91723,"à¸Ńายุ":91724,"åĨįè¿Ľè¡Į":91725,"貪":91726,"lehem":91727,"درس":91728,"besar":91729,"âħ¢":91730,"Ġhinges":91731,"Ġapprehension":91732,"obook":91733,"ä¹ĿçϾ":91734,"æĭĽæīĭ":91735,"disabled":91736,"atiivi":91737,"åĩºåİ»çļĦ":91738,"Ġbidang":91739,"ĠиÑģполÑĮзÑĥÑİÑĤ":91740,"ĠÕ¢Õ¡Õ¼":91741,"ĠTroubles":91742,"çĭ©çĮİ":91743,"ĠBaden":91744,"Ġestudi":91745,"Ġcontentious":91746,"åģľäºĨä¸ĭæĿ¥":91747,"æĹģ人":91748,"ä¸įåľ¨æĦı":91749,"ĠCalling":91750,"Ġmétodos":91751,":t":91752,"Ġgels":91753,"ĠPau":91754,"ĠDiffer":91755,"acho":91756,"Inline":91757,"管çIJĨä½ĵç³»":91758,".tail":91759,"ç¶ĵçIJĨ":91760,"æł¹æľ¬å°±æ²¡æľī":91761,"Ġoctobre":91762,"ĠUtilities":91763,"ĠÑĨели":91764,"æĺ¥èĬĤæľŁéĹ´":91765,"Ġquienes":91766,"Ġdispatched":91767,".result":91768,"bk":91769,"bak":91770,"chwitz":91771,"æĪij们ä¸į":91772,"()),":91773,"æİ¨ç®Ĺ":91774,"åįİå±±":91775,".tar":91776,"èĹı书":91777,"驱åĬ¨åύ":91778,"ĠDeutschen":91779,"Palindrome":91780,"ĠWhitman":91781,"çĥ¬":91782,"Ġназад":91783,"èѽ":91784,"èĭ¦æ¶©":91785,"社交åªĴä½ĵ":91786,"ĠWolfe":91787,"Ġdlou":91788,"èĢĮåħ¥":91789,"å·¥ä½ľæĺ¯":91790,"Ġslang":91791,"ĠзакÑĢÑĭ":91792,"ĠRepubl":91793,"Ġeverlasting":91794,"ĠDiagonal":91795,"Ġjurid":91796,"å®ŀè´¨æĢ§":91797,"æĬīæĭ©":91798,"окÑĢÑĥг":91799,"æķ£çļĦ":91800,"ðĿijħ":91801,"Ġ×Ļ׾×ĵ":91802,"Ġà¹Ģล":91803,"Ġ문íĻĶ":91804,"ĠáĥĵáĥIJ":91805,"voy":91806,"ĠLitt":91807,"Ġнаи":91808,"iteral":91809,"Ġanguish":91810,"ĠгÑĢÑĥппа":91811,"timestamp":91812,".Product":91813,"[{":91814,"stmt":91815,"对æłĩ":91816,"èĩªè´Ł":91817,"çĶ±åĽ½å®¶":91818,"ĠZah":91819,"Ġcentred":91820,"×ķר×IJ":91821,"Skills":91822,"åģļé¢ĺ":91823,"åIJįèijĹ":91824,"ð٧":91825,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":91826,"å¾ģä¿¡":91827,"ĠÑĢеда":91828,"коÑģÑĤи":91829,"à§Ģল":91830,"ĠTexts":91831,"ĠAviv":91832,"Ġgruppo":91833,"ĠWyatt":91834,"ĠÑĢайона":91835,"ĠRCT":91836,"ĠESC":91837,"porte":91838,"åĽŀéģĵ":91839,"embles":91840,"Ġvariances":91841,"ĠSTE":91842,"-Af":91843,"Ġdeduced":91844,"ĠÙħاÙĬÙĪ":91845,"æĺŁæľŁåĽĽ":91846,"æľŁéĻIJåĨħ":91847,"æºľæºľ":91848,"çŀŃè§£":91849,"ĠAlmighty":91850,"Oil":91851,"énd":91852,"ï¼ī#":91853,"Ġsubconscious":91854,"Ġesters":91855,"Ġsimulating":91856,"Ġforearm":91857,"æİ¢å¯»":91858,"ĠBurd":91859,"twenty":91860,"Ġneste":91861,"æĪĸå¤ļ个":91862,"çī¹å¾´":91863,"æľ¨è̳":91864,"Ġorganización":91865,"èĥ¸èħĶ":91866,"ĠHAS":91867,"å®ŀæµĭ":91868,"ä½ĨéĤ£":91869,"Ġestre":91870,"åĽĽæ¬¡":91871,"Ġencodes":91872,"æī¹ç¤º":91873,"è°Īèµ·":91874,"饮çĶ¨æ°´":91875,"主åĬ¨æĢ§":91876,"æłıæĿĨ":91877,"é»ijæļĹä¸Ń":91878,"ĠUri":91879,"ipur":91880,"ició":91881,"Ġplasm":91882,"èŀºéĴī":91883,"Ġcirculatory":91884,"Ġcherish":91885,"Ġdownturn":91886,"uciary":91887,"Lj":91888,"}c":91889,"ĠSelling":91890,"seat":91891,"å¿ĥå¢ĥ":91892,"æĸ¯èĴĤ":91893,"طار":91894,"ĠÙĩÙĨÚ¯":91895,"énom":91896,"ĠOperators":91897,"æIJ¬å®¶":91898,"æ³Įå°¿":91899,"-ord":91900,"_box":91901,"uels":91902,"ĠThorn":91903,"Ġmanic":91904,"зоÑĢ":91905,"Ġfeudal":91906,"ĠShark":91907,"ĠErfahr":91908,"Ġhunted":91909,"Ġpineapple":91910,"Ġinfestation":91911,"ĠÑĦевÑĢа":91912,"-era":91913,"lude":91914,"æĺµ":91915,"ĠGog":91916,"æĸ¹æĸ¹éĿ¢":91917,"å¥Ĺè£ħ":91918,"è´¦éĿ¢":91919,"ĠTakeaways":91920,"ĠKirby":91921,"ç£ħ礴":91922,"Ġunbelievable":91923,"à¸ĵà¸ijà¹Į":91924,"_weight":91925,"rž":91926,"åľ¨æĹ¥å¸¸":91927,"对ä¸į对":91928,"ä¹Łè¯´":91929,"æĪij们èĥ½":91930,"æĢ»æķ°çļĦ":91931,"Ġmultilateral":91932,"ĠاÙĦÙħÙĨت":91933,"ĠÏħÏĢο":91934,"leneck":91935,"ĠvÅ¡echn":91936,"Ġrecourse":91937,"é»ijæ´ŀ":91938,"ÑģиÑĤÑĮ":91939,"Ġfirefighters":91940,"ĠÑħо":91941,"æ¿ĢåıijäºĨ":91942,"Ġeosin":91943,"æľ¬åĬŀæ³ķ":91944,"crew":91945,"ï¼Ľ(":91946,"第ä¸Ģå±Ĭ":91947,"leri":91948,"è¡£æŁľ":91949,"Ġsymbolize":91950,"Ġpmid":91951,"åļĵ":91952,"vertisements":91953,"æĺŁæľŁä¸ī":91954,"uwega":91955,"Ġthrottle":91956,"Ġادار":91957,"åĪĨéĺŁ":91958,"Ġtransp":91959,"çłĶç©¶æĬ¥åijĬ":91960,"æĿİæĸĩ":91961,"Ġতথ":91962,"ĠPoison":91963,"ĠCriticism":91964,"iesta":91965,"Ġoxidase":91966,"ĠHermione":91967,"éªĨ驼":91968,"'una":91969,"Ġcapped":91970,"çļĦæĪĺæĸĹ":91971,"ä¹Łç»Ļ":91972,"Ġblat":91973,"ächen":91974,"å½±éĻ¢":91975,"é»ĦèĬ±":91976,"ç´¢åıĸ":91977,"å᱿̥":91978,"åħ¨éĿ¢å»ºè®¾":91979,"ĠFORE":91980,"侦æİ¢":91981,"éĵĿåIJĪéĩij":91982,"hez":91983,"oupling":91984,"ĠAph":91985,"éducation":91986,"ĠÙħات":91987,"ĠgetId":91988,"Ġimpede":91989,"åĩıæ³ķ":91990,"alese":91991,"Median":91992,"è¿ĶåĽŀå̼":91993,"æĥ¬æĦı":91994,"uomo":91995,"hep":91996,"çļĦçĬ¶åĨµ":91997,"quee":91998,"æľ¬äººçļĦ":91999,"èĭ±è¶ħ":92000,"å®ĺåºľ":92001,"é»ŀé»ŀéłŃ":92002,"Ġcinqu":92003,"ĠPROJECT":92004,"å®ī稳":92005,"åĨįä¸į":92006,"Ġмало":92007,"Ġpolyester":92008,"Ġadenocarcinoma":92009,"alculate":92010,"æĢĤ":92011,"Ġ/>;ĊĊ":92706,"ä¸Ģæĸ°":92707,"Ġhavoc":92708,"建设ä¸Ń":92709,"Ġexisten":92710,"å¼Ģå±ķå·¥ä½ľ":92711,"ĠMorse":92712,"Ġহà¦ļà§įà¦Ľ":92713,"бина":92714,"Histor":92715,"Ġséculo":92716,"Ġmantra":92717,"ĠاÙĦعاÙħØ©":92718,"Ġböjningsform":92719,"Ġà¸Ĺำà¹ĥหà¹ī":92720,"Ġthats":92721,"ĠDug":92722,"ĠRin":92723,"Ġ{:":92724,"便被":92725,"ãģ¨èĢĥãģĪ":92726,"ÐŁÑĢед":92727,"ĠEstimated":92728,"Ġslowdown":92729,"Ġ»Ċ":92730,"è¢ģä¸ĸåĩ¯":92731,"/product":92732,"Ġproponents":92733,"ocyst":92734,"Ġspas":92735,"é«ĺä»·":92736,"protein":92737,"Jonathan":92738,"Ġneurodegenerative":92739,"ĉlong":92740,"Ġreint":92741,"ĠSop":92742,"ĠFt":92743,"appliquer":92744,"åĩłæĿ¡":92745,"диÑĤ":92746,"åĨ°æ·ĩæ·ĭ":92747,"]).ĊĊ":92748,"åľ¨è®¸å¤ļ":92749,"åĽ½åħ¬":92750,"åºĶçŃĶ":92751,"ĠThereafter":92752,"æĮīåħ¶":92753,".prev":92754,"Ġkomputer":92755,"èĢķèĢĺ":92756,".Button":92757,"arck":92758,"ĠnÃŃvel":92759,"ĠWach":92760,"åĽ½éļĽ":92761,"好åĿı":92762,"å¤ĸåĬĽ":92763,"('-":92764,"éĢģåħ¥":92765,"ĠAmbient":92766,"Ġmartyr":92767,"à¸Ļัà¸ģà¹Ģรียà¸Ļ":92768,"Ġrewritten":92769,"ikko":92770,"ĠLehrer":92771,"éĢĻä¸Ģ次":92772,"èĮĥä¾ĭ":92773,"Ġcombating":92774,"Ġáĥ¬":92775,"}$ĊĊ":92776,"international":92777,"_open":92778,"å½ĵå½ĵ":92779,"管çIJĨæ°´å¹³":92780,"incorpor":92781,"à¸Ĺุà¸Ļ":92782,"ĠImam":92783,"Ġprimeros":92784,"éļIJå½¢":92785,"locations":92786,"åİĭç¼©æľº":92787,"oconut":92788,"âŀķ":92789,"à¸ŀัà¸Ļà¸ĺุà¹Į":92790,"è¿Ĥ":92791,"ĠLaz":92792,"ĠGaut":92793,"æĪĸåħ¶":92794,"Ġëĸ":92795,".len":92796,"é»ijå½±":92797,"operations":92798,"æĹ¢ä¸į":92799,"Ġبرد":92800,"Assume":92801,"ç¡ķ士çłĶç©¶çĶŁ":92802,"ĠÙħدÛĮرÛĮت":92803,"enst":92804,"veer":92805,"éta":92806,"ĠпиÑģа":92807,"æĿİæĺİ":92808,"è½»çĽĪ":92809,"åıijçĶŁäºİ":92810,"à¸Ħรà¸Ńà¸ĩ":92811,"ĠباشÛĮد":92812,"æļ«æĻĤ":92813,"Ġresur":92814,"åıijèªĵ":92815,"Ġteasing":92816,"æķĻèĤ²äºĭä¸ļ":92817,"积æŀģæİ¨è¿Ľ":92818,"Ġmetabolite":92819,"Ġfebruar":92820,"?),":92821,"kids":92822,"itability":92823,"æĪIJåįĥ":92824,"缺æįŁ":92825,"çά山":92826,"(width":92827,"Ġmanga":92828,"اؤ":92829,"æķ´è½¦":92830,"rigation":92831,"ĠÄijá»ĵ":92832,"Ġkelompok":92833,"-acre":92834,"Ġlugares":92835,"Ġelicited":92836,"ĠAktiv":92837,"ĠSOCIAL":92838,"ÙĪÙĦاÙĬات":92839,"Ġhacen":92840,"Ġtá»ij":92841,"对æĪijåĽ½":92842,"compl":92843,"ä¸Ģ个å°ıæĹ¶":92844,"ĠShak":92845,"ĠÄįas":92846,"设ç«ĭäºĨ":92847,"Ġsekali":92848,"诵读":92849,"Ġà¦¬à¦Ľà¦°":92850,"ucalyptus":92851,"-angle":92852,"ĠJagu":92853,"åıįéĿ¢":92854,"ëĬĺ":92855,"опÑĢов":92856,"ç©¿åŃĶ":92857,"Ġtoughness":92858,"å¤ĸ交éĥ¨":92859,"hovah":92860,"Ġsg":92861,"Ġsavor":92862,"ivary":92863,"å°ı鸣":92864,"å·²ç»ıåΰäºĨ":92865,"ĠMedina":92866,"ossal":92867,"çĦ¡æ¯Ķ":92868,"èĤ¤èī²":92869,"Ġblooming":92870,"ĠÙĪØ§ØŃدة":92871,"è¾Ĺ转":92872,")।":92873,"è¿ĻæĬĬ":92874,"Ġplaza":92875,"Ġplentiful":92876,"åľ°éĹ®éģĵ":92877,"æľ¬å¹´":92878,"Ġzig":92879,"æıIJ纲":92880,"Ġhistological":92881,"ĠNoel":92882,"ĠSomehow":92883,".Runtime":92884,"Åijk":92885,"ĠSlope":92886,"Ġstacking":92887,"Ġкомна":92888,"Ġpillows":92889,"ĠдÑĢÑĥгими":92890,"ä¹ŁåŃĺåľ¨":92891,"ĠConverting":92892,"Ġskipping":92893,"æ¥Ķ":92894,"Ġbreve":92895,"à¥ģद":92896,"\\}\\)":92897,"ĠMAL":92898,"estimate":92899,"eki":92900,"åĨįåģļ":92901,"çϽèĻİ":92902,"åįĹéĺ³":92903,"Ġmotility":92904,"认为èĩªå·±":92905,"à¸ŀูà¸Ķ":92906,"δεÏĤ":92907,"ξη":92908,"ĠBacterial":92909,"Fol":92910,"Ġmite":92911,"Ġkong":92912,"ÑĢаб":92913,"åİŁä½ľèĢħ":92914,"便å°Ĩ":92915,"ĠManor":92916,"ĠÙĬÙĪÙĨ":92917,"IMAL":92918,"çѾåıij":92919,"æĪIJæľ¬åĴĮ":92920,"Ġoriental":92921,"Ġpreciso":92922,"Ġlibrarian":92923,"Ġдобав":92924,"æĢĿæĥ³æĶ¿æ²»æķĻèĤ²":92925,"imin":92926,"ĠJavier":92927,"weets":92928,"ĠProverbs":92929,"Ġparall":92930,"áĥ®":92931,"ési":92932,".).Ċ":92933,"ë²ł":92934,"லà¯Ī":92935,"ĠEighth":92936,"ç»ıèĦī":92937,"chent":92938,"æĪ¿ä¼ģ":92939,"ĠPolyn":92940,"Ġpositivo":92941,"Ġbibliographical":92942,"ĠAyurved":92943,"Ġsporadic":92944,".rel":92945,"abat":92946,"Ġspecjal":92947,"åįķä½ĵ":92948,"Ġcreatively":92949,"Ġ×IJפשר":92950,"çݰ代åĨľä¸ļ":92951,"ìŀĪëĬĶ":92952,"ĠPets":92953,"ĠLIVE":92954,"大åIJĥ":92955,"ĠValu":92956,"Ancient":92957,"Ġvaria":92958,"ĠEducator":92959,"partition":92960,"ĠTiming":92961,"employees":92962,"BV":92963,"æĶĶ":92964,"åħµçļĦ":92965,"ãģķãģ¾":92966,"utherford":92967,"Ġglossary":92968,"ãģ«å¯¾ãģĻãĤĭ":92969,"Ġnouveaux":92970,"/)ĊĊ":92971,"åĴĮä¸Ģ":92972,"ritos":92973,"æħĪ禧":92974,"ĠÑįÑĤомÑĥ":92975,"Ġközött":92976,".How":92977,"Câu":92978,"YLE":92979,"predict":92980,"tak":92981,"ĉnode":92982,"inities":92983,"ĠYen":92984,"erty":92985,"æį¶":92986,"Ġoverthrow":92987,"Ġrelatable":92988,"axel":92989,"Ġmenace":92990,"Ġdura":92991,"åĸľåºĨ":92992,"Ġ×ijף":92993,"Ġpulsed":92994,"Ġaula":92995,"æĺ¯ç¤¾ä¼ļ":92996,"Ġproactively":92997,"resolve":92998,"Ġadhered":92999,"Ġ×ŀספר":93000,"åħ·æľīä¸Ģå®ļ":93001,"ĠCompat":93002,"èѦåį«":93003,"ĠRedirect":93004,"Ġlitre":93005,"Ġalgún":93006,"roviral":93007,"ĠMartÃŃnez":93008,"ä¸įä¹ı":93009,"Ġcurios":93010,"ãĢĤâĢĿ*":93141,"zf":93142,"chronic":93143,"ĠRang":93144,"使çĶ¨å¯¿åij½":93145,"çķĻåŃĺ":93146,"omsnitt":93147,"Ġpainfully":93148,"Ġprécis":93149,"ĠÑĥÑģловий":93150,"ĠHastings":93151,"Ġclad":93152,"æĶ¹åζ":93153,"空èĻļ":93154,"è¯ĬæīĢ":93155,"æµħæµħ":93156,"ìĻķ":93157,"ĠUNIVERSITY":93158,"ĠCretaceous":93159,"Boy":93160,"ĠNing":93161,"Ġsean":93162,"Ġuur":93163,"ä¿Ŀè´¹":93164,"ä»ĬçĶŁ":93165,"é¾Ļçİĭ":93166,"øm":93167,"Ġspoiled":93168,"Ġзаболеваний":93169,"ĠExpectations":93170,"漩涡":93171,"'elle":93172,"-English":93173,"çļĦæľªæĿ¥":93174,"ĠNEXT":93175,"ĠAdverse":93176,"å¸ĿåĽ½çļĦ":93177,"à§įযাস":93178,"ÐĽÐ¸":93179,"(Ċ":95297,"Ġdne":95298,"Ġexerts":95299,"Ġklin":95300,"illers":95301,"大å°ĨåĨĽ":95302,"æīĢåĪĹ":95303,"æĿ¡å½¢":95304,"Ġcardio":95305,"çĸ¾æİ§":95306,"Ġpropagated":95307,"çļĦå¤ĸè§Ĥ":95308,"ĠDragons":95309,"LW":95310,"çļĦè¿Ļä¸Ģ":95311,"ĠCuc":95312,"ĠDock":95313,"ä¸į认è¯Ĩ":95314,"Ġoutlaw":95315,"æľ¬åħ¬åı¸":95316,"èµ·ä½ľç͍":95317,"缸è²Į":95318,"åī§ç»Ħ":95319,"ä¸įåı¯ç¼ºå°ij":95320,"micron":95321,"Ġsurfing":95322,"-emitting":95323,"ĠFluor":95324,"åľ¨åĽ½å¤ĸ":95325,"åıijè´¢":95326,"æĭĭ":95327,"exchange":95328,"åĽŀ转":95329,"éĿŀå¾Ĺ":95330,"-det":95331,"Ġperiodontal":95332,"रà¥įथ":95333,"ĠSTATUS":95334,"stoffe":95335,"jid":95336,"sty":95337,"ÑĢой":95338,"ä¸Ĭä¸ĩ":95339,"æĿ¥ä¸ª":95340,"åı¯ä»¥èİ·å¾Ĺ":95341,"engo":95342,"æ°ijåľĭ":95343,"azuje":95344,"irected":95345,"ylus":95346,"Ġargent":95347,"_cnt":95348,"Ġcoorden":95349,"çļĦç¡®æĺ¯":95350,"ä¸Ńåįİæ°ijæĹıä¼Łå¤§å¤įåħ´":95351,"ĠÙĨسبة":95352,"ĠHutchinson":95353,"Ġdna":95354,"çĶŁå¾Ĵ":95355,"ĠاÙĦÙĪØ²":95356,"éĩijæĺŁ":95357,"Ġmetode":95358,"Ġerhö":95359,"æŀģçĤ¹":95360,"Ġнее":95361,"ĠÙħØŃÙĦ":95362,"রà§įষ":95363,"ĠبÙĪØ¯ÙĨ":95364,"çį¨ç«ĭ":95365,"ĠвлиÑıние":95366,"æĽ¾åĽ½èĹ©":95367,"Ġwield":95368,"ĠJal":95369,"Ġjäl":95370,"ĠKou":95371,"âĢĶ_":95372,"-sem":95373,"Ġrealiza":95374,"Ġvanity":95375,"æĮ¥èĪŀ":95376,"ĠRomero":95377,"ĠCNY":95378,"Ġenorme":95379,"æµģè¡ĮçĹħ":95380,"ĠNucl":95381,"ĠVinc":95382,"áci":95383,"åIJij大家":95384,"Ġ%.ĊĊ":95385,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":95386,"çݯå¢ĥå½±åĵį":95387,"ÑīеÑģÑĤво":95388,"ULATION":95389,"../../../":95390,".preventDefault":95391,"-ring":95392,"Ġfooth":95393,"Ġ-------------------------------------------------":95394,"ä¿Ŀå̼":95395,"åIJĦä¸į缸åIJĮ":95396,"è¿ľè¿ij":95397,".findById":95398,"Ġacknowledgement":95399,"ì¡Į":95400,"Ġapplause":95401,"Ġhindered":95402,"Ġléka":95403,"(form":95404,")B":95405,")\",Ċ":95406,"ĠBov":95407,"ĠHicks":95408,"ĠEW":95409,"ä¹Łæľª":95410,"åīįåı°":95411,"Ġkeer":95412,"è¾ĵè¡Ģ":95413,"worksheets":95414,"ĠпÑĢеимÑĥ":95415,"×ŀת":95416,"Ġবিশà§ĩষ":95417,"Ġbounding":95418,"rschein":95419,"Ġפ×Ļ":95420,"ĠاÙĦÙħؤÙĦÙģ":95421,"ä¸Ŀ绸ä¹ĭè·¯":95422,"ĠReverend":95423,"ĠCanaan":95424,"å®ķ":95425,"Ġagreg":95426,"空éĹ²":95427,"à°Ĥà°¦":95428,"éĺ²çģ«å¢Ļ":95429,"Ġluminous":95430,"ç¿©ç¿©":95431,".peek":95432,"+F":95433,"inflammatory":95434,"Ġdex":95435,"Ġornaments":95436,"è¿ľæľŁ":95437,"setting":95438,"æĻļå®ī":95439,"ä»£è¡¨ä½ľ":95440,"Ġseguito":95441,"Ġprophetic":95442,"기ëıĦ":95443,"érations":95444,"ĠPandemic":95445,"Ġiçin":95446,"/pre":95447,"ĠRaleigh":95448,"Ġclave":95449,"-bold":95450,"-packed":95451,"Äįky":95452,"ĠTransparency":95453,"ç§Ģæīį":95454,"Ġhotter":95455,"ä¹ŁæľīäºĨ":95456,"æī¬èµ·":95457,"çѹçłģ":95458,"ällen":95459,"ĠSlovakia":95460,"ĠHygiene":95461,"ivre":95462,"ĠGw":95463,"orema":95464,"为使":95465,"Ġaccol":95466,"çĶ»åį·":95467,"opted":95468,"ĠGrave":95469,"Ġinterviewer":95470,"æĹ¶æľŁåĨħ":95471,".em":95472,"/router":95473,"åĪĨå½ķ":95474,"äºİçĤ¹":95475,"ä½įæĸ¼":95476,"Ġзаг":95477,"rogens":95478,",Z":95479,"-Ed":95480,"vast":95481,"onite":95482,"Ġsno":95483,"åħ³ä¸Ĭ":95484,"å½ĵä»ĸ们":95485,"Ġequil":95486,"arski":95487,"truth":95488,"éĺ³æŀģ":95489,"Ġhorrors":95490,"иÑģан":95491,"Ġsprang":95492,"Ġretardation":95493,"\\bar":95494,"Ġtofu":95495,"utrients":95496,"ĠKard":95497,"Ġ**[":95498,"ÏĢί":95499,"Ġnotebooks":95500,"Ġkażde":95501,"缮çŀªåı£":95502,"Voice":95503,"ĠSüd":95504,"osum":95505,"istak":95506,"ĠNied":95507,"Ġباب":95508,"Ġglaring":95509,"anej":95510,"æIJĢ":95511,"è¿ŀè´¯":95512,"éĿŀ常好çļĦ":95513,"æĵįå¿ĥ":95514,"ĠâĦĥ":95515,"éĿĪéŃĤ":95516,"æĦ£ä½ıäºĨ":95517,"Marketing":95518,"ÑĤенÑģив":95519,"();ĊĊ":95745,"romo":95746,"ä¸Ĭåľº":95747,"ikation":95748,"æľįåĬ¡æľºæŀĦ":95749,"Ġexcursion":95750,"ĠAston":95751,"Ġcorte":95752,"ĠOmaha":95753,"UIColor":95754,"ĠSoviets":95755,"ĠãĢĪ":95756,"ĠPÅĻ":95757,"riot":95758,"Ġkennen":95759,"å®ħåŁºåľ°":95760,"ffffff":95761,"ĠплоÑīади":95762,"Ġanálise":95763,"onych":95764,"ulton":95765,"åĵ½":95766,"éĤ£ä¸įæĺ¯":95767,"axios":95768,"ceptual":95769,"Ġposto":95770,"Ġأساس":95771,"Ġvanish":95772,"éĩįçĤ¹æĺ¯":95773,"ĠUltr":95774,"Ġsprayed":95775,"Mess":95776,"Nobody":95777,"Ġicing":95778,"Ġweeping":95779,"表éĿ¢ç§¯":95780,"-support":95781,"Ġprofil":95782,"éϤå¤ķ":95783,"èϽæĺ¯":95784,"ipsych":95785,"cala":95786,"ĠDominic":95787,"?_ĊĊ":95788,"hese":95789,"ocious":95790,"è¿Ļåħ¶ä¸Ń":95791,"æ³ķ西æĸ¯":95792,"Ø·ÙĤØ©":95793,"ðŁĮĢ":95794,"ĠÕİ":95795,"å¥ĩèij©":95796,"çľĭçĿĢæĪij":95797,"è¡ĮæĶ¿è¯ī讼":95798,"Ġmigrating":95799,"à¥įरà¥Ģ":95800,"Ġï¼īãĢĤĊĊ":95801,"ЧÑĤобÑĭ":95802,".line":95803,"anam":95804,"ĠNé":95805,"Ġformule":95806,"ruptcy":95807,"however":95808,"Ġpellet":95809,"ĠSelain":95810,"NonNull":95811,"HN":95812,"unächst":95813,"ĠPATH":95814,"Ġconclusive":95815,"ebel":95816,"æŀģé«ĺçļĦ":95817,"ĠEconomist":95818,"ĠPetitioner":95819,"ĠPRINT":95820,"ëħĢ":95821,"ìĪĺ를":95822,"èģ¯çĽŁ":95823,"Ĺקר":95824,"à¦ŀà§įà¦ľ":95825,"+'":95826,"ä½ĵé¨ĵ":95827,"ç§į群":95828,"Ġformes":95829,"Ġово":95830,"Ġcurly":95831,"ĠDoch":95832,"Ðļом":95833,"رÙģÙĩ":95834,"Ġними":95835,"çļĦæĦıæĢĿæĺ¯":95836,"ĠпÑĢимеÑĢно":95837,"ĠDepartments":95838,"Bright":95839,"Ġcached":95840,"ĠSonic":95841,"Ġshimmer":95842,"issima":95843,"â̦â̦â̦":95844,"Ġseeded":95845,"Ġmergers":95846,"abra":95847,"æľ¬èī²":95848,"æ¯ıä¸ĢæŃ¥":95849,"----------":95850,"สืà¹Īà¸Ń":95851,"classified":95852,"å¸ĮæľĽä½ł":95853,"Ġsinister":95854,"汽车çļĦ":95855,"ĠPotato":95856,"ĠSemantic":95857,"åħįè´¹çļĦ":95858,"STRING":95859,"Ġpula":95860,"å¿ĥæĤ¸":95861,"å®ŀåĪĻ":95862,"ylated":95863,"åĿĩæĺ¯":95864,"åŁİå¸Ĥè§ĦåĪĴ":95865,"触åĬ¨":95866,"Ġcadmium":95867,"ãģĹãģ¾ãģĹãĤĩãģĨ":95868,"Ġinfrastructures":95869,"ĠDodge":95870,"æľºèº«":95871,"awks":95872,"Ġverst":95873,"aiman":95874,"Argent":95875,"ĠÙħÛĮÙĦÛĮ":95876,"-chip":95877,"ĠترÛĮÙĨ":95878,"ä¸İæĹ¶ä¿±":95879,"Hem":95880,"gments":95881,"ĠCors":95882,"ĠBrat":95883,"ĠNEC":95884,"planned":95885,"以ä¸Ģç§į":95886,"å°±æĺ¾å¾Ĺ":95887,"Ġallot":95888,"Ġallerdings":95889,"Ġarrib":95890,"constants":95891,"Ġharmed":95892,"ĠاÙĦØŃÙĤ":95893,"ĠEmployers":95894,"Ġredistribute":95895,"ĠпÑĢодолжи":95896,"Ġgithub":95897,"ĠKult":95898,"é«ĺèĪĪ":95899,"Recipe":95900,"æķ£å¸ĥ":95901,"è½®çļĦ":95902,"åħ³éĶ®çļĦ":95903,"åħ½åĮ»":95904,"龸æ°Ķ":95905,"ĠzmÄĽ":95906,"á»ĥu":95907,"Ġпобе":95908,"nienie":95909,"ĠPued":95910,"ĠDz":95911,"å·¥ä½ľæĸ¹æ¡Ī":95912,"-sector":95913,"ĠÙĦازÙħ":95914,"éĻIJéĩı":95915,"Ġstealth":95916,"ä¸įæĸŃæī©å¤§":95917,"CCA":95918,"Ġpowod":95919,"å¿ĥçIJĨåѦ家":95920,"ç¥ĸå®Ĺ":95921,"ĠSimplify":95922,"stige":95923,"Ġ(),":95924,"æĪijéĿŀ常":95925,"Ġ[[]":95926,"èµ·å±ħ":95927,"æľĢ常è§ģçļĦ":95928,"ä¿¡ä»¶":95929,"头é¢ħ":95930,"ĠSpray":95931,"â̦â̦ãĢį":95932,"ICC":95933,"æ±Łåİ¿":95934,"ĠGeorgetown":95935,"ç£IJ":95936,"æĮijåīĶ":95937,"ynie":95938,"åĩłæĹ¥":95939,"ä¹Ŀ年级":95940,"å®ĪæģĴ":95941,"Ġà¦īদ":95942,"å¹²åĩĢçļĦ":95943,"Creator":95944,"ĠOPEN":95945,"-readable":95946,"ĠfordÃŃt":95947,"ppsala":95948,"ĠاÙĦØ´ÙĬØ®":95949,"waves":95950,"Ġpry":95951,"çļĦ顺åºı":95952,"èĥ«":95953,"社åijĺ":95954,"æ±Ĥè¯ģ":95955,"awia":95956,"大åѦæ¯ķä¸ļ":95957,"ILabel":95958,"ĠاÙĦآخر":95959,"Cele":95960,"ĠCindy":95961,"izm":95962,"дÑĸ":95963,"ä¸İå®ŀè·µ":95964,"Ġacoust":95965,"Ġacadém":95966,"ĠPho":95967,"Ġbuscar":95968,"èµĽåľº":95969,"িà¦ķার":95970,"SiO":95971,"Ġidiom":95972,"ĠÑĤÑĢебÑĥеÑĤÑģÑı":95973,"Eh":95974,"Tensor":95975,"jahr":95976,"voke":95977,"Ġditem":95978,"Ġgm":95979,"åģļå·¥":95980,"ĠMinimal":95981,"Ġmorse":95982,"ר×ij×ķת":95983,"/ep":95984,"atea":95985,"ynch":95986,"horizontal":95987,"ĠThemen":95988,"orting":95989,"ĠGJ":95990,"ubo":95991,"éļıåľ°":95992,"主è¦ģè´Łè´£äºº":95993,"ÙİÙĩ":95994,"éĢıæŀIJ":95995,"é¼ĵæİĮ":95996,"Ïģαγ":95997,"ĠLabels":95998,"ĠCONCLUSION":95999,"ĠMST":96000,"ĠOce":96001,"è¿Ľè¡Įä¸Ģ次":96002,"è¿Ļä¸ªæł·åŃIJ":96003,"å¼ķçͳ":96004,"èĩªå·±çļĦæĥ³æ³ķ":96005,"Adjust":96006,"ĠTwain":96007,"Ġdelaying":96008,"ĠClubs":96009,"ĠRamsey":96010,"è£Ŀç½®":96011,"ÙĨسا":96012,"ĠновÑĭе":96013,"ĠReservoir":96014,"ноги":96015,"åĴĮ第":96016,"ç͵ç«ŀ":96017,"Ġhardening":96018,"ĠBallet":96019,"ĠRené":96020,"ĠMPI":96021,"è¿Ļé¦ĸè¯Ĺ":96022,"åĽłæŀľåħ³ç³»":96023,"Más":96024,"ully":96025,"Ġesses":96026,"AMB":96027,"ൻ":96028,"ä¸ĺéϵ":96029,"ĠÑĥÑĩеÑĤом":96030,"pÃ¥":96031,"Ġadore":96032,"就对":96033,"ileen":96034,"ĠÑģви":96035,"Ġdiscol":96036,"ÏģεÏĤ":96037,"Ġsemantically":96038,"ĠErr":96039,"Phill":96040,"ĠFormats":96041,"æĹĹä¸ĭçļĦ":96042,"èĥĥèĤłéģĵ":96043,"Ġdripping":96044,"\"How":96045,"<$":96046,"rzym":96047,"yte":96048,"Å©":96049,"ĠArche":96050,"åĽĽçĤ¹":96051,"Ġcondol":96052,"åħ¬åħ±åħ³ç³»":96053,"ĠGallagher":96054,"Ġgigg":96055,"å®ŀç͍æĢ§":96056,"ĠKatrina":96057,"provider":96058,"Ġcuidado":96059,"Ġações":96060,"erian":96061,"ĠHorses":96062,"ĠFAA":96063,"ocre":96064,"ĠArb":96065,"åĩłå®¶":96066,"Ġsleepy":96067,"á»ijn":96068,"ãĤĴè¡ĮãģĨ":96069,"Ġelliptical":96070,"Ġejercicio":96071,"å°±è·ij":96072,"ĠVega":96073,".predict":96074,"鸳":96075,"ĠItalians":96076,"çļĦé«ĺæīĭ":96077,"ĠFlexibility":96078,"æģįçĦ¶å¤§æĤŁ":96079,"ĠButterfly":96080,"çļĦåľºæĻ¯":96081,"agric":96082,"ĠDarm":96083,"ĠWIN":96084,"Ġoracle":96085,"manage":96086,"ĠÑĤекÑĥ":96087,"åѸæľĥ":96088,"æĿ¨æŁ³":96089,"æ·±åĮĸæĶ¹éĿ©":96090,"ĠCycling":96091,"ACIÃĵN":96092,"(The":96093,"/inter":96094,"=ï¼Ī":96095,"Cir":96096,"Ġrebut":96097,"ä¸Ģæľ¬ä¹¦":96098,"人æĢ§çļĦ":96099,"ocio":96100,"大åŃĹ":96101,"æĿ¥æĿ¥":96102,"ibre":96103,"awasan":96104,"Ġmusik":96105,"主ä¹īåĴĮ":96106,"ذب":96107,"æĭĽæĥ¹":96108,"ĠPatrol":96109,"éĢıæ°Ķ":96110,"ç®ĬæĢ§":96111,"ĠCrossword":96112,"ycznych":96113,"Ġstereotype":96114,"Ġencuentran":96115,"Ġhodnot":96116,"Holy":96117,"jobs":96118,"Ġmã":96119,"ĠBray":96120,"ä¸Ń度":96121,"allowed":96122,"Ġempez":96123,"Ġesos":96124,"éĸij":96125,"Ġutile":96126,"ણ":96127,"rosa":96128,"Ġbedside":96129,"ĠJewel":96130,"Ġnanometers":96131,"éĢĨåIJij":96132,"ĠVenet":96133,"åIJķå¸ĥ":96134,"Ġ(âĢĵ":96135,"Ġalas":96136,"ĠKand":96137,"ä¿Ŀä½ı":96138,"èĬĤ缮çļĦ":96139,"åį°è¯ģ":96140,"ĠпÑĢоÑĤи":96141,"ĠTout":96142,"Ġvener":96143,"æľīäºĮ":96144,"åĴĮéĻĪ":96145,"ĠвÑĬ":96146,"riture":96147,"ĠZukunft":96148,"åħĪåľ¨":96149,"ĠØ¥ÙĦÙĬÙĩ":96150,"ä¸Ģè§Ĵ":96151,"ĠPDP":96152,"Ġsubsp":96153,"常è¦ĭ":96154,"éĻ¢éĩĮ":96155,"é»ij客":96156,"ç§ĺè¯Ģ":96157,"ĠìĿ´íĽĦ":96158,"Ġtaille":96159,"åĬ¨çī©åĽŃ":96160,"-voltage":96161,"Ġcio":96162,"okines":96163,"æĺİæľĹ":96164,"æĹłåĬ©":96165,"Stra":96166,"Ġmonoton":96167,"ĠExist":96168,"åIJĥæİī":96169,"è¿ĺæľīå°±æĺ¯":96170,"Ġpropelled":96171,"ĠSkinner":96172,"ëŀµ":96173,"ĠалгоÑĢиÑĤ":96174,"Ġparabola":96175,"ĠSprint":96176,"ĠSIP":96177,"ĠTos":96178,"ä¸Ģæŀ¶":96179,"emaker":96180,"å¿ĥæĪ¿":96181,"ĠYORK":96182,"Ġbottled":96183,"综åIJĪèĢĥèĻij":96184,"áŁĭ":96185,"ĠPolytechn":96186,"ÐŁÐ¾Ñģле":96187,"ä¹ŁæĪIJ为":96188,"å¤ļçĤ¹":96189,"Ġcreeping":96190,".âĢĿ#":96191,"Ġlegumes":96192,"ECE":96193,"Ġmarrying":96194,"ĠNotable":96195,".getInstance":96196,"缸åħ³éĹ®é¢ĺ":96197,".Tag":96198,"ektor":96199,"árnÃŃ":96200,"Kr":96201,"Sj":96202,"eon":96203,"Ġaccusing":96204,"æŃ¤ä¸¾":96205,"ĠدÙģ":96206,"Ġpathophysiology":96207,"æŃ¦æĺĮ":96208,"czny":96209,"Ġmoyenne":96210,"Ġì¤Ģ":96211,"ä½łä¸ºä»Ģä¹Ī":96212,"ä½łä¹Łåı¯ä»¥":96213,"天ä¸ĭçļĦ":96214,"ãĢĤâĢĿ(ãĢĬ":96215,"åħ¶ä»ĸåĽ½å®¶":96216,"êts":96217,"Åįng":96218,"_seq":96219,"-products":96220,"å¾®éĩıåħĥç´ł":96221,"Ġinvertebrates":96222,"icule":96223,"Ġalam":96224,"ovasc":96225,"Ġmodulating":96226,"Ġhereafter":96227,"æ»ijåĿ¡":96228,"ĠDerived":96229,"çάä¸Ĭ":96230,"ä¼łéĢĴç»Ļ":96231,".classList":96232,"orschung":96233,"Ġskewed":96234,"Ġdemolition":96235,"äºĨä¸ĭä¾Ĩ":96236,"ä¸ĭè¿°":96237,"åı¯ä»¥è¾¾åΰ":96238,"ĠAlien":96239,"èĩªæĪijä»ĭç»į":96240,"Ġresistivity":96241,"ĠÙħرتب":96242,"ĠApostle":96243,"æ«»":96244,"ĠPAGE":96245,"ĠFighter":96246,"ĠاÙĨتشار":96247,"#ifdef":96248,"Ford":96249,"zett":96250,"Ëļ":96251,"Ġyrs":96252,"ĠBlick":96253,"æľīçļĦæĹ¶åĢĻ":96254,"ĠQString":96255,"驹":96256,"讲æķħäºĭ":96257,"ĠLipp":96258,"èĺĭ":96259,"×ķסף":96260,"Ġväl":96261,"дка":96262,"å°ıåĿĹ":96263,"Ġbacklash":96264,"æĶ¾çĸĹ":96265,"ç´§è¦ģ":96266,"é£ŀèĪŀ":96267,"Ġtenor":96268,"ĠReddy":96269,"驾驶è¯ģ":96270,"Ġflourished":96271,"á»ĩt":96272,"ĠاÙĨÚ¯ÙĦÛĮ":96273,"ĠMikhail":96274,"Ġskim":96275,"лÑĮнÑĭми":96276,"Chars":96277,"(dir":96278,"åĭ¾èµ·":96279,"ĠIhnen":96280,"àªĤàª":96281,"opropyl":96282,"ä¸įæŃ£å¸¸":96283,"å¿ĥæĥĬ":96284,"Ġrobbed":96285,"äºĮ审":96286,"гÑĭ":96287,"Ġ×Ķפר×":96288,"éĴĪåĪº":96289,"ĠSwimming":96290,"çĽ¸å¯¹çļĦ":96291,"é쥿ĦŁ":96292,"ustainability":96293,"æĺĤè´µ":96294,"protocol":96295,"çĪªåŃIJ":96296,"ĠÑĥÑĢовне":96297,"ĠLondres":96298,"åľ¨ä»Ĭ":96299,"Ġdelights":96300,"ĠMinh":96301,"cznej":96302,"è§Ĵ度æĿ¥çľĭ":96303,"çαæĥħçļĦ":96304,"Ġaccentu":96305,":c":96306,"Founded":96307,"SAT":96308,"ĠSous":96309,"ä¸ĢæĭĽ":96310,"è¦ģ约":96311,"æŃ£åĩĨå¤ĩ":96312,"othor":96313,"åIJĦåĽ½çļĦ":96314,"ç¥ŀçµĮ":96315,"Ġauditors":96316,"IMENT":96317,"ĠNorse":96318,"çĵ¦è§£":96319,"Finance":96320,"Ġtahu":96321,"Ġmasculinity":96322,"éĤ£å¤´":96323,"ä½Ĩä¸įæĺ¯":96324,"Ġfecal":96325,"ĠPhyll":96326,"Ġرب":96327,"×Ļפ×Ķ":96328,"Singapore":96329,"GRAPH":96330,"人åı¯ä»¥":96331,"ĠNairobi":96332,"ä¸Ń以":96333,"-study":96334,"èħ¾èħ¾":96335,"çļĦä¸Ńå¹´":96336,"Ġlorsqu":96337,"æ½įåĿĬ":96338,"(async":96339,"Laura":96340,"Ġsushi":96341,"Ġwakes":96342,"ĠmÃł":96343,"笺":96344,"Ġdela":96345,"æĮĩæĺİ":96346,"requests":96347,"Ġinfluencers":96348,"第ä¸īæŃ¥":96349,"åºĬ头":96350,"Ġtimelines":96351,"Ġà¦ħনà§įয":96352,"itim":96353,"-safe":96354,"Heading":96355,"Ġשת":96356,"اشر":96357,"ĠShowing":96358,"ç´§å¼łçļĦ":96359,"Ö·×Ļ":96360,"ĠиÑģÑĤоÑĩ":96361,"ĠHarmon":96362,"Ġelliptic":96363,"usual":96364,"ĠMVC":96365,"ĠroÅĽlin":96366,"èģĶå¸Ń":96367,"çIJĥå½¢":96368,"Ġsatire":96369,"ĠAsthma":96370,"ÐķÐĿÐĺ":96371,"ĠLatvia":96372,"ĠEqs":96373,"ützt":96374,"Ġalrededor":96375,"Metric":96376,"Ġcông":96377,"olang":96378,"ĠDIG":96379,"Ġqueues":96380,"Ġmicrotub":96381,"CHO":96382,"Ġutiliser":96383,"å°ĩæľĥ":96384,"äºĨåĩłåĪĨ":96385,"ابراÛĮÙĨ":96386,"Ġoutfits":96387,"_gen":96388,"Ġchilly":96389,"éria":96390,"ä¹ĭé¦ĸ":96391,"ä½łç»ĻæĪij":96392,"å®ŀå½ķ":96393,"åħ¶äºº":96394,"æŃ£è§Ĩ":96395,"worked":96396,"å¸ĤåľºåĴĮ":96397,"Ġloosen":96398,"åħįè´£":96399,"å̾éĶĢ":96400,"æĺ¨å¤©æĻļä¸Ĭ":96401,"Ġponad":96402,"Ġproyectos":96403,"Ġunification":96404,"Ġglued":96405,"èģĶåĨĽ":96406,"Ġduplicated":96407,"Ġíά":96408,"æķ£åİ»":96409,"秦çİĭ":96410,"æĿĥåĪ©çļĦ":96411,"Ġcompounding":96412,"ĠLyons":96413,"Ġaucun":96414,"Ġadipisicing":96415,"Ġleasing":96416,"Ġà¦ł":96417,"åĨħåIJij":96418,"ĠÙ쨹":96419,"äºĨä¸ĢåĿĹ":96420,"æ¹Ĭ":96421,"ãģ¾ãģ¾":96422,"æĭ¿èijĹ":96423,"Ġstej":96424,"éĢıå½»":96425,"讨论äºĨ":96426,"èĥĥçĻĮ":96427,"ÏģαÏĤ":96428,"Ġantonyms":96429,".]Ċ":96430,"Ġbx":96431,"Ġвд":96432,"ĠSeems":96433,"ĠZucker":96434,"Ġà²Ĩ":96435,"æµĵ度çļĦ":96436,"Ġrectal":96437,"ĠALS":96438,"à¸ķรี":96439,"Ġluogo":96440,"Ġtéto":96441,"Ġwszystkim":96442,"ĠWrestling":96443,"Ġjot":96444,"ikos":96445,"henol":96446,"Ġ_**":96447,"缸éļĶ":96448,"æµģéľ²":96449,"ĠstyleUrls":96450,"çļĦæīĭèĩĤ":96451,"ĠFlashcards":96452,"Ġhastily":96453,"\\langle":96454,"æĺ¯åĴĮ":96455,"为己":96456,"ĠInterfaces":96457,"åĹĶ":96458,"åľĪçļĦ":96459,"åĩºçīĪçī©":96460,"Ġrendre":96461,"æĭĵæīij":96462,"æľīæľºçī©":96463,"ĠAutoCAD":96464,"æµĩçŃij":96465,"åŁİ乡å±ħæ°ij":96466,"Arguments":96467,"Ġмиллионов":96468,"Ġnú":96469,"Ġnehmen":96470,"éķ¿åŃIJ":96471,"лег":96472,"à¹Ģà¸Ľà¹ĩà¸Ļà¸ģาร":96473,"æĦŁæĥħçļĦ":96474,"Ġweten":96475,"MRC":96476,"à¥Ĥल":96477,"ç´§å¯Ĩç»ĵåIJĪ":96478,"å¦Ħæĥ³":96479,"Growth":96480,"Ġlequel":96481,"éĩįéĺ³":96482,"Ġrecap":96483,"æĶ¾ä¸ĭäºĨ":96484,"ĠÙ쨱ÙĬÙĤ":96485,"è´¹åĬĽ":96486,"ORIES":96487,"æ¯į女":96488,"éĿŀ常éĢĤåIJĪ":96489,"é¦ĻçĶľ":96490,"çıłæ±Ł":96491,"Supported":96492,"Ġenergi":96493,"Kir":96494,"ĠIGF":96495,"ethoxy":96496,"æĬĬå°ı":96497,"Ġsola":96498,"Soil":96499,"ائÙī":96500,"ĠпÑĢедела":96501,"津津":96502,"çı¾åľ¨çļĦ":96503,"åıijè¨Ģ人":96504,"弧度":96505,"Developing":96506,"Ġendeavour":96507,"å¤ĸåķĨæĬķèµĦ":96508,"Carm":96509,"çļĦçľĭèijĹ":96510,"ĠBordeaux":96511,"Ġsetback":96512,"ç²ķ":96513,"Ġ...âĢĿ":96514,"æĿ¾äºĨåı£æ°Ķ":96515,"é¤IJåħ·":96516,"ĠGiac":96517,"åľ¨è¿Ļ个æĹ¶åĢĻ":96518,"ĠKitty":96519,"à¸łà¸²à¸§à¸°":96520,"Actor":96521,"arist":96522,"Ġmén":96523,"åĬłåİĭ":96524,"versed":96525,"-moving":96526,"ĠManag":96527,"ĠAntioxid":96528,"าà¸ģาศ":96529,"éĶĢåĶ®çļĦ":96530,"Ġpositif":96531,"ĠHonors":96532,"ASCAR":96533,":id":96534,"Ġbiking":96535,"æĪį":96536,"æµļ":96537,"çŃīé«ĺ":96538,"Ġlookout":96539,"å±±æ¥Ĥ":96540,"åĵªä¸Ģç§į":96541,"Parallel":96542,"ĠExpand":96543,"åľŁåľ°ä¸Ĭ":96544,"Geo":96545,"Ġnhư":96546,"Ġwitty":96547,"\\Entity":96548,"_manager":96549,"climate":96550,"Ġglu":96551,"æļĪ":96552,"brecht":96553,"ër":96554,"ĠConstantine":96555,"ĠÙħجÙĦس":96556,"_iterator":96557,"ʲ":96558,"rozen":96559,"arean":96560,"еннÑĭм":96561,"åŃ¦ä¹łèĢħ":96562,"ĠDarcy":96563,"Ġreverber":96564,"Via":96565,"ĠDAN":96566,"çŃīæİªæĸ½":96567,"ĠlocalStorage":96568,"åĽºæĢģ":96569,"æ´Ľå¤«":96570,"ĠDifficulty":96571,"ĠDurante":96572,"Ġpylori":96573,"ĠSanctuary":96574,"ayashi":96575,"resolution":96576,"羯":96577,"Ġflute":96578,"Ġquestão":96579,"Ġcondições":96580,"Ġnecessário":96581,"å®¶éķ¿çļĦ":96582,"ждÑĭ":96583,"áĥĿáĥij":96584,"ĠNamibia":96585,"Ġmeteorological":96586,"LH":96587,"â̼":96588,"Ġpolo":96589,"åĪĻ该":96590,"Ġlistens":96591,"ónico":96592,"麻辣":96593,"éĥ½æľīäºĨ":96594,"ĠباÙĦس":96595,"ĠPetra":96596,"åŁĭ头":96597,"åŁĭä¼ı":96598,"Ġexplorers":96599,"Ġscratched":96600,"%.Ċ":96601,":_ĊĊ":96602,"]\",":96603,"Ġlangu":96604,"ĠTomb":96605,"Ġenpres":96606,"ÃŃng":96607,"ĠAlone":96608,"ç§ģèĩª":96609,"ãĤ¢ãĤ¤":96610,"réal":96611,")the":96612,"Mental":96613,"YB":96614,"ĠTav":96615,"ĠMim":96616,"ĠоÑĤмеÑĩа":96617,"æłijå¹²":96618,"-Fran":96619,"å½ĵåīį离线":96620,"ĠIllness":96621,"èĤĸåĥı":96622,"ĠTrojan":96623,"ĠÑĪеÑĢан":96624,"teilung":96625,"vac":96626,"ĠCCC":96627,"Ġheats":96628,"Ġjargon":96629,"Ġoby":96630,"rapist":96631,"éĥ½æĺ¯ä»İ":96632,"æ½ľæĦıè¯Ĩ":96633,"اÙĪÙĬØ©":96634,".reset":96635,"Ġivory":96636,"Ġfenomen":96637,"Ġcoffin":96638,".Supp":96639,"åħ«è·¯åĨĽ":96640,"æıIJä¾ĽæľįåĬ¡":96641,"çł´åĿıäºĨ":96642,"ĠWilderness":96643,"ĠØŃدÙĬØ«":96644,"太æŀģæĭ³":96645,"çľĭä¸Ģçľ¼":96646,"ursing":96647,"å®ĺçļĦ":96648,"ĠmiRNA":96649,"θν":96650,"Ġpolygons":96651,"åħ©åĢĭ人":96652,"åĮħåIJ«çĿĢ":96653,"ложение":96654,"ĠElephant":96655,"Mis":96656,"honderd":96657,"æİĻ":96658,"áž":96659,"æĪij们åĨį":96660,"Ġprimi":96661,"鸣类":96662,"fv":96663,"mongoose":96664,"Ġrefle":96665,"ĠControvers":96666,"ĠBerks":96667,"compress":96668,".Home":96669,"èªįçŁ¥":96670,"-el":96671,"}p":96672,"ÙĬØŃ":96673,"Ġdiscontent":96674,"identique":96675,"è¿Ļæł·åŃIJ":96676,"ä¼ĺå¼ĤçļĦ":96677,"Ïĥκ":96678,"åĪ·çīĻ":96679,"Ġauthenticate":96680,"é¡ŀä¼¼":96681,"Ġtoán":96682,"æł¡åĩĨ":96683,"ÏģÎŃ":96684,"Ġpolitica":96685,"ĠMcInt":96686,"çĺĺ":96687,"ãĤĤãģ®ãĤĴ":96688,"\"My":96689,"(with":96690,"[current":96691,"_INT":96692,"ĉstr":96693,"leit":96694,"ĠEFFECT":96695,"ĠInhal":96696,"à§·":96697,"ienti":96698,"ÑĪÑĭ":96699,"å°±æĺ¯æĮĩ":96700,"ĠDevOps":96701,"ajÄħcy":96702,"Ġrecalling":96703,"Pho":96704,"ĠÌ":96705,"ĠMAD":96706,"arsely":96707,"Ġporter":96708,"iteness":96709,"nostÃŃ":96710,"Mort":96711,"çļĦåİŁçIJĨ":96712,"ĠRBC":96713,"å°±é¤IJ":96714,"æĸ°ãģĹãģĦ":96715,"èĢģçİĭ":96716,"ĠMatemat":96717,"ê°Ģì§Ģ":96718,"ĠAML":96719,"çµĦæĪIJ":96720,"Ġfestivities":96721,"Ġbotanical":96722,"ĠPythagorean":96723,"Ġboven":96724,"对éĺµ":96725,"Ġperusahaan":96726,"ĠSeine":96727,"Ġlocalities":96728,"æ²³ä¸ľ":96729,"Inters":96730,"äºĨè§£åĴĮ":96731,"æģIJé¾Ļ":96732,"æĩĤäºĭ":96733,"+\\,\\":96734,"Ġsede":96735,"ĠCatholicism":96736,"ĠTuber":96737,"Ġlire":96738,"åIJĮæĹ¥":96739,"åģļçĿĢ":96740,"Ġcareless":96741,"Chap":96742,"onese":96743,"ĠÑĩаÑģ":96744,"Õ¸Õ¿":96745,"Ġchampagne":96746,"Ġতাà¦ģর":96747,"æīĢ使ç͍çļĦ":96748,"åħ¬æĬ¥":96749,"åıĬæĹ©":96750,"Ġpassa":96751,"åĥıä¸Ģ个":96752,"ĠвÑĭвод":96753,"\"))ĊĊ":96754,"åIJĪçIJĨå®īæİĴ":96755,"Ġfostered":96756,"Ġзаконода":96757,"å¦Ĥä¸ĭåĽ¾æīĢ示":96758,"Ġcrib":96759,"arom":96760,"ĠRoe":96761,"ĠOverse":96762,"èĢģæĹ§":96763,"åıĪéģĵ":96764,"Ġ×Ķ×Ĺ×":96765,"-cells":96766,"λÏİ":96767,"ç»Ŀä¸įæĺ¯":96768,"ĠвÑĭбиÑĢа":96769,".MAX":96770,"å¥ĹæĪ¿":96771,"Ġphilosophies":96772,"Ġregained":96773,"åıĹç²¾":96774,"åĺħ":96775,"classification":96776,"ĠFrantsay":96777,"æī«é»ij":96778,"ç´ħèī²":96779,"Reporting":96780,"ĠاÙĦØ¢ÙĨ":96781,"(content":96782,"uuid":96783,"ĠWalls":96784,"天平":96785,"æľºä¸Ĭ":96786,"社ä¼ļåIJĦçķĮ":96787,"Problems":96788,"éļıåı£":96789,"ãģ¾ãĤĭ":96790,"ä¿¡æģ¯åħ¬å¼Ģ":96791,"æ¦Ĥ念çļĦ":96792,"çĽĪäºı":96793,"á»įn":96794,"ä¸ĥåħ«ç³Ł":96795,"ĠигÑĢÑĭ":96796,"ĠLarsen":96797,"ĠباÙĨ":96798,"éĢīæ¡Ĩ":96799,"Ġ;;":96800,"বিদ":96801,"Ġতà¦ĸন":96802,"绣ä¸ĢæĪĺ线":96803,"simp":96804,"Ġtav":96805,"ĠSapp":96806,"ĠTuring":96807,"ĠÑĥбе":96808,"è§ĴéĢIJ":96809,".world":96810,"ĠngOn":96811,"ĠÃľbers":96812,"Ġisotropic":96813,"ĠTent":96814,"енин":96815,"Ġprofiss":96816,"ĠÑĤÑĥÑĢ":96817,"Activ":96818,"Ïģακ":96819,"à¹Ģà¸ŀราะ":96820,"Ġañ":96821,"ĠbÃ¥":96822,"ĠTerr":96823,"ĠBubble":96824,"ĠUll":96825,"é«ĺ举":96826,"ĠعضÙĪ":96827,"ENTIAL":96828,"èĦ±åı£":96829,"ĠEstud":96830,"Nullable":96831,"Ġrazor":96832,"Ġdiligently":96833,"Ġcreepy":96834,"Ġpauses":96835,"两åĿĹ":96836,"é¦ĭ":96837,"æļĦ":96838,"ÙĪØ±ÙĪØ¨":96839,"çŁŃæĸĩ":96840,"è¡£é£Ł":96841,"åħ³äºİåĬłå¼º":96842,"Ġsurvives":96843,"ĠÑħÑĥ":96844,"à¹īวย":96845,"Ġescalating":96846,"EMAIL":96847,"ĠRobbins":96848,"人æµģ":96849,"ä¸İæĸ¹æ³ķ":96850,"çŃīæĸ¹æ³ķ":96851,"عض":96852,"建åĨĽ":96853,"Ġsred":96854,"ç»ıå¼Ģ":96855,"Ġsublim":96856,"æłĩæĺİ":96857,"çĹħæĤ£èĢħ":96858,"ĠQuiet":96859,"æ¼Ķ说":96860,"skin":96861,"ĠConnecting":96862,"Ġconjugated":96863,"åĨ¤æŀī":96864,"Ġdizzy":96865,"cq":96866,"orra":96867,"Ùĥب":96868,"ĠNewfoundland":96869,"åĨ³ç®Ĺ":96870,"ĠÙĨÙĤÙĦ":96871,"ĠOlsen":96872,"ĠStartup":96873,"Ġstickers":96874,"Soci":96875,"mény":96876,"umably":96877,"è¿Ľåİ»äºĨ":96878,"å·¥ä½ľå¼Ģå±ķ":96879,"Ġfootwear":96880,"ĠподÑħод":96881,"-Americans":96882,"/The":96883,"IOS":96884,"ingt":96885,"ÑĢиÑĤе":96886,"acca":96887,"éĿ¢æĹłè¡¨æĥħ":96888,"ĠOj":96889,"renal":96890,"åıĸåIJį":96891,"ĠSuomen":96892,"\":[":96893,"жиÑĤе":96894,"社ä¼ļ主ä¹īæł¸å¿ĥä»·å̼è§Ĥ":96895,"awsze":96896,"خرج":96897,"ĠíĮ¨":96898,"Ġназвание":96899,"æ¯Ľç»Ĩè¡Ģ管":96900,"ĠDAM":96901,"Ġzahl":96902,"éĤ£åĩłä¸ª":96903,"Ġindis":96904,"Ġsubmar":96905,"ç«ĭå¾·":96906,"èijĨ":96907,"注解":96908,"å§ĭèĩ³":96909,"ãģ¨ãģĨ":96910,"ĠElaine":96911,"éĺ»å°¼":96912,"æĬµè§¦":96913,"æ°¸è¿ľæĺ¯":96914,"çĦĬç¼Ŀ":96915,"ĠÑĢÑĥковод":96916,"Growing":96917,"Ron":96918,"uais":96919,"人åIJį":96920,"æĪijåĪļ":96921,"ä¸ŃåĽ½æ¢¦":96922,"游è¡Į":96923,"úc":96924,"å§IJ夫":96925,"ĠиÑģпÑĭÑĤа":96926,"çĮĤ":96927,"Tek":96928,"umina":96929,"Ġchoke":96930,"æĿ¥ä¹ĭ":96931,"å¸¸åľ¨":96932,"éĢłåĮĸ":96933,"лаÑĤ":96934,"ç»Ń表":96935,"Ġstructuring":96936,"volved":96937,"gw":96938,"{matrix":96939,"Ġdeceptive":96940,"äºĨ大éĩı":96941,"arekin":96942,"建åĬŁ":96943,"Ġcolt":96944,"ĠÙ쨵ÙĦ":96945,"åĽ¢åĽ¢":96946,"Ġeyed":96947,"ĠоÑĤпÑĥ":96948,"eteen":96949,"çļĦåıijå±ķåĴĮ":96950,"æµģç¨ĭåĽ¾":96951,"微信群":96952,"è¡Įéķ¿":96953,"Ġteó":96954,"èµ°åĬ¨":96955,"Ġfamed":96956,"åĮ»æ²»":96957,"Ġassociative":96958,"åĬŁèĥ½æĢ§":96959,"ãĥ¼ãĥĦ":96960,"ĠGentiles":96961,"ĠоÑĨенки":96962,"Ġentanto":96963,"ĠмодÑĥ":96964,"áĥłáĥ":96965,"Ġvisite":96966,"åĩĢèµĦ产":96967,"Ġbanker":96968,"Ġপà§įরব":96969,"åįģä¹ĿæĿ¡":96970,"CatalÃł":96971,"ç¬Ķè®°æľ¬ç͵èĦij":96972,"ĠбÑİдж":96973,"yam":96974,"Ġfy":96975,"ĠWeapons":96976,"Ġdiret":96977,"OTHER":96978,"Ġآثار":96979,"ĠHelper":96980,"èĢIJç͍":96981,"èİīèİī":96982,"/share":96983,"=j":96984,"ĠStrain":96985,"ä¸īæĸ¹":96986,"èĢģå¸Īåľ¨":96987,"æĢªçļĦ":96988,"Ġrobes":96989,"audi":96990,"åĮĪçīĻ":96991,"è¡ĻéŨ":96992,"ĠAUTO":96993,".With":96994,"Hart":96995,"inatal":96996,"çļĦåĵģçīĮ":96997,"ayama":96998,"ÙĨادÙī":96999,"æģĻ":97000,"Ġremnant":97001,"ç»Ļä»ĸçļĦ":97002,"åĨįè¿ĩ":97003,"жеÑĤÑģÑı":97004,"auty":97005,"à¹ģà¸ļ":97006,"ĠCreature":97007,"åij¼åIJ¸åĽ°éļ¾":97008,"Ġdescriptors":97009,"Asp":97010,"æĪijåºĶ该":97011,"Ġmodelos":97012,"ĠпоÑģÑĤе":97013,"æ¿®":97014,"daq":97015,"åIJ¯è¿ª":97016,"ĠRolls":97017,"ĠÐŀÑģоб":97018,"วิà¸ĺีà¸ģาร":97019,"é·¹":97020,"çļĦèµĦæºIJ":97021,"Ġunbalanced":97022,"éĩijåįİ":97023,"-beta":97024,"åį´æľī":97025,"æī¿åħij":97026,"ĠOffers":97027,"-prop":97028,"Ġplugs":97029,"ĠмаÑĢÑĤа":97030,".th":97031,"Ġconical":97032,"ustion":97033,"Ġdesember":97034,"æľįåĬ¡ä½ĵç³»":97035,"ừ":97036,"Ġterl":97037,"lena":97038,"Ġpilgrims":97039,"åħļé£İå»īæĶ¿å»ºè®¾":97040,"Cod":97041,"ннаÑı":97042,"unding":97043,"Ġmesenchymal":97044,"Delay":97045,"çĽ¼æľĽ":97046,"greSQL":97047,"ĠInfections":97048,"ĠSoldier":97049,"ĠTears":97050,"athlon":97051,"ŀ×¢":97052,"åĺĪ":97053,"æĬķåħ¥ä½¿ç͍":97054,".selected":97055,"-modal":97056,"ì²ĺëŁ¼":97057,"ãĤ¨ãĥįãĥ«ãĤ®ãĥ¼":97058,"缮çŀªåı£åijĨ":97059,"ÄĦ":97060,"atars":97061,"ĠVoll":97062,"ä¹ŁéĢIJæ¸IJ":97063,"æ°ijæŃĮ":97064,"太æ¹ĸ":97065,"éľĢè¦ģä¸Ģ个":97066,"Ġreceivable":97067,"ĠScorp":97068,"Ġamplifiers":97069,"Ġhalogen":97070,"Ġdrummer":97071,"-techn":97072,"Ġexpansions":97073,"à¦Ĺà§ģলà§ĭ":97074,"ĠComplementary":97075,"'o":97076,"ĠHari":97077,"Ġresize":97078,"å¿ĥå®ī":97079,"çľ¼äºĨ":97080,"ĠTris":97081,"çĶŁäº§åŁºåľ°":97082,"Quarter":97083,"èĩªçĦ¶åľ°":97084,"æĺ¯åIJ¦ä¼ļ":97085,"royo":97086,"ĠStatist":97087,"-Lab":97088,"ĠÙħدÙĬÙĨØ©":97089,"èĦĸåŃIJä¸Ĭ":97090,"Ġaumentar":97091,"سرعة":97092,"Ġglossy":97093,"Ġtyranny":97094,"IAS":97095,"ĠSEN":97096,"个æ¡Ī":97097,"ç͵æĦŁ":97098,"Ġαι":97099,"åģ¶æķ°":97100,"(?":97101,"LIST":97102,"ĠWohn":97103,"Ġké":97104,"è¿Ļåı¯èĥ½":97105,"achable":97106,"å®ļè¯Ń":97107,"ekom":97108,"è¿Ļä¸ªä¸ľè¥¿":97109,"éľĢæ±ĤåĴĮ":97110,"Ġallegation":97111,"鸣åĦ¿":97112,"ĠPriorit":97113,"åįĶåĬ©":97114,"æĻ¶ä½ĵ管":97115,"ingale":97116,"ĠTad":97117,"Ġkru":97118,"åĨĽå§Ķ":97119,"NAP":97120,"---|---":97121,"Ġétabl":97122,"ĠBowen":97123,"çŃīåIJĮäºİ":97124,"Ġvents":97125,"ĠBÃłi":97126,"agged":97127,"å¿«æĿ¥":97128,"ä¸ĵæĶ¿":97129,"Ġswine":97130,"à¸Ħà¹Īาà¸":97131,"åħŃä¸Ģ":97132,"ĠElm":97133,"à§ģণ":97134,"æ¯Ķè¾ĥ容æĺĵ":97135,"ĠErg":97136,".Local":97137,"ĠAPR":97138,"æIJľæŁ¥":97139,"Ġà¸ģร":97140,"Ġstumble":97141,"istos":97142,"ç͍æĹ¶":97143,"年头":97144,"ĠتÙĩراÙĨ":97145,"Ġrischio":97146,"ĠFarming":97147,"ının":97148,"ĠÑĨенÑĤÑĢ":97149,"Communic":97150,"éĩĮç¨ĭç¢ij":97151,"¤×ľ":97152,"äºĨçĤ¹":97153,"çĻ£":97154,"çĥŃæ°Ķ":97155,"Ġtreatise":97156,"Ġdolls":97157,"穷人":97158,"Ġlobster":97159,"äºĮæīĭæĪ¿":97160,"ĠReproductive":97161,"è¦ģ被":97162,"Ġadherent":97163,"++;ĊĊ":97164,"Ġ![":97165,"Ġ×IJ׾×Ķ":97166,"è´´çݰ":97167,"Ġphenolic":97168,"å̼å¾ĹæĪij们":97169,"Ġdisagreed":97170,"ädagog":97171,"ĠFellows":97172,"Ġnatuurl":97173,"ä¸Ģåīij":97174,"æľ¬ä»¥ä¸º":97175,"çĶ±åĽ¾":97176,"Buttons":97177,"Getty":97178,"ĠDepartamento":97179,"ĠToxicol":97180,"å¯ĦçĶŁèĻ«":97181,"Ġê·¸ëŁ¬ëĤĺ":97182,"ĠHubb":97183,"æĺİæĻ°":97184,"Ġremembrance":97185,"èĥ¶åİŁ":97186,"ĠÎłÎ¿":97187,"ÈĽÄĥ":97188,";\\;\\":97189,"Ġstellt":97190,"abu":97191,"æľīæŃ¤":97192,"inted":97193,"åħ¨çº¿":97194,"ĠAlph":97195,"è¯Ŀåī§":97196,"ç§¯éĽª":97197,"ophore":97198,"çł´å£ŀ":97199,"çͱäºİåħ¶":97200,"(buf":97201,"ĠподÑĢоб":97202,"并没æľīä»Ģä¹Ī":97203,"天èĬ±æĿ¿":97204,"_board":97205,"Ġstres":97206,"ĠHid":97207,"ĠEi":97208,"Ġruss":97209,"çĶŁçģµ":97210,"åıijåĩºä¸Ģ":97211,"ensky":97212,"ophile":97213,"äºīåģļ":97214,"åºĬè¾¹":97215,"اجع":97216,"ĠCrash":97217,"-record":97218,"Ġglycerol":97219,"Ġpics":97220,"Ġgout":97221,"ĠLaut":97222,"è¦ģåѦä¼ļ":97223,"åı¯è¨Ģ":97224,"é¢ĺå¹²":97225,"管çIJĨ模å¼ı":97226,"Ġartigo":97227,"uxe":97228,"åıĮè¾¹":97229,"Ġporcelain":97230,"Ġhomolog":97231,"Ġutilisation":97232,"帮åĬ©ä½ł":97233,"åζéĢłçļĦ":97234,"ä¹Įäºij":97235,"ĠCameroon":97236,"ĠاÙĦÙħرÙĥز":97237,"æĿİä¸ĸæ°ij":97238,"ĠÑĪÑĤ":97239,"æ¯Ķå°Ķ":97240,"ä¼ģä¸ļåĨħéĥ¨":97241,"é»Ħè±Ĩ":97242,"ĠCorb":97243,"ĠÙĪØ§ÙĦØ´":97244,"Ġsunscreen":97245,"/download":97246,"ĠĠĠĠĠĠĠĠĊĠĠĠĠĠĠĠĠĊ":97247,"Ġhurricanes":97248,"Ġallocations":97249,"çĸ¯åŃIJ":97250,"Ġresiduals":97251,"Ġdicho":97252,"Ġharnessing":97253,"Ġhinaus":97254,"Ġgoalkeeper":97255,",@":97256,"Fra":97257,"ĠSPR":97258,"pea":97259,"ĠVamp":97260,"Ġdix":97261,"Ġ).Ċ":97262,"Ġwaterways":97263,"ĠعÙĪØ§ÙħÙĦ":97264,"éϤæģ¶":97265,"mlung":97266,"ĠMonarch":97267,"Ġধর":97268,"Ġdeductible":97269,"'ad":97270,"Ġness":97271,"ä¸įé«ĺåħ´":97272,"Ġclenched":97273,"ĠتÙĤد":97274,"派对":97275,"ĠMoran":97276,"åŁ¹è®ŃæľºæŀĦ":97277,"////////////////////////////////////////////////////////////////":97278,"pone":97279,"heids":97280,"Ġew":97281,"ĠGull":97282,"Ġshalt":97283,"ĠThromb":97284,"è¿ĩèĬĤ":97285,"ONDS":97286,"Ġnameeee":97287,"Ġmusi":97288,"æķħä½ľ":97289,"ä¸įæĸŃå¢ŀåĬł":97290,"é²ľæ´»":97291,"òria":97292,"çļĦéŃħåĬĽ":97293,"ĠCyl":97294,"ĠWyn":97295,"ĠOE":97296,"为åħ¬åı¸":97297,"çĶŁåŃ©åŃIJ":97298,"åħ³ç¾½":97299,"æł¡ä¼ģ":97300,"Ġমন":97301,"çĸ«æĥħå½±åĵį":97302,"-shirts":97303,"Delivery":97304,"ĠtecnologÃŃa":97305,"ÅĦskiego":97306,"anen":97307,"Ġtoch":97308,"åĪĨéĶĢ":97309,"å¿ĥçĶµåĽ¾":97310,"Ġتعد":97311,"ç½ijçIJĥ":97312,"ĠBeet":97313,"Addition":97314,"åĢŁè®°":97315,"боÑĤан":97316,"âłĢ":97317,"^k":97318,"eal":97319,"Ġbim":97320,"ä¸ŃæĮĩåĩº":97321,"Ġtratta":97322,"ä¸İåIJĪä½ľ":97323,"ä»İ天":97324,"åħ³ç³»ä¸Ń":97325,"ĠGuerr":97326,"olyb":97327,"ĠоднÑĥ":97328,"Ġà¹Ģà¸Ħ":97329,"arrollo":97330,"Ġdistintas":97331,"æĪijçªģçĦ¶":97332,"èĩªæĪIJ":97333,"æĪij们èĩªå·±":97334,"äºĮçͲ":97335,"å°ijæŀĹ":97336,"è¿Ļ个æ¶Īæģ¯":97337,"ĠNeptune":97338,"ä¹¡åľŁ":97339,"ĠпÑĢевÑĢа":97340,"åĺīéĿĸ":97341,"ARTMENT":97342,"Ġë¶ģ":97343,"(idx":97344,"à§ĩমà§įবর":97345,"éĦĻè§Ĩ":97346,"typical":97347,"çļĦæ°ijæĹı":97348,"ĠWolver":97349,"peer":97350,"è¦ģèĢĥèĻij":97351,"intosh":97352,"æĹłæŀģ":97353,"ĠکاÙħÙĦ":97354,"ĠÙĨتÛĮجÙĩ":97355,"æŃ¢æįŁ":97356,"ĠTrache":97357,"-wrap":97358,"ĠاÙĦÙĨبات":97359,"ãĥĥãĥģ":97360,"Ġminimization":97361,"Ġபà¯Ĩ":97362,"âĻª":97363,"Ġpobre":97364,"Discuss":97365,"Ġefek":97366,"สูà¸ķร":97367,"Ġaccusation":97368,"Ġerythe":97369,"ĠIncorporated":97370,"inguishable":97371,"Fix":97372,"SQ":97373,"çļĦç»ĵåIJĪ":97374,"ä¼ļè§īå¾Ĺ":97375,"Ġperg":97376,"é«ĺè¶ħ":97377,"æ¯ıç»Ħ":97378,"Ġguitars":97379,"éĢłç¦ı":97380,"令ä»ĸ":97381,"haal":97382,"Ġsynergy":97383,"ä¹Ļéħ¸":97384,"ལ":97385,"強大çļĦ":97386,"æĬ¬é«ĺ":97387,"æŀĿæĿ¡":97388,"Ġsprouts":97389,"设ç«ĭçļĦ":97390,"åĿļå®ŀçļĦåŁºç¡Ģ":97391,"Ġkall":97392,"ä¼ļè¯Ŀ":97393,"å·²äºİ":97394,"Ġmonomers":97395,"éĢłçº¸":97396,"ä¸ĵåįĸ":97397,"æĹı群":97398,"Ġfauc":97399,"Ġgrasslands":97400,"ĠÙħثاÙĦ":97401,"ĠNucleic":97402,"Ġbok":97403,"èĩªåį«":97404,"使ç͍æĸ¹æ³ķ":97405,"صØŃ":97406,"åĽ½å®¶éĺŁ":97407,"è¶ħæłĩ":97408,"Ġcivilized":97409,"×ķ׳×Ļת":97410,"Ġcréer":97411,"ĠPAPERS":97412,"Ġcoercion":97413,"åŃ£åIJİèµĽ":97414,"ĠcÅĵur":97415,"è¾ĵåįµç®¡":97416,"ĠReproduction":97417,"ĠmiÄĻ":97418,"Ġstruktur":97419,"ĠJeanne":97420,"Ġprodutos":97421,"Ġtusen":97422,"=E":97423,"Dalam":97424,"Ġstag":97425,"ĠJol":97426,"ä¿ij":97427,"æ°´ä¸ĭ":97428,"å¾·æĸ¯":97429,"ĠDescriptive":97430,"Ġgeneralize":97431,"åĵ¥ä»¬":97432,"Ġkomm":97433,"×ķ×ĵ×ķת":97434,"ĠPARTICULAR":97435,"Ġtho":97436,"andre":97437,"Ġmetac":97438,"ä¼łæĿ¥çļĦ":97439,"å®īåħ¨å·¥ä½ľ":97440,"表示æĦŁè°¢":97441,"Ġ×¢×ķ×ĵ":97442,"æIJŃæ¡£":97443,"Ġespresso":97444,"Ġinterfacial":97445,"Ġসমà§įপরà§įà¦ķ":97446,"Ġatividade":97447,"_SE":97448,"might":97449,"Ġvows":97450,"æĪijéĤĦ":97451,"åĪĩå¿Į":97452,"åĨĻè¿ĩ":97453,"ESTAMP":97454,"漫天":97455,"æIJħæĭĮåĿĩåĮĢ":97456,"-surface":97457,"Initialize":97458,"æ¯ĶåĪ©æĹ¶":97459,"ĠGreenwich":97460,"Ġì§ĢìĽIJ":97461,"åĮ®ä¹ı":97462,"Fried":97463,"çļĦéĢ»è¾ij":97464,"ĠMOR":97465,"teÅĻÃŃ":97466,"å¹´èĢģ":97467,"çłĶç©¶æĸ¹æ³ķ":97468,"Ġdolphins":97469,"Ġíĸ¥":97470,"Ġsalsa":97471,"Ġinductor":97472,"çİ®":97473,"insured":97474,"åİŁåIJį":97475,"ĠÑĥде":97476,".mp":97477,"ĠобÑīего":97478,".Dis":97479,"ª×Ŀ":97480,"ĠÏĢÏĮ":97481,"Ġgiác":97482,"পà§ģর":97483,"ĠPartnerships":97484,"Anthony":97485,"-ep":97486,"Ġdiabet":97487,"åį³å°ĩ":97488,"äºij端":97489,"鸯":97490,"èģŀè¨Ģ":97491,"æ³ķå®ļ代表人":97492,"Ġtoma":97493,"äºĨä¸ĬæĿ¥":97494,"Ġkron":97495,"ubuntu":97496,"å°ıçĶ·åŃ©":97497,"issements":97498,"пеÑĢе":97499,"scar":97500,"æ¸ħåĩĢ":97501,"à¸Ľà¹Īวย":97502,"ĠDiscord":97503,"ĠÑĤип":97504,"ĠRaphael":97505,"ãĥĥãĤ¯ãĤ¹":97506,"ĠÑĨвеÑĤа":97507,"Pas":97508,"atim":97509,"Ġpony":97510,"stance":97511,"æĺ¯ä¸¤ä¸ª":97512,"ĠRach":97513,"tyw":97514,"ĠZones":97515,"ç¥ŀ社":97516,"ĠPharma":97517,"ĠErasmus":97518,"ĠStatutes":97519,"Translate":97520,"ĠOccur":97521,"ĠÑģооÑĤвеÑĤÑģÑĤвенно":97522,"Ġdruh":97523,"Ġechocard":97524,"ĠíĤ¤":97525,"'esp":97526,"binding":97527,"ĠDund":97528,"ĠDSL":97529,"Ġpreclude":97530,"çľĭåģļ":97531,"æīĵæĪIJ":97532,"空空":97533,"ÑĤиÑĢÑĥ":97534,"åįĥèIJ¬":97535,"ipses":97536,"ĠخاÙĨÙĪ":97537,")ãĢģ(":97538,"áĥĿáĥĽ":97539,"verbose":97540,"ĠSlowly":97541,"ĠÐŁÐµÑĢевод":97542,"Ez":97543,"Ġdusk":97544,"seus":97545,"Ġnebul":97546,"è¿ĻåĽĽä¸ª":97547,"对æīĢæľī":97548,"åħ±åŃĺ":97549,"-front":97550,"è´¨éĩıæİ§åζ":97551,"å¢ŀåĬłå̼":97552,"Ġê°Ŀ":97553,"ĠاÙĦطاÙĤØ©":97554,"溶液çļĦ":97555,"åįĹ京å¸Ĥ":97556,"ĠIncorporating":97557,"ĠRally":97558,"æľĪåľ¨":97559,"ÑħодиÑĤÑĮ":97560,"Ġexporting":97561,"Elsevier":97562,"crow":97563,"qx":97564,"åΰ她":97565,"Ġdisgrace":97566,"å¤ĩæĪĺ":97567,"(code":97568,"Ġpoliceman":97569,"gestellt":97570,"ĠPurdue":97571,"à®ķà®°":97572,"ĠFerry":97573,"Ġdziew":97574,"ĠSuf":97575,"ĠErie":97576,"æīĢéľĢè¦ģ":97577,"ç¬ijèµ·æĿ¥":97578,"çĻ¾è®¡":97579,"åī§çĥĪçļĦ":97580,"à¹Ģà¸ģิà¸Ļ":97581,"åĤ²æħ¢":97582,"åīµä½ľ":97583,"Ġtrabajar":97584,"надÑĨа":97585,"%%%%%%%%%%%%%%%%":97586,"ĠìĥĪë¡ľìļ´":97587,"Ġhouden":97588,"enzen":97589,"Ġpère":97590,"ĠHancock":97591,"çŃī她":97592,"-duty":97593,"ĠÙĥÙĪÙħ":97594,"Ġbinocular":97595,"Evolution":97596,"Ġobsessive":97597,"-/":97598,"hara":97599,"yper":97600,"ĠNIC":97601,"ä¸Ĭè¯ģ":97602,"zeum":97603,"à¸Ĺà¹Īาà¸Ļ":97604,"ĠBecoming":97605,"éĥ½æĺ¯ä¸ºäºĨ":97606,"Ġtoolbar":97607,"disable":97608,"ISTER":97609,"ĠLemmon":97610,"ĠÑģоÑģÑĤоÑıнии":97611,"ĠUtilize":97612,"ZI":97613,"ĠRabb":97614,"以示":97615,"ä¸İ该":97616,"åĬłèµ·æĿ¥":97617,".\";Ċ":97618,"åĪĺæŁIJ":97619,"Ġêµ°":97620,"Ùĩرس":97621,"Ġ걸":97622,"Cursor":97623,"说ä¸Ģ说":97624,"Ġspines":97625,"空èħ¹":97626,"ambre":97627,"ĠاÙĦÙħÙĩ":97628,"éĢĢä¼į":97629,"Ġthinning":97630,"åĭĺæŁ¥":97631,"Ġpratiques":97632,"å°ıä¼Ļ伴们":97633,"éĩįè¦ģ讲è¯Ŀç²¾ç¥ŀ":97634,"emps":97635,"çłĶ究对象":97636,"çĶ»ç¬Ķ":97637,"ĠÙĬØ£":97638,"ĠISIS":97639,"ĠÑĺед":97640,"Ġ×Ķ×Ĵ×ĵ":97641,"Ġpuppet":97642,"ĠTODAY":97643,"sig":97644,"çļĦçģµéŃĤ":97645,"ĠKg":97646,"å·¥åķĨä¸ļ":97647,"fluss":97648,"-most":97649,"ĠÑĩÑĤ":97650,"дика":97651,"Ġstarving":97652,"Ġklub":97653,"zij":97654,"×ŀ×ķ×":97655,"inÄĽ":97656,"Ġwissen":97657,"çļĦè¯ģæį®":97658,"æīĢåħ·æľī":97659,"Ġentangled":97660,"èģĮä¸ļåѦéĻ¢":97661,"ÑĨами":97662,"Ġpalsy":97663,"'Connor":97664,"cancel":97665,"Ġspills":97666,"åĽĽåĢĭ":97667,"Ġapproving":97668,"èĤ²ç§į":97669,"æĸŃå®ļ":97670,"itação":97671,"çĪĨçϼ":97672,"Ġפר":97673,"å®ıè§Ĥè°ĥæİ§":97674,"éĽįæŃ£":97675,"Ġnemen":97676,"ä¸įéĹ®":97677,"Ġquis":97678,"æĸ°æ¨¡å¼ı":97679,"æīĭèīº":97680,"Ġaffiliations":97681,"à¸ŀืà¸Ĭ":97682,"ugo":97683,"STE":97684,"ĠGeographical":97685,"ĠMorales":97686,"迷人çļĦ":97687,"åªĴä½ĵæĬ¥éģĵ":97688,"ç©Ĩæĸ¯":97689,"Programming":97690,"-adjusted":97691,"ĠÑĢаÑģÑĤений":97692,"æľ¬æĥ³":97693,"ä¸īåĨľ":97694,"æĽ´ä¸įèĥ½":97695,"å¼¹èį¯":97696,"ĠκÏħ":97697,"ĠLowell":97698,"Ġmediates":97699,"ĠAstrophysics":97700,"Ġfronte":97701,"fam":97702,"Ġduke":97703,"âĢĻ-":97704,"åΰ头":97705,"ä»İåĵªéĩĮ":97706,"å®ī妮":97707,"Ġconstrain":97708,"让åŃ©åŃIJ们":97709,"离åĪ«":97710,"ĠÙħÙĨظ":97711,"ĠMonaco":97712,"æĽ¸ãģį":97713,"Ġbanning":97714,"ósito":97715,"Ġdisproportionate":97716,":m":97717,"Mut":97718,"ĠTory":97719,"åľ¨çİ°åľº":97720,"thorn":97721,"akses":97722,"éĤ£ç¾¤":97723,"西山":97724,"æĮģä»ĵ":97725,"Ġhandheld":97726,"åıĸä¸ĭ":97727,"é¢Ĩåľ°":97728,"å®īåħ¨æķĻèĤ²":97729,"ĠEmissions":97730,"ä¸įè¿ĩä»ĸ":97731,"γά":97732,"Ġdimost":97733,"ленноÑģÑĤÑĮ":97734,"OWS":97735,"ãĤīãĤĮãģ¦ãģĦãĤĭ":97736,"ólnie":97737,"_\\+":97738,"ĠBuh":97739,"å¤ļ头":97740,"æķĻ主":97741,"å¹¶è¦ģæ±Ĥ":97742,"Ġmateriales":97743,"Ġminded":97744,"ĠOffering":97745,"Ġà¹Ģà¸Ĺ":97746,"ؤاÙĦ":97747,"Ġawaited":97748,"=`":97749,"ĠSerm":97750,"ĠWad":97751,"ĠUPS":97752,"æĪij们åıijçݰ":97753,"äg":97754,"ĠZH":97755,"åıĪä½ķ":97756,"女æİĴ":97757,"夫åŃIJ":97758,"计ç®Ĺç»ĵæŀľ":97759,"æ¶²æĢģ":97760,"åĪĺæµ·":97761,"Ġberdasarkan":97762,"èĥŀèĥİ":97763,"ône":97764,"mapsto":97765,"ãģ¨ãģĦãģ£ãģŁ":97766,"Rt":97767,"Ġcia":97768,"ĠnÃły":97769,"utility":97770,"Ġunrecogn":97771,"èĥ½åĴĮ":97772,"Ġsoort":97773,"ciation":97774,"åħ¬éģĵ":97775,"æĸ°æĹ§":97776,"æĪij们ç͍":97777,"اÙĦØ´":97778,"Ġoriginality":97779,"ĠاÙĦسÙĬ":97780,"æ·±åħ¥åŃ¦ä¹łè´¯å½»":97781,"èĦıèħij":97782,"Ġdepartmental":97783,"çªĹåı£ä¸Ń":97784,"ĠBulls":97785,"Ġinterferon":97786,"ĠÑĤеоÑĢии":97787,"Ġsplicing":97788,"Gib":97789,"arı":97790,"ĠSmy":97791,"Ġlaz":97792,"ĠAlvarez":97793,"ICATIONS":97794,"ĠпÑĢигоÑĤов":97795,"Ġceases":97796,"æķĻåѦè¿ĩç¨ĭ":97797,"宽广":97798,"æĭĴä¸į":97799,"ç»ĻäºĨä»ĸ":97800,"Ġvraag":97801,";import":97802,"bang":97803,"vette":97804,"Ġtels":97805,"Ġprokary":97806,"é«Ļ":97807,"ĠÙħÙĦÛĮ":97808,"Ġдоказа":97809,"ĠAlpine":97810,"æīĵ车":97811,"è£ħä½ľ":97812,"à¸Ħà¹Īา":97813,"ĠÙĤاعدة":97814,"CPA":97815,"Ġbattered":97816,"îĢ":97817,"itha":97818,"bera":97819,"ccio":97820,"æĭļ":97821,"Ġobat":97822,"ĠнаÑĤÑĥÑĢа":97823,"Ġslugg":97824,"ĠSpine":97825,"åĩºçīĪåķĨ":97826,"æĸĩ竳æĿ¥æºIJ":97827,"slice":97828,"èĭįèĿĩ":97829,"ĠPMCID":97830,"ĠÏĩα":97831,"ĠWelch":97832,"Ġincarceration":97833,"èłķåĬ¨":97834,"Inventors":97835,"ĠFITNESS":97836,"ĠTucson":97837,"Bn":97838,"_Q":97839,"xin":97840,"Ġpaj":97841,"åĴĮä¿¡æģ¯":97842,"主讲":97843,"è¿ĺèĥ½å¤Ł":97844,"æ¶ĪçĤİ":97845,"æĬķèµĦåŁºéĩij":97846,"ĠLogical":97847,"Ġreactant":97848,"Congratulations":97849,"çļĦå®¶ä¼Ļ":97850,"毡":97851,"è¿Ľè´§":97852,"Ġдек":97853,"åıijå±ķæĸ¹åIJij":97854,"วรร":97855,"ĠзавеÑĢ":97856,"Ġsequenced":97857,".includes":97858,"Ġovershadow":97859,"çĺĭçĭĤ":97860,"_space":97861,"æĢ§æĺ¯":97862,"Ġrecor":97863,"-cert":97864,"主è¦ģ表çİ°åľ¨":97865,"ĠÐŁÑĢов":97866,"ĠÐĶан":97867,"били":97868,"è¿ĻæĿ¡è·¯":97869,"大大å°ı":97870,"åIJİæİĴ":97871,"ç»ıåķĨ":97872,"Ġoverpower":97873,"交éĽĨ":97874,"çŁ¥éģĵä»ĸ":97875,"ä¸ŃçļĦåľ°ä½į":97876,"ĠØ¢ÙħرÛĮÚ©":97877,"æĶ¹åıĺçļĦ":97878,"schl":97879,"å°¿æ¶²":97880,"Ġretrieving":97881,"ĠاÙĨدازÙĩ":97882,"oplasty":97883,"Ġsynergistic":97884,"IoT":97885,"ĠBok":97886,"ä¸įä¸Ģä¼ļåĦ¿":97887,"ierungs":97888,".command":97889,"管çIJĨæĿ¡ä¾ĭ":97890,"Ġlineages":97891,"лÑıÑİÑĤÑģÑı":97892,"æľīä¸ĢèĤ¡":97893,"èľĴ":97894,"ä¹Łæ²¡æľīä»Ģä¹Ī":97895,"Loaded":97896,"Portal":97897,"ĠдÑĥма":97898,"Ġlodging":97899,"åīĶéϤ":97900,"ĠENGINE":97901,".mean":97902,"åľ¨é©¬":97903,"फ":97904,"ahat":97905,"ĠZan":97906,"åĵį声":97907,"ä¿®çĤº":97908,"Ġtypu":97909,"AGC":97910,"è½°è½°":97911,"Ġá¼IJν":97912,"Ġkondado":97913,"ĠBaxter":97914,",name":97915,"ĠSistem":97916,"使å®ĥ":97917,"å¹³æģ¯":97918,"并被":97919,"รà¸ĸ":97920,"æµ·çĽĹ":97921,"èį¯ä¸ļ":97922,"Ġannuity":97923,"ĠÏĦῶν":97924,"_lines":97925,"vdots":97926,"Ġnär":97927,"ĠTie":97928,"ĠBones":97929,"æĺİ亮çļĦ":97930,"åIJĦåĮº":97931,"åIJĽçİĭ":97932,"ç§ģãģŁãģ¡":97933,"å¥ĭæĸĹ缮æłĩ":97934,"Ġhovering":97935,"Battle":97936,"jou":97937,"ĠmÅĤod":97938,"ĠMuj":97939,"ĠWatching":97940,"formal":97941,"ä¹ĭ举":97942,"à¸Ĺà¸Ķ":97943,"ĠModules":97944,"orz":97945,"éĢīæ°ij":97946,"ĠìĿ¸ê°Ħ":97947,"Ġmarché":97948,"ĠBhag":97949,"Ġbeispielsweise":97950,".Common":97951,"æĹ¥åĩĮæĻ¨":97952,"ĠAllocation":97953,"建çŃijå¸Ī":97954,"رÙĪØ¬":97955,"è¿ŀç»ŃçļĦ":97956,"ĠاÙĦرس":97957,"_report":97958,"ĠCrohn":97959,"ĠÑģозданиÑı":97960,"æłĸæģ¯":97961,"leine":97962,"ĠAUC":97963,"âĢĿ).ĊĊ":97964,"ÃŃamos":97965,"عÙģ":97966,"æĽ´ä½İ":97967,"éĵİ":97968,"жнÑĭе":97969,"à¹ģสà¸Ļ":97970,"Ġcréd":97971,"ĠCarlson":97972,"èŀįåIJĪåıijå±ķ":97973,"Ġerotic":97974,"æĢ»ç®¡":97975,"AMENT":97976,"ĠÑĢеÑĩи":97977,"è°ģæĿ¥":97978,"èĥ¡è¯´":97979,"éĵºåŀ«":97980,"Ġpuedes":97981,"Ġfederation":97982,"ãģªãĤīãģªãģĦ":97983,"}P":97984,"çļĦæĬĹ":97985,"leitung":97986,"æ±ĤãĤģ":97987,"éĻ¢åĨħ":97988,"rands":97989,"ÙİØª":97990,"å»īä»·":97991,"éĹºå¥³":97992,"Ġforeseeable":97993,".ncbi":97994,"Ġnám":97995,"åı¯ä½ľä¸º":97996,"geom":97997,"ĠChir":97998,"å®Įç»ĵ":97999,"书åIJį":98000,"ĠGeophys":98001,"ç§»éϤ":98002,"ĠtenÃŃa":98003,"ĠMcCain":98004,"æ³ķåĬĽ":98005,"æ¸İ":98006,"ritann":98007,"åıĹçģ¾":98008,"oots":98009,"aito":98010,"à¸ļวà¸Ļ":98011,"温æĥħ":98012,"åŃ¦æł¡åĴĮ":98013,"ĠTransplant":98014,"ĠMetz":98015,"ĠPalae":98016,"×ķ×ĵ×Ļ":98017,"ĠKirche":98018,"ãĥĢãĤ¤":98019,"Mix":98020,"}$$Ċ":98021,"çļĦèĢģ人":98022,"olari":98023,"大好":98024,"ÙħÙĩ":98025,"å°ı妹":98026,"å¦ĤæĦ¿":98027,"ĠiP":98028,"ÃŃsk":98029,"éĢłèι":98030,"ĠResume":98031,"afs":98032,"é»Ħæ²¹":98033,"èŀºæ¯į":98034,"akhir":98035,"Ġingenuity":98036,"Dad":98037,"ç±»æ¯Ķ":98038,"Ġmusique":98039,"ublique":98040,"çĶŁäº§è¦ģç´ł":98041,"ĠJanuar":98042,"Ġbioactive":98043,"رارة":98044,"=g":98045,"çļĦé¢Ĩ导":98046,"äºĨä»Ģä¹Ī":98047,"ĠHib":98048,"Ġ\"^":98049,"erequisites":98050,"产ç§ij":98051,"ä½Ĩä¹Łæľī":98052,"Ġpostgraduate":98053,"commons":98054,"Ġembar":98055,"-search":98056,"waarden":98057,"æĪ´åı£ç½©":98058,"tagHelper":98059,"ĠAberdeen":98060,"Ġmagistrate":98061,"Ġdistortions":98062,":String":98063,"ĠCrack":98064,"æµĴ":98065,"avad":98066,"è°§":98067,".Domain":98068,".Title":98069,"Ġintegrative":98070,"ĠCybersecurity":98071,"æĶĢçĻ»":98072,"Few":98073,"Ġpoli":98074,"tole":98075,"ĠHarley":98076,"åIJĮæĦıäºĨ":98077,"йÑĤеÑģÑĮ":98078,"ZL":98079,"Ġsacks":98080,"ÑĢение":98081,"ĠGV":98082,"ç©Ģ":98083,"å¾Īå·®":98084,"remos":98085,"çĭ¬æľīçļĦ":98086,"èιéķ¿":98087,"ĠSalis":98088,"ĠWaterloo":98089,"åįıè®®çļĦ":98090,"Ġstratég":98091,"ĠStere":98092,"Ġkeluarga":98093,"ĠHAR":98094,"ĠSteele":98095,"åģľäº§":98096,"oelect":98097,"çĸıéĢļ":98098,"æıŃå¼Ģ":98099,"_write":98100,"âĢļ¬":98101,"ĠÏĥÏĦον":98102,"èĩĢéĥ¨":98103,"ä¸įèĩªç¦ģ":98104,"Bang":98105,"Dry":98106,"第ä¸īç§į":98107,"ĠHorror":98108,"ĠRhine":98109,"åį°è±¡æ·±åĪ»":98110,"èħ³æŃ¥":98111,"Universit":98112,"Für":98113,"Ġtud":98114,"annten":98115,"åĬłæĪIJ":98116,"Ġtermes":98117,"Ġdao":98118,"Ġmaxima":98119,"Ġinformazioni":98120,"Ġentreprises":98121,"Assuming":98122,"اطع":98123,"æĴĴå¨ĩ":98124,"Ġbroadcasts":98125,"ĠCure":98126,"odoxy":98127,"ĠHanna":98128,"éĥ½çͱ":98129,"åİ»åĵª":98130,"西欧":98131,"ĠدÙĤÛĮ":98132,"Ġخدا":98133,"Ġvalidator":98134,"Ġfifteenth":98135,"ĠPlantae":98136,"Ġбило":98137,"ĠBriefly":98138,"Ġদà§ĩà¦ĸা":98139,"Motion":98140,"æķķ":98141,"Ġcompiling":98142,"ĠAbigail":98143,"-seq":98144,"ĠìŀĪìĿĦ":98145,"æĢ»ä½ĵè§ĦåĪĴ":98146,"ĠDamascus":98147,"profits":98148,"วัà¸Ĵà¸Ļ":98149,"Ġbastard":98150,"ĠHistorically":98151,"ä¸ªåĽ½å®¶":98152,"Ġdepressing":98153,"管çIJĨåѦéĻ¢":98154,"Ġpaperback":98155,"çIJĨè®ºçŁ¥è¯Ĩ":98156,"Ġsnail":98157,"Ġspectroscopic":98158,"ä¿ĿæĮģäºĨ":98159,"æĮ¯å¹ħ":98160,"ĠговоÑĢиÑĤ":98161,"ĠAjax":98162,"_print":98163,"ĠâĮ":98164,"ÑĤÑĮÑı":98165,"оваÑļа":98166,"bitos":98167,"å¯ĴåĨ¬":98168,"klär":98169,"Ġwaived":98170,"Ġút":98171,"æĴ¤åĽŀ":98172,"Ġcompanionship":98173,"-setting":98174,"Ġwiping":98175,"åı¢":98176,"ÙĬص":98177,"Ġindisc":98178,"омен":98179,"лиÑĤе":98180,"ä»ħåľ¨":98181,"æ¡ĥåŃIJ":98182,"à¹Ģหมาะ":98183,"Opening":98184,"ĠдокÑĥменÑĤа":98185,"çĬ¹å¤ªäºº":98186,"彷彿":98187,"sthe":98188,"thi":98189,"izal":98190,"åĩºãģĻ":98191,"æ³ķè¯Ń":98192,"Ġneedy":98193,"æ¯ĶæŃ¦":98194,"åĻĵ":98195,"Ġlandslide":98196,"褲":98197,"ĠAbsolutely":98198,"è¾¼ãģ¿":98199,"ĠÙħÙĦÙĬÙĪÙĨ":98200,"າàº":98201,"(words":98202,"FREE":98203,"hews":98204,"anum":98205,"ä¸Ĭ交":98206,"æľĢ容æĺĵ":98207,"ĠAluminum":98208,"æį¢çĥŃ":98209,"Subscription":98210,"ç¿»æ»ļ":98211,"ĠMori":98212,"ĠNOAA":98213,"ĠRandomized":98214,"ĠBorrower":98215,"Rearrange":98216,"BOSS":98217,"Hill":98218,"Ġalem":98219,"Ġdaddy":98220,"个好":98221,"Ġsails":98222,"æĪij们没æľī":98223,"ä¼ĺåĬ£":98224,"æ²ĻåŃIJ":98225,"æ²Ļæĭī":98226,"Ġshafts":98227,"Ġexpresión":98228,"?!ĊĊ":98229,"Ġtoho":98230,"ĠHJ":98231,"è¿Ľé©»":98232,"Ġводе":98233,"Ġcolect":98234,"çݯ氧":98235,"Ġbietet":98236,"หาย":98237,"è´´åľ¨":98238,"èĭĹæľ¨":98239,"ĠPoet":98240,"Ġrailways":98241,"ĠFarms":98242,"ĠíĻľìļ©":98243,"-\"":98244,"]++;Ċ":98245,"ĠIJ":98246,"Ġatra":98247,"以å®ŀéĻħè¡ĮåĬ¨":98248,"åľ°æľĽçĿĢ":98249,"ĠStations":98250,"Ġparach":98251,"åĨ·éħ·":98252,"ĠзнаÑĤÑĮ":98253,"Instructions":98254,"ായ":98255,"ĠдÑĢжа":98256,"ĠCannabis":98257,"åĴ¬çīĻåĪĩ":98258,"Ġì»´":98259,".rs":98260,"/dev":98261,"Ow":98262,"åĪ°åľº":98263,"Ġpointless":98264,"Ġisolating":98265,"алÑĮной":98266,"Ġkeratin":98267,"_bytes":98268,"aszt":98269,"Ġshunt":98270,"Ġattaining":98271,"åħļ群":98272,"ç¼ĸèĢħ":98273,"prepare":98274,"ĠGlossary":98275,"Ġcriticised":98276,"Ġassembl":98277,"Ġresembled":98278,",in":98279,"-onset":98280,"ĠKurs":98281,"å°ıå·§":98282,"ä»ĸ们说":98283,"Ġlikeness":98284,"æĿĢçļĦ":98285,"aminated":98286,"ĠÐIJмеÑĢи":98287,"å¨ĺçļĦ":98288,"ĠÅĽwiad":98289,"ĠкÑĢÑĥг":98290,"ĠUtilizing":98291,"ĠDresden":98292,"ugno":98293,"åĨįçͱ":98294,"è®°è¿°":98295,"Ġcytochrome":98296,"æĶ»åħĭ":98297,".Path":98298,"pathic":98299,"ĠدÛĮگرÛĮ":98300,",null":98301,"Sets":98302,"reja":98303,"ĠTrap":98304,"Ġvase":98305,"ĠEI":98306,"å±±ç¾Ĭ":98307,"Questa":98308,"Ġ׾׼׾":98309,"Ġкомпании":98310,"NX":98311,"Ġformazione":98312,"ĠQUE":98313,"ĠMarvin":98314,"ĠرÙĨÚ¯":98315,"ĠDisp":98316,"æ¯Ľè¡£":98317,"pañ":98318,"ä¹Į鸦":98319,"Ġesteemed":98320,"abbing":98321,"ĠCubs":98322,"ĠSeparate":98323,"ĠPebrero":98324,".click":98325,"warts":98326,"utting":98327,"ĠEMB":98328,"opi":98329,"è¿ŀå¤ľ":98330,"åįĩå̼":98331,"ĠاÙĦÙħÙħÙĦÙĥ":98332,"à¸Īุà¸Ķ":98333,"ĠпеÑĢеÑħод":98334,"Ġroofing":98335,"Ġinfantil":98336,"วัà¸ķิ":98337,"à¸łà¸²à¸¢à¹ĥà¸Ļ":98338,"ä¸įå°ıäºİ":98339,"Ġengulf":98340,"overrightarrow":98341,"çͲåħ¬åı¸":98342,"ĠSwap":98343,"Ġcoolant":98344,"Ġsacr":98345,"Õ¸ÖĤÕµ":98346,"ĠбеÑĢеменноÑģÑĤи":98347,"ĠKorn":98348,"ÑħождениÑı":98349,"Ġacum":98350,"Ġwaiter":98351,"Ġwidths":98352,"à½ł":98353,"ÙĩدÙģ":98354,"Ġleased":98355,"Ġwee":98356,"夺å¾Ĺ":98357,"æĸ¹ç¨ĭç»Ħ":98358,"Ġ'../../../":98359,"%ï¼ī":98360,"+T":98361,"âĢī":98362,"ĠBord":98363,"è¿Ļä¸įä»ħ":98364,"å°±æĪIJ为":98365,"大å¤ļæķ°çļĦ":98366,".Content":98367,"Multiplication":98368,"ĠJohannesburg":98369,"codes":98370,"ĠBACK":98371,"ikoa":98372,"ategorie":98373,"æŃĮåī§":98374,"à¸Ĺีà¹Īà¸Ķี":98375,"']))":98376,"ĠBetrieb":98377,"-alone":98378,"à§§à§®":98379,"(:,":98380,"Ġimproperly":98381,"'autre":98382,"Ġ×IJ×ľ×£":98383,"-To":98384,"inat":98385,"utdown":98386,"åIJ¡":98387,"ĠPERSON":98388,"quiet":98389,"ĠKG":98390,"éĽĨ约":98391,"å¸Ĥåľºä¸ĬçļĦ":98392,"Ġmaggiore":98393,"Ġingested":98394,"ìĸ´ì§Ħ":98395,"åĩŃçĿĢ":98396,"-acting":98397,"ĠQuadratic":98398,"ĠÑĢеакÑĨии":98399,"มาà¸Īาà¸ģ":98400,"Ġmister":98401,"ĠBism":98402,"Ġsext":98403,"èĥ½ä»İ":98404,"Adj":98405,"éļĶç»Ŀ":98406,"áŀ·":98407,"äºĮåįģä¹Ŀ":98408,"ĠExpenses":98409,"Ġstarred":98410,"Ġétude":98411,"ÙĪØ¬ÙĪØ¯":98412,"ĠÑĢабоÑĤаÑĤÑĮ":98413,"ĠColombian":98414,"Ġfalsely":98415,"Ġtranquility":98416,"Ġsunglasses":98417,"ĠkteÅĻÃŃ":98418,"以åĨħçļĦ":98419,"æĭ´":98420,"æĮģå¹³":98421,"è¿Ļ个æķħäºĭ":98422,"æķĪçİĩåĴĮ":98423,"ĠMelanie":98424,"Õ¥Õ¯":98425,"iators":98426,"ĠNamen":98427,"大æ±Ĺ":98428,"ĠInjection":98429,"ï¼Īï¼īãĢĤĊĊ":98430,"embros":98431,"åĨľä¸ļ大åѦ":98432,"ĠÚ©ÙĨÙĨدÙĩ":98433,"西æĸ¹åĽ½å®¶":98434,"Ġdziecka":98435,"ĠBosch":98436,"ÑĦикаÑĨии":98437,"ë¸Ķ":98438,"ĠstÅĻed":98439,"Ġkosten":98440,"Ġadquir":98441,"å¦Ŀ":98442,"à¤Ļ":98443,"Ġzg":98444,"órd":98445,"Ġcapitals":98446,"æ¶ĪéĢĢ":98447,"Ġelectorate":98448,"Prepare":98449,"Accounts":98450,"Ġlinux":98451,"Ġperkembangan":98452,"ĠMongoDB":98453,"breviations":98454,"Rome":98455,"owaniu":98456,"verg":98457,"Ġflax":98458,"被æįķ":98459,"åį³ä½į":98460,"æĶ¯æķĻ":98461,"çİ°åľ¨å°±":98462,"åį´è¯´":98463,"ÑĤелÑĮном":98464,"ĠNueva":98465,"ĠпÑĢоÑĦилакÑĤи":98466,"\"When":98467,"Tro":98468,"Ġfray":98469,"Ġbola":98470,"ä¸įä¸İ":98471,"ĠRear":98472,"éģŃåΰäºĨ":98473,"Ñļено":98474,"ĠLesser":98475,"Ġ(...)ĊĊ":98476,"Highest":98477,")âĨĴ":98478,"HOME":98479,"ĠMolecules":98480,"astre":98481,"æľ¬æºIJ":98482,"éĩįå¡ij":98483,"å½ĵ好":98484,"å°ĨæĮģç»Ń":98485,"çϽçļĻ":98486,"ĠWorcester":98487,"è¿ĺæĺ¯æĮº":98488,"åºĹéĿ¢":98489,"-Per":98490,"æııè¿°äºĨ":98491,"Ġgrassland":98492,"Ġscraps":98493,"Ġহà¦ļà§įà¦Ľà§ĩ":98494,"+P":98495,"ĠSAC":98496,"ĠSitting":98497,"åĮĸçŰ":98498,"ĠProjek":98499,"身亡":98500,"æ®ĩ":98501,"åŃĺåıĸ":98502,"象éĻIJ":98503,"Ġtotality":98504,"éķĩéķ¿":98505,"éĺ´æļĹ":98506,"ترÙĦ":98507,"Ġsimplistic":98508,"-running":98509,"Justice":98510,"使åij½æĦŁ":98511,"Ġphosphatase":98512,"'all":98513,"çļĦæ¯Ķè¾ĥ":98514,"ĠGOV":98515,"天山":98516,"åİŁåıijæĢ§":98517,"çıŀ":98518,"zaÄĩ":98519,"é»Ħè¿ŀ":98520,"æıIJä¾Ľç»Ļ":98521,"衣衫":98522,"享ç͍":98523,")\\).ĊĊ":98524,"Ġש׳":98525,"CAST":98526,"ಿನ":98527,"ĠSEQ":98528,"ĠÑĨелом":98529,"ĠÑĥÑģÑĤÑĢойÑģÑĤва":98530,"-engine":98531,"/components":98532,"FU":98533,"uner":98534,"åŁºè°ĥ":98535,"Ġxn":98536,"ALA":98537,"ifted":98538,"å®Ŀåīij":98539,"åŁ¹è¨ĵ":98540,"ä¸ĥæĺŁ":98541,"Ġcierto":98542,"ĠJacksonville":98543,"ãĤ¦ãĤ§":98544,"Ġtémo":98545,"ĠLef":98546,"Ġ{},":98547,"对å°ı":98548,"çĪ±åĽłæĸ¯åĿ¦":98549,"ĠØŃÙ쨏":98550,"éĽ¨å¤©":98551,"çļĦçĶŁæ´»æĸ¹å¼ı":98552,"ĠApproval":98553,"-discovery":98554,"ĠавÑĤомаÑĤи":98555,"èµİåĽŀ":98556,"ĠQUESTIONS":98557,"Aa":98558,"ä½łè¿Ļä¹Ī":98559,"åħ¬å°º":98560,"åİ»åIJij":98561,"æĶ¾ä»»":98562,"Ġactivator":98563,"Ġlineback":98564,"ĠQuel":98565,"读è¿ĩ":98566,"Ġsituational":98567,"/details":98568,"ĠDonovan":98569,"æijĩæijĨ":98570,"rijke":98571,"ãĤīãĤĮãģ¾ãģĻ":98572,"íĿ¬":98573,"Ġcest":98574,"Ġhl":98575,"Ġstale":98576,"ĠDzie":98577,"Ġpreface":98578,"头çĽĶ":98579,"Converting":98580,"ç®Ģæĺİ":98581,"Ġpolitely":98582,"ĠGeV":98583,"äºİæĺ¯ä»ĸ":98584,"PLAY":98585,"Suppl":98586,"æĴĩåĺ´":98587,"ड़":98588,"ĠHindus":98589,"ÙĪÙĬÙĥبات":98590,"_helper":98591,"Ġвода":98592,"ĠØ£ÙĩÙĦ":98593,"Ġfacade":98594,"ĠاÙĦتأ":98595,"çļĦéĩįè¦ģåĽłç´ł":98596,"éĤ®å¯Ħ":98597,"ạng":98598,"باشد":98599,"Rn":98600,"xon":98601,"åħ¨åĨĽ":98602,"Ġsecondly":98603,"Ġfondo":98604,"两大类":98605,"à¸Ħà¹Īะ":98606,"}C":98607,"çļĦè®Ńç»ĥ":98608,"æĶ¤":98609,"кÑĬ":98610,"æīĢåģļ":98611,"Ġpochod":98612,"åıĹ访":98613,"ÏĦικÏĮ":98614,"даÑĩи":98615,"å¸Ĥåľºä¸»ä½ĵ":98616,"èĥĮå¾Į":98617,"ĠWilkins":98618,"æijĦåĥıæľº":98619,"ĠизмеÑĢениÑı":98620,"idus":98621,"è¿ĩä½İ":98622,"æĪij们çľĭåΰ":98623,"ä»ĸ们è¿ĺ":98624,"Ġcrept":98625,"ĠدÛĴ":98626,"åĽ´æĶ»":98627,"åºŁæ°Ķ":98628,"åħļå§Ķå§Ķåijĺ":98629,"ĠLectures":98630,",!":98631,"uitive":98632,"ĠPNG":98633,"å®¶éķ·":98634,"itekt":98635,"ĠRecht":98636,"ä½ĨéļıçĿĢ":98637,"åħĥ代":98638,"ä¼łè®°":98639,"Ġجدا":98640,"楼æĪ¿":98641,"éĸĭåķŁ":98642,"/dl":98643,"ãĤĪãģŃ":98644,"ÃŃnas":98645,"ĠDouglass":98646,"cutta":98647,"াষà§įà¦Łà§įর":98648,"referentziak":98649,"HJ":98650,"Oracle":98651,"idious":98652,"ä¸Ģæ´¾":98653,"Ġoutskirts":98654,"ç»ĵè¯Ĩ":98655,"ymb":98656,"ĠâĢĺâĢĻ":98657,"ãģĽãĤĭ":98658,"Requirements":98659,"ĠBethlehem":98660,"/~":98661,"_TH":98662,"Ġfprintf":98663,"çļĦå¿«":98664,"ĠPocket":98665,"ĠRMS":98666,"Ġformato":98667,"ledged":98668,"è¿°èģĮ":98669,"ĠÙĬÙĪ":98670,"ç¹³":98671,"Ġwelke":98672,"ĠCampo":98673,"ãĥ³ãĥĢ":98674,"åŀĤ缴äºİ":98675,"ĠмÑĥзе":98676,"åįĶæľĥ":98677,"ĠDentistry":98678,"éĹŃä¸Ĭçľ¼çĿĽ":98679,"ĠÙ¾ÚĺÙĪÙĩØ´":98680,"gli":98681,"enko":98682,"Ġsifat":98683,"ouw":98684,"Ġwithheld":98685,"èİĺ":98686,"ĠÑģила":98687,"åĪĨéĴŁå·¦åı³":98688,"Genesis":98689,"ándose":98690,"æ±ķ头":98691,"Ġdazzling":98692,"Ġciento":98693,"igual":98694,"æĿ¥å½¢å®¹":98695,"Ġspazio":98696,"åıĪ以":98697,"æĸĻåΰ":98698,"Ġsubjectivity":98699,"APPL":98700,"ĠÑģоÑħ":98701,"ĠLuigi":98702,"æĢĿç»´èĥ½åĬĽ":98703,"Ġoddly":98704,"ï¼ģï¼ģï¼ģĊĊ":98705,"Ġà¸Ħุà¸ĵ":98706,"Ġsuccinct":98707,"Ġrampant":98708,"ĠEstablishing":98709,"çķĻå®ĪåĦ¿ç«¥":98710,"Ġzombie":98711,"çļĦåĩłä¸ª":98712,"ĠTanner":98713,"عÙī":98714,"Ġposición":98715,"红çģ¯":98716,"Ġvoit":98717,"OTT":98718,"emplos":98719,"å̾åŁİ":98720,"_RES":98721,"ĠIcelandic":98722,"ĠLaurie":98723,"å¿ĥå¾ĭ失常":98724,"çĺĻçĹĴ":98725,"ĠPfe":98726,"åľ¨å¼¹åĩºçļĦ":98727,"ĠArter":98728,"ç½Ĺ伯çī¹":98729,"Ġnightmares":98730,"ÐłÐ°Ñģ":98731,"漫漫":98732,"ĠAuthorities":98733,"è´¢æĶ¿å±Ģ":98734,"سÙħبر":98735,"éļĬä¼į":98736,"latest":98737,"ĠHBV":98738,"Ġheparin":98739,"Ġthal":98740,"Ġjohn":98741,"Ġmeadow":98742,"ĠReception":98743,"efeller":98744,"Ġcheering":98745,"shown":98746,"Ġapan":98747,"å´ĩé«ĺçļĦ":98748,"Ġলà§ĩà¦ĸ":98749,"Ġdiverted":98750,"Ġetxek":98751,"Vous":98752,"rů":98753,"ĠMMA":98754,"ĠLakers":98755,"Ġretreated":98756,"-san":98757,"Ú©ÛĮÙĦ":98758,"è¨Ģæĥħ":98759,"èĩ´æŃ»":98760,"èİ«è¿ĩäºİ":98761,"Ġ×Ļש×":98762,"æĬ±èijĹ":98763,"Ġ['./":98764,"å¤ļ项å¼ı":98765,"-users":98766,"olone":98767,"ä¸įå̼å¾Ĺ":98768,"izadas":98769,"ĠProportion":98770,"常人":98771,"ĠSeasons":98772,"Uns":98773,"drawal":98774,"Ġfutur":98775,"ĠUncertainty":98776,"Pont":98777,"Ġbib":98778,"Ġandra":98779,"Ġmayores":98780,"è¿ĺæľī许å¤ļ":98781,"çĶļèĩ³åı¯ä»¥":98782,"软çļĦ":98783,"ĠPresidents":98784,"年轻人çļĦ":98785,"Ġjunio":98786,"Cf":98787,"èĢĮç«ĭ":98788,"æ¸ħçļĦ":98789,"å¾Īå¤ļäºĭæĥħ":98790,"é¡¿äºĨ":98791,"Ġréponse":98792,"ç¼ĸè¾ijåύ":98793,"æīĢå¾ĹçļĦ":98794,"âľĵ":98795,"ĠConsultation":98796,"ĠTranslated":98797,"ĠRosenberg":98798,"ä¸įèĢIJçĥ¦":98799,"uracies":98800,"ä»ĸçªģçĦ¶":98801,"-node":98802,"Ġwavelet":98803,"ĠPROP":98804,"ÃŃsica":98805,"ЧÑĤо":98806,"è¨Ĭæģ¯":98807,"èī°èĭ¦å¥ĭæĸĹ":98808,"Ġhaya":98809,"quina":98810,"ä»ĸåıª":98811,"æ¸ħå»ī":98812,"\\)).":98813,"ĠPluto":98814,"ĠElon":98815,"å¸Įçī¹åĭĴ":98816,"ĠNowadays":98817,"çģ¯åħ·":98818,"ç°¸":98819,"à¸łà¸±":98820,"Ġreticulum":98821,"(#":98822,"Viol":98823,"stral":98824,"ĠRNS":98825,"ä½ıå¤Ħ":98826,"碩":98827,"Ġvoi":98828,"ĠÑĦоÑĤ":98829,"Ġalienation":98830,"ĠAdvocacy":98831,"Ġintrinsically":98832,".Not":98833,"ĠJh":98834,"åİ»åĵªéĩĮ":98835,"Ġservicio":98836,"à¸Ĭุม":98837,"-CD":98838,"ĠADP":98839,"ÑĢовано":98840,"ấy":98841,"ĠÑĤеÑĢми":98842,"ĠLifetime":98843,"Cases":98844,"Ġreak":98845,"igte":98846,"Ġdelving":98847,"Ġexecutor":98848,"лÑĥа":98849,"MSO":98850,"ĠAnalyse":98851,"ĠповÑĭÑĪен":98852,"Literal":98853,"Ġsanctioned":98854,"Som":98855,"Susan":98856,"Ġguts":98857,"Ġisto":98858,"å¾Ĺå¾Ī好":98859,"æľ¬èĬĤ课":98860,"Ġoffsets":98861,"åĽĽåĪĨ":98862,"è¿ĺæľī个":98863,"æĬĹè¡¡":98864,"Ġcomputerized":98865,"Ġcastell":98866,"ĠSchematic":98867,"ä½£éĩij":98868,"çĹħèϫ害":98869,"belt":98870,"Ġluce":98871,"è¦ģåĪĩå®ŀ":98872,"hatikan":98873,"åĮħåĮħ":98874,"è¾ĥå¼±":98875,"å¤įåİŁ":98876,"Ġدراسة":98877,"Ġpurposeful":98878,"'or":98879,"Cass":98880,"Ticket":98881,"Ġdinners":98882,"raga":98883,"ĠbeforeEach":98884,"è§Ħ模åĮĸ":98885,"çŁĽçĽ¾çºłçº·":98886,"çĽ£çĿ£":98887,"Ġmaioria":98888,"-jud":98889,"pont":98890,"Ġnomenclature":98891,"ĠFDI":98892,"ĠHeck":98893,"Ġsimul":98894,"Ġdoesnt":98895,"æĶ¹ç͍":98896,"дав":98897,"Ġdoute":98898,"å·¦ä¸Ĭ":98899,"ئÛĮ":98900,"ìĦ±ìĿ´":98901,"ĠCSI":98902,"/Day":98903,"Ġscraping":98904,"碳水åĮĸåIJĪçī©":98905,"ĠWAR":98906,"æľĢ主è¦ģçļĦ":98907,"عÙĨ":98908,"ĠØŃسب":98909,"keywords":98910,"iyah":98911,"Ġshoreline":98912,"SavedPoint":98913,"DATE":98914,"ilh":98915,"ĠFuzzy":98916,"Ġhumane":98917,"Ġtransformers":98918,"Ġcomprehensively":98919,"trecht":98920,"ला":98921,"Ġdelegated":98922,"çħİçĨ¬":98923,"ĠChoices":98924,"Ġsincerity":98925,"ĠheiÃŁt":98926,"#line":98927,"_FL":98928,"Ġfps":98929,"ĠLets":98930,"åĴĦ":98931,"å·¥ä½ľè¦ģæ±Ĥ":98932,"çļĦ人éĻħ":98933,"Ġplacements":98934,"é¢Ħå¤ĦçIJĨ":98935,"Ġproblemi":98936,"ĠпÑĢоÑĤÑı":98937,"æĺ¯åIJ¦æĺ¯":98938,"缼å¼Ģ":98939,"orbidity":98940,"жаÑĤ":98941,"ávÄĽ":98942,"åįĶèѰ":98943,"Ġtremendously":98944,"ĠÑģвидеÑĤелÑĮ":98945,"åģľç͵":98946,"Ġlatitudes":98947,"кÑĥлÑı":98948,"Ġtitration":98949,"sexual":98950,"ç»Ļ人以":98951,"ĠGradient":98952,"WEB":98953,"]he":98954,"Ġmarty":98955,"Ġflamm":98956,"éľı":98957,"社éķ¿":98958,"åıĪéĹ®":98959,"Ġзол":98960,"ãĤĴ使ç͍":98961,"μι":98962,"ĠWarwick":98963,"SetSavedPoint":98964,"à¤ķार":98965,"Ġcarta":98966,"ĠзаданиÑı":98967,"Ġdécada":98968,"Ġebenfalls":98969,"ä¸į妥":98970,"actually":98971,"Ġmeglio":98972,"åĵ§":98973,"ĠEnrique":98974,"ĠнеÑĥ":98975,"æ¼Ķä¹ł":98976,"âĢĶâĢĶâĢĶâĢĶĊĊ":98977,"Ġশর":98978,".*ĊĊ":98979,"Ġinconsistency":98980,"ç¡®ç«ĭäºĨ":98981,"Ġunrestricted":98982,"Ġblossom":98983,"å§Ĭ妹":98984,"-Christian":98985,"ĠSIL":98986,"设å®ļçļĦ":98987,"åħīåIJĪ":98988,"обе":98989,"æĭīåΰ":98990,"æĻ¯æ°Ķ":98991,"Ġhoop":98992,"顺åĪ©å®ĮæĪIJ":98993,"fus":98994,"ĠNec":98995,"Ġadel":98996,"éĢļåIJij":98997,"ελ":98998,"ĠChristi":98999,"Ġpasa":99000,"CEP":99001,"æľīæīĢæĢĿ":99002,"ä¸įçĶ¨è¯´":99003,"Ġpuissance":99004,"ĠWatkins":99005,"ĠMandela":99006,"ĠMandarin":99007,"à¹Ģà¸Ħราะ":99008,"Ġescala":99009,"Investig":99010,"Ġextraordinarily":99011,"ĠCone":99012,"ĠMá":99013,"ĠFas":99014,"åĴĮçݯå¢ĥ":99015,"ĠUW":99016,"ä¸İ大":99017,"ä»»æķĻ":99018,"æ¡Īæĥħ":99019,"aptop":99020,"Ġdiseño":99021,"æĺ¥éĽ¨":99022,"oudre":99023,"اÙģÙĬ":99024,"å¹»è§ī":99025,"é¸ŃåŃIJ":99026,"çĿĢçľ¼äºİ":99027,"ĠблагодаÑĢÑı":99028,"ÎĴ":99029,"ä¸ĭé¢Į":99030,"好èݱåĿŀ":99031,"表çİĩ":99032,"Ġ×IJ×Ĺת":99033,"æijĦæ°ı":99034,"Entries":99035,"ĠPsalms":99036,"ĠDestiny":99037,"ĠPamela":99038,"ãĢĤï¼īĊ":99039,"åIJİå°Ĩ":99040,"èĩ³é«ĺ":99041,"Challenge":99042,"çİ°åľ¨æĪij":99043,"æ±Łæ³½":99044,"Quando":99045,"ĠSupervision":99046,"Ġ×ŀ×IJ×ķ×ĵ":99047,"Ġdeciduous":99048,"ilver":99049,"Ġvite":99050,"çĶŁå¹³":99051,"ĠThé":99052,"åIJĮä½į":99053,"×ķ×Ļ×Ļ×Ŀ":99054,"Ġautores":99055,"Ġpastors":99056,"iosync":99057,"ĠاÙĦÙĤدر":99058,"Offer":99059,"ĠPaso":99060,"Ġfotograf":99061,"Ġuninterrupted":99062,"Virginia":99063,"nage":99064,"Ġmailed":99065,"ĠRhet":99066,"éĤ£ä¸¤ä¸ª":99067,"å¼łä¸ī":99068,"medio":99069,"Ġunequiv":99070,"软åĮĸ":99071,"Ġзнаком":99072,"Ġblossoms":99073,"orov":99074,"urricular":99075,"ĠUTF":99076,"Ġdataframe":99077,"Reilly":99078,"éĿŀ常é«ĺ":99079,"Ġdirección":99080,"Ġreferencia":99081,"ষà§įà¦Ł":99082,"à§ĥতি":99083,"ĠÐľÐ¸Ñħа":99084,"СÑĤановниÑĪÑĤво":99085,"Ġprueba":99086,"zwe":99087,"Ġdude":99088,"ĠRican":99089,"æ°´æ·±":99090,"æĬĬä¸Ģ个":99091,"ĠEquilibrium":99092,"丹çͰ":99093,"åij½ä»¤è¡Į":99094,"ÃŃmbol":99095,"ĠпÑĢÑıмоÑĥголÑĮ":99096,"à¹ģà¸ľà¸Ļ":99097,"Ļàµįà´":99098,"iony":99099,"ä¸į顺":99100,"ĠWinners":99101,"gev":99102,"å¾Ĺå½ĵ":99103,"Ġзаме":99104,"Ġprecarious":99105,"Ġà¦¨à¦¿à§Łà§ĩ":99106,"è±ĨæµĨ":99107,"Ġtutta":99108,"Ġcyclists":99109,"æµģåĬ¨èµĦéĩij":99110,"Ġ'@/":99111,"Ġocas":99112,"ĠHighest":99113,"Ġevacuated":99114,"ĠÙħÙĤدار":99115,"æĺ¯å¦ĤæŃ¤":99116,"å§ĭçµĤ":99117,"à§Ģদà§ĩর":99118,"tzmann":99119,"Ġembarking":99120,"ä¸įåĴĮ":99121,"å·¥ä½ľæľºåζ":99122,"Ġpathetic":99123,"ĠLeaving":99124,"ĠPhantom":99125,"æ¥ļåĽ½":99126,"æĥĬéĨĴ":99127,"Ġambiance":99128,"缼çļĦ":99129,"交æµģä¼ļ":99130,"Ġwoody":99131,"ĠEURO":99132,"è¿Īè¿Ľ":99133,"æľĢæĸ°ç«łèĬĤ":99134,"Ġzircon":99135,"ván":99136,"ĠLarger":99137,"Ġ\"\")Ċ":99138,"ĠKup":99139,"å¸Ĥ人æ°ijæĶ¿åºľ":99140,"eya":99141,"è§ģæķĪ":99142,"ä¼Ĭå§ĭ":99143,"ãĥ©ãĥ³":99144,"ĠExtensive":99145,"ĠExpressible":99146,"Ġcomum":99147,"-business":99148,"ANO":99149,"æī¾å·¥ä½ľ":99150,"ਮ":99151,"ĠMathemat":99152,"Ġjackets":99153,"Ġemptiness":99154,"Ġdemeanor":99155,"cash":99156,"Ġrant":99157,"ĠAltra":99158,"åıĪæ²¡æľī":99159,"Ġaversion":99160,"åĪĿ审":99161,"Ġswore":99162,"ĠDisyembre":99163,"å®ģåİ¿":99164,"Ġপà§įরয়":99165,"Ġpooling":99166,"ĠPlatforms":99167,"è©¢åķı":99168,"ĠÑģамоÑģÑĤоÑıÑĤелÑĮно":99169,"mq":99170,"olome":99171,"ä»Ĭå¤ľ":99172,"ĠDepos":99173,"_folder":99174,"è¿Ķæł¡":99175,"Ġinjecting":99176,"ované":99177,"Ġprophylaxis":99178,"Bow":99179,"åħ¨åħļ":99180,"Ġfeces":99181,"åįģåĩłå¹´":99182,"Ġrefurb":99183,"Expr":99184,".Post":99185,"éĹ»åΰ":99186,"ÐļÐIJ":99187,"Definitions":99188,"çļĦæĸ¹å¼ıæĿ¥":99189,".short":99190,"{sub":99191,"çݰå¦Ĥä»Ĭ":99192,"Ġprojector":99193,"Ġsafest":99194,"Ġá¼Ħ":99195,"Ġbattalion":99196,"Ġsesuatu":99197,"Ġvære":99198,"Sed":99199,"çļĦèģĮä¸ļ":99200,"ĠEtymology":99201,"Ġhawk":99202,"éħįæľī":99203,"èĩªå·±çļĦ身ä½ĵ":99204,"Ġplantes":99205,"åĨ²å¤©":99206,"-evolving":99207,"误导":99208,"å³°ä¼ļ":99209,"रण":99210,"ÙIJÙĬÙĨ":99211,"Ġstoichi":99212,"Ġpermanente":99213,"Ġnodding":99214,"ĠPASS":99215,"ĠHors":99216,"åľ¨å½ĵåľ°":99217,"çŁ¥åIJįçļĦ":99218,"æį¢è¨Ģä¹ĭ":99219,"ĠØ´Ùħار":99220,"åĪ¶åº¦åĮĸ":99221,"limp":99222,"Ġà¦Ĩদ":99223,"Ġসরà¦ķার":99224,"Ġprojektu":99225,"\"][\"":99226,"Sender":99227,"icar":99228,"åIJįå½ķ":99229,"Ġbuen":99230,"é£İå¯Ĵ":99231,"潺":99232,"ĠÏĦὴν":99233,"ä¿ĿæĬ¤å¥½":99234,"çļĦæĹ¶éĹ´åĴĮ":99235,"èħ°éĹ´":99236,"Ġalcohols":99237,"Ġgénero":99238,"ĠÑģимпÑĤомÑĭ":99239,"ĠBeitrag":99240,"roplasty":99241,"Ġyacht":99242,"Ġkup":99243,"çĶŁçĶŁçļĦ":99244,"économ":99245,"лев":99246,"বà§įয":99247,"æļ´èºģ":99248,"Ġdefeats":99249,"-feira":99250,"çľĭä½ľæĺ¯":99251,"tid":99252,"Ġuni":99253,"éĢłè¡Ģ":99254,"è·Łéŀĭ":99255,"atoon":99256,"伤çĹķ":99257,"åįģäºĮæĮĩ":99258,"çĮİ人":99259,"ĠконеÑĩно":99260,"Ġtamaño":99261,"Friend":99262,"tol":99263,"Ġtroll":99264,"Ġsú":99265,"Ġstumbling":99266,"ĠGud":99267,"Ġinvading":99268,"ä¸įèĥ½è®©":99269,"ä»·æł¼ä¸Ĭ涨":99270,"åijĪçı¾":99271,"IOException":99272,"滿æĦı":99273,"ĠRooms":99274,"ĠKonstant":99275,"vara":99276,"ĠHeads":99277,"proble":99278,"Ġتبد":99279,"ŀף":99280,"å¼łæĸĩ":99281,"ç»Ħç»ĩäºĨ":99282,"æ²³çļĦ":99283,"è¡¥æķij":99284,"Ġhomestead":99285,"Ġcertify":99286,"åĶĩè§Ĵ":99287,"åľ°çIJĥä¸Ĭ":99288,"Ġreflexive":99289,"Ġconteú":99290,"TK":99291,"Ġmappings":99292,"ĠTack":99293,"æľīæĪIJ":99294,"ĠInhibition":99295,"æĮĩåĩºäºĨ":99296,"ytest":99297,"产ä¸ļéĽĨ群":99298,"Ġcmp":99299,"æĬĺä¸į":99300,"Ġoptimally":99301,"åı¦ä¸ĢåįĬ":99302,"ització":99303,"æģ°åΰ":99304,"ĠÑģлÑĥÑĩаев":99305,"ĠCroatian":99306,"asio":99307,"ĠCups":99308,"ĠDSP":99309,"andemic":99310,"åħ¥åĬĽ":99311,"Ġsystemat":99312,"anea":99313,"ĠOrch":99314,"Ġterreno":99315,"ĠобÑģ":99316,"çĽijåIJ¬":99317,"Ġâĸ½":99318,"Ġ×ĸ׼":99319,"Ġê°ľëħIJ":99320,"nden":99321,"ĠTrit":99322,"åľ¨åīįéĿ¢":99323,"Ġinvocation":99324,"ĠLease":99325,"rmann":99326,"åħįè²»":99327,"Ġodk":99328,"çĴŀ":99329,"à¥Ģन":99330,"èħ¿ä¸Ĭ":99331,"æĿľé¹ĥ":99332,"ç»ŀçĹĽ":99333,"ĠSoldiers":99334,"Ġseep":99335,"åݻ年çļĦ":99336,"عÙħÙĦ":99337,"Thirty":99338,"ä¸ĩ象":99339,"شرة":99340,"رÙģØª":99341,"æī£æĬ¼":99342,"ĠPromote":99343,"ĠMcGill":99344,"ropractic":99345,"-icons":99346,"çĤľ":99347,"ucos":99348,"ohm":99349,"Ú¯ÙĪ":99350,"ĠRelay":99351,"Ġبرابر":99352,"åľ¨è¿Ļåľº":99353,"ĠÙħرة":99354,"ĠBolshe":99355,"æĥĭæĥľ":99356,"GK":99357,"Ġlapse":99358,"ĠCCS":99359,"ĠPlays":99360,"æľªå®Į":99361,"ponen":99362,"ĠParan":99363,"Ġaspire":99364,":d":99365,"Ġcactus":99366,"çļĦæĪ¿åŃIJ":99367,"opera":99368,"à®ĩ":99369,"\",ĊĊ":99370,"ç§ijæ¯Ķ":99371,"Õ¶Õ¥ÖĢÕ¨":99372,"onomia":99373,"ĠMcCorm":99374,"Ġperpetrators":99375,"Ġtöbb":99376,"ĠAccommod":99377,"Ġmisunderstandings":99378,"Ġjat":99379,"è¾į":99380,"å°Ĩä»ĸ们":99381,"Ġdemikian":99382,"à¸ļู":99383,"ettlement":99384,"å¹¼èĭĹ":99385,"俩人":99386,"Ġepidemi":99387,"ĠContributor":99388,"ĠDissertation":99389,"Ġempre":99390,"appers":99391,"еÑĢов":99392,"ä½ĽéĻĢ":99393,"丽ä¸Ŀ":99394,"блиÑĨа":99395,"ĠSelecting":99396,"developer":99397,"ĠChilean":99398,"ĠIllustration":99399,"ÑĭдÑĥ":99400,"ĠStur":99401,"Ġduż":99402,"ä¸ĵä¸ļ人士":99403,"Objectives":99404,"àµįà´ļ":99405,"सम":99406,"CharArray":99407,"åŁºåĽłç»Ħ":99408,"æ²§æµ·":99409,"ĠMackenzie":99410,"ĠwpÅĤyw":99411,"ç¼ħæĢĢ":99412,"为é¦ĸçļĦ":99413,"Bull":99414,"Kate":99415,"Ġdrown":99416,"æľīåĢĭ":99417,"å¿¡":99418,"clo":99419,"èĩªä¹ł":99420,"Ġevoc":99421,"çϽå±ħæĺĵ":99422,"Ġkeadaan":99423,"åħ´å»º":99424,"æĩĤçļĦ":99425,"çĤ¼åζ":99426,"åħĦå¼Łå§IJ妹":99427,"Ġlymphatic":99428,"(height":99429,"dling":99430,"alignment":99431,"Ġdni":99432,"Ġkval":99433,"owered":99434,"ä¸ĩèĤ¡":99435,"Ġimprov":99436,"à¥įड":99437,"Ġodm":99438,"Ġentrev":99439,"Preferences":99440,"Ġê´Ģíķľ":99441,"λεÏħ":99442,"ĠGlacier":99443,"Ġaccretion":99444,"Ġthorn":99445,"åľ¨æ¯ı个":99446,"Ġkodea":99447,"åĴĮæľī":99448,"actin":99449,"æĦı念":99450,"æ°Ķ缸":99451,"ĠAbnormal":99452,"å¸Ĥåľºè§Ħ模":99453,"ihak":99454,"viser":99455,"延误":99456,"Ġ×ķש":99457,"ĠBelize":99458,"Ġgroep":99459,"Ġliberalism":99460,"ĠÑĦÑĥнкÑĨий":99461,"REFIX":99462,"ικοί":99463,"cw":99464,"|^{":99465,"orin":99466,"Ġrin":99467,"å®ļåŀĭ":99468,"ervative":99469,"ä¸Ģ个åŃĹ":99470,"engagement":99471,"лава":99472,"COOH":99473,"Ġà¦ıà¦ĸন":99474,"ĠViral":99475,"èµıæŀIJ":99476,"åĪĽå»ºçļĦ":99477,"Ġপà§įরà¦ķাশ":99478,"Ġpertains":99479,"ÏĮÏĦηÏĦα":99480,"Ġtl":99481,"ä»ĸä¹Łä¸į":99482,"çĻ«":99483,"Ġflere":99484,"Ġflung":99485,"Ġparticulièrement":99486,"åŁİåįĹ":99487,"çĭ¬åѤ":99488,"ĠاÙĦتس":99489,"åįĸç»Ļ":99490,"ĠTablespoon":99491,"Ġczasu":99492,"Ġjelas":99493,"ĠСевеÑĢ":99494,"ĠRutgers":99495,"idio":99496,"ĠMord":99497,"è¿ĺ对":99498,"äºĮåı·":99499,"éĵ¿":99500,"çİĭ大":99501,"Ġgoverns":99502,"æłijç§į":99503,"æĺ¯åIJ¦åı¯ä»¥":99504,"à¹Ģà¸Ķืà¸Ńà¸Ļ":99505,"Ġfrecuencia":99506,"Ġruthless":99507,"Ġreopen":99508,"Ġalte":99509,"æľºæŀª":99510,"éļıå¿ĥ":99511,"表示çļĦ":99512,"éĻIJåζäºĨ":99513,"以æŃ¤æĿ¥":99514,"æıīäºĨ":99515,"ĠBronx":99516,"Ġmyeloid":99517,"ĠEinsatz":99518,"ĠAten":99519,"ĠWage":99520,"è¦ģ大":99521,"ï¼ļâĢĺ":99522,"áss":99523,"å¹¶å°±":99524,"ĠDataFrame":99525,"實è¸IJ":99526,"Ġhypoten":99527,"Ġmoistur":99528,"ĠÂłĠÂłĠÂł":99529,"ĠFelipe":99530,"itioners":99531,"缴çļĦ":99532,"女åŃIJçļĦ":99533,"太éļ¾":99534,"æĺ¥è¿IJ":99535,"æ²Ĵäºĭ":99536,"âĨµ":99537,"ĠÏĢαÏģ":99538,"è®¤çľŁèIJ½å®ŀ":99539,"ĠRodney":99540,"éħ¿éħĴ":99541,"ĠDemonstr":99542,"-Cola":99543,"ĠSlavery":99544,"èĢĮåIJĮ":99545,"æķ°æ¬¡":99546,"Ġcarers":99547,"ÅĽni":99548,"ĠÕ¹":99549,"ĠAnnounce":99550,"ĠPraxis":99551,"æĴ°ç¨¿":99552,"-general":99553,"Magic":99554,"ĠженÑīин":99555,"ĠMiscellaneous":99556,"åĻ©æ¢¦":99557,"SIM":99558,"rekt":99559,"Ġtratar":99560,"å¦Ĥåīį":99561,"é«ĺ楼":99562,"åIJĦçıŃ":99563,"çļĦä¸Ģå®ļ":99564,"ä¸Ģ缴éĥ½":99565,"åĵ²çIJĨ":99566,"Ġdeuxième":99567,"ĠIterator":99568,"(view":99569,"Ġregrets":99570,"enged":99571,"upmu":99572,"ĠTrigger":99573,"åĨľæŀĹ":99574,"è¯ĹéĽĨ":99575,"éĸĵçļĦ":99576,"Counting":99577,"Registered":99578,"Ġitaliani":99579,".resolve":99580,"Tam":99581,"hare":99582,"é«ĺæĸ¯":99583,"âĢĶâĢĿ":99584,"ĠZust":99585,"',$":99586,"Ġavalan":99587,"ä¸įä¼ļæĺ¯":99588,"Ġstressing":99589,"ãģıãĤīãģĦ":99590,"ĠSupplier":99591,"ĠLearner":99592,"Ġcorporal":99593,"迫害":99594,"침":99595,"Styled":99596,"ĠÙħشخص":99597,"ĠTrainer":99598,"ĠTudor":99599,"Ġremuneration":99600,"/<":99601,"Either":99602,"bidden":99603,"mur":99604,"è··":99605,"课åīį":99606,".font":99607,"æİ¢æŁ¥":99608,"اضر":99609,"ĠelsÅij":99610,"ĠиÑģполÑĮзÑĥÑİÑĤÑģÑı":99611,"åħĪéĶĭ模èĮĥ":99612,"Ġundist":99613,"ĠÙĦÙĤ":99614,"åį¡éĢļ":99615,"åĢĴéĹŃ":99616,"Ġbrilliantly":99617,"ailleurs":99618,"Ġjub":99619,"åIJĦéĥ¨":99620,"εÏħ":99621,"Eventually":99622,"ĠKK":99623,"èĢĮ她":99624,"ysÅĤ":99625,"åĬłåĢį":99626,"ĠDele":99627,"Ġinsensitive":99628,"æĪĺä¸Ń":99629,"ĠбеÑĢ":99630,"ĠÙĥتب":99631,"çIJĨè§£äºĨ":99632,"Ġcovari":99633,"æ¼Ĥæµģ":99634,"Ġà¶´":99635,"ĠFatigue":99636,"ä¸Ŀ毫没æľī":99637,"Ġinflow":99638,"ĠجÙĨÚ¯":99639,"æĺ¨å¤ľ":99640,"ç¨İåĬ¡æĢ»å±Ģ":99641,"department":99642,"Variables":99643,"Ġextermin":99644,"èĢħåı¯":99645,"Ġprova":99646,"Ġhelfen":99647,"åıĺçݰ":99648,"ĠPlatinum":99649,"Ġpopulate":99650,"Ġsummons":99651,"ieta":99652,"åıijçĶŁçļĦäºĭæĥħ":99653,"Ġবà§ĥ":99654,"æľ±çĨ¹":99655,"تÙħد":99656,"Ġkitchens":99657,"ãĥģãĤ§":99658,"ĠBurning":99659,"ongsTo":99660,"ĠзнаÑĩиÑĤелÑĮно":99661,"奥æŀĹåĮ¹":99662,"çļĦæıIJé«ĺ":99663,"ĠLOW":99664,"ĠOlig":99665,").#":99666,"èĢĮåħ¶":99667,"ä½įä¸Ĭ":99668,"-si":99669,"newcommand":99670,"è³ľ":99671,"Ġconfiguring":99672,"Ġhallmark":99673,"çĽĨèħĶ":99674,"ĠкÑĢаÑĤ":99675,"Ġmotivates":99676,"Ġsqueezing":99677,"ĠRespir":99678,"Jour":99679,"rification":99680,"}')Ċ":99681,"ĠWoo":99682,"èĩ§":99683,"Ġacclaim":99684,"Ġ#ĊĊ":99685,"èģĶæĥ³åΰ":99686,"ÄħÄĩ":99687,"ĠMedication":99688,"à´³":99689,"Ġdiseased":99690,"Ġbarang":99691,"ĠÛĮعÙĨÛĮ":99692,"ĠReflex":99693,"áĥĶáĥ¡":99694,"Ġsubstitutions":99695,"çĶŁæĹ¥å¿«ä¹IJ":99696,"æµĵæµĵçļĦ":99697,"Ġprogres":99698,"ĠNomin":99699,"没æľīéĤ£ä¹Ī":99700,"è®©ä½łçļĦ":99701,"Ġmultit":99702,"Ġcalculators":99703,"Ġmicroenvironment":99704,"æįĨç»ij":99705,"Ġkidnapped":99706,".+":99707,"Domin":99708,"_true":99709,"Ġlø":99710,"essere":99711,"رت":99712,"cls":99713,"é«ĺåĪĨåŃIJ":99714,"èĩªå·±è¦ģ":99715,"è£ħåľ¨":99716,"Ġtimetable":99717,"ĠاÙħرÙĪ":99718,"Ġtrespass":99719,"Interestingly":99720,"ĠAdvancement":99721,"FV":99722,"Lam":99723,"ĠMk":99724,"ĠHinter":99725,"azan":99726,"Ġchanger":99727,"-stud":99728,"æĦıè§ģåĴĮ建议":99729,"å¼·åĮĸ":99730,"Ġneuros":99731,"Generate":99732,"ĠFacilit":99733,"ĠGruppe":99734,"Ġbezpie":99735,"Ġdernière":99736,"ĠMeetings":99737,"ĠDISTRICT":99738,"-road":99739,"ä¹ĭ交":99740,"ä¹ĭæģ©":99741,"ĠComes":99742,"两ä¸ī":99743,"à¹Ħà¸ĭ":99744,"Ġconvertible":99745,"ĠDeveloped":99746,"Ġtangled":99747,"çļĦå½¢çĬ¶":99748,"ĠWrap":99749,"åĴĮå®ŀè·µ":99750,"å¦Ĥèĭ¥":99751,"Ġ×Ķ×§×":99752,"æĿİåŃIJ":99753,"åįĩèĩ³":99754,"éĻĪçļ®":99755,"ç©¿è¡£":99756,"è¬Ļ":99757,"æľīä»Ģä¹Īåħ³ç³»":99758,"éĴ»äºķ":99759,"ĠAuschwitz":99760,"ĠRouting":99761,"payload":99762,"ç¬ĶèĢħ认为":99763,".active":99764,"aroo":99765,"ĠاصÙĦ":99766,"ĠReinh":99767,"åıĬçŃĶæ¡Ī":99768,"Ġacab":99769,"æµ·å°Ķ":99770,"áĥĴ":99771,"Keyboard":99772,"endez":99773,"à¸Ľà¸£à¸°à¸Īำ":99774,"éļ¾ä»¥ç½®ä¿¡":99775,"ĠOsborne":99776,"Ãītat":99777,"superscriptsubscript":99778,"ĠNathaniel":99779,"(options":99780,"alera":99781,"Ġreused":99782,"ä¸į详":99783,"sev":99784,"说ä¸Ģä¸ĭ":99785,"Ġfeud":99786,"çŁ³åŃIJ":99787,"ĠAbdel":99788,"cols":99789,"laid":99790,"Ġrhymes":99791,"ĠPHYS":99792,"çĿģå¼Ģçľ¼çĿĽ":99793,"çIJĨèµĶ":99794,"reeze":99795,"death":99796,"ÏĦÏİν":99797,"Ġglances":99798,"ารà¸ĵ":99799,"ĠArchitects":99800,"rende":99801,"æĸľçİĩ":99802,"åķĨåĬ¡éĥ¨":99803,"ĠدÙĩÙĨد":99804,"Ġvertebrae":99805,"(iv":99806,"Ġcé":99807,"好æ¯Ķ":99808,"ĠÙĨد":99809,"æĭ¿åİ»":99810,"ä¸ĩåħĥ以ä¸Ĭ":99811,"ĠÙħÙģÙĩ":99812,",Q":99813,"ongru":99814,"дÓĻ":99815,"éĤ£ä¸Ģ天":99816,"æīĢ以她":99817,"Ġthinly":99818,"Ġfonte":99819,"Ġ구조":99820,"Jn":99821,"_ms":99822,"åľ¨å¸Ĥ":99823,"Ġraging":99824,"ãģ®åł´åIJĪ":99825,"Ġrequer":99826,"Ġterrest":99827,"ëĬIJ":99828,"å¯Ĵé£İ":99829,"׾×Ĵ":99830,"åħ³éĶ®åľ¨äºİ":99831,"Paragraph":99832,"æĬµæī£":99833,"çĶľåĵģ":99834,"ĠCatalunya":99835,"ächlich":99836,"à¸Ľà¸ģà¸ķิ":99837,"à¹Ģà¸ģษà¸ķร":99838,"&=":99839,"ĠFN":99840,"è¿Ļ个çĶ·äºº":99841,"èĬ±æľŁ":99842,".Sprintf":99843,"Ġmotherhood":99844,"ÐĿи":99845,"ĠOrthop":99846,"ĠszkoÅĤy":99847,"ÃĶ":99848,"idou":99849,"äºİ人":99850,"çĿĢ她çļĦ":99851,"çŃīéĥ½":99852,"Ġphantom":99853,"çĹħæ°Ĺ":99854,"eteria":99855,"ĠScand":99856,"ĠPauline":99857,"Ġἡ":99858,"×ķ×ij×ķת":99859,"ĠTaipei":99860,"衬æīĺ":99861,"ĠHolden":99862,"Ġoutsider":99863,"çķľçī§ä¸ļ":99864,"Ġapprenticeship":99865,"ĠDebbie":99866,"icating":99867,"Ġlizards":99868,"Ġvyp":99869,"ayat":99870,"æĭ®":99871,"ä¸ĩè¾¾":99872,"è¿ĻäºĽäºĭæĥħ":99873,"åĽ¾çīĩæĿ¥æºIJ":99874,"ĠNiagara":99875,"è¾ĥä½İçļĦ":99876,"-price":99877,"}b":99878,"幡":99879,"iax":99880,"å±ķä¼ļ":99881,"åŀĭä¼ģä¸ļ":99882,"ATIC":99883,"-tri":99884,".token":99885,"åī¯åİ¿éķ¿":99886,"Ġbuffet":99887,"çļĩå¸ĿçļĦ":99888,"Ġmismos":99889,"ĠÑĢаÑģÑģÑĩиÑĤÑĭ":99890,"Ġecclesiastical":99891,")y":99892,"heer":99893,"Ġnimi":99894,"以å®ŀçݰ":99895,"Ġdij":99896,"æŃ¥æŀª":99897,"åī¯äº§åĵģ":99898,"-stat":99899,".Min":99900,"æ³ķå¾ĭåĪ¶åº¦":99901,"åĽłç´łçļĦå½±åĵį":99902,"æĽ¿ä»ĸ":99903,"éĩįè¦ģçļĦæĦıä¹ī":99904,"Ġtacit":99905,".HashMap":99906,"Ġsuficiente":99907,"Ġsuelo":99908,"åĩºå¾ģ":99909,"å͝å¿ĥ":99910,"PathVariable":99911,"æ¡ĥæºIJ":99912,"æ¯ģäºĨ":99913,"Ġepidermal":99914,"ĠAxel":99915,"(client":99916,"_mean":99917,"essler":99918,"ç͍å°ı":99919,"Ġemper":99920,"cyd":99921,"çŁ¥éĿĴ":99922,"ä¸ĩèĥ½":99923,"åĬŁèĢĹ":99924,"éļ¾å¾ĹçļĦ":99925,"{{{":99926,"Entities":99927,"æĻºèĥ½åζéĢł":99928,"ĠìĪĺíĸī":99929,"Ġpermis":99930,"Ġrentals":99931,"ĉtmp":99932,"ĠвелиÑĩинÑĭ":99933,"à¹ģà¸Ĺà¸Ļ":99934,",ooo":99935,"_prefix":99936,"ä»¥æľŁ":99937,"Ġemits":99938,"å½ĵä¸ĭçļĦ":99939,"æľºç¼ĺ":99940,"çĸŁ":99941,"å¾ħ人":99942,"æĿ±æĸ¹":99943,"跨度":99944,"ĠNanop":99945,"ðŁĴ°":99946,"Ġdiscreet":99947,"à¸ŀัà¸Ļà¸ĺà¹Į":99948,"ĠQUESTION":99949,"Ġciencia":99950,"ĠLTE":99951,"æĪijåIJ¬":99952,"æĪijæĺ¯ä¸Ģ个":99953,"就以":99954,"Ġwillen":99955,"ĠStabil":99956,"åĮĸéªĮ":99957,"éĩįç͍":99958,"æĹłæĿĥ":99959,"ç¾İå¦Ļ":99960,"ç§ijåįı":99961,"Ġdonna":99962,"Ġpotrebbe":99963,"第ä¸Ģéĥ¨åĪĨ":99964,"ä¸įèĥ½æ»¡è¶³":99965,"èĤ¿åĿĹ":99966,"Ġsesame":99967,"noÅĽciÄħ":99968,"éĴ¢çŃĭæ··åĩĿåľŁ":99969,"ĠHolidays":99970,"Ġrethink":99971,"ĠServing":99972,"ldon":99973,"ĠDeposit":99974,"产çĶŁå½±åĵį":99975,"ĠÑĢазÑĢÑĥ":99976,"æľĢç»Īè¿ĺæĺ¯":99977,"Ġitaliana":99978,"åħ¸åŀĭæ¡Īä¾ĭ":99979,"Ġcrabs":99980,"å¸ĪèĮĥåѦéĻ¢":99981,"ĠlÃŃder":99982,"éĽĮæ¿Ģç´ł":99983,"ĠPeggy":99984,"/)Ċ":99985,"|}":99986,"teral":99987,"ĠJem":99988,"Ġsubcontract":99989,"اÙĦس":99990,".Spring":99991,"éĿĴèıľ":99992,"Ø·ÙĬع":99993,"_card":99994,"roidery":99995,"æ·¡åĮĸ":99996,"Ġthrives":99997,"éĶ»éĢł":99998,"Ġpúblicas":99999,"è¶ħ声波":100000,"æĻ®æ´±èĮ¶":100001,"éĤ¯éĥ¸":100002,"berta":100003,"Ġabiotic":100004,"Ġtrailed":100005,"ä½ľç͍æĺ¯":100006,"å®ŀæĸ½ç»ĨåĪĻ":100007,"å·¥ä¸ļåĩºçīĪ社":100008,"çī¹çĤ¹åĴĮ":100009,"ĠjejÃŃ":100010,"+-+-+-+-":100011,"Ġouders":100012,"obacillus":100013,"ĠMemorandum":100014,"ĠDEVELOPMENT":100015,"(child":100016,"niki":100017,"ä¸Ģ个æĸ°":100018,"Ġbetre":100019,"èĢģçι":100020,"Ġeras":100021,"Ġhumiliation":100022,"ircular":100023,"åΤåĪ«":100024,"çĮ®ç»Ļ":100025,"Ġszá":100026,"ĠUNC":100027,"avl":100028,"ĠXY":100029,"ĠXing":100030,"å¾ĢæĹ¥":100031,"ĠAbril":100032,"ाध":100033,"ĠÑĢеÑĪи":100034,"ĠÑģÑĤанов":100035,"ä»İèĢĮ导èĩ´":100036,"ĠEXT":100037,"æĺĤæī¬":100038,"Ġnhất":100039,"ãģ»ãģ¨":100040,"ĠгипеÑĢ":100041,"ĠпоÑĩемÑĥ":100042,"à¹Ģà¸Ħราะหà¹Į":100043,"NGC":100044,"Ù«":100045,"ä½łè¿Ļæĺ¯":100046,"åīįåįģ":100047,"ове":100048,"失äºĨ":100049,"ĠBlogs":100050,"ä½Ĩæĺ¯ä»ĸ们":100051,"Ġantigu":100052,"ĠÙĥÙĪØ±Ø©":100053,"以ä¸ĭåĩł":100054,"िप":100055,"ìĭľíĤ¤":100056,"Ġcomplainant":100057,"ĠзаÑīиÑĤÑĭ":100058,"Ġgénéralement":100059,"Ġ측":100060,"Ġcac":100061,"çļĦ巨大":100062,"Ġtol":100063,"åѦè¯Ĩ":100064,"Ġhelpers":100065,"æİĴ便":100066,".................................":100067,"Religion":100068,"æĪĺæĸĹæľº":100069,"æ¡ĤæŀĿ":100070,"à§Ĥম":100071,"ĠìķĦëĭ":100072,"Ó©ÑĢ":100073,"à¸ŀุà¸Ĺà¸ĺ":100074,"atm":100075,"Ġbart":100076,"etcode":100077,"ĠCholesterol":100078,"Ġsurged":100079,"ospatial":100080,"ä¸ĸçķĮç»ıæµİ":100081,"URY":100082,"èĤīè´¨":100083,"æķ´ä¸ªè¿ĩç¨ĭ":100084,"ĠEssentials":100085,"Ġbé":100086,"çļĦåΰæĿ¥":100087,"ctype":100088,"æİ¥éĢģ":100089,"ĠPrzy":100090,"åĽ¢èģļ":100091,"Ø·ÙĨÙĬ":100092,"ç©¿èijĹ":100093,"Ġآز":100094,".output":100095,"ĠSalvation":100096,"忽æĤł":100097,"Ġpunitive":100098,"ç¬¬åĽĽæ¬¡":100099,"æĸ¹ç¨ĭ为":100100,"ãĤªãĥ³":100101,"ĠاÙĦÙĪØ·ÙĨÙĬØ©":100102,"Ġĉĉ":100103,"upaten":100104,"æijĴ":100105,"è¿ijçϾ":100106,"æĪ¿åŃIJçļĦ":100107,"ÑĤÑĭм":100108,"åĿļæĮģä¸įæĩĪ":100109,"å¿įèĢħ":100110,"è°ĭæ±Ĥ":100111,"ĠMiriam":100112,"Ġlaminate":100113,"FIN":100114,"Treat":100115,"arach":100116,"izando":100117,"Ġsoi":100118,"еÑĤеÑĢ":100119,"èĩ´çĻĮ":100120,"Albert":100121,"賬":100122,"å¦Ĥä½ķçľĭå¾ħ":100123,"é¤ĵ":100124,"ĠMoist":100125,"ĠпÑĢодÑĥкÑĤов":100126,"ĠHaitian":100127,"ĠRaspberry":100128,"wasser":100129,"åľ¨æĸ°çļĦ":100130,"Ġunidad":100131,"Ġappart":100132,"ä¿Ŀ驾":100133,"»ØĮ":100134,"ĠEdmond":100135,"Ġbully":100136,"ĠStreets":100137,"PPPP":100138,"èĤ¾çĤİ":100139,"ĠHalifax":100140,"ĠFriendship":100141,"competitive":100142,"ĠAdjusted":100143,"ĠاÙĦدراسة":100144,"ĠZusammenh":100145,"Wis":100146,"eating":100147,"Ġsuture":100148,"ĠRX":100149,"好书":100150,"Ġtransmissions":100151,"Ġcaric":100152,"ç³»ç»Łåľ°":100153,"à¸Īีà¸Ļ":100154,"缮åīįåľ¨":100155,"ĠÙĪØ§ÙĦÙĤ":100156,"æľīä¸Ģ段":100157,".reverse":100158,"æĢ»ä½ĵä¸Ĭ":100159,"uginosa":100160,"Ġprefixes":100161,"ĠмаÑģÑģÑĭ":100162,"(email":100163,"ĠIMD":100164,"ĠHogan":100165,"Ġintoler":100166,"Ġzacz":100167,"éĢļãĤĬ":100168,"西路":100169,".mock":100170,"Ġжена":100171,"ĠKepler":100172,"Ġsheltered":100173,"ä½łçŁ¥éģĵåIJĹ":100174,"ÅĽciej":100175,"Ġglycogen":100176,"bv":100177,"Ġdisple":100178,"Ġknowingly":100179,"éĹ®é¢ĺäºĨ":100180,"ìĹĩ":100181,"Ġinitiates":100182,"å®Įåħ¨ä¸įåIJĮ":100183,"è¾ĵåħ¥çļĦ":100184,"ĠARC":100185,"Ġindelible":100186,"moment":100187,"Ġวัà¸Ļ":100188,"esimal":100189,"å·¥ä½ľè¿Ľè¡Į":100190,"边形çļĦ":100191,"}\\)\\(":100192,"æĺ¯ä¸ĢéŨ":100193,"åIJĮæĹ¶å¯¹":100194,"ĠModer":100195,"Ġsurnames":100196,"ĠWARRANTY":100197,"æ·Ħåįļ":100198,"Harm":100199,"gels":100200,"Ġpep":100201,"Ġyearning":100202,"æĪij们就åı¯ä»¥":100203,"ärm":100204,"emset":100205,".address":100206,"corpor":100207,"Ġtransplanted":100208,"ĠtysiÄĻcy":100209,"ĠëģĿ":100210,"Ġinteroperability":100211,"ĠCen":100212,"Ġvene":100213,"лÑijн":100214,"è¦ģåħħåĪĨ":100215,"å¤ļå±Ĥ次":100216,"Ġ','":100217,"天ä¹ĭ":100218,"Ġtrays":100219,"åĪĩ身":100220,"çªģèµ·":100221,"EMPL":100222,"æ»ij稽":100223,"渡è¿ĩ":100224,"Redis":100225,"locale":100226,"Ġutilizando":100227,"ĠíĻľëıĻ":100228,"ĠSiemens":100229,"Ġfret":100230,"ĠFK":100231,"åIJİä¼ļ":100232,"éĤ£å°ı":100233,"ĠConcerning":100234,"é¦ĸéķ¿":100235,"æĶ¿æ²»å®¶":100236,"Ġfreshness":100237,"||||":100238,"HasColumn":100239,"ç¥Īæ±Ĥ":100240,"Ġaand":100241,"Ġkitt":100242,"ugas":100243,"æŃ¤æ³ķ":100244,"æĬĢå¸Ī":100245,"-doped":100246,"åŃ¦ä¹łæĪIJ绩":100247,"ç͍æĪ·åIJį":100248,"ĠUNIT":100249,"éŁ³ä¹IJä¼ļ":100250,"çļĦæ°Ķè´¨":100251,"ĠÑĢоÑģÑĤа":100252,"-client":100253,"ĠRÃŃo":100254,"akak":100255,"ä¸Ńåı¯ä»¥":100256,"å°±ç»Ļ":100257,"Ġallotted":100258,"é¾Ī":100259,"è¯·åľ¨":100260,"}\\)/":100261,"avigate":100262,"å¿ĺè¨ĺ":100263,"ĠANN":100264,"Remark":100265,"财产å®īåħ¨":100266,"ĠAlternate":100267,"ĠÑģÑĤÑĢане":100268,"Ġgemacht":100269,"Ġtossing":100270,"žitÃŃ":100271,"»ê²Į":100272,"edited":100273,"ĠBihar":100274,"è¿Ļ表æĺİ":100275,"å¤ļåľ°":100276,"ĠRept":100277,"平庸":100278,"确信":100279,"جاÙĨ":100280,"æ´Ĺå¹²åĩĢ":100281,"اÙĩÙĬÙħ":100282,"Ġknob":100283,"Corporate":100284,"ĠLEVEL":100285,"è©ķåĥ¹":100286,"ãĥ¯ãĥ¼ãĤ¯":100287,"Ġnewborns":100288,"ุษยà¹Į":100289,"§×©":100290,"-bel":100291,"é£Łè°±":100292,"æĭīå¼ĢäºĨ":100293,"è¿Ļæĺ¯ä»ĸ":100294,"ÐľÑĭ":100295,"Characters":100296,"Ġprzyczyn":100297,"Accessed":100298,"\"S":100299,"Lot":100300,"¦×Ļ":100301,"icu":100302,"ĠHahn":100303,"çī¹åĬ¡":100304,"ĠSeñ":100305,"æīįæľīåı¯èĥ½":100306,"ç´§æī£":100307,"ĠLaud":100308,"ãģĭãģij":100309,"à¸Ĭà¸Ńà¸ļ":100310,"Ġhubungan":100311,"Ġcocktails":100312,"Ġbounty":100313,"çļĦé£İæł¼":100314,"ä¸įåŃķ":100315,"ä¹ŁåĪ«":100316,"ç³ł":100317,"ä¿Ŀè´¨":100318,"Ġguer":100319,"شاء":100320,"èĩªçĶ±è´¸æĺĵ":100321,"Ġgroaned":100322,"åı¹äºĨä¸Ģåı£æ°Ķ":100323,"寥寥":100324,"Ġbuzzing":100325,"Ġtë":100326,"为客æĪ·":100327,"åĴĮæĶ¹è¿Ľ":100328,"Ġbioc":100329,"ĠDispatch":100330,"幸åŃĺ":100331,"Ġà¦Ĩà¦ľ":100332,"å¾IJå¾IJ":100333,"æĢĴäºĨ":100334,"ĠfontWeight":100335,"è§£æĶ¾æĢĿæĥ³":100336,"ĠЦенÑĤ":100337,"ĠGastroenterol":100338,"Ġlabyrinth":100339,"DOC":100340,"orh":100341,"ĠcÃŃm":100342,"ĠinÃŃcio":100343,"ĠSb":100344,"ĠSGD":100345,"ĠTung":100346,"ansky":100347,"çIJ°":100348,"creases":100349,"Ġsubter":100350,"ĠAno":100351,"ãģ®ãĤĪãģĨãģª":100352,"ç±»åĴĮ":100353,"æ¸ħæľ«":100354,"èµ°åħ¥":100355,"åı²å¯Ĩ":100356,"Meeting":100357,"å¹½çģµ":100358,"éĨīäºĨ":100359,"ÐĽÐĺ":100360,"Ġermög":100361,"lán":100362,"ĠMAS":100363,"Ġuuid":100364,"ĠKT":100365,"åĬĽéģĵ":100366,"åĮºåĮº":100367,"è´¢ç¨İ":100368,"帮åĬ©æĪij们":100369,"Ġwrongly":100370,"겨":100371,"ĠBuddy":100372,"×ķ×ĵ×Ļ×Ŀ":100373,"åı¹æ°Ķ":100374,"ĠBuckingham":100375,"ĠParadox":100376,"Ġffilm":100377,"éĤ£æĹ¶çļĦ":100378,"ĠZr":100379,"å·®é»ŀ":100380,"çģŃç»Ŀ":100381,"主é¢ĺåħļ":100382,"ĠOfficials":100383,"Ġdwellings":100384,"Nos":100385,"ĠLESS":100386,"æīĢåŃ¦æł¡":100387,"å¼Ģ端":100388,"éĤ£åĿĹ":100389,"ä¹IJåĽ¢":100390,"ä¸ĵåĪ©çĶ³è¯·":100391,"Ġanteced":100392,"åĺĹ試":100393,"ĠàªĽà«ĩ":100394,"çļĦæ¯ĶèµĽ":100395,"Ġcommas":100396,"åıĹéĺ»":100397,"æľįå½¹":100398,"Ġmencap":100399,"Ġconcepto":100400,"CTS":100401,"Ġrendah":100402,"OVER":100403,"éŁ¿èµ·":100404,"ĠSubsidi":100405,"ĠاÙĦاÙĥت":100406,"Herm":100407,"eck":100408,"ĠCPA":100409,"à¦Ŀ":100410,"åıijå±ķ为":100411,"લ":100412,"logs":100413,"ä¸ĵä¸ļ课":100414,"_TEST":100415,"å®ŀè´¨ä¸Ĭ":100416,"Ġgeometries":100417,"observed":100418,"HAM":100419,"riko":100420,"Ġheure":100421,"Ġsoma":100422,"-Saxon":100423,"Ġfastened":100424,"chery":100425,".project":100426,"Ġcsak":100427,".with":100428,"Fax":100429,"_]":100430,"Ġial":100431,"ĠTalm":100432,"Ġdisordered":100433,"ertools":100434,"ĠSpending":100435,"å¾®é£İ":100436,"ĠÙĬÙĥ":100437,"lightly":100438,"substant":100439,"ç¿°æŀĹ":100440,"Ġprejudices":100441,"CopyWith":100442,".«":100443,"increase":100444,"ĠCarly":100445,"大头":100446,"ĠEnrollment":100447,"çįħ":100448,"æľ¬èº«å°±":100449,"Ġheterosexual":100450,"ĠJonah":100451,"ಾನ":100452,"飵åij³":100453,"querque":100454,"ampsia":100455,"opathological":100456,")·":100457,"çļĦç»ĦåIJĪ":100458,"ĠPQ":100459,"Ġprojets":100460,"ĠVALUE":100461,"åĪĨéĥ¨":100462,"Ġuphe":100463,"Ġscrit":100464,"Ġpowerless":100465,"Ġsingly":100466,"Ġsammen":100467,"ĠÐŁÑĢави":100468,"è°Īä¸įä¸Ĭ":100469,"ãĤ¹ãĥĿ":100470,"zoa":100471,"Ġemphasised":100472,"Ġextremities":100473,"Ġdeterrent":100474,"Ġvernacular":100475,"Ug":100476,"cannot":100477,"Ġhizo":100478,"Ġjeg":100479,"liczba":100480,"åIJĥèį¯":100481,"ç»ĵæŀľä¸º":100482,"Ġcoordin":100483,"Ġramifications":100484,"ãĤ«ãĥ«":100485,"ĠMindfulness":100486,"ĠаÑĢÑħиÑĤек":100487,"ĠOunce":100488,"CHANTABILITY":100489,"LX":100490,"otemporal":100491,"å¹´å¹³åĿĩ":100492,"åľ°éĿ¢ä¸Ĭ":100493,"પ":100494,"ichtet":100495,"Ġsacra":100496,"Ġtubig":100497,"éĻĤ":100498,"Ġданном":100499,"å¼ĵç®Ń":100500,"Labour":100501,"Ġexplosives":100502,"ĠSEE":100503,"arnish":100504,"ĠVisible":100505,"å±ħæ°ijçļĦ":100506,"Ġpossessive":100507,"åĪijäºĭæ¡Īä»¶":100508,"à§ĩলà§ĩ":100509,"Ġmög":100510,"ĠÑĢодиÑĤелей":100511,"Damage":100512,"AxisAlignment":100513,"ĠScrib":100514,"ĠTons":100515,"åΰæĻĤåĢĻ":100516,"çģ«èħ¿":100517,"èijĹæľī":100518,"ánica":100519,"Emma":100520,"ĠORGAN":100521,"ĠÑĤиÑģÑı":100522,"尤为éĩįè¦ģ":100523,"Ġaneurysm":100524,"ĠSainte":100525,"charts":100526,"عÙĦÙħ":100527,"Ġslapped":100528,"éĢĻéĩĮ":100529,"æŃ£å¸¸äºº":100530,"ĠPhilips":100531,"ĠFreddie":100532,"ĠProsper":100533,"uling":100534,"ĠInclusive":100535,"éĽij":100536,"лайн":100537,"ĠÙĦÙĩÙħ":100538,"Seed":100539,"ĠStrings":100540,"éĥijå·ŀå¸Ĥ":100541,"æĺ¯éĿŀ常éĩįè¦ģçļĦ":100542,"Ġgehört":100543,"arod":100544,"Ġkota":100545,"ĠStoff":100546,"ç¶Ļ":100547,"financial":100548,"}d":100549,"Ġduc":100550,"igrants":100551,"ĠKins":100552,"æīĢç§°":100553,"æ¯Ķåħ¶ä»ĸ":100554,"Ġdeflect":100555,"лÑĮÑı":100556,"ãĤĴãģĬ":100557,"ĠBois":100558,"ائج":100559,"è¶³å¤ŁäºĨ":100560,".header":100561,"Ou":100562,"tur":100563,"ĠÉĻ":100564,"Ġsón":100565,"ĠESR":100566,"åĴĮåIJİ":100567,"ä½ľå¼Ĭ":100568,"èĩªåªĴä½ĵ":100569,"å¿ĥåŃĺ":100570,"registered":100571,"logos":100572,"ÐŁÐ¾Ð»":100573,"à¶§":100574,"jeto":100575,"Ġcropping":100576,"Ġmolte":100577,"ĠÑĢода":100578,"ؤÙĦ":100579,"Ġsummarizing":100580,"ĠвозÑĢаÑģÑĤе":100581,"Ġlumière":100582,"Ġaleg":100583,"Ġincess":100584,"ĠAES":100585,"ĠCAB":100586,"Ġhaze":100587,"à¹ķ":100588,"åĩºçϼ":100589,"ä¹ĭèĻķ":100590,"çĿĢåij¢":100591,"æĥħåķĨ":100592,"ä»ĸ们å°Ĩ":100593,"åĽ´æ£ĭ":100594,"é¢ijè°±":100595,"åĢŁéĴ±":100596,"Ġutilised":100597,"ìĭĿìĿĦ":100598,"à¤ľà¤¼":100599,"é«ĺå°Ķ夫":100600,".tt":100601,"Ald":100602,"Council":100603,"Ġ_{\\":100604,"Insp":100605,"-men":100606,"Exerc":100607,"Leod":100608,"Ġcounteract":100609,"Ġ§§":100610,"Ġburgl":100611,"Ġwrinkles":100612,"ĠآزÙħاÛĮØ´":100613,"æĺ¯å®ŀçݰ":100614,"Ġunpopular":100615,"ä¸ĭå²Ĺ":100616,"ÃŃme":100617,"áĢľ":100618,"åįĥæĸ¹":100619,"_full":100620,"Се":100621,"ĠProtective":100622,"Generation":100623,"ĠTanaka":100624,"Ġdemolished":100625,"Ġanisotropy":100626,"()Ċ":101229,"MBA":101230,"ĉĠĠĠĠĠ":101231,"Ġcaching":101232,"iglio":101233,"Ġquattro":101234,"å¤ļåľ¨":101235,"Ġnuma":101236,"ãģªãģľ":101237,"Ġgenomics":101238,"Ġ×ijפ":101239,".Api":101240,"ĠLawyers":101241,"সà¦Ĥ":101242,"Ġtrigonometry":101243,"ÐľÐ¸":101244,"luor":101245,"Ġê·¸ê²ĥ":101246,"åĽ½åľŁèµĦæºIJ":101247,"ĠабÑģолÑİÑĤ":101248,"cée":101249,"ä¸Ńè·¯":101250,"为åħĪ":101251,"Ġmeu":101252,"ilevel":101253,"并讲è¯Ŀ":101254,"æĬĢæľ¯çļĦåıijå±ķ":101255,"æ´»åĬ¨çİ°åľº":101256,"bolt":101257,"ĠcapacitÃł":101258,"çļĦæĹ¥åŃIJéĩĮ":101259,"ĠÑģлово":101260,"Ġenpresak":101261,"\"])":101262,"otroph":101263,"ĠDiverse":101264,"ĠHao":101265,"ĠTheological":101266,"大人çļĦ":101267,"Ġpolyn":101268,"å°ijåĦ¿":101269,"è¯Ńå½ķ":101270,"è¿ľå¤ĦçļĦ":101271,"夫人çļĦ":101272,"Ġbehaved":101273,"Ġà¦ķব":101274,"Ġnorthwestern":101275,"Ġdescendant":101276,"ĠDarren":101277,"å¸ħåĵ¥":101278,"æĦŁåħ´è¶£çļĦ":101279,"Ġcomposting":101280,"Ġtattoos":101281,"ĠwÅĤaÅĽci":101282,"ĠRebellion":101283,")',":101284,"Farm":101285,"ĠSik":101286,"idt":101287,"ĠNabi":101288,"ç͍è¿ĩ":101289,"èĢĮä¾Ĩ":101290,"å¾ĪçŁŃ":101291,"ĠСÑĢед":101292,"æĪIJåĬŁçİĩ":101293,"örter":101294,"è¡ĮæĶ¿è®¸åı¯":101295,".Buff":101296,"åĵŃçĿĢ":101297,"ĠCastillo":101298,"éĦ°":101299,"Ġayudar":101300,"Flight":101301,"pies":101302,"alers":101303,"ĠCyrus":101304,"æľīä¸īç§į":101305,"éĥ½å¥½":101306,"åĪ©çī©":101307,"éĢģäºĨ":101308,"ĠISP":101309,"Ġbesøkt":101310,"Ġpokud":101311,"ëł¥ìĿĦ":101312,"à¸ķัวà¸Ńยà¹Īาà¸ĩ":101313,"-transform":101314,"à¸Ķูà¹ģล":101315,"Ġoutrageous":101316,"ANGUAGE":101317,"&C":101318,"fran":101319,"{l":101320,"Ġdá»ĭ":101321,"ä¼ļå¢ŀåĬł":101322,"Ġ[{":101323,"ĠReactive":101324,"å¹³éĿľ":101325,"ĠÙĪØ²Ø§Ø±":101326,"ĠAndhra":101327,"Ġverific":101328,"ĠMcGu":101329,"ĠPowerful":101330,"absent":101331,"Ġunofficial":101332,"ĠоÑĤноÑĪение":101333,"Ġocz":101334,"Ġmio":101335,"robi":101336,"ĠlÃŃm":101337,"ulner":101338,"ĠLorem":101339,"reeks":101340,"åıįèħIJ":101341,"ä¸ĩåIJį":101342,"commands":101343,"ाष":101344,"Ġrevolving":101345,"Ġpretended":101346,"æ¶Īè´¹ç¨İ":101347,"ç»ĨèĥŀåĨħ":101348,"اØŃد":101349,"ÖĢÕ¯":101350,"Avatar":101351,"ĠUttar":101352,"@media":101353,"PGC":101354,"åľ¨å¾Ī大ç¨ĭ度ä¸Ĭ":101355,"èĥ½è¾¾åΰ":101356,"ĠاÙĦاخ":101357,"__Ċ":101358,"Ġprefrontal":101359,"åIJĮé¾Ħ":101360,"她å¾Ī":101361,"æĬĬä»ĸçļĦ":101362,"é£İæ³¢":101363,"armee":101364,"ï¼ļâĢľâ̦â̦âĢĿĊĊ":101365,"è¯ķä¸Ģè¯ķ":101366,"çĶ·ç¯®":101367,"ktions":101368,"设计æĸ¹æ¡Ī":101369,"-growth":101370,"bao":101371,"ĠÚ¯ÙĪØ´":101372,"Ġplugged":101373,"Ġhijo":101374,"Ġë²Ķ":101375,"Ġfishery":101376,"everything":101377,"ĠDodgers":101378,",input":101379,"ĠKne":101380,"ÃŃcula":101381,"ĠTraff":101382,"Ġfootnote":101383,"ĠÑĩаÑģа":101384,"åľĸçīĩ":101385,"æĸijçĤ¹":101386,"è©ķè«ĸ":101387,"á»ĥn":101388,"Ġfacilitation":101389,"});":101390,"ĠlÃ¥":101391,"ĠGav":101392,"ĠпалÑĮ":101393,"×ķ×Ĺ×":101394,"Ġescl":101395,"æīĵåŃĹ":101396,"æīįæľĥ":101397,"Ġskincare":101398,"ä¸įåIJĮç±»åŀĭçļĦ":101399,"åıĮä¾§":101400,"伤æ®ĭ":101401,"mmol":101402,"ĠMoroccan":101403,"Ġtendons":101404,"ÐļÐŀ":101405,"starting":101406,"ÐķТ":101407,"Ġpueda":101408,"ĠCorey":101409,"ĠмаÑĤеÑĢиалÑĭ":101410,"ĠfÃŃsico":101411,"LOBAL":101412,"Ġmnie":101413,"Ġ(£":101414,"chol":101415,"åIJĦ乡éķĩ":101416,"ĠGlu":101417,"åģıåĥ»":101418,"Ġauthorship":101419,"Ġpelig":101420,"lsx":101421,"à¥ģन":101422,"à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩà¸Īาà¸ģ":101423,"Ġbrochure":101424,"<%@":101425,"treatment":101426,"Ġurs":101427,"ĠLiet":101428,"ä»ĸä¸Ģçľ¼":101429,"Ġzna":101430,"å¹¶åı¯":101431,"è§īå¯Ł":101432,"åŃ¦ä¹łä¸Ń":101433,"Ġ×ŀ׳×":101434,"åĶIJä¸ī":101435,"اÙĩÛĮ":101436,"åħļçļĦé¢Ĩ导":101437,"说è¯ĿçļĦ":101438,"ĠMicrowave":101439,"ĠÔ¿":101440,"(queue":101441,"raven":101442,"ä¹Łè¡Į":101443,"çªģåħĢ":101444,"ĠDesire":101445,"æĿ¥è¯´æĺİ":101446,"åīªçº¸":101447,"辦çIJĨ":101448,"Ġ×ij×©×ł×ª":101449,"Rp":101450,"coding":101451,"mese":101452,"sales":101453,"ĠICP":101454,"æĺ¯æĮī":101455,"ï¼Łï¼ģâĢĿ":101456,"åıªåı¯æĥľ":101457,"Ġdonating":101458,"ĠDeuter":101459,"Pero":101460,"Ġcách":101461,"个头":101462,"ĠOrte":101463,"playing":101464,"alfa":101465,"å·´åħĭ":101466,"ÑĢÑĭй":101467,"tableView":101468,"浩çĢļ":101469,"ĠWalay":101470,"Ġjoules":101471,"ĠAlbanian":101472,"æĺ¯æĪij们çļĦ":101473,"Ġalap":101474,"ä¹ŁåĴĮ":101475,"éĶŃ":101476,"Ġbacktrack":101477,"ĠFrans":101478,"çĭĤ欢":101479,"ĠHorace":101480,"Ġscarlet":101481,"Ġróżnych":101482,"ĠÑģлиÑĪком":101483,"еб":101484,"ĠJóz":101485,"éĢĤäºİ":101486,"DataType":101487,"Ġmutated":101488,"washer":101489,"Ġgigabits":101490,"Ġsubtracted":101491,"Ġpriesthood":101492,"fasst":101493,"Ġmathematicians":101494,"ĠHanoi":101495,"ä½łæľī没æľī":101496,"ĠChannels":101497,"ahoo":101498,"æıIJæĹ©":101499,"Ġespect":101500,"åħīå½±":101501,"çľ¼èĬ±":101502,"Ġopts":101503,"å¼ķæĿ¥":101504,"ĠÐļол":101505,"ĠDecimals":101506,"æļ´æ¶¨":101507,"æĤłçĦ¶":101508,"Õ¸Õ¬":101509,"delimited":101510,"Kondado":101511,"\\varphi":101512,"antor":101513,"éģ´":101514,"-political":101515,"ĠرسÙħ":101516,"ĠPresidency":101517,"olaire":101518,"èĢIJåıĹ":101519,"æĺ¯ä»Ģä¹Īæł·çļĦ":101520,"Ġnecesidades":101521,"wek":101522,"даÑĤ":101523,"Closing":101524,"èϽçĦ¶è¯´":101525,"Ġsnails":101526,"aksa":101527,"instruction":101528,"ণà§ĩর":101529,"義åĭĻ":101530,"Ġdagli":101531,"Von":101532,"ив":101533,"Ġanthem":101534,"åľ¨åIJĦç§į":101535,"ittance":101536,"éĢīèĩª":101537,"åıijå±ķéĺ¶æ®µ":101538,"à¸Ħะ":101539,"ä¸ĢèάèĢĮè¨Ģ":101540,"ä»·æł¼ä¸º":101541,"Ġunsuitable":101542,"ĠAsteroid":101543,"ĠWinnipeg":101544,",len":101545,"PRE":101546,"ĠTiffany":101547,"ĠLester":101548,"ãģ§ãģĤãģ£ãģŁ":101549,"Ġbooming":101550,"红å°ĺ":101551,"äºijéĽ¾":101552,"Ġsamo":101553,"Andy":101554,"Ġپار":101555,"ĠTextbook":101556,"ĠVisiting":101557,"ĠпеÑĢеÑģе":101558,"æĿ°åĩºçļĦ":101559,"Ġà¹Ģว":101560,"à¯ĩà®°":101561,"Ġtrimming":101562,"Ġarquitect":101563,"ĠBulldogs":101564,"ĠÙħØ´Ú©ÙĦات":101565,"Ġsubdued":101566,"ĠتاثÛĮر":101567,"alien":101568,"ä¸ĢæİĴ":101569,"Ġagama":101570,"ĠÑģмож":101571,"æľºæĻº":101572,"Ġsete":101573,"书çĶŁ":101574,"åĦ¿ç§ij":101575,"Ġdaya":101576,"Ġlegge":101577,"ç¾İåĽ½æĢ»ç»Ł":101578,"ĠPetit":101579,"Ġintellectually":101580,"ĠSensory":101581,"decision":101582,"ĠÑĪколоваÑļа":101583,"_COMP":101584,"ĠMercer":101585,"Ġanecdotes":101586,"któber":101587,"anat":101588,"ĠPoc":101589,"Ġwasher":101590,"èĢĮä½ł":101591,"åıĬçļĦ":101592,"åıĪ称为":101593,"ĠIndirect":101594,"ĠListe":101595,"structures":101596,"æĮºå¥½":101597,"详ç»ĨäºĨè§£":101598,"Ġcustod":101599,"Ġdereg":101600,"ĠHeavenly":101601,"าà¸Ĥà¸Ńà¸ĩ":101602,"Ġpatriotism":101603,"EK":101604,"Xu":101605,"erad":101606,"ilaren":101607,"angkat":101608,"é£İåIJij":101609,"è¶³çļĦ":101610,"ĠAngelo":101611,"åĢĴä¸ĭ":101612,"ĠIsraelis":101613,"ðĿIJ¶":101614,"Coordinate":101615,"-exec":101616,"à¹Ģสà¹īà¸Ļ":101617,".assertTrue":101618,"Ġconcerted":101619,"ç¶łèī²":101620,"Ġevaporated":101621,"Ġchrome":101622,"ĠKolkata":101623,"ĠDeaf":101624,"èĢĥå®ĺ":101625,"åıªæľīå½ĵ":101626,"åľŁåľ°çļĦ":101627,"Ġpandemia":101628,"ĠHubert":101629,"éģ®æİ©":101630,"Ġmencapai":101631,"Gran":101632,"itely":101633,"ĠLOT":101634,"ä¹ĭæĢ¥":101635,"othic":101636,"äºĨä¸Ģçīĩ":101637,"âĪĴ(":101638,"å°Ħé¢ij":101639,"ĠÙ¾ÙĨج":101640,"èĢ³çĽ®":101641,"æıĴæīĭ":101642,"ĠPoem":101643,"Ġà´¤":101644,"Ġfod":101645,"çĶŁæ°£":101646,"ä½ľæ³ķ":101647,"llen":101648,"çĦ¯":101649,"ุà¹Īà¸ĩ":101650,"éŁ³ä¹IJçļĦ":101651,"æĮĩæłĩä½ĵç³»":101652,"Ġreproducing":101653,"_LIST":101654,"ãĤ¦ãĥł":101655,"Ġnghìn":101656,"è¡Ģ红èĽĭçϽ":101657,".pr":101658,"heon":101659,"äºĨåĩłä¸ª":101660,"èĩ¼":101661,"ĠUIImage":101662,"ä¸ĩå²ģ":101663,"ç¬ijè¯Ń":101664,"ĠSchultz":101665,"оки":101666,"Ġmiddleware":101667,"ä¸ī个代表":101668,"ä¸¥æł¼æİ§åζ":101669,"åĪºæ¿ĢæĢ§":101670,"continence":101671,"çļĦè¾ĵåĩº":101672,"omét":101673,"chuk":101674,"Ġ\\;":101675,"Ġà¤ĺ":101676,"ÅĤam":101677,"divisions":101678,"Ġloge":101679,"Ġduoden":101680,"ÙĩاÙĬØ©":101681,"ademia":101682,"Ġpenicillin":101683,"Ġpropel":101684,"ὺ":101685,"Ġturmeric":101686,"Ġcytotoxicity":101687,"Ġponieważ":101688,"ĠConditional":101689,"Ġmellom":101690,"README":101691,"âĢįâĢį":101692,"ĠãĢķ":101693,"Ġmarm":101694,"ĠMULT":101695,"ĠFACT":101696,"Ġkidd":101697,"ä»ĸå¿ĥéĩĮ":101698,"Ġtrunks":101699,"å°ıå®Ŀ":101700,"Institut":101701,"tenir":101702,"Ġresulta":101703,"Ġessas":101704,"itets":101705,"Ġeject":101706,"implement":101707,"ĠLama":101708,"åĴĮç»´æĬ¤":101709,"æĮĻ":101710,"æīĢéĢī":101711,"ĠReis":101712,"管路":101713,"ä¸įä»ħåľ¨":101714,"奶éħª":101715,"chenko":101716,"Ġcatchment":101717,"ĠFreund":101718,"çłĤç³ĸ":101719,"ĠìłķëıĦ":101720,"ĠVirol":101721,"å¹³åĩ¡çļĦ":101722,"è¿Ħä»Ĭ为æŃ¢":101723,"Karl":101724,"è¦ģ强åĮĸ":101725,"妩":101726,"Ġperovsk":101727,"éĥ¨éļĬ":101728,"beam":101729,"Ġbreakout":101730,"æĭįæĶĿ":101731,"ĠSolved":101732,"æ»´æ°´":101733,"Ġrupee":101734,"ĠVanessa":101735,"çī§å¸Ī":101736,"ãĤ³ãĥŁ":101737,"Ġcontraception":101738,"ĠRubber":101739,"Ġ문ìĦľ":101740,"iwers":101741,"ãĥķãĤ¡ãĤ¤ãĥ«":101742,"(username":101743,"Gn":101744,"æĪijæĦŁåΰ":101745,"вÑĭй":101746,"åĽ½åº¦":101747,"Ġdetalles":101748,"(\"@":101749,"è¿Ľè¡ĮåĪĨç±»":101750,"æŃ»è§Ĵ":101751,"ĠFlags":101752,"Ġsemiconductors":101753,"ĠлиÑĩноÑģÑĤи":101754,"ĠMemories":101755,"onnaise":101756,"ĠبÙĪØ§Ø¨Ø©":101757,"ĠPrak":101758,"âĢĿâĨĴ":101759,"istro":101760,"Ġcurls":101761,"ç»Ħç»ĩé¢Ĩ导":101762,"Ġtonic":101763,"ĠPAS":101764,"Ġleans":101765,"Animals":101766,"naeus":101767,"สีà¹Ī":101768,"ĠÙĨب":101769,"ä½ľç͍åĴĮ":101770,"ÖĢÕ¡Õ¶":101771,"ĠSupplies":101772,"ĠAttend":101773,"Ġpeuple":101774,"å¸Ĥå§Ķ常å§Ķ":101775,"ĠFeminist":101776,"åĹ¡åĹ¡":101777,"åķ§åķ§":101778,">Ċ":102602,"ì¿":102603,"ä¸įæĶ¯æĮģ":102604,"ebra":102605,"ofen":102606,"اÙģØªÙĩ":102607,"(year":102608,"ÈĻti":102609,"Ġnostri":102610,"Ġwilayah":102611,"Ġosserv":102612,"entos":102613,"ãĢĤ###":102614,"ĠFilters":102615,"大è·Į":102616,"çľĭä¸įæĩĤ":102617,"ĠProspects":102618,"åĽŀæĹı":102619,"ä»ĸ们已ç»ı":102620,"öff":102621,"äºĨä¸Ģèά":102622,"zeichen":102623,"éŁ³èĬĤ":102624,"ĠChristensen":102625,"_pop":102626,"Ġstolet":102627,"ìĿ¸ìĿĺ":102628,"èĺ¿":102629,"Ġsanity":102630,"Ġkoji":102631,"Ġpemerintah":102632,"+S":102633,"arrays":102634,"Ġgenders":102635,"Ġvara":102636,"à¸ģว":102637,"精油":102638,"åį³ä»¥":102639,"çĮ¥":102640,"è¿ĺæĺ¯ä¸Ģ个":102641,"稳æĢģ":102642,"åıªèĥ½ç͍":102643,"तà¤ĥ":102644,"Ġétat":102645,"è¾£çļĦ":102646,"ç§ijçłĶæĪIJæŀľ":102647,"ளà¯įள":102648,"Graphics":102649,"西éĥ¨åľ°åĮº":102650,"Ġrooftop":102651,"åĮĪçīĻåĪ©":102652,"Nich":102653,"poor":102654,"Ġcx":102655,"ĠVERY":102656,"ä¿Ŀç¨İ":102657,"ç¡®æľī":102658,"第ä¸ĢåĢĭ":102659,"ĠCalc":102660,"èĦijä¸Ń":102661,"((-":102662,"Ġ__('":102663,"ĠEndangered":102664,"é³Į":102665,"辨æŀIJ":102666,"×ijר×Ķ":102667,"-blood":102668,"ĠWiener":102669,"Ġanisotropic":102670,"\"));ĊĊ":102671,"Ġmong":102672,"Ġexcret":102673,"philis":102674,"æľ¬éĥ¨":102675,"å¼ı计ç®Ĺ":102676,"åĥıæĺ¯åľ¨":102677,"åĮ»æľ¯":102678,"ANI":102679,"ĠPrés":102680,"ĠMonaster":102681,"ĠвÑĭÑģÑĤÑĥпа":102682,"تراض":102683,"Usuario":102684,"transition":102685,".edit":102686,"vana":102687,"sticks":102688,"oland":102689,"ĠDish":102690,"ä¼ļè®¡æł¸ç®Ĺ":102691,"Ġrustic":102692,"ĠпоÑģÑĤоÑıнно":102693,"Ġáĥ¡áĥIJáĥ":102694,"åłķèIJ½":102695,"-ho":102696,"qz":102697,"onent":102698,"ĠdziÄĻki":102699,"oulli":102700,"ä»»æľŁ":102701,"Ġseres":102702,"ï¿¥":102703,"Ġimprob":102704,"åī¯è¯į":102705,"è¯Ńè¨ĢæĸĩåŃĹ":102706,"Ġ×ĶÖ·":102707,"ä¹Łåı¯ä»¥ç͍":102708,"ĠLicensing":102709,"æ£Ģå¯Łéķ¿":102710,"ĠThermod":102711,"Implemented":102712,"'Or":102713,"etako":102714,"ĠSST":102715,"æĪĽ":102716,"绫":102717,"Ġappellants":102718,"åºĶä»İ":102719,"exc":102720,"å¹³åľ°":102721,"Ġcheque":102722,"åĢĴå¡Į":102723,"èϽçĦ¶æľī":102724,"Ġechoing":102725,"踪迹":102726,"Functional":102727,"ĠدÙĩÛĮد":102728,"بØŃØ«":102729,"Ġprzeciw":102730,"åĽŀè¿ĩç¥ŀæĿ¥":102731,"ĠطبÛĮعÛĮ":102732,"ª×¨":102733,"ĠSod":102734,"umna":102735,"åĩºæ¸¸":102736,"ccan":102737,"Ġtrast":102738,"æīĢå¼ķèµ·çļĦ":102739,"ottes":102740,"Ġloft":102741,"Ġempires":102742,"¡×Ĵ":102743,"Ġkinases":102744,"Ġdangerously":102745,"Ġadultos":102746,"Ġhampered":102747,"ÑĤеÑĤа":102748,"èĭ¥å¹²ä¸ª":102749,"industrial":102750,"Ġepochs":102751,"#,":102752,"cional":102753,"Ġbinge":102754,"çļĦåħ±åIJĮ":102755,"entar":102756,"åľ¨åĴĮ":102757,"ewel":102758,"Ġcož":102759,"顾åıĬ":102760,"paRepository":102761,"ĠNorwich":102762,"éģµçħ§":102763,"isième":102764,"åĮªæµħ":102765,"æĸĩçī©ä¿ĿæĬ¤":102766,"ĠWebsites":102767,"Ġtij":102768,"Ġaang":102769,"Ġfencing":102770,"ĠBoul":102771,"ĠWolves":102772,"çŃīåħ¶ä»ĸ":102773,"arked":102774,"éļ¾ä¸įæĪIJ":102775,"Ġorigine":102776,"Ġimmoral":102777,"(-\\":102778,"ĠGrill":102779,"ĠниÑĩего":102780,"èĦļè¸ıå®ŀåľ°":102781,"鸳鸯":102782,"Ġforsk":102783,"ĠOll":102784,"æĿ¥ç͵":102785,"Ġpresque":102786,"åĪļèIJ½":102787,"ĠMarketplace":102788,"Accordingly":102789,"Ġmoonlight":102790,"Ġר×Ĵ":102791,"åĨ¶çĤ¼":102792,"Ġdiligent":102793,"ĠAppropriate":102794,"'er":102795,"Ġdred":102796,"è¿Ļé¢Ĺ":102797,"ä¸ľå±±":102798,"çĶ²éª¨":102799,"Ġcyclone":102800,"讲解äºĨ":102801,"Ġneutrophils":102802,"ĠArbitration":102803,"-occur":102804,"_device":102805,"rochemical":102806,"Ġnäch":102807,"åŃIJæĽ°":102808,"åħ»çļĦ":102809,"Ġshortcuts":102810,"纤ç»Ĩ":102811,"éĶ¦ç»£":102812,"ÄŁi":102813,"å¤įåIJĪæĿIJæĸĻ":102814,"ĠViktor":102815,"à¹ģà¸Ĥà¹ĩà¸ĩ":102816,".ro":102817,"_err":102818,"ĠDane":102819,"ĠKJ":102820,"à¸ļาล":102821,"Ġرأ":102822,"å¯ĨéĹŃ":102823,"ĠMcMahon":102824,"èͼ":102825,"æŃ£å¸¸å·¥ä½ľ":102826,"èĢĥè¯ķçļĦ":102827,"ĠPractitioner":102828,"ç½²åIJį":102829,"arshall":102830,"Ġbanquet":102831,"ä¸ŃéĹ´çļĦ":102832,"_BU":102833,"ÖīĊ":102834,"ĠDermatol":102835,"Islamic":102836,"ĠоÑģигÑĥÑĢа":102837,"Ġpigeon":102838,"æľīåĬŁ":102839,"Ġpatented":102840,"Ġtechnologie":102841,"æ¶Īçĺ¦":102842,"äºī端":102843,"Ġnorma":102844,"Unity":102845,"è¿Ľä¸ĢæŃ¥åıijå±ķ":102846,"ĠSioux":102847,"Ġadjour":102848,"ĠмоÑĩе":102849,"yczÄħ":102850,"Ġassaults":102851,"/default":102852,"[B":102853,"ifol":102854,"ĠHG":102855,"ĠHert":102856,"åľ¨ä¸»":102857,"ĠGron":102858,"å°±æĦıåij³çĿĢ":102859,"è¿Ľè¡ĮçłĶç©¶":102860,"INTS":102861,"textit":102862,"á»Ĺ":102863,"Ġairplanes":102864,"åĿļæŀľ":102865,"æ½°":102866,"Ġà¦ķরà§įম":102867,"ç¢İçŁ³":102868,"Ġthankfully":102869,"ĠCrossref":102870,"íı¬íĬ¸":102871,"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM":102872,"सà¤Ĥà¤ĸà¥įया":102873,"Ġtelegraph":102874,"æĥ³åģļ":102875,"à¥ī":102876,"åŁºè´¨":102877,"risy":102878,"Ġfaithfulness":102879,"ĠHolz":102880,"Ġtitik":102881,"Bench":102882,"éŁĵåľĭ":102883,"ĠLorentz":102884,"ĠtÅĻeba":102885,"ĠÙħربÙĪØ·":102886,"Cli":102887,"HOW":102888,"Ġfide":102889,"ĠAlem":102890,"书å±ĭ":102891,"strut":102892,"åı¤å·´":102893,"Ġeyew":102894,"Ġrichly":102895,"-examination":102896,"Ġcosmological":102897,"Ġwegen":102898,"orske":102899,"proxy":102900,"ĠIsles":102901,"Ġpracticable":102902,"饮åĵģ":102903,"Systems":102904,"ĠjurÃŃd":102905,"-performing":102906,"Ġdiaspora":102907,"ĠInequality":102908,"éĤ£çīĩ":102909,"Increase":102910,"Ġentrenched":102911,"åķĨç͍":102912,"æµĭ温":102913,"å¾·åľĭ":102914,"Ġstoryt":102915,"èĥĮ书":102916,"æĽ¾æĺ¯":102917,"åĵĪèIJ¨åħĭ":102918,"尿管":102919,"Ġän":102920,"éĹ²æļĩ":102921,"å¼Ģåı£è¯´éģĵ":102922,"Absolutely":102923,".Input":102924,"Kg":102925,"Zur":102926,"fäh":102927,"Õ°":102928,"reve":102929,"unner":102930,"åľ°ä¸Ńæµ·":102931,"æ°´çĵ¶":102932,"æķĻåŃ¦å·¥ä½ľ":102933,"ãģ£ãģĭãĤĬ":102934,"ï½ħ":102935,"åıĬæĹ¶åıijçݰ":102936,"è¾ŀåħ¸":102937,"ிரà¯ģà®":102938,"ä¸Ńä¹ĭéĩį":102939,"Favorite":102940,"fills":102941,"ĠOy":102942,"ostÃŃ":102943,"å¦Ĥä¸Ģ":102944,"ä¸įæĺ¯ä½ł":102945,"Ġbookstore":102946,"ãĤĤãģĤãĤĬãģ¾ãģĻ":102947,"éļIJæĢ§":102948,"Ġmedicina":102949,"à§ĩলা":102950,"ĠComplaint":102951,"ĠÑįлеменÑĤÑĭ":102952,"Ġhelicopters":102953,"ä¸Ń级人æ°ijæ³ķéĻ¢":102954,"Credentials":102955,"æĭĸæĭīæľº":102956,"Ġcape":102957,"ĠHector":102958,"以åĩıå°ij":102959,"æ¯Ķä¸įä¸Ĭ":102960,"ylie":102961,"çIJĨ论çłĶç©¶":102962,"lamide":102963,"Ġaurait":102964,"æĬ¤èĤ¤åĵģ":102965,"Ġprakty":102966,"cj":102967,"rÄħ":102968,"}[/":102969,"additional":102970,"便åı¯ä»¥":102971,"_dataset":102972,"èĩªçĦ¶äºº":102973,"à¹Ĥà¸Ń":102974,"ĠÙħعÙĨ":102975,"ĠиÑģпол":102976,"ĠKurdish":102977,"Ġasparagus":102978,"ĠUltrasound":102979,"ĠÙħصطÙĦØŃات":102980,"(action":102981,"punk":102982,"ĠCognition":102983,"ĠTheresa":102984,"Ġquark":102985,"Ġpatter":102986,"Ġopere":102987,"Ġ%Ċ":102988,".pk":102989,"åĪĨ享ä¸Ģä¸ĭ":102990,"ĠتØŃد":102991,"ç´§ç´§çļĦ":102992,"Õ¡Õ£ÖĢ":102993,"-admin":102994,"mort":102995,"pile":102996,"Ġdagen":102997,"tev":102998,"dez":102999,"ĠÑģва":103000,"checks":103001,"åħ¥ãĤĮ":103002,"æīĵåİĭ":103003,"åĽłä¸ºè¿ĻäºĽ":103004,"Ġbehaving":103005,"Sept":103006,"奥ç§ĺ":103007,"豪éŨ":103008,"Ġluas":103009,"Ġunmistak":103010,"ĠGREAT":103011,"ĠзнаÑĩений":103012,"Jy":103013,"MCA":103014,"Ġnuit":103015,"Ġcomplicate":103016,"Ġunnamed":103017,"åĬ¨éĩı":103018,"-solid":103019,"-critical":103020,"สูà¹Ī":103021,"Ġbeneficios":103022,"ाà¤ĥ":103023,"Ġrouted":103024,"Ġdilation":103025,"çļĦ主è§Ĥ":103026,"乡æĿijæĹħ游":103027,"Ġleisurely":103028,"ĠÙħÙ쨵":103029,"æ¶ħæ§ĥ":103030,"Asc":103031,"Fab":103032,"Ġdva":103033,"ĠTitus":103034,"adis":103035,"çĽ¸å£°":103036,"æĺĵçĩĥ":103037,"Ġbestselling":103038,"hrte":103039,"é½IJåĽ½":103040,"åĪ¥çļĦ":103041,"ĠInspire":103042,"æijĶåĢĴ":103043,"Essential":103044,"ĠاÙĦاØŃÙħر":103045,"aksanakan":103046,"Ġprotruding":103047,"ä¹ħèĢĮä¹ħä¹ĭ":103048,"igneur":103049,"çľĭæ¸ħæ¥ļ":103050,"æľĢ常è§ģ":103051,"å°±æĺ¯ä¸Ģç§į":103052,"িয":103053,"Ġolds":103054,"àµĬ":103055,"Ġmisinterpret":103056,"ç¾İåħĥçļĦ":103057,"ç²ĺè¿ŀ":103058,"ĠTelecommunications":103059,"Ġslicing":103060,"Ġdul":103061,"ï¼³":103062,"ĠFis":103063,"merzen":103064,"ĠShiva":103065,"åįĹè·¯":103066,"è¯Ńè°ĥ":103067,"åĬ¿åĬĽçļĦ":103068,"æ¼ĶæĪı":103069,"æĢĿæĥ³å®¶":103070,"Ġhistori":103071,"Ġconsultancy":103072,"Û±Û´":103073,"à¸ĭี":103074,"Untuk":103075,"Ġà¦Ľà¦¿à¦²à§ĩন":103076,"bron":103077,"çļĦæĶ¹åıĺ":103078,"abets":103079,"usted":103080,"ä¹IJçļĦ":103081,"Ġgevo":103082,"ç´§éĹŃ":103083,"åĺ´è¾¹":103084,"illaume":103085,"ĠIVF":103086,"åĤ¬çľł":103087,"Ġquadru":103088,"ĠForests":103089,"Ùħسار":103090,"×Ļפ×ķר":103091,"å̤ãĤĴ":103092,"çµĮé¨ĵ":103093,"acie":103094,"ĠFIELD":103095,"æĪijåı¯æĺ¯":103096,"omegal":103097,"éŁ³éĩı":103098,"æ²¹çĤ¸":103099,"åĽºæľīçļĦ":103100,"uelas":103101,"éļĨèµ·":103102,"ĠХа":103103,"apitre":103104,"Baker":103105,"HCl":103106,"{item":103107,"iru":103108,"ĠEternal":103109,"uby":103110,"å°±è§ģ":103111,"xties":103112,"Ġpasso":103113,"hematica":103114,"Adult":103115,"严éĺ²":103116,"helps":103117,"çīĽæ´¥":103118,"ĠVIEW":103119,"ĠкаÑĢÑĤи":103120,"Ġnagy":103121,"éģĹæĨ¾çļĦæĺ¯":103122,"ĠJurassic":103123,"æ¤ľæŁ»":103124,"Ġenseign":103125,"Rachel":103126,"qe":103127,"ĠSø":103128,"Ġgf":103129,"ĠвокÑĢÑĥг":103130,"æĪĸ以":103131,"oxys":103132,"èģĶè°Ĭ":103133,"ulação":103134,"è¿Ļå°±è¦ģæ±Ĥ":103135,"ĠÙħدت":103136,"à¸ķà¹Īà¸Ńà¹Ħà¸Ľ":103137,"Ľ×Ļ×Ŀ":103138,"omach":103139,"ĠGK":103140,"大æ´ĭ":103141,"ä¸Ĭ身":103142,"Increasing":103143,"ĠÙĨÚ©":103144,"ilihan":103145,"ленно":103146,"-real":103147,"Ġrollers":103148,"ĠTimur":103149,"ĠCardiovasc":103150,"idelijk":103151,"ä¹Łåıª":103152,"å°ıäºĮ":103153,"çŃī离åŃIJ":103154,"è°ĥåΰ":103155,"вании":103156,"genstein":103157,"Ġboa":103158,"读åΰ":103159,"Ġillusions":103160,"âge":103161,"èĻļå®ŀ":103162,"Ġvegetative":103163,"çݰå®ŀ主ä¹ī":103164,"Ġrailroads":103165,"Ġsigue":103166,"èĤļåŃIJéĩĮ":103167,"Ġë¹ĦêµIJ":103168,"ĠGemini":103169,"ĠDiplom":103170,"Ġtungsten":103171,"اÙĪÛĮ":103172,"è¦ģé¢Ĩ":103173,"ĠYor":103174,"ĠAlain":103175,"ĠWhites":103176,"Ġhardy":103177,"ãģ£ãģŁãĤĬ":103178,"áŁĨ":103179,"ĠÙħستÙĤ":103180,"Ġbewild":103181,"Ġankles":103182,"缸è¾ĥäºİ":103183,"MID":103184,"onim":103185,"çļĦ妻åŃIJ":103186,"stup":103187,"ĠTodos":103188,"illar":103189,"éĥ½ä¸įè¦ģ":103190,"éĤ£è¾¹çļĦ":103191,"à§įণ":103192,"åĮĹ约":103193,"guest":103194,"çİĦå¹»":103195,".substr":103196,"éŀŃçĤ®":103197,".na":103198,"Eye":103199,"_shape":103200,"xF":103201,"äºĨ她çļĦ":103202,"thest":103203,"ارÙĬØ©":103204,"ginx":103205,"ĠPayPal":103206,"{document":103207,"Ġannoyance":103208,"çļĦç»ıèIJ¥":103209,"Ġchoked":103210,"为ç͍æĪ·":103211,"Ġusur":103212,"ĠArten":103213,"Ñİз":103214,"ĠIsra":103215,"িà¦ı":103216,"ĠPrussia":103217,"ĠAmes":103218,"çĨĦçģŃ":103219,"ä¿Ŀ驾æĬ¤èĪª":103220,"poke":103221,"Ġbian":103222,"Ġforging":103223,"æľĢéļ¾":103224,"undai":103225,"æľªå°½":103226,"åĽ½éĻħä¸Ĭ":103227,"Ġqualitatively":103228,"é¢ĨåŁŁä¸Ń":103229,"ĠPresented":103230,"/âĪĴ":103231,"Mb":103232,"lach":103233,"Ãķ":103234,"æĪijåľĭ":103235,"ÑĤиÑĩеÑģкие":103236,"Ġgroupes":103237,"纳德":103238,"à¥ĩष":103239,"Ġöffent":103240,"নà§įড":103241,"Ġuphill":103242,"Ġuczniów":103243,"znej":103244,"heesta":103245,"unu":103246,"ĠPp":103247,"ä¸Ĭæĸĩ":103248,"æīįä¼ļæľī":103249,"ä¾Ľæļĸ":103250,"åŃ¦æł¡æķĻèĤ²":103251,"ĠJeÅĽli":103252,"ாம":103253,"æ´ªèįĴ":103254,"ĉĉĊĉĉĊ":103255,"×ķצר":103256,"Ġirregularities":103257,"åįģäºĮæĮĩèĤł":103258,"arek":103259,"ĠVOC":103260,"ÑĢиди":103261,"Ġdownwards":103262,"åķĨäºĭ":103263,"象æ£ĭ":103264,"Ïĥκε":103265,"åħ³ç³»åĴĮ":103266,"åı¥ä¸Ń":103267,"whatever":103268,"CTC":103269,"çļĦåİŁåĽłæĺ¯":103270,"Ġfructose":103271,",but":103272,"alais":103273,"Ġnude":103274,"ä¸įæĢĿ":103275,"ĠUh":103276,"ç»ı纬":103277,"è®°å½ķçļĦ":103278,"çĶĺå¿ĥ":103279,"Ġgeworden":103280,"Ġارزش":103281,"uddled":103282,"stoffen":103283,".ForeignKey":103284,"wania":103285,"è¿ĺç͍":103286,"åĽŀ车":103287,"声åĬ¿":103288,"ĠÙĦب":103289,"лÑıем":103290,"arcane":103291,"ĠFrança":103292,"ä»Ĭ天æĪij们":103293,"å½Ĵæ¡£":103294,"ĠÑģамÑĭм":103295,"watering":103296,"Ġbekend":103297,"kB":103298,"çļĦçĹħ人":103299,"ĠCarth":103300,"ĠPty":103301,"éĹ´è°į":103302,"ä½Ĩä»ĸçļĦ":103303,"Ġelke":103304,"åij¨è¾¹çļĦ":103305,"(top":103306,"æİ¢åºĹ":103307,"åºŃ审":103308,"ĠFeather":103309,"æķ¬è¯·":103310,"ĠоÑģновном":103311,"Followers":103312,"Ġë§İìĿ´":103313,"ĠAthena":103314,"Ġì¦Ŀê°Ģ":103315,")R":103316,"=lambda":103317,"fighter":103318,"çļĦçŃĸçķ¥":103319,"Ġenim":103320,"ulfide":103321,"è¦ĸéł»":103322,"Ġabstracts":103323,"Ö¸×IJ":103324,"ĠTelecom":103325,"çĨıé϶":103326,"bouw":103327,"ĠMesopotamia":103328,"VN":103329,"çļĦå®ŀçݰ":103330,"elage":103331,"ĠLips":103332,"大åºĨ":103333,"è¦ģ害":103334,"åIJĮå±ħ":103335,"æĢ§çĶŁæ´»":103336,"าà¸ĵ":103337,"åħ»çĮª":103338,"æŃ»ç¥ŀ":103339,"æĬķèµĦé¡¹çĽ®":103340,"å¤ı侯":103341,"Ġdrog":103342,"Ġmécan":103343,"忽è§ĨäºĨ":103344,"Ġmultimodal":103345,"ĠTrouble":103346,"ĠRegistrar":103347,"ĠاÙĦÙ쨱ÙĨس":103348,"à°¿à°Ĥà°ļ":103349,"ĠGospels":103350,"Ġsenc":103351,"unay":103352,"åħ¨éĻ¢":103353,"æĸ°å¥ĩ":103354,"è´£å¤ĩ":103355,"ðĿĺ":103356,".category":103357,"é£ŀæĿ¥":103358,"åģıå¿ĥ":103359,"Ġdependable":103360,"¤×Ĵ":103361,"CMC":103362,"ĠTranslations":103363,"AWS":103364,"ĠBaba":103365,"ä¸İåѦçĶŁ":103366,"æĹłèĢ»":103367,"å®¹é¢ľ":103368,"åĩłåı¥è¯Ŀ":103369,"Ġrestricts":103370,"Ġobjectivity":103371,"ĠзаÑĢабоÑĤ":103372,"ĠجÙĪØ§ÙĨ":103373,"çĨŁçŁ¥":103374,"ĠProcessor":103375,"Ġverbally":103376,"Ġaeruginosa":103377,"ĠÑģозна":103378,"Ġdevastation":103379,"åį°ç¬¬å®ī":103380,"dire":103381,"ĠNokia":103382,"ĠKarena":103383,"Ġìº":103384,"ä¸īçľģ":103385,"engk":103386,"两个å°ıæĹ¶":103387,"èħ¿éĥ¨":103388,"çħ¤å±Ĥ":103389,"Ġcodon":103390,"Ġ×Ķ×IJ×Ĺר":103391,"Ġeuph":103392,"ĠRicky":103393,"oggles":103394,"ãģ®ä¸Ńãģ§":103395,"Ġfiduciary":103396,"ìµľ":103397,"ourism":103398,"é¡Ķ":103399,"ä¸ľè·¯":103400,"ĠClan":103401,"羣çļĦ太":103402,"ĠSerra":103403,"ĠAntwort":103404,"马ä¸Ĭå°±è¦ģ":103405,"Ġpublique":103406,"å©·å©·":103407,"辩è¯ģæ³ķ":103408,"Difficulty":103409,"zust":103410,"ĠFruits":103411,"å¹´éī´":103412,"她åİ»":103413,"马éĩĮ":103414,"Ġaffluent":103415,"orthand":103416,".fetch":103417,"×ķ×ĵת":103418,"-Jones":103419,"Ġaffectionate":103420,"Ġdoubly":103421,")âĢĻ":103422,"ĠSIN":103423,"ĠKopf":103424,"åζæ³ķ":103425,"æ¯Ķ为":103426,"æĽ´è¿Ľä¸ĢæŃ¥":103427,"è¯¥åĽ½":103428,"áĢħ":103429,"åij³åĦ¿":103430,"Ġcooker":103431,"Ġtallest":103432,"ĠобнаÑĢÑĥжи":103433,"Ġoktóber":103434,"ĠMSC":103435,"Ġheres":103436,"htar":103437,"为群ä¼Ĺ":103438,"åıĹåĤ·":103439,"ĠEmmy":103440,"å¯Įäºİ":103441,"åıªè¦ģèĥ½":103442,"ãģ§ãģĻãģĭ":103443,"Ġhepatocellular":103444,"Organic":103445,"åѦçĿĢ":103446,"å°ıå±±":103447,"Ġbli":103448,"Ġresposta":103449,"å®ĥå°±":103450,"INING":103451,"ĠSchre":103452,"ĠContrary":103453,"çĬ¯äºº":103454,"IRD":103455,"ĠÑĢезÑĥлÑĮÑĤаÑĤов":103456,"ĠØ£ØŃÙħد":103457,".company":103458,"Ġconsommation":103459,"ĠÑĤеÑĢапи":103460,"Ġhuvud":103461,"T":103787,"wege":103788,"identally":103789,"Ġcellar":103790,"-circle":103791,"çĥĪçģ«":103792,"ĠCourage":103793,"rahydro":103794,"Ġbipartisan":103795,"prav":103796,"arbe":103797,"ĠNug":103798,"ä¸ĩç§ij":103799,"ĠÙģØ´Ø§Ø±":103800,"æĿ¾æĩĪ":103801,"pertensive":103802,"èĪĴç¼ĵ":103803,"åģ·ç¬ij":103804,"ÑīаÑĤÑĮ":103805,"ä»İå°ıå°±":103806,")\")Ċ":103807,"Morgan":103808,"gere":103809,"çļĦåĨħéĥ¨":103810,"Ġtoh":103811,"Ġjapon":103812,"ळ":103813,"decimal":103814,"Ġpoids":103815,"æ¡ģ":103816,"ĠeconomÃŃa":103817,"端åŃIJ":103818,"ä¹ĭåIJİå°±":103819,"宫éĩĮ":103820,"é«Ķç³»":103821,"Ġneutrino":103822,"Ġбел":103823,"ãģĿãĤĮãĤĴ":103824,"åı¯æĢľçļĦ":103825,"-De":103826,"-images":103827,"=input":103828,"çī©åĬĽ":103829,"mson":103830,"avec":103831,"æĶ¯æī¿":103832,"_detail":103833,"ĠØŃÛĮ":103834,"hunderts":103835,"ĠCoefficient":103836,"'Ar":103837,"ĠCoke":103838,"رÚĺÛĮ":103839,"ÙĪØªØ±":103840,"arka":103841,"Ġeram":103842,"ĠPlatz":103843,"æ£Ģåĩº":103844,"读åĩº":103845,"ĠMedications":103846,"å¥Ĺ管":103847,"RAINT":103848,"Ġতà§ģল":103849,"åIJĪçIJĨåľ°":103850,"ĠMagnesium":103851,"à®±à¯įà®ķ":103852,"åĪ©çµ¦":103853,":\")Ċ":103854,"Ġyummy":103855,"ä»ĸæĢİä¹Ī":103856,"天å°Ĭ":103857,"åĨį说äºĨ":103858,"ĠEnemy":103859,"Ġdigested":103860,"Ñģкими":103861,"èıľçļĦ":103862,"æķ´çIJĨäºĨ":103863,"Ġprodotti":103864,"adaptive":103865,"ĠЯндекÑģ":103866,"Ġসà¦Ĥà¦Ĺà§įরহà§ĩর":103867,"ĠSalisbury":103868,"çĦĸ":103869,"è·º":103870,"æĸĩåĪĽ":103871,"äºĮä¸ĸ":103872,"ecimal":103873,"æĪĺ绩":103874,"ĠاÙĦÙħرض":103875,"wherein":103876,"æĢªæĪij":103877,"ãģłãģ¨":103878,"FirstName":103879,"èĤ¡ä»½åζ":103880,"بÙĬÙĤ":103881,"åĦĴåѦ":103882,"Ġeradicate":103883,"Serve":103884,"Ġtipping":103885,"Ġmère":103886,"ĠWür":103887,"æľīä½į":103888,"Ġamigos":103889,"ieres":103890,"มà¸ŀ":103891,"亲çĶŁ":103892,"Ø«ÙħاÙĨ":103893,"æ²Ĵæľī人":103894,"éĻĦåĴĮ":103895,"Ġheaters":103896,"åīijæ°Ķ":103897,"Ġà°°":103898,"ĠMadras":103899,"ĠCicero":103900,"à¹Ģศรษà¸IJ":103901,"ĠÅij":103902,"orrelation":103903,"åľ°éĿ¢çļĦ":103904,"社ä¼ļæ²»çIJĨ":103905,"صائ":103906,"ĠпÑĢивÑĭ":103907,"ÙħاÙĬØ©":103908,"оÑĤоÑĢÑĭе":103909,"Ġà¦ħà¦Ĥশ":103910,"TRAN":103911,"è»įäºĭ":103912,"ĠNigel":103913,"மிழ":103914,"ĠCOMPANY":103915,"ĠاÙĦÙĩÙĨد":103916,"à²ķà³įà²":103917,"PW":103918,"oub":103919,"Ġkreat":103920,"ĠKnot":103921,"ÙħÙĬ":103922,"Ġdecad":103923,"ĠShiv":103924,"åij¨åΰ":103925,"éĩĩç͍çļĦæĺ¯":103926,"auri":103927,"èιåijĺ":103928,"ĠAnnu":103929,"-Ray":103930,"ĠLibert":103931,"Ġgladly":103932,"Ġcoexistence":103933,"Measurement":103934,"Ġaλ":103935,"ĠAeron":103936,"æľīä»·å̼":103937,"identification":103938,"è¿ĻäºĽéĥ½":103939,"ĠEmotions":103940,"(body":103941,"Ġnonex":103942,")$.":103943,"ĠValidate":103944,"pil":103945,"uire":103946,"åĴĮåĪĽæĸ°":103947,"æľĢãĤĤ":103948,"该书":103949,"å¼ķèĦļ":103950,"ĠCommittees":103951,"adders":103952,"êtes":103953,"èľ·":103954,"æľ«æľŁ":103955,"szág":103956,"Ġconceivable":103957,"ktionen":103958,"Ġorchestr":103959,";<":103960,"vendor":103961,"educt":103962,"çļĦèĩªå·±":103963,"ĠWarn":103964,"Ġorganising":103965,"Ġأج":103966,"Clark":103967,"éķĩçĹĽ":103968,"Ø«ÙĤ":103969,"Ġmerry":103970,"模åŀĭä¸Ń":103971,"ĠÚĨÙĨÛĮÙĨ":103972,"Ġprecipitated":103973,"-plugin":103974,"ëłĪìĿ´":103975,"Ġawakened":103976,"Ġdisguised":103977,"çĥŁèĬ±çĪĨ竹":103978,".They":103979,"\\Controller":103980,"ĨãĤ£":103981,"Ġtp":103982,"Ġwx":103983,"ĠSSS":103984,"Ġmein":103985,"ceding":103986,"chnen":103987,"Ġdecidedly":103988,"纵深":103989,"é³ĸ":103990,"丢å¼ĥ":103991,"Georgia":103992,"èĭ±éĩĮ":103993,"çķĻæľī":103994,".task":103995,"irming":103996,"ogenicity":103997,"åıij表æĹ¥æľŁ":103998,"-viol":103999,"ä¸Ģéģĵéģĵ":104000,"Ġnanoparticle":104001,"ãģ¨ãģªãĤĬãģ¾ãģĻ":104002,"ĠXCTAssert":104003,".awt":104004,"Ġoily":104005,"ĠWochen":104006,"ĠKiel":104007,"portal":104008,"weisen":104009,"åģ¥åº·çĬ¶åĨµ":104010,"æĽ¸è¨ĺ":104011,"ĠUlrich":104012,"Ġналогов":104013,"èľ¿èľĴ":104014,"ĠоÑģигÑĥÑĢаÑļе":104015,"ĠAks":104016,"åĴĮåİĨåı²":104017,"Ġshouts":104018,"Ġunidentified":104019,"éĻ¢èIJ½":104020,"æĶ¯åĩºçļĦ":104021,"Alle":104022,"å®ĭæ±Ł":104023,"Ġliar":104024,"Ġscripting":104025,"ĠÙģÙĩÙĪ":104026,"æºĿéĢļ":104027,"ĠVaughan":104028,"wali":104029,"ĠSloan":104030,"ĠMendoza":104031,"veau":104032,"ĠKran":104033,"æĹ¥ãģ®":104034,"è°ĥçļ®":104035,"Å¡ka":104036,"åı«ä½ľ":104037,"-states":104038,"ĠAccum":104039,"ĠμL":104040,"大è§Ħ模çļĦ":104041,"Nd":104042,"Ġinclusions":104043,"çļĦçĶ·åŃIJ":104044,"utm":104045,"Ġgör":104046,"Ġdeceive":104047,"ĠLies":104048,"Ġkultur":104049,"大巴":104050,"Ġeten":104051,"çŁ³æĿIJ":104052,"ĠÙĨشر":104053,"-grown":104054,"Ġvarie":104055,"bz":104056,"folder":104057,"{split":104058,"Ġforza":104059,"ihilation":104060,"給äºĪ":104061,"çĤ¼ä¸¹":104062,"ĠCDT":104063,"ponses":104064,".decode":104065,"Ġpantry":104066,"Ġdoenças":104067,"Ġimpoverished":104068,"abal":104069,"ĠRK":104070,"åı¯æĪij":104071,"ensively":104072,"社ç§ij":104073,"Ġfilaza":104074,"Ġconverters":104075,"çĹĽçĤ¹":104076,"ĠDepot":104077,"ilité":104078,"è¶ĬæĿ¥è¶Ĭé«ĺ":104079,"Ġsiebie":104080,"åĪĨå¸ĥçļĦ":104081,"Ġclarifying":104082,"æħĮå¿Ļ":104083,"ĠÛģÛĴ":104084,"Ġrelics":104085,".il":104086,"Ġais":104087,"æĸĵ":104088,"ĠDijk":104089,"æıIJæĮ¯":104090,"Ġdireitos":104091,"ãĤĤãģĤãĤĭ":104092,"çĿ¡å¾Ĺ":104093,"ç«¥åŃIJ":104094,"çĵľåŃIJ":104095,"WEEN":104096,"Ġнемного":104097,"åĴ§åĺ´":104098,"ĠнепоÑĤпÑĥним":104099,"%s":104100,"":104285,":G":104286,"ĠMHC":104287,"ä¸ĢèĬĤ":104288,"Ġyy":104289,"å¤ĸè²Į":104290,"INI":104291,"伤æĦŁ":104292,"åºĵéĩĮ":104293,"aphore":104294,"_TIME":104295,"ĠпомоÑīÑĮ":104296,"Criteria":104297,"Beginning":104298,"Ġconvol":104299,"Ġalde":104300,"åľ¨å½ĵåīį":104301,"æīĭæı¡":104302,"Ġemailed":104303,"设置æľī":104304,"Ġthermally":104305,"ĠÑĢабоÑĤаеÑĤ":104306,"ĠConsolidated":104307,"ĠоÑĤноÑģÑıÑĤÑģÑı":104308,"@app":104309,"TING":104310,"Ġome":104311,"ĠRt":104312,"ĠاÙģØª":104313,"Ġpreclinical":104314,"çĿĢèī²":104315,"Ġblush":104316,"éĹ®ä¸ĸ":104317,"COME":104318,"ç¡®å®ļ为":104319,"Ġlabelling":104320,"Ġpairwise":104321,"èĿ¦":104322,"Ġfingerprints":104323,"ĠDiesel":104324,"Millis":104325,"ĠапÑĢелÑı":104326,"Minn":104327,"nose":104328,"Ġcem":104329,"çļĦåŃ£èĬĤ":104330,"asilkan":104331,"大èĤĨ":104332,"Ġmeine":104333,"Ġiff":104334,"Ġ+(":104335,"å¸ĤæĶ¿åįı":104336,"×Ļ×ij×":104337,"ĠProficiency":104338,"ÑĢиÑĤÑĮ":104339,"ä¼ģä¸ļ对":104340,"ĠPlut":104341,"èIJ½èĦļ":104342,"ä¸ĥ彩":104343,".Att":104344,"ï½ı":104345,"Ġtomu":104346,"湿çĸ¹":104347,"âĿ·":104348,"Ġfootprints":104349,"èħĮåζ":104350,"idic":104351,"çĽĤ":104352,"ideon":104353,"Ġspruce":104354,"ĠAdel":104355,"窩":104356,"åĬŀäºĨ":104357,"สึà¸ģ":104358,"à¸Ħุ":104359,"หาà¸ģ":104360,"åŬ":104361,"åºĶç͍ä¸Ń":104362,"ведиÑĤе":104363,"社åĮºçļĦ":104364,"Ġshotgun":104365,"æĥ§æĢķ":104366,"Ġpancreatitis":104367,"Ġintermediaries":104368,"{:":104369,"enzo":104370,"Ġejection":104371,"ĠDop":104372,"Ġtransgress":104373,"äºĶåĪĨéĴŁ":104374,"ãĢį(":104375,"_copy":104376,"åĢŁåĬ©äºİ":104377,"Compared":104378,"Ġabandoning":104379,"ĠPEOPLE":104380,"ĠHazel":104381,"Ġgegenüber":104382,"Ġplagiarism":104383,"Ġbry":104384,"stern":104385,"ĠCPS":104386,"Ġdeven":104387,"æľī帮åĬ©":104388,"ĠGug":104389,"Ġì½":104390,"ç»´åħĭ":104391,"ĠFlame":104392,"èµĽè½¦":104393,"ĠÙĥÙĨ":104394,"ĠмаÑģÑĤеÑĢ":104395,"ĠاÙĦØŃرب":104396,"åIJ¯åĬ¨ä»ªå¼ı":104397,"ĠEncryption":104398,"ĠSERVICE":104399,"":104487,"éĴ¢ç»ĵæŀĦ":104488,"æķ¸åѸ":104489,"ĠSeminary":104490,"Ġmammary":104491,"Awesome":104492,"ĠteorÃŃa":104493,"ĠAmendments":104494,"-media":104495,"/is":104496,"IOR":104497,"Ġwicket":104498,"ĠRotation":104499,"èĢĮåĬªåĬĽ":104500,"зонÑĤа":104501,"å·¥ä½ľåİŁçIJĨ":104502,"麾":104503,"åİĨç»ĥ":104504,"æį¢ä¸Ĭ":104505,"æ»ijåĿĹ":104506,"Prev":104507,"ĠHelps":104508,"ÙĦاÙĬا":104509,"å·®å¼ĤåĮĸ":104510,"çīµè¿ŀ":104511,"è¿Ļéĥ¨åī§":104512,"ĠCHARACTER":104513,"Ġcomorbidities":104514,"Ġdizer":104515,"人影":104516,"éĩįåĽŀ":104517,"Ġskirts":104518,"Ġinsbesondere":104519,"åĩłå¼ł":104520,"Ġéx":104521,"ĠÙĨÙĬ":104522,"Ġ?>ĊĊ":104523,"èĢĥè¯ķæĪIJ绩":104524,"é¼»æ¶ķ":104525,"warf":104526,"ĠNSF":104527,"ĠвÑĭполн":104528,"Psychology":104529,".Res":104530,"KING":104531,"RUN":104532,"ulman":104533,"æķ°å¹´":104534,"Ġaccuse":104535,"Ġelas":104536,"Ġsonic":104537,"ĠподвеÑĢ":104538,"ĠÑĩелÑĥ":104539,"ĠBacillus":104540,"Ġfinanzi":104541,"çļĦèĥĮå½±":104542,"ĠCzy":104543,"inkl":104544,".mat":104545,"ç®ĢéĻĭ":104546,"çĸijåķı":104547,"Ġguarding":104548,"znym":104549,"Ġpropagating":104550,"à¹Ģà¸Ķิม":104551,"ราà¸ļ":104552,"è¾½éĺĶ":104553,"Ġsedimentation":104554,"Ġwszystkie":104555,"aer":104556,"IJ׾":104557,"anthrop":104558,"ĠTeg":104559,"igal":104560,"Ġmenores":104561,"è¶Ĭéķ¿":104562,"æĸ¹å¼ıæĺ¯":104563,"ÙĨدگاÙĨ":104564,"ammu":104565,"-hours":104566,".wait":104567,"Ġobligatory":104568,"éĴ»è¿Ľ":104569,"æķµäºº":104570,"Known":104571,"ĠSick":104572,"æľīåģ¿":104573,"Ġadicional":104574,"ĠاÙĦÙĪÙĦاÙĬات":104575,"çŃīæľīåħ³":104576,"Ġblister":104577,"åŃĹæł·":104578,"Ġbiographical":104579,"ojen":104580,"-quarters":104581,"ĉĠĠĠĠ":104582,"Ġire":104583,"ĠPorsche":104584,"ä¸į大äºİ":104585,"åľ¨ç͍":104586,"çľĭç͵影":104587,"жноÑģÑĤÑĮ":104588,"å̼æĺ¯":104589,"æŀĹåĩ¡":104590,"èĩªå·±çļĦåĬĽéĩı":104591,"Attack":104592,"ï¼Ŀï¼Ŀ":104593,"cfg":104594,"ĠÑĨик":104595,"æľīåĬĽåľ°":104596,"ĠEBITDA":104597,"Ġapprentice":104598,"ĠMERCHANTABILITY":104599,"Pow":104600,"çļĦéĥ½":104601,"çļĦèī²å½©":104602,"ĠNU":104603,"ä¸ŃçĶŁ":104604,"Ġcoached":104605,"äºļå½ĵ":104606,"δη":104607,"Ġìķł":104608,"éŃĤéŃĦ":104609,"ĠEpile":104610,"ĠPRACT":104611,"æĹºåŃ£":104612,"ĠCruc":104613,"Ġsailor":104614,"åĬ¨äººçļĦ":104615,"ä¸İå°ı":104616,"çł·":104617,"Ġscree":104618,"让人们":104619,"æ¸ħæ¸ħæ¥ļæ¥ļ":104620,"çľ¼éĥ¨":104621,"-produced":104622,"éĢļ常ä¼ļ":104623,"Ġdiverses":104624,"èĬ¬åħ°":104625,"маÑħ":104626,"é¦Ļæ²¹":104627,"Ġجاء":104628,"à¸Ħวà¸ļà¸Ħ":104629,"èģĺä»»":104630,"èī¾ä¼¦":104631,"åıĤè°ĭéķ¿":104632,"Ġhs":104633,"neu":104634,"她éĥ½":104635,"Ġchemin":104636,"elska":104637,"Ġcourte":104638,"Ġpredatory":104639,"secured":104640,"ä¼¯æł¼":104641,"èĤĿèĤ¾":104642,"Ġcomputationally":104643,"Û²Û°Û±":104644,"=\\)":104645,"É«":104646,"Ñĵ":104647,"Ġvyd":104648,"è¿Ľæ°Ķ":104649,"é«ĺç´łè´¨":104650,"ç¾İçϽ":104651,"kei":104652,"á»ħ":104653,"áncer":104654,"Ġdealership":104655,"ĠBreath":104656,"umbersome":104657,"欢è¿İ大家":104658,"ĠMidnight":104659,"ĠCEOs":104660,"Ġdreaded":104661,"ĠCreed":104662,"terror":104663,"ĠNost":104664,"Ġraped":104665,"éŁ³ç¬¦":104666,"Ġartistry":104667,"Ġidiopathic":104668,"ноÑģÑĤÑĢан":104669,"-amino":104670,"Ġuncontroll":104671,"ĠÑĢекомендÑĥеÑĤÑģÑı":104672,"Grace":104673,"ĠtÅĻÃŃ":104674,"ä¸Ģå°ģ":104675,"ĠKib":104676,"ä½łéĢĻ":104677,"åīįåįĬ":104678,"äºĶä½į":104679,"æĬķåIJij":104680,"éϤæķ°":104681,"çIJĥèĽĭçϽ":104682,"ĠاÙĦÙħÙĤاÙĦ":104683,"Ġlasci":104684,"èĥĨæ±ģ":104685,"åĨľæ°ijçļĦ":104686,"Ġprosecuted":104687,"Ġkurz":104688,"Ġextrinsic":104689,"ionate":104690,"ĠHN":104691,"ç¬ijåĵŃ":104692,"Ġwindy":104693,"å®ģå¸Ĥ":104694,"ÑĢовка":104695,"æĶ¾åľ¨å¿ĥä¸Ĭ":104696,"çĬ¯ç½ªåĪĨåŃIJ":104697,"ĠнапÑĢавлениÑı":104698,"ĠĠĠĠĠĠĠĠĊĊ":104699,"ĠBER":104700,"æĽ´æĸ¹ä¾¿":104701,"Ġdecays":104702,"ä¸ĵåijĺ":104703,"è´¹çİĩ":104704,"afx":104705,".card":104706,"åĩĢåľŁ":104707,"çĽ¸ä¿¡èĩªå·±":104708,"ĠÑģпек":104709,"Ġprzem":104710,"datum":104711,"纯粹çļĦ":104712,"%e":104713,"\\operatorname":104714,"_return":104715,"Ġ����":104716,"ĠSutherland":104717,"ĠCaus":104718,"ç͍å®ŀéĻħè¡ĮåĬ¨":104719,"ĠLeón":104720,"é¢ĦåºĶåĬĽ":104721,"ĠSheridan":104722,"Ġbullish":104723,"ĠìĹ°ê²°":104724,"Ġlenguaje":104725,"ĠtÄĽch":104726,"ĠSVM":104727,"atek":104728,"ĠFrey":104729,"ä»ĸæĺ¯ä¸Ģ个":104730,"说ä¸įåĩº":104731,"åħŃä¸ĥ":104732,"Ġborderline":104733,"ĠвоÑģп":104734,"ĠÑĤÑĢÑĥдов":104735,"Ġdischarging":104736,"elmÃ¤ÃŁ":104737,"Ġfaux":104738,"Ġyolk":104739,"ĠLoud":104740,"çģ«åħī":104741,"åºĶ该æľī":104742,"巴马":104743,"è¯Ĺä¸Ń":104744,"ìĤ¬ë¥¼":104745,"crime":104746,"Ġtrabalh":104747,"Ġreplicates":104748,"à®¾à®Łà¯įà®Ł":104749,"ĠÙĪØ²ÙĨ":104750,"/al":104751,"Ġà¹Ģม":104752,"illiam":104753,"Ġunethical":104754,"Ġdisdain":104755,"æŃ£åĪĻ":104756,"ĠUnlimited":104757,"满洲":104758,"éħ¸ç¢±":104759,"Ġgigabytes":104760,"çĪ·çη奶奶":104761,".Org":104762,"Trip":104763,"Ġtám":104764,"åı¯æĺ¯ä¸Ģ":104765,"ä¹Łè§īå¾Ĺ":104766,"Ġenth":104767,"ĠGuess":104768,"religious":104769,"ĠÙĥÙĨت":104770,"éĢ£æİ¥":104771,"eedback":104772,"ĠYounger":104773,"ç¾ŀè¾±":104774,"kowo":104775,"xA":104776,"league":104777,"ĠCout":104778,"åΰæĿ¥çļĦ":104779,"ä¸ĭ设":104780,"Ġrefrigeration":104781,"æŀĹåľ°":104782,"Ġnulla":104783,"Ġhonoured":104784,"æ°ijæĹıæĸĩåĮĸ":104785,"ĠConfidential":104786,"èĪĴéĢĤçļĦ":104787,"ĠNicholson":104788,"Ġsorg":104789,"ĠisValid":104790,"Ġkitten":104791,"Ġseconde":104792,"earning":104793,"Ġsoient":104794,"æĸ°çīĪ":104795,"Ġrestraints":104796,"èĤ¡æľ¬":104797,"Ġargon":104798,"åįķä½į为":104799,"ä»İèĢĮè¾¾åΰ":104800,"給她":104801,"äºĭä¸ļéĥ¨":104802,"uencias":104803,"ĠTuple":104804,"ĠAquinas":104805,"¶ģ":104806,"inox":104807,"æ¯ĶæĭŁ":104808,"äär":104809,"Ġdistracting":104810,"ĠZer":104811,"åıĸæĿIJ":104812,"æĹ¶éĹ´åİ»":104813,"æĪĺèΰ":104814,"çļĦäººåľ¨":104815,"-fuel":104816,"åħ«åįĥ":104817,"Ġসà¦ķল":104818,"è·ijæĿ¥":104819,"Ġindustrialized":104820,"Artikel":104821,"Certainly":104822,"$/":104823,"çļĦ表éĿ¢":104824,"èµ·èĪŀ":104825,"å¹²åĬ²":104826,"Ġdivider":104827,"миÑĢа":104828,"Ġcitoy":104829,"Ġfigs":104830,"èĪĴçķħ":104831,"ĠпÑĢеде":104832,"-dimethyl":104833,"Ġmonstrous":104834,"Ġwhim":104835,"æ³ķåľĭ":104836,"Ġsoal":104837,"åģµ":104838,"าà¸į":104839,"Ġesas":104840,"ç¥ŀåĨľ":104841,"å¸Ī妹":104842,"entionally":104843,"ĠUSING":104844,"ĠParade":104845,"Ùıر":104846,"åIJ¾å°Ķ":104847,"uwen":104848,"è¿Ŀ约éĩij":104849,"ZF":104850,"atype":104851,"Ġinco":104852,"ĠSES":104853,"odot":104854,"åĬ¨å¼¹":104855,"Ġsubnet":104856,"Ġ:)Ċ":104857,"殼":104858,"anyahu":104859,"主è¦ģé¢Ĩ导":104860,"åıĮè¯Ń":104861,"æ¯įçĮª":104862,"éĤ£ä¹Ī好":104863,"Ġmashed":104864,"ĠBrune":104865,"Ġattractiveness":104866,"ðĿIJº":104867,"æĮºæĭĶ":104868,"Ġconvened":104869,"ĠAlfonso":104870,"ĠобÑĬекÑĤа":104871,"Ġastonished":104872,"ĠÐŁÐ¾Ð¿Ð¸Ñģ":104873,"ĠBose":104874,"åΰè¿Ļ个":104875,"-shell":104876,"attach":104877,"Ġ}ĊĊĊĊ":104878,"åı³èĦļ":104879,"é²ľç¾İ":104880,"ĠBalanced":104881,"è¡°å¼±":104882,"लà¥Ģ":104883,"Ġklasy":104884,"ĠDIRECT":104885,"Ov":104886,"ĠICS":104887,"Ġcoefic":104888,"ç»Ħå§Ķä¼ļ":104889,"ĠZoe":104890,"ãģĻãģIJ":104891,"Ġidiosync":104892,"æĭħå¿ĥçļĦ":104893,"âijłâij¡":104894,"/profile":104895,"Ġleveraged":104896,"ENSION":104897,"è¿ĻäºĭåĦ¿":104898,"æĹłç§ģå¥īçĮ®":104899,"Ġsowohl":104900,"CHE":104901,"ĠMenge":104902,"Ġbye":104903,"geo":104904,"ä¸įæĺ¯åIJĹ":104905,"æĬķ篮":104906,"æĮīçIJĨ":104907,"ĠاÙĦÙħÙĥتب":104908,"èĢģå¸Ī说":104909,"Ġsuspicions":104910,"åħ¬åħ±åĪ©çĽĬ":104911,"Ġfacilitator":104912,"çĵ¶ä¸Ń":104913,"Ġreproducible":104914,"èı²åĪ©":104915,"ĠDanielle":104916,"Ġenormously":104917,"缺çĤ¹æĺ¯":104918,"bags":104919,"ĠAVA":104920,"antry":104921,"ĠYue":104922,"交æıĽ":104923,"à°¶":104924,"é¡¹çĽ®åĴĮ":104925,"äºĴåĪ©":104926,"帮çĿĢ":104927,"Figs":104928,"Ġczyt":104929,"Ġthirteenth":104930,"æĥħæĵį":104931,"ä¿Ŀæľī":104932,"æīĵåľ¨":104933,"liber":104934,"æŀĹèĤ¯":104935,"Ġreducir":104936,"EDMF":104937,"Clip":104938,"Ġtotalmente":104939,"è¯ĹçļĦ":104940,"Leader":104941,"Ġroadway":104942,"Ġsnaps":104943,"ÐĴÑĤ":104944,"Ġwaarbij":104945,"ç¼ĺçͱ":104946,"åĿłèIJ½":104947,"+:":104948,"Ġpóź":104949,"riosis":104950,"ĠKrit":104951,"cli":104952,"ĠSeat":104953,"Ġért":104954,"ĠCHANGE":104955,"Ġhinted":104956,"measured":104957,"qvist":104958,"onet":104959,"Ġstares":104960,"Ġglared":104961,"ç²¾æ°Ķ":104962,"Ġbiologists":104963,"ĠСÑĢ":104964,"èĥĮ离":104965,"ĠWeston":104966,"çļĦé«ĺéĢŁ":104967,"ĠSSH":104968,"ĠпÑĢиÑĩинÑĭ":104969,".Resource":104970,"åľ¨è¿Ļ次":104971,"кнÑĥ":104972,"åĽŀçļĦ":104973,"åįĹæŀģ":104974,"loed":104975,"-repeat":104976,"åĵŃç¬ij":104977,"Ġruby":104978,"ĠAdjustment":104979,"ĠNervous":104980,"quartered":104981,"Ġcálculo":104982,"Ġoblasti":104983,"ĠSten":104984,"Ġappare":104985,"åĩłå¹´åīį":104986,"liwe":104987,"EDER":104988,"ä»ħéĻIJäºİ":104989,"à¯ģள":104990,"binom":104991,"Ġwithdrawing":104992,"-termin":104993,"ĠhPa":104994,"ä½ľçŃĶ":104995,"ä¹ĭæľ¯":104996,"áĢ®":104997,"ä¸ĥä¸ĥ":104998,"ĠPreheat":104999,"æŃĮé¢Ĥ":105000,"Ġkilo":105001,"Ġunsupervised":105002,"马åħĭæĢĿæģ©æł¼æĸ¯":105003,"大åĬĽæİ¨è¿Ľ":105004,"Ġrivol":105005,"qc":105006,"ureen":105007,"perc":105008,"yser":105009,"ambiente":105010,"ĠнеÑĦ":105011,"紧缩":105012,"غاز":105013,"é¡¶ä¸Ĭ":105014,"'];ĊĊ":105015,"éĹŃå¡ŀ":105016,"guid":105017,"Ġscramble":105018,"EDMFunc":105019,"enan":105020,"égal":105021,"aterally":105022,"éĢļåĪĻ":105023,"åıĬåºĶç͍":105024,"Ġphishing":105025,"æŀĦæĥ³":105026,"-max":105027,"æľĥä¸įæľĥ":105028,"å¾ģæĸĩ":105029,"æĽ´å¤ļ人":105030,"èĴĻçī¹":105031,"Ġartefacts":105032,"ĠAlessandro":105033,"Įĵ":105034,"åı¯å¥¹":105035,"é«ĺç¨ĭ":105036,"spinal":105037,"ванием":105038,"åIJĥçĤ¹":105039,"à¸Īิ":105040,"嫦":105041,"Ġê°ĸ":105042,"Ġwiden":105043,"ĠFullEDMFunc":105044,"Ġamazingly":105045,"à¸ģัà¸ļà¸ģาร":105046,"ĠLagrangian":105047,"ocomplete":105048,"-ranked":105049,"Acknowledg":105050,"Ġbât":105051,"Ġprocur":105052,"ĠVod":105053,"æĬĬéĴ±":105054,"Ġdecrypt":105055,"å¦ĤæŀľéľĢè¦ģ":105056,"å¾·è¡Į":105057,"ziako":105058,"éģĶæĪIJ":105059,"Ġsekarang":105060,"ĠlÃłm":105061,"éķ¿æĹ¶éĹ´çļĦ":105062,"ĠسرطاÙĨ":105063,"润æ»ijæ²¹":105064,"ä¸Ńéķ¿æľŁ":105065,"æ³ĵ":105066,"Ġevils":105067,"稳稳":105068,"Ġмеж":105069,"Ġhairy":105070,"CLUDE":105071,"ĠÚ¯ÙĦ":105072,"ãģĪãģ¾ãģĻ":105073,"utterstock":105074,"ä¹Ķæľ¨":105075,"ĠPraha":105076,"æĸ°åĨłçĸ«æĥħ":105077,"ÅĦstw":105078,"ĠÙĪØ±Ø²Ø´":105079,"-empty":105080,".Any":105081,"zki":105082,"ä¸Ģ缮":105083,"ä¸įæĺ¯ä¸ºäºĨ":105084,"é¢Ħä¹ł":105085,"é£ŀåİ»":105086,"èĩªçĦ¶çݯå¢ĥ":105087,"ĠÐIJнд":105088,"olicies":105089,"å¤ļå°ij个":105090,"ç͵åŃIJä¿¡æģ¯":105091,"æĨĶ":105092,"ãĤ¢ãĤ¯":105093,"ĠBragg":105094,"Ġtriplet":105095,"Ġanglisy":105096,"Ġlaminated":105097,"(CH":105098,"[lower":105099,"Ġngaran":105100,"æķ°æį®ä¸Ńå¿ĥ":105101,"Getter":105102,"evolution":105103,"ä¸ĭéĻįåΰ":105104,"çĬ¯ç½ªè¡Į为":105105,"æģĴæĺŁ":105106,"Ġalarmed":105107,"ouin":105108,"Ġinmate":105109,"artifact":105110,"表ä¸ŃçļĦ":105111,"measures":105112,"arenta":105113,"ĠAppearance":105114,"éĿŀ常å¤ļ":105115,"Ġkinematic":105116,"Ġâĸ¶":105117,"ĠRESUM":105118,"Tokens":105119,"ĠвÑĢаÑĩ":105120,"éter":105121,"ĠUnc":105122,"ĠMead":105123,"Ġcreatinine":105124,"Ġprized":105125,"çĩİ":105126,"çİ©åĦ¿":105127,"èįĴåĶIJ":105128,"ĠÚ©ÙĨترÙĦ":105129,"ĠпеÑĢвÑĭÑħ":105130,"ĠاÙĦÙħرأة":105131,"Degree":105132,"é¡¿äºĨé¡¿":105133,"(search":105134,"heen":105135,"Ġlame":105136,"Ġvii":105137,"ĠBMP":105138,"æĹ³":105139,"æľīæĪij":105140,"åľ°çĽĺ":105141,"ecia":105142,"ãģĮãģĤ":105143,"ĠElk":105144,"Ġobservance":105145,"Interactive":105146,"软件çļĦ":105147,"ĠBarnett":105148,"ÅĪuje":105149,"VIRON":105150,"ĠAlejandro":105151,"^.":105152,"tro":105153,"ĠNissan":105154,"ahs":105155,"æĹłæĤĶ":105156,"ĠClint":105157,"æºIJåľ°":105158,"ಶ":105159,"ambiguous":105160,"Ġangst":105161,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":105162,"ratos":105163,"åĬŀåħ¬å®¤ä¸»ä»»":105164,"czyÄĩ":105165,"纪å§ĶçĽijå§Ķ":105166,"ĠBMJ":105167,"-used":105168,".yml":105169,"Were":105170,"labor":105171,"ĠGros":105172,"ĠKomb":105173,"yming":105174,"-gly":105175,"æľĿçļĦ":105176,"ĠSeriously":105177,"OLS":105178,"ĠOuts":105179,"ковой":105180,"Ġmultipliers":105181,")V":105182,"resi":105183,"Ġhavet":105184,"è¿ĩæĪij":105185,"å¾Īæ·±":105186,"ITCH":105187,"-prem":105188,"ĠСол":105189,"ĠÙĦÙĦس":105190,"ĠIslander":105191,"èī²å½©çļĦ":105192,"ĠÃĸsterreich":105193,"æµ·åįĹçľģ":105194,"(unsigned":105195,"arono":105196,"ĠMTV":105197,"Ġinterne":105198,"ajn":105199,"_____ĊĊ":105200,"ĠLongitudinal":105201,"(GL":105202,"oteric":105203,"ĠبÛĮÙħار":105204,"ĠFUNCTION":105205,"æĺ¯åIJĮ":105206,"æīĪ":105207,"okk":105208,"ovement":105209,"ANSW":105210,"åıĭ好çļĦ":105211,"Ġgeologic":105212,"夫åIJĽ":105213,"ĠMonteneg":105214,"Ġת×Ĺ":105215,"ĠFerreira":105216,"ĠдейÑģÑĤвий":105217,"buster":105218,"ĠIw":105219,"ä¸Ģåıij":105220,"åĴĮçľģ":105221,"åij»":105222,"å¯¹æľªæĿ¥":105223,"åıĹé¨ĵ":105224,"æĪĺç¥ŀ":105225,"éģĵ路交éĢļå®īåħ¨":105226,"车è¾ĨçļĦ":105227,"缸åıįçļĦ":105228,"ĠBartlett":105229,"ĠBBQ":105230,"åĺŁåĺŁ":105231,"ĠÙħÙĨØ·ÙĤØ©":105232,"Ġположение":105233,"ĠÑħÑĥдоже":105234,"inyl":105235,"Ġdunes":105236,"æĸĽ":105237,"ĠFoley":105238,"ĠWuhan":105239,"Ġperched":105240,"åĽ½åIJĽ":105241,"å®¶éŨåı£":105242,"æ³ķçŃī":105243,"æĸ°æĶ¿":105244,"Ġdemarc":105245,"วาม":105246,"ãĤĴç͍":105247,"Ġfinalized":105248,"ä½Ľåĥı":105249,"ĠÐĵÑĢа":105250,"Ġcrackers":105251,"Ġoatmeal":105252,"Ġexhilarating":105253,"=np":105254,"fÃŃ":105255,"ĠPett":105256,"ĠBö":105257,"**}":105258,"æīĵå®Į":105259,"èĬĤåζ":105260,"ĠзеÑĢ":105261,"úcar":105262,"ĠPreferences":105263,"æī§è¡Įå®ĺ":105264,"ĠPersonally":105265,"Ġenvelopes":105266,"ĠLepidoptera":105267,"å±Ĭä¸īä¸Ńåħ¨ä¼ļ":105268,"ĠRiding":105269,"è¿ĺè¡Į":105270,"æıIJéĢŁ":105271,"ONES":105272,"ktet":105273,"ĠизÑĥÑĩа":105274,"çī¹åΫ好":105275,"æĺ¾ç¤ºçļĦ":105276,"éħ¶çļĦ":105277,"Ġsadd":105278,"elor":105279,"adjusted":105280,"ä¸ĢæĽ²":105281,"ĠÃŀ":105282,"Ġreliant":105283,"å°ĨæĿ¥çļĦ":105284,"æ¡¿":105285,"ĠÙĦÙĥÙĦ":105286,"ĠساÛĮر":105287,"ç´§è·Ł":105288,"Ġsituação":105289,"Ġnaturales":105290,"åį°åζ":105291,"Ġmerid":105292,"().__":105293,"Ġgarrison":105294,"rachten":105295,"Ġhectometers":105296,"Ġincarcerated":105297,"bble":105298,"}z":105299,"atine":105300,"ĠKuz":105301,"ï¼ī-":105302,"æł¹çļĦ":105303,"Ġعص":105304,"quele":105305,"å¿ħ须以":105306,"åħ¶ä¸Ńä¹ĭä¸Ģ":105307,"logging":105308,"ULO":105309,"ĠConseil":105310,"ì³IJ":105311,"Ġmoor":105312,"ĠEre":105313,"ĠNLR":105314,"æĪij以åīį":105315,"大åIJ¼":105316,"ä¸īå°º":105317,"ĠTooth":105318,"ambi":105319,"ĠÙĨÙ쨱":105320,"AMI":105321,"ĠAnalyses":105322,"ĠобÑĢазование":105323,"ĠProcurement":105324,"Ġnatürlich":105325,"çļĦä¿¡å¿ĥ":105326,"Ġinvoking":105327,"æĪĺæľº":105328,"åİ¿å¿Ĺ":105329,"Ġpastures":105330,"两个åŃ©åŃIJ":105331,"ĠANT":105332,"åı¸æ³ķå±Ģ":105333,"å°½åı¯èĥ½åľ°":105334,"Ġinteressante":105335,"Ġziekte":105336,"utnya":105337,"æľīåĽĽ":105338,"Ġabl":105339,"gegeven":105340,"ä¸İæľ¬":105341,"åŃ¦ä¹łæĸ¹æ³ķ":105342,"ENTITY":105343,"æĢİä¹Īæł·äºĨ":105344,"रà¥įम":105345,"-added":105346,"Nin":105347,"Ġvyr":105348,"å°ıä¸ī":105349,"被çĽĹ":105350,"Ġcarve":105351,"è§ģæŃ¤":105352,"Ġнай":105353,"使ç͍çļĦæĺ¯":105354,"Ġpractised":105355,"лое":105356,"ĠHayden":105357,"ĠопеÑĢаÑĨии":105358,")ãĢģãĢĬ":105359,"XS":105360,"nol":105361,"Ġwelt":105362,"ä»®":105363,"ä¸ĢéĶ®":105364,"Ġprog":105365,"ĠLAT":105366,"Ġatlas":105367,"è¿Ļåī¯":105368,"Ġphased":105369,"ाà¤Ī":105370,"ĠFinch":105371,"Ġmisdeme":105372,"Ġirritating":105373,"é£²é£Ł":105374,"åµĮåħ¥å¼ı":105375,"Bloom":105376,"Ġrozwoju":105377,"Hans":105378,"hg":105379,"etCode":105380,"âĢĿï¼ģ":105381,"ä¸īèģĶ":105382,"è´Ńåħ¥":105383,"à¸Ĥà¸ĵะ":105384,"æ³ķå¾ĭæı´åĬ©":105385,"_mat":105386,"Ġâĸª":105387,"åįķä¸ĢçļĦ":105388,"Edition":105389,"Ġcpu":105390,"Ġbitten":105391,"Ġinexperienced":105392,"etro":105393,"uric":105394,"ĠвÑĸ":105395,"Ġmicroprocessor":105396,"åı¯æĺ¯ä»ĸ":105397,"ä¸įçŁ¥éģĵèĩªå·±":105398,"ĠDistinguished":105399,"æįŁå®³èµĶåģ¿":105400,"_REQUEST":105401,"çĸ¤çĹķ":105402,"ZM":105403,"ä¸Ģä¸įå°ıå¿ĥ":105404,"å¤ļä¸ĩåħĥ":105405,"éĤ£åı¥":105406,"åıªåĽł":105407,"ÙĥÙİ":105408,"arsch":105409,"Ġscrewed":105410,"ĠاÙĦØŃÙĥÙĪÙħ":105411,"ĠÙĦÙĦÙĨ":105412,"碰ä¸Ĭ":105413,"åIJĦ个çݯèĬĤ":105414,"çļĦåľ°çĤ¹":105415,"levels":105416,"patterns":105417,",......":105418,"rj":105419,"Ġfumes":105420,"owano":105421,"åΰä¸Ģèµ·":105422,"çħ§çĿĢ":105423,"åĸľè¿İ":105424,"ihi":105425,"é¼ĵåĭµ":105426,"åĪĽå»ºå·¥ä½ľ":105427,"Ġбио":105428,"Statistical":105429,"Ġìĸ¸ìĸ´":105430,"Kent":105431,"Ĺ×Ļ×Ŀ":105432,"â̹":105433,"ĠAlarm":105434,"æīĵåĪĨ":105435,"æĶ¶è§Ĩ":105436,"Ġprofond":105437,"ĠبÙĩتر":105438,"-Four":105439,"Ġcomponentes":105440,"éĶĢåĶ®éĩı":105441,"Ġliquef":105442,"ÙĬÙħÙĬ":105443,"Ġpetitioners":105444,"åĿŁå¢ĵ":105445,"\"।":105446,"Ġdps":105447,"ĠCada":105448,"ĠKall":105449,"å·¥çļĦ":105450,"ç±»èį¯çī©":105451,"åı·ä¸º":105452,"ĠاÙĦÙħعد":105453,"Ġcondenser":105454,"ĠPolo":105455,"ä¹ĭéĹ´åŃĺåľ¨":105456,"Ġdrawers":105457,"canvas":105458,"ìĭľê°Ħ":105459,"åĤ»çĵľ":105460,"Ġfresco":105461,"ĠCONCLUSIONS":105462,"ĠTrie":105463,"ä¸įä»İ":105464,"Ġchic":105465,"Ġprer":105466,"Ġinterrelated":105467,"ä»Ģä¹Īéĥ½æ²¡æľī":105468,"æŁ¥çIJĨ":105469,"ĠAPPE":105470,"Ġolives":105471,"Ġglucocortic":105472,"éĸ¢éĢ£":105473,"Ġ_________":105474,"ĠAufgabe":105475,"é»ĺé»ĺçļĦ":105476,"à§įদà§įর":105477,"Ġinterchangeably":105478,"Pra":105479,"ĠBorders":105480,"ĠBootstrap":105481,"ĠHare":105482,"ĠSchiff":105483,"Ġbiochemistry":105484,"arrer":105485,"Ġberry":105486,"ÙĦاÙĥ":105487,".resize":105488,"\\+\\_\\+":105489,"ĠngOnInit":105490,"=<":105491,"HCO":105492,"Nz":105493,"Ġaes":105494,"Ġseams":105495,"å¦Ĥæŀľä¸įèĥ½":105496,"åıijçĶŁçİĩ":105497,"éĻįä½İæĪIJæľ¬":105498,"лекÑĤÑĢо":105499,"æİ¥è¿ijäºİ":105500,"Ġmehrere":105501,"Ġjewellery":105502,"ĠÙĪØ¹ÙĦÙī":105503,"Ġangiography":105504,"Ġgird":105505,"人ä¼ļ":105506,"Ġgenerality":105507,"ĠPrima":105508,"Ġcollide":105509,"çĥĪæĹ¥":105510,"Ġdarkened":105511,"Ġ×IJ×ķ×ŀר":105512,"ä¹Ļéħ°":105513,"ImageView":105514,"ĠTaxonomy":105515,"лÑĭм":105516,"Ġdysplasia":105517,"Ġjewels":105518,"ĠнаблÑİда":105519,"Ġstabbed":105520,"Ġneurotransmitter":105521,"سطس":105522,"ĠLark":105523,"ĠHowell":105524,"ĠзапÑĥ":105525,"emptive":105526,"Ġdimethyl":105527,"guess":105528,"纵è§Ĥ":105529,"åĭĴæĸ¯":105530,"ĠBernie":105531,"ĠпоÑĤÑĢеби":105532,"ĠâĶľ":105533,"ĠtvÃ¥":105534,"ĠwartoÅĽci":105535,"Ġlaatste":105536,"çļĦå®£ä¼ł":105537,"ĠPus":105538,"formin":105539,"åĨį审":105540,"åIJ¬ä¸įæĩĤ":105541,"æľįåĬ¡æ°´å¹³":105542,"-coding":105543,"à¥įà¤Ń":105544,"ĠPreface":105545,"justice":105546,"ĠÐĹдеÑģÑĮ":105547,"μοÏĤ":105548,"çĪ¬èµ·æĿ¥":105549,"ĠNigerians":105550,"ĠInitiatives":105551,"ĠÑĢайон":105552,"========================================================================":105553,"Sant":105554,"nights":105555,"Ġwody":105556,"ĠnÄĥm":105557,"åħ¨é¢Ŀ":105558,"Ġflaming":105559,"ãģŁãĤī":105560,"è¿Ļä¸ĢæĹ¶æľŁ":105561,"ç§»é»ĺ":105562,"ĠCompiled":105563,"ä¹ŁæľīæīĢ":105564,"Ġunspecified":105565,"Ġdwind":105566,"æģ¢å¤įåΰ":105567,"Ġapartheid":105568,"Ġdilat":105569,"ordenatuak":105570,"anggap":105571,"Ġlaparoscopic":105572,".TabIndex":105573,"Fest":105574,"igas":105575,"Ġdoel":105576,"ĠполÑĮзÑĥ":105577,"çŃīåįķä½į":105578,"ï¼ģï¼ģâĢĿĊĊ":105579,"å®īåį±":105580,"åįĬåĪĨ":105581,"ç¦ıå¾·":105582,"ĠAngus":105583,"NumberOf":105584,"Ġszem":105585,"ĠContractor":105586,"Ġunleash":105587,"Berg":105588,"Xt":105589,"_command":105590,"arren":105591,"ĠSich":105592,"群èIJ½":105593,"Clone":105594,"æĬ¢å¤º":105595,"ĠAudrey":105596,"ç»§æī¿äºĨ":105597,"Ġpacient":105598,"Ġcrowns":105599,"provide":105600,"Ġimpecc":105601,"ĠÑģказаÑĤÑĮ":105602,"ĠICM":105603,"segment":105604,"Ġkebutuhan":105605,"å¤ļåıij":105606,"æ±ĤçĶŁ":105607,"士åįĴ":105608,"æīįèĥ½ä½¿":105609,"βο":105610,"ĠUNITED":105611,"posted":105612,"åĽĽä¸ªæĸ¹éĿ¢":105613,"(NO":105614,"_ALL":105615,"ĠDome":105616,"åıªè¦ĭ":105617,"çĬ¶è¯Ń":105618,"Isn":105619,"åĨ¬èĩ³":105620,"çļĦå½±åĵįåĬĽ":105621,"à¹Īาà¸Ļัà¹īà¸Ļ":105622,"oclass":105623,"Ġtypedef":105624,"âĴ":105625,"Ġsagen":105626,"ĠArag":105627,"Ġyks":105628,"phans":105629,"зÑĸ":105630,"è·¯åŃIJ":105631,"ĠпоÑĢа":105632,"ĠEditions":105633,"æĿ̿ή":105634,"åį±éĻ©æĢ§":105635,"Ġ$$Ċ":105636,"Ġserialize":105637,"ÑģÑĤÑĥплениÑı":105638,"pause":105639,"ä¸Ģæ°Ķ":105640,"è°¤":105641,"æľĢå¼Ģå§ĭ":105642,"Ġjustices":105643,"ç§ijå°Ķ":105644,"ĠScouts":105645,"è¸ī":105646,"Ġש׳×Ļ×Ŀ":105647,"Ġreflexes":105648,"ç²¾ç¥ŀæĸĩæĺİ建设":105649,"LISH":105650,"ä½ĥ":105651,"ĠOch":105652,"zeh":105653,"ĠAppend":105654,"åݿ人æ°ijæĶ¿åºľ":105655,"ĠÙĥØ«":105656,"Ġবির":105657,"ĠÑĤелеÑĦон":105658,"Ġpytest":105659,"ä¸ĢæĻĥ":105660,"iei":105661,"ciÄħ":105662,"ĠномеÑĢ":105663,"å¸Ĥä¸Ń":105664,"олÑİ":105665,"Ġرابط":105666,"å·´æĭī":105667,"ĠTransformer":105668,"ellett":105669,"ানà§ĭ":105670,"ĠUkrain":105671,"Ġligaments":105672,"æī¹åĩĨçļĦ":105673,"ãĥįãĥĥãĥĪ":105674,"ként":105675,"ĠSpotlight":105676,"niejsze":105677,"ĠBurgess":105678,"Ġhypothalamus":105679,"Ġtb":105680,"ĠFiona":105681,"Ġleaching":105682,"ijos":105683,"анг":105684,"DPI":105685,"ĠÄįlov":105686,"Ġkillers":105687,"Ġcommissioning":105688,"Ġhospice":105689,"Koordenatuak":105690,"Ġjulio":105691,"ĠðĿľ":105692,"ĠPLEASE":105693,"ĠEusk":105694,"ä¼łæĿ¥äºĨ":105695,"Ġresta":105696,"Ġsiete":105697,"èŀ¨":105698,"æ¿Ģè¿Ľ":105699,"åı¦ä¸ĢåĢĭ":105700,"ĠìĻķ":105701,"Ġaptitude":105702,"Ġlignin":105703,"Ġunifying":105704,"çĶŁåľ¨":105705,"دÛĮد":105706,"好åķ¦":105707,"æĥ³ä½ł":105708,"åĪĻçͱ":105709,"èįīçļĦ":105710,"বà§ĩন":105711,"Ġgranddaughter":105712,"achev":105713,"åıªèĥ½è¯´":105714,"éĢļå¸¸åľ¨":105715,"ئات":105716,"Ġtakich":105717,"ளà¯Ī":105718,"تÙĬجة":105719,"à®®à¯įப":105720,"Ġgrips":105721,"åĬ´åĥį":105722,"goto":105723,"haupt":105724,"ĠLec":105725,"isecond":105726,"Ġregel":105727,"åıĬåIJĦ":105728,"åıªç®¡":105729,"æ¯Ķæ¯Ķ":105730,"æĬĬæĪijçļĦ":105731,"venirs":105732,"ลà¹īà¸Ńม":105733,"ĠзаÑĤ":105734,"审å®ļ":105735,"_filename":105736,"Ġalternativa":105737,"casts":105738,"ª×ŀש":105739,"ково":105740,"ç¦ħå¸Ī":105741,"åºŁå¼ĥçī©":105742,"olaryng":105743,"ĠBout":105744,"ä¹ĭ计":105745,"没说":105746,"Ġhumankind":105747,"åĨĽä¸Ń":105748,"ĠRepublik":105749,"Ġadjusts":105750,"zieh":105751,"ĠExpend":105752,"Ġsickle":105753,"çŃ¾è®¢çļĦ":105754,"Ġmagnetization":105755,"Ġinquired":105756,"Ġsluggish":105757,"donald":105758,"xv":105759,"itty":105760,"Ġprou":105761,"å°±çŃīäºİ":105762,"ä¹Łå¤ļ":105763,"ovar":105764,"Ġzape":105765,"Ġbioge":105766,"Ġdocente":105767,"Beck":105768,"______________________________":105769,"à¥ĭऽ":105770,"ĠCardiology":105771,"ãĤĤãģ®ãģ§ãģĻ":105772,"ĠKristen":105773,"ĠÑĸн":105774,"Ġhistórico":105775,"Ġimplica":105776,"Ġiniciativa":105777,"Joint":105778,"kraft":105779,"ĠHike":105780,"åľ¨éĢĻ裡":105781,"Ġclassifiers":105782,"çĸĿ":105783,"åıĪæĥ³":105784,"ĠExeter":105785,"ä¹¦çĽ®":105786,"äºīæĸĹ":105787,"contacts":105788,"ä¹Ŀæ±Ł":105789,"åºĹå®¶":105790,"ÐŁÐµÑĢ":105791,"æ®Ĭä¸įçŁ¥":105792,"Ġcatchy":105793,"æĸĩæĺİå®ŀè·µ":105794,"èħIJæľ½":105795,"-limiting":105796,"ilidad":105797,"ä¸ĢæĹłæīĢ":105798,"ä¸ĢæĢĶ":105799,"Ġusia":105800,"Ġbusinessmen":105801,"Ġcrumbs":105802,"åĭķåĬĽ":105803,"绿åı¶":105804,"unker":105805,"Ġrapidement":105806,"Ġrainwater":105807,"åĩŃ空":105808,"ĠTorino":105809,"ĠShelby":105810,"ĠErm":105811,"Ġseura":105812,"Ġrogue":105813,"åij¨çļĦ":105814,"马桶":105815,".getMessage":105816,"expand":105817,"integr":105818,"ÃŃcÃŃch":105819,"ä¸Ģ大æĹ©":105820,"ã썿ĢĿãģĨ":105821,"Ġpuncture":105822,"ĠPhenomen":105823,"Oi":105824,"_option":105825,"cic":105826,"mberg":105827,"Ġbekerja":105828,"主張":105829,"ĠбеÑĤ":105830,"-talk":105831,"empuan":105832,"hasil":105833,"Ġsuitcase":105834,"åĦĺ管":105835,"ĠÑħолод":105836,"-na":105837,"Ġscler":105838,"stva":105839,"æµľ":105840,"å¹´å¤ľ":105841,"ghum":105842,"æĹ¥æ¶Īæģ¯":105843,"Ġfréqu":105844,"åĩºçݰéĹ®é¢ĺ":105845,"æĸĩä»¶åĴĮ":105846,"Ġmatchup":105847,"ĠRaise":105848,"çĻºè¡¨":105849,"íĮħ":105850,"ĠWoolf":105851,"ystyrene":105852,"ĠRai":105853,"ÑĢока":105854,"å¤ļç±³":105855,"Ġ+#":105856,"ĠAnast":105857,"æ±Ĥå©ļ":105858,"æĢ»èĢĮè¨Ģä¹ĭ":105859,"arno":105860,"ä¸ŃåĽ½ç¤¾ä¼ļç§ijåѦ":105861,"èĬ±å²Ĺ":105862,"biological":105863,"åħģ許":105864,"LastName":105865,"าà¸Ĭà¸Ļ":105866,"åĵ¥ä¼¦æ¯Ķäºļ":105867,"Ġstump":105868,"rowed":105869,"ĠXYZ":105870,"attia":105871,"åĨĽç͍":105872,"ĠÙĩÙī":105873,"èĶ»":105874,"Ġburge":105875,"æĤīå°¼":105876,"Ġeclectic":105877,"æ¼ıæĸĹ":105878,"ĠActiveRecord":105879,"Ġnestled":105880,"Ġsquadron":105881,"consulté":105882,"ÙħÙĤاÙĦÙĩ":105883,"leon":105884,"ĠEhr":105885,"ĠFilipp":105886,"selection":105887,"ĠKish":105888,"Ġprett":105889,"ç¥ŀéŃĤ":105890,"æĢ»ä¸įèĥ½":105891,"Ġvolumen":105892,"ĠرÙĪØ¯":105893,"Ġconcentric":105894,"Ġinspectors":105895,"Ġmediums":105896,"Ġbulls":105897,"Ġrepublican":105898,"實éļĽä¸Ĭ":105899,"Ġpamphlet":105900,"stal":105901,"unia":105902,"ĠPew":105903,"æĪijæŃ£åľ¨":105904,"大æĢĴ":105905,"å°±å¤ŁäºĨ":105906,"Ġ{/*":105907,"åľ°èªª":105908,"便æIJº":105909,"Ġbenches":105910,"UTES":105911,"umbuhan":105912,"ÐŁÐµÑĢе":105913,"λλα":105914,"ccal":105915,"é«ĺ产":105916,"建åįİ":105917,"常ä½ı":105918,"羣æĥ³":105919,"æĭ¿åĩºäºĨ":105920,"æ²īå¯Ĥ":105921,"ĠDeco":105922,"â̲)":105923,"æ¸IJåıĺ":105924,"expressed":105925,"缩åĩı":105926,"åļı":105927,".findAll":105928,"åľĺé«Ķ":105929,"propylene":105930,"è°ħè§£":105931,"ĠnM":105932,"Ġredefine":105933,"ĠMif":105934,"æ°´åĬ¡":105935,"Ġxu":105936,"Ġدائ":105937,"åĿĩåºĶ":105938,"Ġ×ij×ĸ":105939,"Ġpleural":105940,"ĠìĿ´ë£¨":105941,"Ġontwikkeling":105942,"ĠBevölker":105943,"ZB":105944,"vars":105945,"Ġmeadows":105946,"æŃ¤è¨Ģ":105947,"åıįèħIJè´¥":105948,"å¢ŀåİĭ":105949,"ALES":105950,"åı¶å¤©":105951,"æĽ²åŃIJ":105952,"師çζ":105953,"Ġê³³":105954,"çĤ¸èį¯":105955,"Ġprzeb":105956,"×IJ×Ļ":105957,"_settings":105958,"difference":105959,"stel":105960,"ĠBrowning":105961,"Ġcreación":105962,"ç¬ijåĺ»åĺ»":105963,"Ġexcursions":105964,"Ġmolé":105965,"/th":105966,"ZC":105967,"ieÅĦ":105968,"æķĻå§Ķ":105969,"éŨä¸Ĭ":105970,"æĮģä¹ĭ以":105971,"é£İå°ļ":105972,"èİħ":105973,"overning":105974,"Ġsupermarkets":105975,"Ġprofessores":105976,"Ġspecialties":105977,"ĠParte":105978,"gyz":105979,"æŃ£å¸¸è¿IJè¡Į":105980,"umerate":105981,"Ġsynapses":105982,"Ġhabitantes":105983,"ĠSignals":105984,"赫å°Ķ":105985,"ĠترÙĥ":105986,"'Am":105987,"ĠEch":105988,"åΰéģĶ":105989,"ágenes":105990,"æł¡å¯¹":105991,"Ġumbil":105992,"鹦":105993,"ãģ¦ãģĦãģªãģĦ":105994,"森æŀĹåħ¬åĽŃ":105995,"Ġproduto":105996,"à¸ŀรà¹īà¸Ńม":105997,"èĺĭæŀľ":105998,"(status":105999,".InputStream":106000,":b":106001,"BERS":106002,"esson":106003,"),[":106004,"Ġarty":106005,"æľºæĪ¿":106006,"×Ļ×ŀ×Ļ×Ŀ":106007,"Ġsco":106008,"Revised":106009,"Ġinfe":106010,"èİ·æī¹":106011,"Ġaccountants":106012,"Ġquieter":106013,"Ġcampaigning":106014,"éĽĨä¸Ńäºİ":106015,"áĢºáĤ":106016,"Ġvineyard":106017,"Ġkasag":106018,"arendra":106019,"Fern":106020,"ĠCrest":106021,"æľīæĺİæĺ¾":106022,"ĠUppsala":106023,"对身ä½ĵ":106024,"æµ·æ·Ģ":106025,"Ġtestes":106026,"çłĶåѦ":106027,"ĠPrat":106028,"Ġcondizioni":106029,"ĠоÑĤÑģ":106030,"踵":106031,"OPE":106032,"è´¦åįķ":106033,"หà¸Ļà¹Īวย":106034,"åIJĮåѦ们çļĦ":106035,"æĿijæ°ij们":106036,"æĹłæķ°æ¬¡":106037,"éĵĥ声":106038,"emment":106039,"äºĨåĩºä¾Ĩ":106040,"Ġquarry":106041,"ĠCalcutta":106042,"ĠØ®ÙĪØ§ÙĨ":106043,"ĠMarta":106044,"çĶľç¾İ":106045,"gré":106046,"æĬĽåĩº":106047,"å¼Ĺåħ°":106048,"Ġ×Ķ×¢×ķ׾×Ŀ":106049,"ĠInformal":106050,"imide":106051,"ĠCri":106052,"ĠKond":106053,"Ġzit":106054,"ecal":106055,"主è¦ģåİŁåĽł":106056,"esehen":106057,"(train":106058,"_non":106059,"宫çļĦ":106060,"imbledon":106061,"Ġ×Ĺ×Ļ×Ļ×Ŀ":106062,"åħ¬å®īéĥ¨":106063,"batis":106064,"CREMENT":106065,"ĠпÑĢогÑĢамм":106066,"Ġmistakenly":106067,"Victoria":106068,"Courses":106069,"pail":106070,"大çĹħ":106071,"é«ĺçĥŃ":106072,"Ġæ":106073,"æĸ¹æ³ķ论":106074,"bladder":106075,"ä»»ä½ķæĹ¶åĢĻ":106076,"积æŀģåľ°":106077,"åįĸçļĦ":106078,"ĠRadar":106079,"Ġontological":106080,"åĵ¼åĵ¼":106081,"Ġundermining":106082,"ĠBrewer":106083,"Republican":106084,"é½IJå¿ĥåįıåĬĽ":106085,")i":106086,"ĠWD":106087,"ä½ľåĵį":106088,"Ġdisabling":106089,"è·¤":106090,"Ñĩке":106091,"æĹłåĬŁ":106092,"æĻĤçļĦ":106093,"Ġnoviembre":106094,"èĨľçļĦ":106095,"ĠSamson":106096,"Ġrulings":106097,"ä¸īè§Ĵæ´²":106098,"CAM":106099,"}',":106100,"ĠSrin":106101,"akings":106102,"大河":106103,"对æķ´ä¸ª":106104,"å¹´å¹¼":106105,"她便":106106,"ä½İéĢŁ":106107,"èĭıå®ģ":106108,"åĢĴåľ°":106109,"Ġgraphically":106110,"Ġútil":106111,"Ġrupees":106112,"çī§åľº":106113,"anthus":106114,"Ġvineyards":106115,"(Context":106116,"Ġhires":106117,"ä¸įä¸ĭåİ»":106118,"äºĨ声":106119,"Ġnewfound":106120,"Ġsuppressor":106121,"èĢĥåīį":106122,"meier":106123,"ÏĢÎŃ":106124,"Ġcausas":106125,"viamente":106126,"Ġcontraind":106127,"áĥĿáĥľ":106128,"ĠدرÛĮاÙģØª":106129,",U":106130,"_term":106131,"bole":106132,"warning":106133,"udget":106134,"Ġclases":106135,"ä½łä»Ĭ天":106136,"éħįéŁ³":106137,"追æĿĢ":106138,"åĭķæīĭ":106139,"ÐļТ":106140,"ନ":106141,"Ġscreenings":106142,"ĠáĥĹ":106143,"Whereas":106144,"VPN":106145,"authors":106146,"ĠFaces":106147,"çĶŁçĶ£":106148,"ÑıÑĢ":106149,"说åΰåºķ":106150,"å¼Ģè£Ĥ":106151,"åħ¥èĤ¡":106152,"çĹ¿":106153,"æĶ¶è´§":106154,"ç±»æİ¨":106155,"çĮĸ":106156,"æĿİäºij":106157,"-Med":106158,"Ġದ":106159,"Ġrepetitions":106160,"Çİo":106161,"ĠCanton":106162,"Ġethnographic":106163,"Ġclerical":106164,"æ¯ĭ庸":106165,"ĠCohort":106166,"æī«é»ijéϤæģ¶":106167,"Ġtast":106168,"çļĦå§¿æĢģ":106169,"ĠHalle":106170,"èĩªä»¥ä¸º":106171,"æĪij们è¿ĺæĺ¯":106172,"ç¾İ满":106173,"ĠNotFound":106174,"ç»ĵæŀĦä¸İ":106175,"æīįèĥ½åľ¨":106176,"Ġپاسخ":106177,"ĠOutreach":106178,"åįģåĪĨéĩįè¦ģ":106179,"ĠëĮĢìĥģ":106180,"ä¾į女":106181,"ĠпÑģиÑħи":106182,"åľ£è¯ŀèĬĤ":106183,"äºĨåı£æ°£":106184,"drug":106185,"eric":106186,"ä¸ĢéĹ®":106187,"Ġkét":106188,"åı¯è´µ":106189,"ĠKirst":106190,"ĠاÙĩ":106191,"æĶ¶ç´§":106192,"æħµ":106193,"ĠدÙĨداÙĨ":106194,"主è¦ģ表çݰ为":106195,"è¡£è¢ĸ":106196,"稳åİĭ":106197,"Ġfaible":106198,"Ġmoderna":106199,"Ġ×ij׾×":106200,"UIKit":106201,"éģ¥è¿ľçļĦ":106202,"ĠTalks":106203,"ĠReturning":106204,"rupal":106205,"ç¾ħæĸ¯":106206,"-peer":106207,"Ġlze":106208,"uny":106209,"ĠPOW":106210,"ä¸Ĭ好":106211,"ÑĩÑĤ":106212,"Ġzim":106213,"èİ«æµĭ":106214,"ĠGrü":106215,"리ìĬ¤":106216,"Ġcolonel":106217,"æľīä»Ģä¹Īäºĭ":106218,"wiata":106219,"Ġaerodynamic":106220,"Ġvraiment":106221,"Ġculmination":106222,"/form":106223,"ĠFRE":106224,"æľīæĻĤ":106225,"Ġkho":106226,"ä»ĸæĿ¥":106227,"æ®ĥ":106228,"交æĦŁ":106229,"ä¸ŃåĽ½æĶ¿åºľ":106230,"åįĹå¼Ģ":106231,"åij¼å£°":106232,"ĠMatlab":106233,"à±įà°ª":106234,"ĠاÙĦصÙĨ":106235,"èŁ¾":106236,"檢測":106237,"輸åĩº":106238,"Tokyo":106239,"ĠCrowley":106240,"Ġbends":106241,"ĠAlley":106242,"竳çļĦ":106243,"ĠÑĤвеÑĢ":106244,"Ġradially":106245,"ĠBaroque":106246,"çĺ¦èĤī":106247,"ĠDowns":106248,"Ġcontrôle":106249,"è§ĴèIJ½éĩĮ":106250,"ĠpoczÄħt":106251,"Ġphysicists":106252,"Ġতà§Īরি":106253,"(add":106254,"baby":106255,"اÙĥÙĦ":106256,"Ġconex":106257,"ĠChop":106258,"inken":106259,"Ġinvaders":106260,"è´¨éĹ®":106261,"ĠSpinal":106262,"ç»´åIJ¾å°Ķ":106263,"åºĹ主":106264,"Ġsavvy":106265,"ĠADS":106266,"***ĊĊ":106267,"ĠÑĢекоменда":106268,"á¿ĸÏĤ":106269,"_body":106270,"zure":106271,"reys":106272,"Ġsø":106273,"Ġdext":106274,"ĠLage":106275,"对ä¸ĢäºĽ":106276,"Ñĩено":106277,"ĠSprach":106278,"è¡Ģç¼ĺ":106279,"lingu":106280,"enca":106281,"èµĦæºIJåħ±äº«":106282,"upported":106283,"γÏī":106284,"Ġ×ij×Ļ":106285,"ä¸Ĭä¸ĭåĬŁå¤«":106286,"éĨ«å¸«":106287,"Ġllevar":106288,"ĠÑģоглаÑģно":106289,"(models":106290,"stelle":106291,"ĠSEL":106292,"ĠAAI":106293,"ĠHarcourt":106294,"ĠVEGF":106295,"æĭĹ":106296,"ĠStain":106297,"éĢļç͍çļĦ":106298,"ĠPlanned":106299,"ĠNotwithstanding":106300,"鼨ä¸Ń":106301,"Ġdiminu":106302,"Ġzeit":106303,"Artigo":106304,"å¾Ĺåĩºç»ĵ论":106305,"Ġexpeditions":106306,"ĠSorting":106307,"lipid":106308,"gui":106309,"íį¼":106310,"Ġpozy":106311,"Ġsimile":106312,"åIJ¬åIJİ":106313,"észet":106314,"å·´æĸ¯":106315,"Ġnovas":106316,"ä¼ļè®®çļĦ":106317,"奥çī¹":106318,"Ġsubtly":106319,"è¡°èIJ½":106320,"ĠBotanical":106321,"Ġíĺķíĥľ":106322,"bardziej":106323,"å®īä¸ľå°¼":106324,".access":106325,"Zw":106326,"ÅĨ":106327,"对æĪij们çļĦ":106328,"éĩijé»Ħ":106329,"Ġwatery":106330,"åıĤåĨĽ":106331,"æ½¢":106332,"Ġparticipantes":106333,"labeled":106334,"ĠÐŃÑĤа":106335,"Ġê²ĥìŀħëĭĪëĭ¤":106336,"æĮªç͍":106337,"Ġlibertad":106338,"Ġhypertensive":106339,"çĶŁæĬ½":106340,"ĠKow":106341,"æ³ķåѦéĻ¢":106342,"å¾Ĺå¿«":106343,"Ġexpanse":106344,"åĮ»çĻĤ":106345,"addad":106346,"Ġtotaling":106347,"ĠشرÙĪØ¹":106348,"ĠинÑĤенÑģив":106349,"Ġproxies":106350,"ä¸Ģ对ä¸Ģ":106351,"æĸ¹æĸ¹éĿ¢éĿ¢":106352,"*}Ċ":106353,"Ġtaman":106354,"rição":106355,"ĠNFC":106356,"Ġrere":106357,"Ġzaz":106358,"æĥħä¸įèĩªç¦ģ":106359,"Ñħал":106360,"Ġâ«":106361,"åģļåĩĨå¤ĩ":106362,"Ġinfek":106363,"æĬĹçĻĮ":106364,"Ġreflectance":106365,"ĠاÙĦعرض":106366,"ĠOffset":106367,"å°ĬèĢħ":106368,"å¿łå¿ĥ":106369,"Ġjakie":106370,"леÑĤи":106371,"Powered":106372,"ĠVanderbilt":106373,",O":106374,"baren":106375,"Ġfx":106376,"Ġisomer":106377,"Ġpolem":106378,"å·¥ä½ľä¸Ĭ":106379,"èĬĤ度":106380,"Completion":106381,"ISON":106382,"ĠAmbro":106383,"缴æİ¥åľ¨":106384,"Ġpsychotic":106385,"é£Łåĵģèį¯åĵģ":106386,"ĠDieser":106387,"带头人":106388,"ĠоÑĤноÑģиÑĤÑģÑı":106389,"dostÄĻp":106390,"Ġaç":106391,"ĠDose":106392,"å¾Īå¥ĩæĢª":106393,"Ġsomm":106394,"èles":106395,"Ġnatureza":106396,"gorit":106397,"èĤºåĬ¨èĦī":106398,"Ġthermostat":106399,"×ŀספר":106400,"Ġ----.":106401,"Ġsuperconducting":106402,"æ±Łæ³½æ°ij":106403,"_ct":106404,"fake":106405,"Ġbaja":106406,"ombre":106407,"ä¸įå±Ī":106408,"äºĨåĽŀåİ»":106409,"ĠStor":106410,"è¿ĩä¸Ģ次":106411,"æĹ¶éĹ´éķ¿":106412,"/how":106413,"Ġdebilitating":106414,"殿åłĤ":106415,"Ġcirculate":106416,"Ġisotopic":106417,"Ġводой":106418,"Ġsire":106419,"Ġbw":106420,"ĠReceptor":106421,"Ġpekerja":106422,"æľĪåŃIJ":106423,"æ°Ķåĸĺ":106424,"Ġconfounding":106425,"rosive":106426,"å°įä»ĸ":106427,"ĠFinished":106428,"Ġwallpaper":106429,"à¤Ĥà¤Ĺ":106430,"ĠÙħشاÙĩ":106431,"ĠConservatives":106432,"Ġinteriors":106433,"anked":106434,"åħ±æĢ§":106435,"ä¼ĺ缺çĤ¹":106436,"æĢİä¹ĪåĨĻ":106437,"ĠINDU":106438,"Ġcliente":106439,"ëĿ¼ìĿ´":106440,"空æ°Ķè´¨éĩı":106441,"è¡ĹéģĵåĬŀäºĭå¤Ħ":106442,"ĠSSC":106443,"Ġperitoneal":106444,"æĸĩéĢī":106445,"äºĨä¸ĢåIJį":106446,"åĽ¢ä¼Ļ":106447,"_PR":106448,"ĠоÑĤвеÑĤÑģÑĤвен":106449,"ĠFPGA":106450,"Romans":106451,"ĠClarendon":106452,"Ġanteriores":106453,"ĠprzykÅĤad":106454,"economics":106455,"Ġauster":106456,"Ġpuesto":106457,"asome":106458,"statt":106459,"ĠDile":106460,"Ġnotwend":106461,"å¸ĤæķĻèĤ²å±Ģ":106462,"ERING":106463,"æĿİ天":106464,"ä¼¼æĺ¯":106465,"ÙĪÙĤÙģ":106466,"Ġdysfunctional":106467,"使ãģĦ":106468,"tsy":106469,"é£İåĴĮ":106470,"-integ":106471,"æĹ¢å®ļ":106472,"æīįèĥ½çľŁæŃ£":106473,"éĢī项ä¸Ń":106474,"æķ°ç»Ħä¸Ń":106475,"Ġponer":106476,"ĠChamberlain":106477,"itäts":106478,"輩åŃIJ":106479,"ĠмоÑīноÑģÑĤÑĮ":106480,"ĠEntrepreneur":106481,"ĠжидкоÑģÑĤи":106482,"ĠDend":106483,"Ġhefty":106484,"æĹ¶æīį":106485,"Ġintervie":106486,"ämp":106487,"bygg":106488,"ského":106489,"å²ĽçļĦ":106490,"ĠкоÑĢи":106491,"Transactions":106492,"é£Ľæ©Ł":106493,"å¾Īå°ijæľī":106494,"igtausend":106495,"_profile":106496,"Singleton":106497,"ãģ¨ãĤĤãģ«":106498,"Ġeigene":106499,"Ġtoughest":106500,"escap":106501,"å¤ļè§ģ":106502,"ç»ĵ转":106503,"ĠSele":106504,"dispatch":106505,"éļIJç§ĺ":106506,"çݰ代社ä¼ļ":106507,"(point":106508,"Beautiful":106509,"ëŁ½":106510,"Understand":106511,"Ġ×ª×ł×":106512,"以å¾ĢçļĦ":106513,"Ġtrasform":106514,"åĨłçĬ¶åĬ¨èĦī":106515,"Ġsensitivities":106516,"Ġhamp":106517,"ä¸Ģåıį":106518,"æĺ¯æľ¬":106519,"ä¾Ĺ":106520,"家裡":106521,"æ¯ıä¸ĢåĢĭ":106522,"Ġpowerhouse":106523,"ä½İæĶ¶åħ¥":106524,"Ġintroductions":106525,"werking":106526,"Ġnanos":106527,"uldade":106528,"측":106529,"thumbnail":106530,"俨çĦ¶":106531,"ĠCIP":106532,"æĬľ":106533,"-situ":106534,"Ġforeclosure":106535,"å®Ŀå¦Ī":106536,"θο":106537,"Compact":106538,"ĠRockefeller":106539,"Ġfavourites":106540,"/=":106541,"Ġsilt":106542,"çļĦè¯į":106543,"缮ä¸į":106544,"Ġentrar":106545,"山人":106546,"ĠPlast":106547,"端起":106548,"è½®èι":106549,"ĠÑĤан":106550,"Ġcivilisation":106551,"ÑĢовании":106552,"-kil":106553,"Ġoverturned":106554,"Ġmasonry":106555,"ĠпÑĢоÑĤиво":106556,"iÅ¡":106557,"ĠHAL":106558,"ä¸Ĭãģ®":106559,"çŃīèħ°":106560,"ĠArx":106561,"客家":106562,"èĭ¥éĿŀ":106563,"ÙĬÙĨÙĬØ©":106564,"çľīå¿ĥ":106565,"ÏĥÏĦική":106566,"ÑģÑģии":106567,"ä¸Ńå°ıåѦçĶŁ":106568,"象å¾ģçĿĢ":106569,"ä¼ĺèī¯ä¼łç»Ł":106570,"ĠÑģÑĥммÑĭ":106571,"/ui":106572,"MJ":106573,"Sounds":106574,"daily":106575,"çļĦæĸ¹éĴĪ":106576,"unek":106577,"åı¯è§Ĥ":106578,"ç¾Ķ":106579,"åħ³åı£":106580,"questa":106581,"Ġdinam":106582,"ĠPassing":106583,"åĴ¨è¯¢æľįåĬ¡":106584,"à¦¾à¦ľà¦¾à¦°":106585,"Ġinterruptions":106586,"Ġterdiri":106587,"Ġhurdle":106588,"#print":106589,"grant":106590,"ĠPRI":106591,"æĪijä¸Ģ个":106592,"Ġunten":106593,"åħ¶ä¸ī":106594,"åIJį稱":106595,"Ġdiscut":106596,"ÄįÃŃslo":106597,"(solution":106598,"rafish":106599,"ĠваÑĪ":106600,"ÙĪØ²Ùĩ":106601,"æ¸Ĺåĩº":106602,"ĠÑģамого":106603,"è·ªä¸ĭ":106604,"Ġcrawled":106605,"ĠRhein":106606,"ĠVolkswagen":106607,"æķĻ诲":106608,"Ġcommunes":106609,"第ä¸ĢæľŁ":106610,"è¿ĺæĺ¯ä¸ª":106611,"Ġmarco":106612,"ä¿ĥè¿Ľä½ľç͍":106613,"})\\]":106614,"olkien":106615,"Ġrelativistic":106616,"ĠпомогаеÑĤ":106617,"codeline":106618,"itiva":106619,"Ġfern":106620,"illac":106621,"åĴĮå¿ĥçIJĨ":106622,"Ġardu":106623,"产äºİ":106624,".sign":106625,"Ġbiologist":106626,"ĠPeruvian":106627,"éķĩä¸Ĭ":106628,"Immun":106629,"Classifier":106630,"ĠClearing":106631,"ĠPlanting":106632,"Ġminimalist":106633,"ĠCovered":106634,"Ġprosthetic":106635,"为ä¸Ģä½ĵçļĦ":106636,"Ġ무ìĹĩ":106637,"GRAPHY":106638,"Ġquirky":106639,"ĠÑģопÑĢовож":106640,"è±Įè±Ĩ":106641,"?\",":106642,"kých":106643,"ĠWand":106644,".slf":106645,"é¢Ĩ头":106646,"éľĢè¦ģç͍":106647,"ÏĢÏīÏĤ":106648,"Ġbrood":106649,"èµ°äºĨåĩºæĿ¥":106650,"ì¹ł":106651,"ĠBegriff":106652,"xz":106653,"æľīåĪ«":106654,"æĪijä¸Ģ个人":106655,"ÙĪØ§Ùħ":106656,"ĠStd":106657,"äºĨä¸Ģ座":106658,"ĠÙĬÙħ":106659,"})_{":106660,"è´¡çĮ®åĬĽéĩı":106661,"Ġprotesting":106662,"âĻĢ":106663,"ĠглÑĥбок":106664,"Mand":106665,"_us":106666,"amins":106667,"æĺ¯åħ¨":106668,"ĠHabits":106669,"æŃ£äº¤":106670,"Ġmenurut":106671,"],\"":106672,".Check":106673,"Ġscientifique":106674,"æŁıæĭī":106675,"Ġmetaphysics":106676,"è©ķä¼°":106677,"Ġgauche":106678,"ĠStreaming":106679,"ĠÑģвеÑĤа":106680,"Ġepistemic":106681,"stice":106682,"ĠGry":106683,"ä¸İåīį":106684,"ebu":106685,"Ġgla":106686,"çļĦä¸Ģéĥ¨":106687,"ä½Ĩæĺ¯è¿Ļ":106688,"çĤºä»Ģ麽":106689,"åŃĺåľ¨éĹ®é¢ĺ":106690,"partner":106691,"Attendance":106692,"ektion":106693,".yaml":106694,"ĠEugen":106695,"iatrists":106696,"ĠcientÃŃfica":106697,"Ġ커":106698,"Ġmalignancies":106699,"ĠØ£ÙĬضاÙĭ":106700,"ĠÑĤолÑīи":106701,"Äĺ":106702,"Ġcatt":106703,"Ġcumbersome":106704,"igor":106705,"ariables":106706,"Ġremorse":106707,"Ġgeval":106708,"æ²īæ²ī":106709,"å¨ģæµ·":106710,"ĠÑıк":106711,"測å®ļ":106712,"æķĻ室éĩĮ":106713,"ĠKyiv":106714,"ĠÙħÛĮØ´ÙĪÙĨد":106715,"ulkner":106716,"ĠDisponÃŃvel":106717,".An":106718,"uously":106719,"ä¸įæ¼ı":106720,"åĴĮåįİ":106721,"ä¸Ĭ讲":106722,"ĠsetUp":106723,"Ġmultiv":106724,"åIJ«éĩıçļĦ":106725,"Ġpitchers":106726,"Ġdictator":106727,"ĠAFTER":106728,"Ġlát":106729,"æľīæĦŁ":106730,"æķĺ":106731,"rukt":106732,"æľ¬å½ĵ":106733,"Ġstrony":106734,"æ¯ı亩":106735,"Ġgrowled":106736,"ĠâĨĹ":106737,"æ¼Ķåĵ¡":106738,"对äºİæĪij们":106739,"ç¿»å¼Ģ":106740,"Ġperspectiva":106741,"اØŃب":106742,"Ġboycott":106743,"Ġર":106744,"ĠWinchester":106745,"callback":106746,"çİ©æĦıåĦ¿":106747,"%/":106748,"Besk":106749,"_month":106750,"ĉcolor":106751,"ĠPOT":106752,"ocultural":106753,"Ġobsz":106754,"ĠبÛĮر":106755,"ampaign":106756,"è¨Ģè¾ŀ":106757,"å¾®ç²Ĵ":106758,"akening":106759,"ëŀľ":106760,"鼶åĶ®åķĨ":106761,"abolismo":106762,"Ġenvisaged":106763,"ématiques":106764,"ĠFrankenstein":106765,"urangi":106766,"ĠPEM":106767,"åľ¨æ°´ä¸Ń":106768,"æĹ¶ä»»":106769,"Ġ'Ċ":106786,"?...":106787,"Winner":106788,"hap":106789,"Ġith":106790,"alance":106791,"ä¸įéĩįè¦ģ":106792,"ĠHaf":106793,"ĠWies":106794,"大åıĺ":106795,"epa":106796,"çŃīå·®":106797,"æľĢç®ĢåįķçļĦ":106798,"Ġ\\(+":106799,"Ġcleft":106800,"Ġverbe":106801,"çĺª":106802,"Ġbesoins":106803,"缸äºĴåħ³ç³»":106804,"ĠHawthorne":106805,"ĠNeeded":106806,"å·¥åķĨæĪ·":106807,"ĠجÙĩاÙĨÛĮ":106808,"æ¶Īè²»èĢħ":106809,"Nil":106810,"rush":106811,"raut":106812,"ä¸ĭæľī":106813,"ÑĤием":106814,"æ²³ä¸Ń":106815,"_session":106816,"ÙİÙijØ©":106817,"ĠØ«ÙĦاثة":106818,"alto":106819,"ouz":106820,"Ġ[`":106821,"æ¯ıæĿ¡":106822,"ĠResidence":106823,"ãģĹãĤĪãģĨ":106824,"ĠâĪ£":106825,"èģļé¤IJ":106826,"ĠRadiol":106827,"æĬĢèĥ½çļĦ":106828,"Ġ׼×ŀ×Ķ":106829,"riority":106830,"ĠMiddles":106831,"ĠCorrespondence":106832,"mals":106833,"Ġbyli":106834,"ä¸İç¾İåĽ½":106835,"ASON":106836,".getLogger":106837,"æľĿå¤ķ":106838,".Act":106839,"ĠDiocese":106840,"Ġfrail":106841,"Ġtrova":106842,"Ġcoveted":106843,"å¦ĸç²¾":106844,"éªĤéģĵ":106845,"Ġaucune":106846,"Ġdisobedience":106847,"Ġindistinguishable":106848,"Ġợ":106849,"enarios":106850,"stuff":106851,"romycin":106852,"доÑĢ":106853,"سد":106854,"Ġraj":106855,"çıı":106856,"Ġafores":106857,"åľ£æ¯į":106858,"Ġiceberg":106859,"ÑģÑĤвием":106860,"Ġнового":106861,"é§ħ":106862,"èĤĨèĻIJ":106863,"ĠинÑĦоÑĢмаÑĨиÑİ":106864,"Ġpleasantly":106865,"اگر":106866,"ĠDura":106867,"ĠNASCAR":106868,"Ġsucks":106869,"è¿ĽéĢĢ":106870,"æŃ£ç»Ł":106871,"ä¿¡çļĦ":106872,"Ġmetri":106873,"ĠAprès":106874,"ĠInterstate":106875,"Ġgestión":106876,"jeno":106877,"picture":106878,"æĺ¯ç¬¬ä¸Ģ":106879,"ä¸įçŃīäºİ":106880,"Ġrarity":106881,"éĩįéĩįçļĦ":106882,"Ġfilings":106883,"å¤ı天çļĦ":106884,"ıs":106885,"ãĥĪãĥ©":106886,"Õ¡Õ¶Õ¡Õ¯":106887,"Ġcommercials":106888,"Ġ׳ק":106889,"ĠÑģобиÑĢа":106890,"Ġtweede":106891,"/\"Ċ":106892,"Coun":106893,"Ice":106894,"_In":106895,"Ġpapa":106896,"ä¸įèĭŁ":106897,"æľīå¤ļç§į":106898,"ĠимÑĥ":106899,"Ġwatered":106900,"Ġmiembros":106901,"ĠborderRadius":106902,"ĠSupports":106903,"浩çī¹":106904,"èĢģ年人çļĦ":106905,"ä¾¿å®ľçļĦ":106906,"ĠBahamas":106907,"Ġìĺģìĸ´":106908,"ĠTerritories":106909,"Ġfondamentale":106910,"Ġsacrificial":106911,":v":106912,"XO":106913,"Ġtại":106914,"ĠBoll":106915,"ĠJans":106916,"usten":106917,"Ġsoff":106918,"undering":106919,"Ïģεί":106920,"Ġnegativity":106921,"缴æİ¥ä»İ":106922,"MMA":106923,"鼨çļĦ":106924,"æĦŁè§īåΰäºĨ":106925,"ĠâĨĴĊ":106926,"ÑģаÑħ":106927,"à¹ĥà¸Ĭà¹Ī":106928,"Ġdecomposed":106929,"-employed":106930,"Ġ```Ċ":106931,"æµĵéĥģçļĦ":106932,"(as":106933,"ĠPWM":106934,"åı¯åĪ©ç͍":106935,"Ġsprite":106936,"Ġinterloc":106937,"Ġoffre":106938,"éĢīäºĨ":106939,"å¦Ĥæŀľç͍":106940,"å©ķ":106941,"礼æľį":106942,"Assets":106943,"áték":106944,"奴æīį":106945,"ãģĿãģĨãģ§ãģĻ":106946,"ĠzostaÅĤa":106947,"Mate":106948,"oises":106949,"ï¼Į(":106950,"Ġtoim":106951,"ĠFury":106952,"angun":106953,"assay":106954,"å¿ĥè£ı":106955,"Ġunderv":106956,"ĠналиÑĩие":106957,"Ġchangement":106958,"notification":106959,"ç»Ħç»ĩå½¢å¼ı":106960,"Äĩi":106961,"Ġhomogeneity":106962,"ĠìĹħ":106963,"è¯ģåΏåħ¬åı¸":106964,"ĠHonolulu":106965,"天çĦ¶çļĦ":106966,"à´¿à´¯":106967,"温æŁĶçļĦ":106968,"Ġvertebrate":106969,"ĠاÙĤتصادÛĮ":106970,"æĺ¯åħ¨åĽ½":106971,"éĩįç½®":106972,"Ġcoleg":106973,"ãĢĭ;":106974,"ymoon":106975,"-mot":106976,"Ġleftovers":106977,"åį°åº¦çļĦ":106978,"鼷æĸ¯":106979,"ĠCourtney":106980,"ĠDirac":106981,"Ġμl":106982,"表达èĥ½åĬĽ":106983,"ĠاÙĦÙĤاÙĨÙĪÙĨ":106984,"-Nine":106985,"ĠProtocols":106986,"ÑĥбеÑĢ":106987,"ĠпÑĢоÑĨеÑģÑģов":106988,"åľ¨å¤ļ":106989,"æĿ¥å¤ĦçIJĨ":106990,"ccia":106991,"ä¸»é£Ł":106992,"æľĪèµ·":106993,"ัล":106994,"è£ħçĿĢ":106995,"è©Ń":106996,"Orig":106997,"ĠTHEY":106998,"æ¾¹":106999,"ä¼´å¥ı":107000,"اÙ쨱":107001,"å¯¾å¿ľ":107002,"Ġcoexist":107003,"ĠCasp":107004,"å°±å½ĵ":107005,"对è¿Ļç§į":107006,"Ñħан":107007,"Ġdetta":107008,"Ġbackups":107009,"æĭīæī¯":107010,"poz":107011,"éĽªçļĦ":107012,"ä»ģä¹ī":107013,"uestra":107014,"æľīçĤ¹åĥı":107015,"Ġnitro":107016,"å¹´åīįå·²åĽŀçŃĶ":107017,"Ġunderwear":107018,"invasive":107019,"Ġetymology":107020,"Ġthalam":107021,"iquant":107022,"çŃĶåį·":107023,"à´±":107024,".CO":107025,"Ġberarti":107026,"ä¸įå°ijçļĦ":107027,"æĢĿèĢĥåĴĮ":107028,"Ġdecompose":107029,"ĠÏĢÏģοÏĥ":107030,"à¹Ģศษ":107031,"Ġnauczyci":107032,"ä¸įæĢ¥":107033,"igna":107034,"åIJĮ为":107035,"â̦âĢĿĊ":107036,"ranet":107037,"/my":107038,"ãģªãģĬ":107039,"åħ¶ä»ĸåľ°æĸ¹":107040,"åıªæĺ¯æĥ³":107041,"aderie":107042,"å·¥ä¸ļçĶŁäº§":107043,"ĠÑģкла":107044,"ĠPropagation":107045,"ĠÑĩаÑģÑĤноÑģÑĤи":107046,"ÿ":107047,"人è¦ģ":107048,"Ġà¦IJ":107049,"геÑĤи":107050,"Ġservo":107051,"Ġدرس":107052,"æĿ¡ä»¶ä¸ĭçļĦ":107053,"çϼåĭķ":107054,"麻å°Ĩ":107055,"اÙĤÙĦ":107056,"Ġalphabetical":107057,"Ġpercorso":107058,"ĠWarszawa":107059,"Ġhymns":107060,"Nearly":107061,"ĠToby":107062,"ä»ĸå¦Ī":107063,"å¹´ç´Ģ":107064,"ä¸ĭéĻIJ":107065,"æµģåħī":107066,"åı¤èij£":107067,".Click":107068,"äºĨè§£çļĦ":107069,"åħ¸æķħ":107070,"以ä¸ĭæľīæľŁå¾ĴåĪij":107071,"Ġwildfires":107072,"slash":107073,"Ġazimuth":107074,"åĬłå¿«äºĨ":107075,"éľįå°Ķ":107076,"Tomorrow":107077,"Ġë°°ìĹ´":107078,"fluidic":107079,"lya":107080,"è¯ĥ":107081,"Ġhaste":107082,"ĠStrict":107083,"neck":107084,"ĠкÓĢ":107085,"Ġeserc":107086,"Ġdurations":107087,"线ä¸Ĭ线ä¸ĭ":107088,"Ġeredet":107089,"buff":107090,"ĠSint":107091,"Ġunordered":107092,"ibaba":107093,"Ġmanoe":107094,"æıIJåįķ":107095,"жÑĥÑĤ":107096,"preter":107097,"çĶļæĺ¯":107098,"BILE":107099,"é«ĺä¸Ńæķ°åѦ":107100,"Ġvivre":107101,"ĠDiscovering":107102,"ĠмеÑģÑıÑĨа":107103,"ĠPOLICY":107104,"ĠÐĵеÑĢма":107105,"Ġcioè":107106,".ba":107107,"ìį¨":107108,"ĠJury":107109,"Ġ\"]":107110,"æ³ķåŃIJ":107111,"çĸĬ":107112,"ĠDeployment":107113,"ä¹īå·¥":107114,"çĥŃå¤ĦçIJĨ":107115,"åįķä½įåĴĮ个人":107116,"ĠÏĦá½°":107117,"æĺ¯åIJ¦éľĢè¦ģ":107118,"ĠìĿ´ë¥¼":107119,"çļĦæĸ¹æ³ķæĺ¯":107120,"Ġdegenerate":107121,"ĠFungi":107122,".»ĊĊ":107123,"ĠRCA":107124,"Ġ$ĊĊ":107125,"ĠNewark":107126,"Ġhardwood":107127,"ĠINPUT":107128,"Ġhablar":107129,"åºĶç͍åΰ":107130,"Ġpretreatment":107131,"建çŃijä¸ļ":107132,"æĭĶåĩº":107133,"Ġoversees":107134,"Ġ×ķ׾×Ķ×":107135,"ĠPreventing":107136,"注è§ĨçĿĢ":107137,"ĠMultiplying":107138,"_ac":107139,"ä¸Ĭ大åѦ":107140,"对大":107141,"ä½łæķ¢":107142,"æľ¬ä½į":107143,"Ġevade":107144,"à´ķàµįà´ķ":107145,"iftung":107146,"åĿ¦çϽ":107147,"Ġguaranteeing":107148,"èĪīè¡Į":107149,"ĠQUAL":107150,"Ġrapporto":107151,"industry":107152,"/us":107153,"AIR":107154,"Sac":107155,"Ġresurgence":107156,"Ġacuity":107157,"ĠدÙĨ":107158,"лаÑĢÑĥ":107159,".success":107160,"款è§Ħå®ļ":107161,"หา":107162,"κή":107163,"æĽ¾æľī":107164,"offee":107165,"æ¹ĸåĮº":107166,"Ġfolly":107167,"ĠConflicts":107168,"aucer":107169,"Ġmocking":107170,"ĠÃģl":107171,"æĬµæĬ¼æĿĥ":107172,"ĠмеÑģÑıÑĨ":107173,"Ġemptied":107174,"/acs":107175,"Dt":107176,"zko":107177,"ĠPhe":107178,"Ġunnecessarily":107179,"å°ıå±ĭ":107180,"Ġmodifiers":107181,"ĠÙĪØ®":107182,"-lact":107183,"Ġkgf":107184,"Started":107185,"anasia":107186,"Dashboard":107187,"Ġpizz":107188,"ĠFarn":107189,"Ġkang":107190,"å°±å¾Ģ":107191,"ualitas":107192,"Ġindem":107193,"ĠÙ쨱Ùħ":107194,"æĴ¸":107195,"ÑģÑĤаве":107196,"é¡»çŁ¥":107197,"éħ¸çĹĽ":107198,"Ġréel":107199,"Ġsolidified":107200,"ĠObtain":107201,"饰åĵģ":107202,"Ġimmunoglobulin":107203,"ĠMosque":107204,"Ġmulticenter":107205,"工伤ä¿ĿéĻ©":107206,"ĠнаÑģÑĤоÑıÑīее":107207,"/Object":107208,"rinnings":107209,"ä¸Ģå¹ķ":107210,"Ġzain":107211,"èĤ²ãģ¦":107212,"温度为":107213,"çħ®çĨŁ":107214,"ĠинÑĤегÑĢа":107215,"{":108207,"leasing":108208,"Ġstär":108209,"ä¸Ģéľĩ":108210,"ĠRund":108211,"ĠRama":108212,"ynes":108213,"æĮģæľī人":108214,"项è§Ħå®ļ":108215,"å¤įçĽĺ":108216,"å¯Įç¿ģ":108217,"åħ°çī¹":108218,"Ġrhin":108219,"BBB":108220,"ảy":108221,"ãĥĿãĥ¼ãĥĪ":108222,"Ġnostalgic":108223,"/etc":108224,"çļĦäºĨè§£":108225,"Ġquid":108226,"对人çļĦ":108227,"说åIJ§":108228,"æĸ°éĻĪ代谢":108229,"羣è¦ģ":108230,"ĠâĨĺ":108231,"Ġhonours":108232,"Ġseedling":108233,"à¸ģรà¸ĵà¹Į":108234,"ĠPatienten":108235,"Faculty":108236,"Ġmisconception":108237,"'W":108238,"_SP":108239,"jg":108240,"enstein":108241,"ĠIra":108242,"è®®ç¨ĭ":108243,"åį¡ç½Ĺ":108244,"Ġprotección":108245,"(task":108246,"åįı议书":108247,"éĨīéħĴ":108248,"ĠREADING":108249,"JL":108250,"otene":108251,"ĠweiÃŁ":108252,"Ġcontender":108253,"çī¹ä¹Ī":108254,"ĠBooking":108255,"éļĨéļĨ":108256,"ĠFAIL":108257,"Õ¸Õ´":108258,"Ġvýznam":108259,"ĠبÙĪØ¯ÙĨد":108260,"åĩºçı¾äºĨ":108261,"Ġgambar":108262,"ĠмоделÑĮ":108263,"-notch":108264,"Nine":108265,"ĠBanc":108266,"å¤ļå§¿":108267,"æľĪåĪĬ":108268,"被æĿĢ":108269,"éϤèįī":108270,"ĠWorkshops":108271,"çĹĺçĹĺ":108272,"Ġtorsion":108273,"ĠCaldwell":108274,"Bab":108275,"¤×ķת":108276,"çļĦçŁĽçĽ¾":108277,"ĠTLS":108278,"Ġprostitution":108279,"ĠKuala":108280,"ickou":108281,"ĠUnable":108282,"ĠExams":108283,"çļĦä¸Ģéģĵ":108284,"æŃ¦çİĭ":108285,"ĠPrepar":108286,"æ±īæŃ¦å¸Ŀ":108287,"Ġmapa":108288,"Ġà¦Ĩশ":108289,"Ġà¦ħত":108290,"organism":108291,"Ġwestward":108292,"ä»ĸèĩªå·±çļĦ":108293,"ĠKapit":108294,"ÑĪеÑĤ":108295,"æ²»æ³ķ":108296,"ä½ķ为":108297,"Ġinsgesamt":108298,"Ġrestorative":108299,"è½»èĢĮæĺĵ":108300,"äºĪéĺ²":108301,"ä¸įçŁ¥éģĵ为ä»Ģä¹Ī":108302,"ائع":108303,"Ġhydrothermal":108304,"åĩīçν":108305,"Ġnuove":108306,"Ġstylistic":108307,"Ġcongregations":108308,".validation":108309,"ipika":108310,"æŃ¦åĪĻ":108311,"Ġfrontiers":108312,"Ġ×IJ×£":108313,"建çŃij设计":108314,"ĠViewer":108315,"竹åŃIJ":108316,"碳æİĴæĶ¾":108317,"ĠDuchess":108318,"ĠìĿĺíķ´":108319,"å¥łåŁº":108320,"Budget":108321,"administration":108322,"ãĤµãĥ¼ãĥĵ":108323,"ĠAppalachian":108324,"'ind":108325,"Ġsizable":108326,"Ġpious":108327,"Ġbows":108328,"ä¸ĭæĦıè¯ĨçļĦ":108329,"åīįæİĴ":108330,"åıĺæ³ķ":108331,"æīįæĻº":108332,"ØŃÙĤ":108333,"éļıç¬Ķ":108334,"ĠSystemic":108335,"æ²īéĹ·":108336,"æĹħ游å±Ģ":108337,"æŀĿåı¶":108338,"Ġcompensatory":108339,"å±ħçĦ¶æĺ¯":108340,"Ġcoincided":108341,"Ġkalba":108342,"æ°´å¹³çļĦæıIJé«ĺ":108343,"ĠÙħؤس":108344,"Ġzodiac":108345,"ĠsÃŃntomas":108346,"Ġbarbecue":108347,"cpu":108348,"声声":108349,"Ġindividualism":108350,"(password":108351,"Ġà¤ķालà¥ĩ":108352,"Ġstuffing":108353,"রà§įà¦Ĺ":108354,"numbered":108355,"ĠSNR":108356,"Õ¡ÖĢÕ£":108357,"ĠCONTIN":108358,"பà¯įà®ªà®Łà¯įà®Ł":108359,"-throughput":108360,"ĠCandida":108361,"Ġíļ¨ê³¼":108362,"Ġexter":108363,"ä¸Ńè¿ĺ":108364,"Ġroaring":108365,"Ġzi":108366,"åŁºçĤ¹":108367,"Ġmined":108368,"溥":108369,"éĺ²çĽĹ":108370,".tag":108371,"çĻ»è¨ĺ":108372,"ä¸Ĭæµ·çļĦ":108373,"å¹³åĿĩæ°´å¹³":108374,"èĥ¸éĹ·":108375,"ĠáĥĶ":108376,"ĠPleasant":108377,"Ġgreetings":108378,"Metal":108379,"çļĦæŃ¥ä¼IJ":108380,"omos":108381,"agree":108382,"ä¹ĭå¢ĥ":108383,"å°Ĩä¸Ģ个":108384,"常ç͍äºİ":108385,"è·¯äºĨ":108386,"èĸı":108387,"å°ļæĹł":108388,"ä¸ĭéĿ¢å°±":108389,".columns":108390,"quate":108391,"ĠKale":108392,"ĠzwiÄĻks":108393,"ĠRealm":108394,"çŁ¥åħ¶":108395,"çͱåħ¶":108396,"Ġinflamed":108397,"è°Īå¿ĥ":108398,"îne":108399,"ä¸įä¹ħåīį":108400,"延伸åΰ":108401,"Ġhierarchies":108402,"à¹ģà¸Ļะ":108403,"Ġdegenerative":108404,"วิà¸Ĺยาศาสà¸ķรà¹Į":108405,"Ġbrainstorming":108406,":f":108407,"Ġtonal":108408,"Ġendlessly":108409,"æķħå±ħ":108410,"æİ§åζäºĨ":108411,"éĻĪæŁIJ":108412,"ĠPrecipitation":108413,".My":108414,"æĬ±èµ·":108415,"éĵ¸éĵģ":108416,"Ġvincul":108417,"Ġwholesome":108418,"Ġcittad":108419,"insp":108420,"Ġgendered":108421,"ä¸Ģåijĺ":108422,"åĴĮåľ°æĸ¹":108423,"-finals":108424,"ĠGoa":108425,"ĠErl":108426,"-reactive":108427,"Ġbudgetary":108428,"بÙĬعة":108429,"Ġमह":108430,"}/{":108431,"Ġencapsulation":108432,"ĠÐĵÐŀСТ":108433,"æĮģä¹ĭ以æģĴ":108434,"ieth":108435,"çļĦèĥĮåIJİ":108436,"Ġ|>":108437,"æĿ¥æĦĪ":108438,"è¿ĺæķ¢":108439,"ï¼ī+":108440,"èĢģæ±ī":108441,"nsics":108442,"Ġfamiliarize":108443,"Ġnavbar":108444,"åŁºæľ¬ä¸Ĭæĺ¯":108445,"Ġacetone":108446,"Ġabsorber":108447,"ĠدÙĬسÙħبر":108448,"ĠDangerous":108449,"ç©Ĩæĸ¯æŀĹ":108450,".Integer":108451,"dra":108452,"Ġstigmat":108453,"Ġuc":108454,"=\"{":108455,"请ä¸įè¦ģ":108456,"ĠзаÑĢа":108457,"Ġapabila":108458,"visions":108459,"ĠFeuer":108460,"岩æµĨ":108461,"ĠконÑĦ":108462,"çļĦ好åĿı":108463,"Ġcigar":108464,"ĠSprinkle":108465,"Ġantidepressants":108466,"iard":108467,"åľ¨ä»Ĭ天":108468,"ä¸Ĭæī¬":108469,"ultures":108470,"å¤įéĢīæ¡Ĩ":108471,"-den":108472,"满天":108473,"é¦ĸå®¶":108474,"æĸĩåĮĸä¸İ":108475,"Ñĩика":108476,".full":108477,"(mm":108478,"mati":108479,"ĠEarthquake":108480,"åºĨåħ¸":108481,"ĠBerk":108482,"éªij车":108483,"Ġà¦īà¦ł":108484,"Ġම":108485,"someone":108486,"ĠJessie":108487,"æĢĿæĥ³æĶ¿æ²»å·¥ä½ľ":108488,"responsive":108489,"ĠStruggle":108490,"junt":108491,"elos":108492,"ulam":108493,"uncia":108494,"ĠWEEK":108495,"åħ¥åĽŃ":108496,"éĩijæĸ¯":108497,"awar":108498,"ÙĥÙĦØ©":108499,"วี":108500,"вига":108501,"ä»»ä½ķäºĭæĥħ":108502,"å½Ĵ宿":108503,".Body":108504,"çļĦæĸ¹å¼ıè¿Ľè¡Į":108505,"Ġabsentee":108506,"ĠëıĻìķĪ":108507,"âĪĻâĪĻ":108508,"æĵĤåı°":108509,"×Ļ׾×ĵ×Ļ×Ŀ":108510,"Ġeconómico":108511,"PVC":108512,"Ġstalled":108513,"ĠPek":108514,"ieuse":108515,"çĭī":108516,"åŀĽ":108517,"è¿Ļç§įæĥħåĨµä¸ĭ":108518,"ytet":108519,"ê³¼íķĻ":108520,"ĠCauchy":108521,"ĠUniversitas":108522,"è´¢åĬ¡çĬ¶åĨµ":108523,"æŁIJç§įæĦıä¹īä¸Ĭ":108524,"ĠBioinformatics":108525,"`.ĊĊ":108526,"erer":108527,"Ġrete":108528,"Ġexhort":108529,"arki":108530,"ĠHeading":108531,"tted":108532,"ajärvi":108533,"缴æİ¥ç͍":108534,"Ġarchaic":108535,"æķ°åŃĹç»ıæµİ":108536,"æĶ¯éĥ¨ä¹¦è®°":108537,"ç¥Ńåı¸":108538,"Ġnajle":108539,"Ġmejores":108540,"Ġsubmits":108541,"ĠнапÑĢÑıжение":108542,"Ġadsorbed":108543,"@RequestMapping":108544,"ĠMales":108545,"ĠKier":108546,"Ġwills":108547,"Ġteatro":108548,"åIJĮéģĵ":108549,"æįº":108550,"åĽłçĹħ":108551,"çİĭ室":108552,"éĢĻæĻĤåĢĻ":108553,"çīĮçħ§":108554,"বা":108555,"Ġsettles":108556,"-Two":108557,"Attention":108558,"×Ļ׳×ķ×ļ":108559,"ĠTobias":108560,"Ġeconó":108561,"IAM":108562,"¨ìĸ´":108563,"Ġà¸Ķà¹īวย":108564,"å°ıéĿĴ":108565,"èĢĮå¼ķèµ·":108566,"å¦Ĥå±±":108567,"ãģ®ãģĵãģ¨":108568,"Ġcorals":108569,"åıĸåħ¶":108570,"æĿ¡çĽ®":108571,"å¸ĥæĸĻ":108572,"éł¼":108573,".Clear":108574,"blich":108575,"ναÏĤ":108576,"æīĵéĢłçļĦ":108577,"ÑĢованнÑĭе":108578,"Ġmucous":108579,"ĠExamining":108580,"Ġconcede":108581,"Probability":108582,"ĠÐŁÐµÑĢеводÑĩик":108583,"-entry":108584,"ĺ×Ļ":108585,"Ġdj":108586,"icill":108587,"Ġanastom":108588,"Ġindia":108589,"ächt":108590,"Ġsegue":108591,"æľī人认为":108592,"éĶĢåĶ®äººåijĺ":108593,"æ¯ı个人éĥ½æľī":108594,"ĠدÙĪØ±Ø§ÙĨ":108595,"ategorized":108596,"ĠÑĤÑĢебÑĥеÑĤ":108597,"Ġگزارش":108598,"为好":108599,"ĠYE":108600,"ç¾£":108601,"Ġgraffiti":108602,"ĠIndus":108603,"Ġболи":108604,"Ġобо":108605,"ä»»ä½ķ人éĥ½":108606,"Ġcapacidade":108607,"paths":108608,"Ġ×Ķ×ŀת×":108609,"ĠNeuropsych":108610,"ĠMascul":108611,"Ġhonorary":108612,"Ġà¦īপর":108613,"anov":108614,"Ġbfs":108615,"uclease":108616,"æ·±èĢķ":108617,"ĠاÙĦÙħختÙĦÙģ":108618,"Ġantipsych":108619,"ĠDesarrollo":108620,"ĠобÑĥÑģ":108621,"Ġdistributive":108622,"IMAGE":108623,"Ġgrandma":108624,"æ·¡æ¼ł":108625,"Ġtempérature":108626,"æĵ¦äºĨ":108627,"à¸Ħรà¸Ńà¸ļ":108628,"èľĤçªĿ":108629,"ĠPropag":108630,"ĠLaurel":108631,"Ġbangsa":108632,"Ġingenious":108633,"ĠCummings":108634,"åĩºä¸įç©·":108635,"对åħ¶ä»ĸ":108636,"Ġdeme":108637,"Ġautopsy":108638,"Ġscheduler":108639,"åįijå¾®":108640,"ĠнеобÑħодимоÑģÑĤи":108641,"éĿĴå²Ľå¸Ĥ":108642,"ĠInvolvement":108643,")arg":108644,"<_":108645,"å¿¿":108646,"è¿ĻèĬĤ课":108647,"åħ¬ç§ģ":108648,"æļĹæ·¡":108649,"éĵ¶è¡Į贷款":108650,"мое":108651,"åľ¨æŃ¤æľŁéĹ´":108652,"ÙĪÙĦÙĪØ¬ÙĬا":108653,"ëłĩê²Į":108654,"Ġabaixo":108655,"_div":108656,"presa":108657,"Ġcair":108658,"çļĦçIJĨæĥ³":108659,"æĿ¥åĪĨæŀIJ":108660,"åĪĩè®°":108661,"Ġmengatakan":108662,"浪费æĹ¶éĹ´":108663,"-governmental":108664,"åĩºåı°äºĨ":108665,"Ġupholding":108666,"ĠиÑİнÑı":108667,"âļł":108668,"=V":108669,"NES":108670,"Ġnl":108671,"stasy":108672,"adat":108673,"ĠWATER":108674,"Ġ_.":108675,"é¦Ģ":108676,"ĠcrÃŃtica":108677,"UTO":108678,"Ġodors":108679,"Ġmisplaced":108680,"ĠUniversité":108681,"ĠRupert":108682,"ắc":108683,".ms":108684,"Ġced":108685,"ĠFj":108686,"ĠFiling":108687,"å®ļæł¼":108688,"reden":108689,"Ġphage":108690,"åħĪçŁ¥":108691,"Ġterminates":108692,"Ġsemaine":108693,"èļĿ":108694,"åĩĮ天":108695,"ĠHandler":108696,"Ġимени":108697,"Investment":108698,"è½»èĢĮæĺĵ举":108699,"Demon":108700,"ĠCGFloat":108701,"ifton":108702,"ĠVince":108703,"achsen":108704,"ÃŃte":108705,"åıªé¡¾":108706,"çĥŃçģ«":108707,"Ġসà§įà¦ķ":108708,"æī¿è½½åĬĽ":108709,".iter":108710,"Ġgull":108711,"ĠCair":108712,"Ġitch":108713,"ä»ĸå¼Ģå§ĭ":108714,"Ġاعت":108715,"Ġعدة":108716,"ÑģÑĤаÑı":108717,"گز":108718,"èĦļæīĭ":108719,"ĠÑħÑĢони":108720,"aternion":108721,"ï¼ħãĢĤ":108722,"Ġanxieties":108723,"ĠJesuit":108724,"Ġweiteren":108725,"ĠYankee":108726,"Ġialah":108727,"$)":108728,"(label":108729,"ĠMethyl":108730,"Ġкожи":108731,"强硬":108732,"äºĨä¸ĢæĿ¯":108733,"鲫":108734,"å±ĭéĿ¢":108735,"ç¨Ģæľī":108736,"ĠSmaller":108737,"èĬĿ士":108738,"Ġ{}\".":108739,"ä»İå·¦":108740,"éĢīåĮº":108741,"Ġdonkey":108742,"Ġqualifies":108743,"ÐŁÑĢ":108744,"æ½ľç§»é»ĺ":108745,"looking":108746,"Ġinstructive":108747,"Ġgratis":108748,"ĠGranada":108749,"Ġagonists":108750,"Ġdissatisfied":108751,"ç»ļ丽":108752,"{itemize":108753,"(Exception":108754,"Noun":108755,"çļĦåı«":108756,"åīįç¨ĭ":108757,"ĠAdmissions":108758,"èµ°ä¸ĭåİ»":108759,"νή":108760,"å¸ĥ满":108761,"åij¼åĸĬ":108762,"Ġaxon":108763,"Ġgenesis":108764,"ïs":108765,"ĠSpectroscopy":108766,"æ´ĭ溢çĿĢ":108767,"Ġ(,":108768,"ĠVag":108769,"ä¿¡å°ģ":108770,"Ġcuriously":108771,"çľ¼çĿģ":108772,"éĵ¶å·Ŀ":108773,"èĹıæĹı":108774,"Ġmidday":108775,"Ġmembaca":108776,"-TV":108777,"Ġpollination":108778,"ĠLiberia":108779,"ĠSimplified":108780,"Spect":108781,"éticos":108782,"astal":108783,"ĠVitt":108784,"ä½łçļĦ人":108785,"她ä»İ":108786,"ç¾İåij³çļĦ":108787,"-course":108788,"æķħæĦıçļĦ":108789,"Ġbanda":108790,"mesh":108791,"æĶ¹éĿ©åıijå±ķ":108792,"åį§åºĬ":108793,"ĠBirch":108794,"Ġpollutant":108795,"ĠболÑĮÑĪое":108796,"Ġসরà§įব":108797,"ĠDwight":108798,"ĠDudley":108799,".Execute":108800,"hore":108801,"Ġfris":108802,"olfo":108803,"ĠCGI":108804,"Ġbegg":108805,"Ġperv":108806,"ĠStati":108807,"onsumsi":108808,"æĹłæįŁ":108809,"ĠSTATEMENT":108810,"ç¼ĵåĨ²åĮº":108811,"Ġajudar":108812,"-land":108813,"/es":108814,"pw":108815,"çļĦä¿¡åı·":108816,"ĠAAC":108817,"çĤ¹çĿĽ":108818,"äºĶéĩij":108819,"Ġfiltr":108820,"ECs":108821,"inoza":108822,"è¡Į为èĥ½åĬĽ":108823,"ĠMerch":108824,"_stmt":108825,"ĠпопиÑģ":108826,"ç¾İæľ¯åѦéĻ¢":108827,"ĠShelter":108828,"ĠDeficient":108829,"ĠSyllabus":108830,"DAR":108831,"Ġlicht":108832,"ĠWink":108833,"ĠInvention":108834,"è¿Ļ个æĸ¹æ³ķ":108835,"Ġrealist":108836,"ãģ¨åIJĮãģĺ":108837,"çļ®ä¹¦":108838,"à¸¸à¸Ľ":108839,"å°įäºĨ":108840,"ĠAfterwards":108841,"è®¾ç½®åľ¨":108842,"åħ¨éĿ¢å»ºæĪIJ":108843,"ĠMicrobial":108844,"ĠAttendance":108845,"Ġconformational":108846,"Ġ×Ķ×IJ×ĵ×Ŀ":108847,"æĶ»åĿļæĪĺ":108848,"ç§īæĮģ":108849,"ĠزÛĮرا":108850,"ĠÑįкÑģплÑĥа":108851,"Kol":108852,"qm":108853,"Ġaku":108854,"ĠMok":108855,"ĠFake":108856,"Ġkary":108857,"ĠpoÅĽ":108858,"项ç¨İé¢Ŀ":108859,"èħĵ":108860,"Ġbiocom":108861,"éĥ¨åĪĨç»ĦæĪIJ":108862,"åĢĴéĢĢ":108863,"Ġpengh":108864,"æķ´ä¸ªä¸ĸçķĮ":108865,"Ġequilibrio":108866,"ì͍":108867,"æĸĩåĩŃ":108868,"Ġwhence":108869,"åºĶéĤĢ":108870,"ç¾İè²Į":108871,"ีà¸ŀ":108872,"修缮":108873,"Ġredress":108874,"å¾ĦåIJij":108875,"ĠBrenda":108876,"numara":108877,"Ġprépar":108878,"å·«å¸Ī":108879,"ĠÑĥÑĢавнений":108880,"ĠпоÑĢÑıдке":108881,"Ġphilanthropic":108882,"Ġpédagog":108883,"LIB":108884,"Ġmely":108885,"ĠAlive":108886,"лик":108887,"INCT":108888,"åĨħ容æĺ¯":108889,"åĿIJä¸Ĭ":108890,"ĠInterim":108891,"Ġsnapping":108892,"éľĩæħij":108893,"å®ĩèĪª":108894,"ÐķÐł":108895,"Ġбалан":108896,"ĠAufgaben":108897,"ĠطراØŃÛĮ":108898,"ĠCHEM":108899,"_limit":108900,"ĠNess":108901,"Ġspar":108902,"åĻİ":108903,"ĠImmediate":108904,"Ġfrantic":108905,"Ġপদ":108906,"Ġalternately":108907,"Ġréflex":108908,"年代ä¸ŃæľŁ":108909,"Ġzwier":108910,"richten":108911,"ĠبØŃÙĬرÙĩ":108912,"Ġvigilance":108913,"å¢ŀæ·»äºĨ":108914,"沦为":108915,"BASE":108916,"ĉS":108917,"ĠLift":108918,"ä¸Ĭ楼":108919,"æİ¥æīĭ":108920,"头ä¸ĬçļĦ":108921,"åħŃ级":108922,"æĦıè§ģ建议":108923,".appendChild":108924,"ĠBowman":108925,"ĠиÑģÑĤоÑĢиÑı":108926,"à´¿à´ķàµįà´ķ":108927,"ĠдвÑĥмÑı":108928,"ĠVegetable":108929,"为æıIJé«ĺ":108930,"ä»ĸçĶļèĩ³":108931,"æĹ¥åİĨ":108932,"ĠÙĪØ¢":108933,"velope":108934,"ÏĦÏİ":108935,"áĥĹ":108936,"ä¿¡æģ¯æľįåĬ¡":108937,"ABB":108938,"è¿Ļä¹Ī好":108939,"幸äºı":108940,"اØŃÙĬØ©":108941,"ĠBrotherhood":108942,"ĠÑģÑĤаÑĤÑĮе":108943,"abhäng":108944,"ĠAlicia":108945,"æłªå¼ıä¼ļ社":108946,";\">.ĊĊ":109471,"Ocean":109472,"mah":109473,"ĠIBD":109474,"ĠCKD":109475,"ĠLoy":109476,"avu":109477,"concat":109478,"Ġspanned":109479,"å±Ĥåĩºä¸įç©·":109480,"èĦ¸éĥ¨":109481,"Ġbomber":109482,"åįłæľīçİĩ":109483,"ĠBoundaries":109484,"骨质çĸıæĿ¾":109485,"NPC":109486,"Ġsiano":109487,"Ġmasing":109488,"ĠLors":109489,"æĢ§åİŁåĪĻ":109490,"лаÑı":109491,"å¯ĴåĨ·çļĦ":109492,"纸巾":109493,"Ġdissolving":109494,"Ġfolgenden":109495,"ĠCamden":109496,"ĠSchemes":109497,"èĭ¥å¹²éĹ®é¢ĺçļĦ":109498,"ë¹ĦìĬ¤":109499,"ĠëĬIJ":109500,"ĠÑģопÑĢоÑĤивление":109501,"alakip":109502,"hner":109503,"ä¸Ģéĸĭå§ĭ":109504,"بÛĮÙĨ":109505,"æ¶ĪæĿĢ":109506,"严å¯Ĵ":109507,"å¹²éĥ¨çļĦ":109508,"Ġignite":109509,"ä¸ģé¦Ļ":109510,"алÑĮнÑĭÑħ":109511,"Ġcroissance":109512,"è´¢æĶ¿éĥ¨éŨ":109513,"ĠECB":109514,"財åĭĻ":109515,"Ġdeteriorated":109516,"Ġrosemary":109517,"ĠICA":109518,"åľ©":109519,"Ġvárias":109520,"اں":109521,"ĠVLAN":109522,"ãĥ¶":109523,"ENDS":109524,"ĠContacts":109525,"altres":109526,"Ġrooting":109527,"Ġrevoked":109528,"ä¹±ä¸ĥåħ«ç³Ł":109529,"éĺħ读çIJĨè§£":109530,"straÃŁe":109531,"HDL":109532,"Ġelegans":109533,"nippet":109534,"æľŁæľ«èĢĥè¯ķ":109535,"Ġbrokerage":109536,"èĬ¹èıľ":109537,"ucceeded":109538,"Rd":109539,"Ġsockets":109540,"æĺ¯ä¼ļ":109541,"avÃŃa":109542,"Ġdisillusion":109543,"ĠChanged":109544,"Ġroy":109545,"åı¯ä»¥åıijçݰ":109546,"Ġcornea":109547,"ĠÑĢаÑģÑĤи":109548,"деб":109549,"ĠEuropea":109550,"åī§åĽ¢":109551,"ĠqualitÃł":109552,"åģıçα":109553,"æĦĪæĿ¥æĦĪ":109554,"åĿ¦è¯ļ":109555,"ĠCooke":109556,"ĠMidlands":109557,"夸å¥ĸ":109558,"Ġrefreshed":109559,"ĠPunkt":109560,"Ġdisgusting":109561,"ĠÑĦÑĢанÑĨÑĥ":109562,"ĠCatar":109563,"iging":109564,"ĠRecreational":109565,"æĶ¹æĢ§":109566,"Ġcosto":109567,"亿ä¸ĩ":109568,"ĠÐłÐ¸":109569,"Ġalcoholism":109570,"ĠBulk":109571,"Ġکاربر":109572,"ówno":109573,"Ġà¦Ĩপনি":109574,"ĠаÑĤмоÑģ":109575,"vival":109576,"erin":109577,"orbit":109578,"رØŃ":109579,"æĶ¶éٳ":109580,"æł¹æ²»":109581,"æĹ¶éĹ´å¤įæĿĤ度":109582,"éĿĴéľīç´ł":109583,"ç³»ç»Łä¸ŃçļĦ":109584,"ĠMeadows":109585,"ÑĦÑĢи":109586,"ĠGeol":109587,"æĮīçħ§è§Ħå®ļ":109588,"Ġstriped":109589,"å¼ĹéĩĮ":109590,"Ġunderserved":109591,"CAL":109592,"Åŀ":109593,"atasi":109594,"ĠDY":109595,"Ġphénom":109596,"aski":109597,"ĠTranscription":109598,"Ġsegurança":109599,"åijĬè¯īäºĨ":109600,"æĬ¬éłŃ":109601,"çļĦçī¹çĤ¹æĺ¯":109602,"Ġpúblicos":109603,"ĠØ¢ÙħÙĪØ²Ø´ÛĮ":109604,"ĠMOSFET":109605,"ĠFörder":109606,"mml":109607,"æĸ¹æł¼":109608,"åģ½":109609,"åIJĮåIJį":109610,"å¿«å¿«":109611,"Revenue":109612,"çļ®éŀĭ":109613,"ombonana":109614,"IndexOf":109615,"æł¸å¿ĥç«ŀäºīåĬĽ":109616,"ĠNormandy":109617,"Ġabbreviations":109618,"ainting":109619,"Ġresumes":109620,"ĠVE":109621,"Ġpreprint":109622,"åIJĦåľ°åĮº":109623,"Ġзави":109624,"Ġcastles":109625,"алÑĮно":109626,"çİĦå®Ĺ":109627,"Ġepidermis":109628,"ĠздÑĢав":109629,"Ġtess":109630,"arita":109631,"Ġimpar":109632,"ÙĪÙĬÙĨ":109633,"车ä¼ģ":109634,"åı«å¥¹":109635,"Ġcontacto":109636,"建çŃijçī©çļĦ":109637,"ĠÐĶоба":109638,"houd":109639,"jans":109640,"ĠBAC":109641,"éĤı":109642,"visiae":109643,"Ġש×Ĺ":109644,"æĬĸåĬ¨":109645,"Ġmerciful":109646,"ĠимÑı":109647,"Ġrůzn":109648,"Ġintrusive":109649,"Ġমাধà§įযমà§ĩ":109650,"ĠPact":109651,"ä¸į说è¯Ŀ":109652,"ĠEMA":109653,"threshold":109654,"Ġjauh":109655,"Ġsubdivided":109656,"ĠExclusive":109657,"åĪĩãĤĬ":109658,"ophones":109659,"ÙİØ§ÙĨ":109660,"Ġnominees":109661,"Ġžád":109662,"ĠPathway":109663,"Ġvibrational":109664,"à¹Ħà¸Łà¸Ł":109665,"ĠÙ쨱ÙĩÙĨÚ¯ÛĮ":109666,"ãĤ¸ãĤ§ãĤ¯ãĥĪ":109667,"åĵŃç¬ijä¸įå¾Ĺ":109668,"Harvard":109669,"Ġcarts":109670,"ä¸įçIJĨè§£":109671,"Ġrite":109672,"ä¹ĭä½į":109673,"ungi":109674,"æľĿæ°Ķ":109675,"纸å¸ģ":109676,"ĠاÙĦÙĥÙĪÙĬÙĥبات":109677,"LAY":109678,"ĠKomment":109679,"Ġmetaphorical":109680,"Ġunsatisfactory":109681,"à¹Ģหลà¹Īาà¸Ļีà¹ī":109682,"CBD":109683,"Nap":109684,"Ġwissenschaft":109685,"Ġbanners":109686,"ĠGins":109687,"ĠdoÅĽwiad":109688,"åIJİç»§":109689,"_{(":109690,"ungg":109691,"èĮ²":109692,"ä»ĸ们没æľī":109693,"æ¸ħæ¸ħ":109694,"ยาว":109695,"hofer":109696,"blr":109697,"æ·±åħ¥è´¯å½»èIJ½å®ŀ":109698,"à½Ķ":109699,"-Jan":109700,"Ġintrospection":109701,"ĠMarianne":109702,"ä¸Ģ模ä¸Ģæł·":109703,"ĠSle":109704,"idl":109705,"akkan":109706,"ä¹ĭçζ":109707,"Ġ<ĊĊ":109708,"ĠChau":109709,"éĤ£åı¥è¯Ŀ":109710,"çļĦä¸Ģåľº":109711,"ĠValve":109712,"ĠErreferentziak":109713,"-Be":109714,"ä»ĵä½į":109715,"ä¿¡çĶ¨ç¤¾":109716,"ì¶©":109717,"ê¹Ģ":109718,"Ġadenosine":109719,"native":109720,"wares":109721,"ä¸Ģä¼Ĺ":109722,"ä¸Ĭå°Ĩ":109723,"èĢĮ论":109724,"Ġ\\%":109725,"ducted":109726,"æĹłè®ºåľ¨":109727,"æĥłå·ŀ":109728,"ĠгÑĢÑĥпп":109729,"-hydrox":109730,"vang":109731,"ĉdb":109732,"ĠsÃŃmbol":109733,"Ġbik":109734,"Ġmalle":109735,"åıijæķ£":109736,"ĠStato":109737,"Ġiombonana":109738,"بÙĪØ¨":109739,"æĹłåĩł":109740,"proper":109741,"Ġacima":109742,"oxox":109743,"åύçŃī":109744,"ç»Ĩå°ı":109745,"ĠÑģÑĤаÑĤÑĥ":109746,"ظة":109747,"compared":109748,"Ġjudgements":109749,"destination":109750,"ĠSaxon":109751,"^^^^^^^^":109752,"dur":109753,"ĠCCR":109754,"ĠMSS":109755,"为åīįæıIJ":109756,"è¦ģå®ŀçݰ":109757,"riches":109758,"樵":109759,"ĠExamine":109760,"éĹ¨æ´¾":109761,"ĠQuelle":109762,"éϷ害":109763,"Ġformalism":109764,"LOY":109765,"Ġdigitale":109766,"à¸ķัวà¹Ģà¸Ńà¸ĩ":109767,"วà¹Īาà¸Īะ":109768,"æį§çĿĢ":109769,"Ġë§Įëĵ¤ìĸ´":109770,"SHA":109771,"ĉdefault":109772,"Ġtrophic":109773,"æĪijæĸ¹":109774,"ä¹Łå¹¶ä¸į":109775,"åĨħè¡£":109776,"éĴ´":109777,"空äºĨ":109778,"валиÑģÑĮ":109779,"اÛĮع":109780,"åıĤä¸İçļĦ":109781,"Ġcircumvent":109782,"èĢIJçģ«":109783,"éĥ½ä¼ļ被":109784,"诺夫":109785,"èį·åı¶":109786,"instagram":109787,"Ġrozm":109788,"å±łå®°":109789,"ä»Ĩ人":109790,"à¸ķัà¹īà¸ĩà¹ģà¸ķà¹Ī":109791,"Judge":109792,"§×ľ":109793,"ä¼İ":109794,"åľ°é»ŀ":109795,"天河":109796,"便å¼Ģå§ĭ":109797,"端çĤ¹":109798,"æĿĢèĻ«":109799,"æīĺ马æĸ¯":109800,"BCD":109801,"\\,=\\,":109802,"ĠEXAM":109803,"àµģà´¨àµįà´¨":109804,"Ġpneumatic":109805,"Ġá½ħ":109806,"Ġosmotic":109807,"Ġtranscendental":109808,"Ġëĭ¤ìĿĮê³¼":109809,"ĠÐĿикола":109810,"Ġcaractérist":109811,"ĠManc":109812,"ĠHul":109813,"Ġjuego":109814,"Ġcaries":109815,"-large":109816,"ĠScrabble":109817,"altet":109818,"èĥ¶çīĩ":109819,"缸åºĶåľ°":109820,"\\mid":109821,"inj":109822,"Ġexcl":109823,"就已ç¶ĵ":109824,"åĪĨæķ°çļĦ":109825,"ĠWeapon":109826,"çĸ½":109827,"åħĪ秦":109828,"оÑĢÑĤа":109829,"æĸŃç»Ń":109830,"ANDS":109831,"å±ħ士":109832,"ãģĵãģ¡ãĤī":109833,"ĠCourtesy":109834,"èĢĹæĹ¶":109835,"大éĥ¨åĪĨçļĦ":109836,"ĠECONOM":109837,"ĠÑĢиÑģк":109838,"enschaften":109839,"Ġchuckle":109840,"åķªåķª":109841,"ĠдоÑģÑĤига":109842,"ĠScarlet":109843,"Ġstromal":109844,"Ġlily":109845,"veget":109846,"äºĨè¿Ľåİ»":109847,"ĠRn":109848,"ellus":109849,"年齡":109850,"åºĶèĢĥèĻij":109851,"Ġpee":109852,"ĠAnagram":109853,"è£ħä¸Ĭ":109854,"çģ«çĤ¬":109855,"ECO":109856,"åħħçĽĪ":109857,"ç¶±":109858,"票价":109859,"æĥĬ天":109860,"çĥŁçļĦ":109861,"Ġغذا":109862,"ìĿ¼ìĹIJ":109863,"Ġcategorization":109864,"Ġnahil":109865,"çĽijæĬ¤äºº":109866,"Ġmisfortune":109867,"Ġophthalm":109868,"нок":109869,"ĠDus":109870,"Ġkettle":109871,"èĢĮå¤į":109872,"Ġinvitations":109873,"ĠкнÑı":109874,"è¡Ģéĩı":109875,"ÑĢÑĥÑİÑĤÑģÑı":109876,"à´¦":109877,"è²ŀ":109878,"Ġsteels":109879,"èįīæľ¬":109880,"ç»Īäºİåľ¨":109881,"Ġdisperse":109882,"éĽ¾æ°Ķ":109883,"Ġdiket":109884,"ç»Ĵæ¯Ľ":109885,"Ġimpeachment":109886,"ĠToulouse":109887,"Ġnexus":109888,"Sold":109889,"eis":109890,"otis":109891,"ä¸Ńæ·»åĬł":109892,"ä»İåħ¶":109893,"жного":109894,"Ġrunt":109895,"κλη":109896,"åħ«æĪĴ":109897,"Ġexcite":109898,"ä¸ĥ大":109899,"Ġchecker":109900,"å²ģçļĦæĹ¶åĢĻ":109901,"ĠÐļÑĢаÑģ":109902,"Ġà¦Ĩন":109903,"HPV":109904,"Ġdentists":109905,"Koordin":109906,"ĠοÏĢοί":109907,"(ST":109908,"Military":109909,"ĠMSN":109910,"è§£å¯Ĩ":109911,"-loving":109912,"ÙĨدر":109913,"Ġforgiving":109914,"ĠновÑĭй":109915,"ĠBotswana":109916,"ĠLionel":109917,"ĠWnt":109918,"ĠNahr":109919,"ä¹ĭæŃĮ":109920,"æİ¨åΰ":109921,"åħ«è§Ĵ":109922,"æĤ¨åľ¨":109923,"ë¦Ń":109924,"ÉĻm":109925,"Ġtuvo":109926,"Ġaccorded":109927,"ĠزÛĮاد":109928,"ĠدÙĪÙĦت":109929,"å«£çĦ¶":109930,"Ġclerks":109931,"EQU":109932,"Robin":109933,"ĉin":109934,"Ġcinc":109935,"çļĦæııè¿°":109936,"stars":109937,"ĠSlim":109938,"oway":109939,"ä¸ªé¡¹çĽ®":109940,"clampsia":109941,"æĸ°éĥİ":109942,"åĪĹä¼ł":109943,"çijĻ":109944,"绿çģ¯":109945,"Ġoptimizer":109946,"cycling":109947,"огÑĢаÑĦ":109948,"Ġglobale":109949,"åįļçī©é¤¨":109950,"othyroidism":109951,"OOOO":109952,"溯æºIJ":109953,"Ġabrasive":109954,"Ġpalavras":109955,"Ġintoxication":109956,"Kam":109957,"Ġquaint":109958,"avoir":109959,"æŀľçľŁ":109960,"ìĿµ":109961,"ranj":109962,"åΰäºĨä¸Ģ个":109963,"EventHandler":109964,"ĠبÙĨاء":109965,"itarianism":109966,"ĠCristina":109967,"Ġinexplic":109968,"Ġtreadmill":109969,"ĠOphthalmol":109970,"Ġnahilalakip":109971,"=âĪij":109972,"ĠTweet":109973,"estanden":109974,"ipiko":109975,"åIJİ被":109976,"èĢĮæĦŁåΰ":109977,"Ġobes":109978,"两é¢Ĺ":109979,"Ġcaroten":109980,"åħīæĿŁ":109981,"-mi":109982,"ä¿®çļĦ":109983,"notice":109984,"å°¼åı¤":109985,"Ġনà§ĩà¦ĩ":109986,"Ġpraising":109987,"ĠδÏį":109988,"Ġpoisoned":109989,"emperaturen":109990,"ĠPatriot":109991,"ĠÙĬÙĤÙĪÙħ":109992,"_->":109993,"D":109994,"Ġlanc":109995,"ĠTyson":109996,"ĠFU":109997,"liction":109998,"å°ıèĬĤ":109999,"ritu":110000,"å±ŀå®ŀ":110001,"Ġidols":110002,"¡×Ŀ":110003,"Ġsembl":110004,"éĹªçĥģçĿĢ":110005,"Ġtind":110006,"Ġñ":110007,"\">&":110008,"_down":110009,"Ġethically":110010,"çŀªçĿĢ":110011,"TiO":110012,"Ġsarebbe":110013,"/create":110014,"\\log":110015,"jor":110016,"çļĦä¼ĺç§Ģ":110017,"ĠAar":110018,"ĠBarg":110019,"ĠLargest":110020,"Ġuid":110021,"spin":110022,"Ñİда":110023,"ÑĤиÑĤе":110024,"Ġأغ":110025,"缸åħ³å·¥ä½ľ":110026,"ĠISS":110027,"ìķĶ":110028,"ickson":110029,"Ġübers":110030,"à¤ķà¥ĩ":110031,"Ġreforming":110032,"åĨ¥æĥ³":110033,"ĠابرÙĬÙĦ":110034,"Ġcomedian":110035,"Lith":110036,"bite":110037,"zum":110038,"atÄĥ":110039,"æĺ¯æĢİæł·çļĦ":110040,"æľīæĥħ":110041,"æĶľ":110042,"дки":110043,"Ġspecs":110044,"Ġerh":110045,"åįĬæķ°":110046,"ĠContoh":110047,"Ġপড়":110048,"Ľ×ł×¡":110049,"éĿ¢å¯¹çĿĢ":110050,"Namespace":110051,"Ġoverlaps":110052,"天空ä¸Ń":110053,"ĠÑģемÑĮи":110054,"æŁijæ©ĺ":110055,"ĠодновÑĢеменно":110056,"Ġacht":110057,"ĠCSP":110058,"åºĶäºĪ":110059,"æµģè¿ĩ":110060,"}})":110061,"è°ĪåıĬ":110062,"éľĩé©ļ":110063,"ĠпÑĢедÑĭдÑĥ":110064,"ĠRamirez":110065,"åŁĶ寨":110066,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤики":110067,"Ġpornography":110068,"inib":110069,"ĠидеÑĤ":110070,"Ġinflection":110071,"Ġintelect":110072,"rags":110073,"ðĿijĢ":110074,"ãģıãģ¨":110075,"éľĢæ±Ĥéĩı":110076,"Ġtransformational":110077,"Ġcrooked":110078,"Ġaccomplishing":110079,"Ġbolst":110080,".Repository":110081,"'article":110082,"ĠRails":110083,"èĩªå·±èĥ½":110084,"è°ĥçļĦ":110085,"å·²ç»ı没æľī":110086,"ĠPrasad":110087,"Ġapologies":110088,"paul":110089,"/base":110090,"-compliance":110091,"ãĥ¡ãĥ¼ãĤ¸":110092,"á±®":110093,"Ġhygien":110094,"èŃ¦ç¤ºæķĻèĤ²":110095,"rvatski":110096,"-industrial":110097,"ä¸į注æĦı":110098,"åĴĮåĪĺ":110099,"Ġzoon":110100,"å¸ĤåĨħ":110101,"efa":110102,"她就æĺ¯":110103,"ä¼ģä¸ļä¸Ń":110104,"éĺŁåľ¨":110105,"á»§":110106,"ä¹Łæĺ¯è¿Ļæł·":110107,"åĨ·æĪĺ":110108,"æĽ¾è¢«":110109,"åı¸æ³ķè§£éĩĬ":110110,"combined":110111,"ĠÑģеÑĢде":110112,"Ġfenó":110113,"~\\":110114,"ĠMás":110115,"çĽ¸çº¦":110116,"åύçī©":110117,".Split":110118,"à´¨àµįà´":110119,"åĿĩçͱ":110120,"æŃ¢åĴ³":110121,"Ġconcussion":110122,"æ±īä¸Ń":110123,"æ·±åħ¥äººå¿ĥ":110124,"iód":110125,"Ġதà¯Ĭ":110126,"ifié":110127,"ĠRodrigo":110128,"(read":110129,"stro":110130,"ĠTurns":110131,"oders":110132,"åĽ½ãģ®":110133,"å¾Īå¼Ģå¿ĥ":110134,"äºĮéĥİ":110135,"åIJijæĹ¥":110136,"Thinking":110137,"æĶ¾èĤĨ":110138,"Ġ):Ċ":110139,"è¯ij为":110140,"ç²¾åĩĨæī¶è´«":110141,"ÙıÙĪÙĨÙİ":110142,"(ret":110143,"ĠSime":110144,"ä¸ī两":110145,"ç¾İåĽ¢":110146,"è´¨åŃIJ":110147,"éľĢè¦ģèĢĥèĻij":110148,"rachen":110149,"ĠGeomet":110150,"éĤ£ä¹Īå°±":110151,"æ¯ı天çļĦ":110152,"enziale":110153,"Ġoverwhelm":110154,".backgroundColor":110155,"CMS":110156,"Ft":110157,"GRE":110158,"PID":110159,"ĠSä":110160,"éģĽ":110161,"ÏĦζ":110162,"åĮħéĩĮ":110163,"ĠContain":110164,"å¾ģåħĨ":110165,"Ġparticipar":110166,"Ġredshift":110167,"Ġmerk":110168,"ç§ĭæ°´":110169,"änner":110170,"è°±åĨĻ":110171,"Ġbioavailability":110172,"Ġà¦īদà§įà¦":110173,"Ġcannabin":110174,"ĠINTERNATIONAL":110175,"ĠHeinz":110176,"ĠاÙĦإسÙĦاÙħÙĬØ©":110177,"fem":110178,"Ġeczema":110179,"ĠFon":110180,"ĠGina":110181,"ÑĭÑĢ":110182,"phs":110183,"è¿Ľè¡Įè°ĥæŁ¥":110184,"éĽĨä½ĵçļĦ":110185,"Ġcoupons":110186,"åģľæĶ¾":110187,"çīĽå¸Ĥ":110188,"å¼±èĢħ":110189,"é«ĶçļĦ":110190,"èĩ³å°ijè¦ģ":110191,"leqslant":110192,"åµĮå¥Ĺ":110193,"allocate":110194,"ĠÑģÑĤÑĥденÑĤов":110195,"-medium":110196,"Md":110197,"_Id":110198,"ŀ×Ļת":110199,"Ġläng":110200,"ä¸į大çļĦ":110201,"verd":110202,"æĹ¶éļĶ":110203,"ĠVos":110204,"å°ıåģ·":110205,"Ġmodulator":110206,"genre":110207,"éĿŀ常ãģ«":110208,"æ¿ĢæĺĤ":110209,"à¹Ĥà¸ļ":110210,"åıªè¦ģæĪij们":110211,"äºİæĺ¯å°±":110212,"ĠAvg":110213,"æĬ¬çľ¼":110214,"Estat":110215,"ãģĺãģ¦":110216,"ĠпÑĢоизведениÑı":110217,"Ġdisplacements":110218,"Ðĥ":110219,"Ġlatt":110220,"ä»ĸåıijçݰ":110221,"å¾ĹæĦıçļĦ":110222,"Ġstudie":110223,"Ġrayon":110224,"Ġflocks":110225,"车çªĹ":110226,"_pass":110227,"ĠPresidente":110228,"Ġwarmly":110229,"æī«åľ°":110230,"ãĤ¤ãĥ³ãĤ¿":110231,"мÑıÑĤи":110232,"Ġthirsty":110233,"Occupation":110234,"[â̦]":110235,"<'":110236,"ä¸Ńãģ®":110237,"Ġbutcher":110238,"éķ¿åº¦çļĦ":110239,"adek":110240,"ĠZi":110241,"Ġcarácter":110242,"å®ĥæĺ¯ä¸Ģç§į":110243,"çĻ½äºº":110244,"koz":110245,"æģĭæĥħ":110246,"èģ½è¦ĭ":110247,"Ġlurking":110248,"Ġzvý":110249,"Ij":110250,"Ġcedar":110251,"oucester":110252,"cego":110253,"ĠDove":110254,"ä»ĸè¿Ļ个":110255,"ä¿ł":110256,"åģķ":110257,"å½¢æħĭ":110258,"liÅ¡":110259,"æķĻèĤ²åĩºçīĪ社":110260,"ÙĦÙĬد":110261,"endeley":110262,"transl":110263,"ÙħÙĪØ§Ø·":110264,"赤åŃĹ":110265,"çķĻä¸ĭæĿ¥çļĦ":110266,"ĠìłĦì²´":110267,"Ġendometrial":110268,"\"s":110269,"-ranking":110270,"Youth":110271,"utzt":110272,"adah":110273,"opers":110274,"Ġ\"":111050,"Ġ()ĊĊ":111051,"éon":111052,"дова":111053,"Ġdenomination":111054,"OSH":111055,"è½®æľº":111056,"åĶIJåĥ§":111057,"Ġsweats":111058,"æīĭæľºçļĦ":111059,"Ġcircumferential":111060,"èı²èı²":111061,"ĠUnterst":111062,"Ġberkembang":111063,"Ġprogeny":111064,"ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ":111065,"ĠBolshevik":111066,"Ott":111067,"ÆĴ":111068,"ollection":111069,"Ġdelect":111070,"Ġundocumented":111071,"å¼Ģåħĥ":111072,"æĸĩåºĵ":111073,"erti":111074,"centric":111075,"çĹħèıĮ":111076,"çİĭæ°ı":111077,"æĿ¿æĿIJ":111078,"åĸĦæģ¶":111079,"Plug":111080,"èİ·å¾ĹæĦŁ":111081,"inputs":111082,"èĻļå¿ĥ":111083,"ĠGreenberg":111084,"NaN":111085,"ĠErgebnisse":111086,"Ġutensils":111087,"åĵ½åĴ½":111088,"/add":111089,"Candidate":111090,"Shel":111091,"dimensional":111092,"ĠCrab":111093,"agland":111094,"ä½Ĩè¿Ļ个":111095,"åĪļä»İ":111096,".fe":111097,"ĠDisadvantages":111098,"ÑĤивное":111099,"伯伯":111100,"muir":111101,"Ġyellowish":111102,"Ġdeformity":111103,"Ġamygdala":111104,"ainya":111105,"Ġplais":111106,"ä¹Łåıĺå¾Ĺ":111107,"æĢ§æĪĸ":111108,"ç©İ":111109,"Ġclassific":111110,"Ġconsiderar":111111,".services":111112,"æľ¨åħ°":111113,"Dean":111114,"_front":111115,"Across":111116,"æĪij们åı¯ä»¥çľĭåΰ":111117,"Ġvitamina":111118,"æģ°å½ĵçļĦ":111119,"Ġacquaintances":111120,"Ġhø":111121,"Ġisomorphism":111122,"áh":111123,"æŃ¤ä¹¦":111124,"ullo":111125,"æĦŁè§īå¾Ī":111126,"Ġbottleneck":111127,"Behind":111128,"æľ±å¾·":111129,"ÙĦÙĥترÙĪÙĨ":111130,"ãĢĭï¼ĮãĢĬ":111131,"çĤĴèĤ¡":111132,"æĸijæĸĵ":111133,"æĢľæĤ¯":111134,"å··éģĵ":111135,"Ġforcibly":111136,"Nig":111137,"æĸ¹åĿĹ":111138,"æĪij们åħļ":111139,"valho":111140,"认åĩº":111141,"çİĭæĸĩ":111142,"æ¦Ķ":111143,"åıijçĶŁæĹ¶":111144,"æĭĽå¼ı":111145,"({'":111146,"ĠIncreases":111147,"Ġwhispering":111148,"ĠPumpkin":111149,"Ġsubmarines":111150,"ĠGEO":111151,"éĿ¢å¸¦":111152,"anything":111153,"ĠDei":111154,"åıĸèĢĮ":111155,"ĠзаÑĢÑı":111156,"波士":111157,"ĠاÙĦعراÙĤ":111158,"Ġboarded":111159,"ĠSalon":111160,"ĠLogistic":111161,"åĽŀçŃĶéĹ®é¢ĺ":111162,"رÙĪØ¨":111163,"åįģäºĮäºĶ":111164,"å°ĺåľŁ":111165,"æį·å¾Ħ":111166,"ĠElles":111167,"祥åĴĮ":111168,"Ġdentistry":111169,",sizeof":111170,"atians":111171,"ĠGret":111172,"ĠJedi":111173,"ĠKnee":111174,"æķ°çϾä¸ĩ":111175,"Ġbacklog":111176,"åıijå±ķä¸İ":111177,"Ġcosta":111178,"èĭ¥èĥ½":111179,"_depth":111180,"ë¶Ģë¶Ħ":111181,"éļ¾éģĵæĺ¯":111182,"Ġপà§įরতি":111183,"erning":111184,"quil":111185,"åIJįæĢĿ":111186,"é£İ顺":111187,"æ¯ıæĻļ":111188,"hesion":111189,"å᡿ĸ¯":111190,"লা":111191,"çīĽæİĴ":111192,"Apparently":111193,"æijĩæ»ļ":111194,"utenberg":111195,"accio":111196,"ĠÑĤеÑĢÑĢиÑĤоÑĢи":111197,"Ġdátum":111198,"christ":111199,"essor":111200,"ĠNicht":111201,"é«ĺéĽħ":111202,"ajat":111203,"åħĥç¥ŀ":111204,"è®°ä½ıäºĨ":111205,"è¿ŀçݯ":111206,"onaldo":111207,"ÙĦاÙĪÙĩ":111208,"Ġrubble":111209,"ĠÎļλιÏĦικÏĮÏĤ":111210,"ĠPolygon":111211,"Ġescolas":111212,"(on":111213,"-CO":111214,".OK":111215,"Mos":111216,"équence":111217,"对å®ĥ":111218,"Ġà¦Ŀ":111219,"çŃīæľįåĬ¡":111220,"ĠArun":111221,"ĠAsians":111222,"è½¬åŁºåĽł":111223,"ç²¾æĺİ":111224,"Ġreducer":111225,"é£ŀåΰ":111226,"ĠÕĬ":111227,"æľĢåIJİçͱ":111228,"×ij×Ļר":111229,"ĠTurbo":111230,"Ġgestured":111231,"çļĦåŁºæľ¬åİŁåĪĻ":111232,"ĠHoriz":111233,"elijkheid":111234,"Ġprésident":111235,"ĠBLACK":111236,"æĥħæ³ģä¸ĭ":111237,")_,":111238,",max":111239,"Biblical":111240,"IJר":111241,"ĠSamm":111242,"Ġentreg":111243,"éĿŀåIJĮ":111244,"管çIJĨè§Ħå®ļ":111245,"Ġump":111246,"çͰå¾Ħ":111247,".Try":111248,"Ġnoticeably":111249,"Ġowls":111250,"gravity":111251,"èĤĭ骨":111252,"Ġemancipation":111253,"-formed":111254,"Kudos":111255,"Vy":111256,"broad":111257,"nomin":111258,"ilded":111259,"ä¸īçŃī":111260,"æīĭæŀª":111261,"å¤ĸå¸ģ":111262,"çī¹å¼Ĥ":111263,"ucia":111264,"Ġpublicado":111265,"tsky":111266,"nesses":111267,"åįĹæľĿ":111268,"èĭ¥æľīæīĢæĢĿ":111269,"Ġnecessidade":111270,"Ñĺан":111271,"éģ¿éĻ©":111272,"Ġ]];":111273,"fluoro":111274,"Ġdominion":111275,"è᡿¼¾":111276,"Ġdiscloses":111277,"ĠسبÙĬÙĦ":111278,"Ġencontra":111279,"Ġeinges":111280,"ä¸Ń西åĮ»":111281,"Ġgiovani":111282,"fighting":111283,"Ġà¸Ĺำ":111284,"ĠoÊ»":111285,"åŃIJæĺ¯":111286,"ĠChil":111287,"æķĻæĪij":111288,"ĠÙĩزار":111289,"Ġlois":111290,"Ġhomens":111291,"ĠWilly":111292,"ĠмоменÑĤа":111293,"phinx":111294,"Ġprzeprowad":111295,"(By":111296,"_run":111297,"_images":111298,"zee":111299,"lywood":111300,"âĢĿ-":111301,"天ä¸ĬçļĦ":111302,"Ġoperable":111303,"ĠпÑĢивеÑģÑĤи":111304,"à¹Ħหà¸Ļ":111305,"ÏĮμε":111306,"å¤ļå°ij次":111307,"ç¦ģ令":111308,"Ġcytometry":111309,"ìĿĮìĿĦ":111310,"ä¸Ģä»¶äºĭæĥħ":111311,"ĠCheryl":111312,"relationships":111313,"-dessus":111314,"Ġaryl":111315,"reis":111316,"ĠFAT":111317,"erea":111318,"Ġemakume":111319,"åĪĻå°Ĩ":111320,"ãģĦãģ¯":111321,"Ġnonverbal":111322,"çŁ³åĿĹ":111323,"Ġbadania":111324,"ãĤ¹ãĥļ":111325,"æ¡Įæ¤ħ":111326,"ĠTHER":111327,"è·ĮåĢĴ":111328,"Ġê·¸ëŀĺ":111329,"رÛĮÙĩ":111330,"áĥĶáĥĽ":111331,"advanced":111332,"ĠкоÑĢÑĢе":111333,"麻çĥ¦äºĨ":111334,"Ġtriumphs":111335,"Ġexcavations":111336,"ĠгеогÑĢаÑĦи":111337,"ĠPharisees":111338,"ĠSized":111339,"iega":111340,"ĠVargas":111341,"Ġzusamm":111342,"åIJįæĽ°":111343,"åı£æ¸´":111344,"Ġcorrig":111345,"ĠZeng":111346,"æİĴçIJĥ":111347,"ä¸ĢäºĽå°ı":111348,".Action":111349,"Ġfrontline":111350,"Ġcarbide":111351,"evidence":111352,"æĬ¢åįł":111353,"åIJIJèķĥ":111354,"ĠWoody":111355,"è¿ĽæŃ¥çļĦ":111356,"ĠLatitude":111357,"å¾Īæľīè¶£":111358,"çijŁçijŁ":111359,"ĠEDTA":111360,"Ġredirected":111361,"ĠÑįлеменÑĤ":111362,"Ġgusts":111363,"Shares":111364,"Ġransomware":111365,"ĠPueblo":111366,"ä»ĸå°±ä¼ļ":111367,"Clin":111368,"åŁºæľ¬åĬŁ":111369,"ÙĪØ¯ÛĮ":111370,"åĪ¶åº¦å»ºè®¾":111371,"SEO":111372,"èģĶç³»åľ¨ä¸Ģèµ·":111373,"ĠPortable":111374,"ĠespÃŃ":111375,"à¸īัà¸Ļ":111376,"ophyte":111377,"ä»ĬåĽŀãģ¯":111378,"ĠbÄĽhem":111379,"ĉis":111380,"teren":111381,"为å¸Ī":111382,"ĠKiev":111383,"ĠStall":111384,"ĠDeux":111385,"ç§ijæķĻ":111386,"æķĻèĤ²ä¸İ":111387,"ACION":111388,"为äºĨå®ŀçݰ":111389,"='\"":111390,"ĠGeneralized":111391,"Ġmengambil":111392,"çļĦå¿ĥä¸Ń":111393,"Ġsitio":111394,"ĠпÑĢодÑĥкÑĤÑĭ":111395,":\\\\":111396,"jie":111397,"vera":111398,"ermont":111399,"Thor":111400,"å®ĥä¸įä»ħ":111401,"å¸ĥéĩĮ":111402,"æĿĢæİī":111403,"-BY":111404,"èĭ±åĽ½äºº":111405,"Ġpenggunaan":111406,"ëIJĺì§Ģ":111407,"ĠдейÑģÑĤвие":111408,"ĠÙĪÙĥذÙĦÙĥ":111409,"(at":111410,"Ġpijn":111411,"Ġflaps":111412,"åķ®":111413,"被éªĹ":111414,".src":111415,"è¿ŀ绵":111416,"Advertising":111417,"Ġtenir":111418,"Ġsequestration":111419,"Ġaufge":111420,"åIJ¬åΰè¿Ļè¯Ŀ":111421,"ĠGalactic":111422,"Ġadversaries":111423,"interno":111424,"âĸijâĸij":111425,"CBA":111426,"ã³":111427,"åİ»åIJĥ":111428,"Ġobdob":111429,"ĠZhe":111430,"Ġniez":111431,"ĠALJ":111432,"?âĢĻâĢĻ":111433,"locals":111434,"ĠسازÛĮ":111435,"Ġanglais":111436,"ĠкомпÑĮÑİÑĤеÑĢ":111437,"Dw":111438,"minton":111439,"Ġdunk":111440,"çĶ¥":111441,"Ġwithhold":111442,"istique":111443,"æĪIJ群":111444,"ovia":111445,"Ġheral":111446,"éľ¹":111447,"è§£æķij":111448,"西西":111449,"Ġavalia":111450,"Ġ×Ķף":111451,"Ïģκ":111452,"ACG":111453,"Ġ×IJ×ķת×Ŀ":111454,"ĠзанÑıÑĤиÑı":111455,"ĠErgebnis":111456,"Ġincompetent":111457,"---------------+---------------+":111458,"Ei":111459,"æĪ»":111460,"Ġunimportant":111461,"ä»ĸè·Ł":111462,"ĠKita":111463,"èĩªè´£":111464,"èĢħãģĮ":111465,"ç¥ŀç¶ĵ":111466,"åĽĽä¸ªäºº":111467,"ĠMeal":111468,"鼶件çļĦ":111469,"Ġbottlen":111470,"åĵŃ声":111471,"Ġdoubtless":111472,"Ġvenir":111473,"ĠпеÑĢвÑĭе":111474,"Digits":111475,"غÙĨاط":111476,"ĠMereka":111477,"<(":111478,"Bucket":111479,"ĸন":111480,"Ġnenh":111481,"æĭ¡":111482,"å¥½äºĽ":111483,"Ù쨧ÙĤ":111484,"广度":111485,"æłij人":111486,"æĽ¾è¯´":111487,"ĠVerizon":111488,"Ġaxons":111489,"Ġদà§ĩà¦ĵ":111490,"Ġappreciating":111491,"Ġlecturers":111492,"çĽĨæł½":111493,"Ġînt":111494,"ĠJahres":111495,"Ġhelmets":111496,"Balt":111497,"_host":111498,"ÑĤова":111499,"ueva":111500,"Ġzin":111501,"॰":111502,"ÙĥÙĬÙĨ":111503,"ĠÙĨزد":111504,"Ġregularization":111505,"Ġrész":111506,"Ġহয়à§ĩ":111507,"_STATUS":111508,"Ġominous":111509,"ĠاÙĦÙħختÙĦÙ쨩":111510,"+g":111511,"_dev":111512,"xg":111513,"ĠTres":111514,"ipend":111515,"ĠKaj":111516,"å°ĨæĪij":111517,"Ġphần":111518,"å¿«åľ°":111519,"ĠSuitable":111520,"ĠControlling":111521,"ĠNej":111522,"eningkatan":111523,"Ġoriginalen":111524,"Appl":111525,"RequestBody":111526,"à¸ľà¸´à¸§":111527,"å¦ĸéŃĶ":111528,"ĠìļĶìĨĮ":111529,"ĠпÑĢинимаÑĤÑĮ":111530,"éļıå¤Ħåı¯è§ģ":111531,"=_":111532,"Gary":111533,"rÃŃan":111534,"Ġcân":111535,"chien":111536,"Ġanorexia":111537,"ĠDOT":111538,"ĠDienst":111539,"perform":111540,"èĢĮä¸İ":111541,"åıĪä¸įèĥ½":111542,"è¿IJç͍çļĦ":111543,"raltar":111544,"Ġаналоги":111545,"ĠPeripheral":111546,"ĠProgramm":111547,"Ġaufgrund":111548,"Ġtaas":111549,"èĤĿ硬åĮĸ":111550,"深度åŃ¦ä¹ł":111551,"Ġsingularity":111552,"Mul":111553,"_dec":111554,"Ġbaker":111555,"Ġпли":111556,"ĠÙģÙĦا":111557,"èĬ±åºı":111558,"åĨ³èĥľ":111559,"-mat":111560,"çģ«ä¸Ĭ":111561,"èŀįåªĴä½ĵ":111562,"Ġел":111563,"å¤ľæĻ¯":111564,"ë¡ľìĦľ":111565,"ık":111566,"Ġastrology":111567,"Ġúj":111568,"uggestion":111569,"Democratic":111570,"Electrical":111571,"Ġclamping":111572,"Ġacompañ":111573,"^i":111574,"ĠIMM":111575,"ieber":111576,"ĠLeather":111577,"éĢļè¿ĩåľ¨":111578,"è½®æ¤ħ":111579,"understanding":111580,"ÏİÏĤ":111581,".year":111582,"Ġunsettling":111583,"ĠBrittany":111584,"#>":111585,"ĺר":111586,"æĹ¶å¿ħé¡»":111587,"å°±æ¯Ķè¾ĥ":111588,"lesh":111589,"ĠReservation":111590,"çĶŁæ´»åŀĥåľ¾":111591,"окой":111592,"以ä¸ĭåĩłç§į":111593,"èģĶç³»æĪij们":111594,"ĠCHF":111595,"ĠاÙĦبد":111596,"ĠминеÑĢа":111597,"çĵ¦å°Ķ":111598,"Ġcerevisiae":111599,"ĠاÙĦاÙĨت":111600,".IsNullOr":111601,"Ġjovens":111602,"qb":111603,"Ġpung":111604,"ivät":111605,"herson":111606,"à¤ī":111607,"Ġmonastic":111608,"转åĢº":111609,"为äºĨè§£åĨ³":111610,"è¯įç»Ħ":111611,"Ġopportunistic":111612,"ãĤĬè¿Ķ":111613,"ĠSlug":111614,"åħļåijĺçļĦ":111615,"å¥½å¥½åľ°":111616,"å¯ĵè¨Ģ":111617,"Ġdeliberation":111618,"ĠdziaÅĤania":111619,"Fed":111620,"Wrap":111621,"oie":111622,"åı¼":111623,"ĠScheduling":111624,"ĠTape":111625,"aguchi":111626,"ĠFTC":111627,"Ġketer":111628,"åĴĮåıijå±ķçļĦ":111629,"completed":111630,"ĠTeatro":111631,"Ġpostulated":111632,"Ġvele":111633,"åĪ·åĪ·":111634,"ĠMontréal":111635,"çī¯":111636,"Ġarbitrator":111637,"iczne":111638,"Ġartean":111639,"ĠForecasting":111640,"ĠположениÑı":111641,"Ġíıīê°Ģ":111642,"ĵ¨":111643,"ĠTrom":111644,"ĠPDE":111645,"åĴĮæ°Ķ":111646,"计ç¨İ":111647,"åIJijåĨħ":111648,"aleur":111649,"ĠkeV":111650,"åĨ³ä¸į":111651,"çĶļèĩ³æľī":111652,"ativamente":111653,"Ġparlament":111654,"-loaded":111655,"Ġparietal":111656,"failure":111657,"人åij½":111658,"å¾Īæĸ¹ä¾¿":111659,"áĥĻ":111660,"ĠBeirut":111661,"Ġcontentment":111662,"Ġrespectfully":111663,"ADI":111664,"Ġmicroarray":111665,"ĠReligions":111666,"ĠEncoding":111667,"Samuel":111668,"ÙĴÙħÙı":111669,"åĬ¨ä¸įåĬ¨":111670,"decode":111671,"Ġzusätz":111672,"Ġlongtemps":111673,"anyol":111674,"æĹ©çŁ¥éģĵ":111675,"åį¡çī¹":111676,"追æį§":111677,"modium":111678,"Ġogran":111679,"Ġliens":111680,"ç«Ļåľ¨éĤ£éĩĮ":111681,"Instagram":111682,"........................................................................................................................":111683,"íĬ¹":111684,"িলà§ĩন":111685,"á¿·":111686,".ToInt":111687,".concat":111688,"Ġaristocratic":111689,"ĠÑĩеÑĤвеÑĢ":111690,"çļĦçľ¼åħī":111691,"ĠHire":111692,"Ġsubpo":111693,"Ġlinea":111694,"ficas":111695,"Ġ`/":111696,"sequential":111697,"å¤ľç©º":111698,"zieÄĩ":111699,"egeri":111700,"åłĨæĶ¾":111701,"Relation":111702,"Ġspráv":111703,"effects":111704,"Ġmobilize":111705,"ĠÑĦакÑĤи":111706,"/libs":111707,"ĠÑģÑĤоÑĢонÑĥ":111708,"ĠмÑĥзÑĭка":111709,"ĠباÙĦإضاÙ쨩":111710,".Instance":111711,"\\cap":111712,"ĠFAR":111713,"clar":111714,"æĸ°æ¬¾":111715,"Ã¥t":111716,"ĠÙĤÙĦب":111717,"è¡ĮåĬ¨çļĦ":111718,"رÙĪØ·":111719,"νοι":111720,"乾淨":111721,"Ġdismissing":111722,"Ġרצ":111723,"ç̾":111724,"ĠManufacturer":111725,"ĠAwesome":111726,"gis":111727,"çļĦ设å¤ĩ":111728,"Ġconsoles":111729,"ÑĤеÑĢеÑģ":111730,"Ġstandby":111731,"失信":111732,"èĤ¡æģ¯":111733,"ĠамеÑĢикан":111734,"河谷":111735,"ĠGeophysical":111736,"æķĻåŃ¦æ¥¼":111737,"ÙIJر":111738,"奥æĸ¯åį¡":111739,"åĴĮè°IJçļĦ":111740,"ĠdostÄĻp":111741,"Triangle":111742,"Ġwynik":111743,"ĠEpidemiol":111744,"ĠGriffiths":111745,"ĠAman":111746,"Ġplc":111747,"åŃ¦æľŁçļĦ":111748,"Ġsurm":111749,"Ġcaliber":111750,"Ġrestraining":111751,"å·®çķ°":111752,"çĽ¸ä¿¡æĪij":111753,"ĠTwentieth":111754,"ĠARTICLE":111755,"áĢŃá̝áĢĦáĢºáĢ":111756,"ĠسرÙħاÛĮÙĩ":111757,"ĠGSM":111758,"ooky":111759,"å°Ĩå®ĥ们":111760,"è§£æĥij":111761,"ĠзÑĥ":111762,"第ä¸Ģ大":111763,"ÑĢÑĥжи":111764,"é¡¿é¥Ń":111765,"Manchester":111766,"æļĸåĴĮ":111767,"Ġspotting":111768,"য়à§ĩ":111769,"Ġnodal":111770,"ÑĴе":111771,"çľĭå¾ĹåĩºæĿ¥":111772,"Zs":111773,"Ġmute":111774,"abord":111775,"تج":111776,"åĬ¨æ¤įçī©":111777,"å°Ĩè¿Ļ":111778,"å·¥ä½ľéĿ¢":111779,"åıĸèĪį":111780,"ĠShadows":111781,"ggen":111782,"Ġpossui":111783,"regional":111784,"æıIJä¾ĽåķĨ":111785,"èĨº":111786,"ÑĢоваÑĤÑĮÑģÑı":111787,"åıijè¡¨åľ¨":111788,"Ġunderside":111789,"kia":111790,"å°Ĩ使":111791,"éĢģåΰäºĨ":111792,"亦称":111793,"orphic":111794,"--------------------------------------------------------------------------------":111795,".float":111796,"_real":111797,"perate":111798,"åħ¨éĿł":111799,"æİĴç»ĥ":111800,"å±ħä¸Ń":111801,"ĠConsistency":111802,"Ġanimaux":111803,"ĠFunny":111804,"FLD":111805,"ĠترکÛĮ":111806,"Ġharmonics":111807,"Ġdeteriorating":111808,"Ġdisponibles":111809,"dividers":111810,"ĠíĹĪ":111811,"Oral":111812,"etimes":111813,"æ¯Ķ以åīį":111814,"Ġporcent":111815,"steht":111816,"å®Ĺå¸Ī":111817,"Ġpictorial":111818,"Ġanimais":111819,"ĠÑģилÑĮно":111820,"ł×Ļ×Ļף":111821,"Ġਮ":111822,"Ġmöchte":111823,"èĥ¡æ¤Ĵç²ī":111824,"ZV":111825,"zünd":111826,"æĹ¶æĹ¥":111827,"rande":111828,"-numbers":111829,"æ´Ľæĸ¯":111830,"èĤ¡ç¥¨çļĦ":111831,"Monochromatic":111832,"IZED":111833,"çŀªå¤§äºĨ":111834,"ĠFederico":111835,"ĠLinguistic":111836,"Ġeradication":111837,".activity":111838,"Freedom":111839,"kken":111840,"Ġlor":111841,"vermel":111842,"ĠGarten":111843,"ĠLea":111844,"textrm":111845,"åı·åĴĮ":111846,"Ġaffords":111847,"ĠساÛĮت":111848,"ĠرÙĤÙħ":111849,"åĹļ":111850,"أت":111851,"Ġempath":111852,"Numbermatics":111853,"å¿ħè¦ģæĿ¡ä»¶":111854,"Ġguesses":111855,"Ġjurisprudence":111856,"Guess":111857,"à¦Ńাব":111858,"ĠTribal":111859,"à¹Ģà¸Ĭิà¸ĩ":111860,"depending":111861,"âŃIJâŃIJ":111862,"WARD":111863,"zj":111864,"Ġcependant":111865,"Ġvá»ģ":111866,"ä¸įè¯Ĩ":111867,"photos":111868,"Ġblinking":111869,"à°¹":111870,"åı·æ¥¼":111871,"Ġnucleation":111872,"æģĴå®ļ":111873,"æľºæ¢°è®¾å¤ĩ":111874,"ikoak":111875,"ĠsavedInstanceState":111876,"inosaur":111877,"çļĦçݯå¢ĥä¸Ń":111878,"ĠBermuda":111879,"Hell":111880,"ĠTc":111881,"ĠBANK":111882,"Ġalmac":111883,"Ġsoar":111884,"说æĸĩ":111885,"Ġinterconnect":111886,"hereal":111887,"undos":111888,"èµ°è¿ĩçļĦ":111889,"Ġprojective":111890,"æ¯ĽåĪ©":111891,"ĠCampos":111892,"ç«ĭåλ就":111893,"capac":111894,"Ġdévelopper":111895,"ĠÑģвеÑĤло":111896,"Ġlineno":111897,"ĠOrdinance":111898,"EJ":111899,"socket":111900,"Ġdeceived":111901,"opies":111902,"ÙĥاÙħ":111903,"à¹ģวà¸Ķ":111904,"Ġquantization":111905,"ĠCommunists":111906,"Ġtaal":111907,"Ġagreeable":111908,"Ġsarcoma":111909,"Ġà¤Ĩहà¥ĩ":111910,"ĠíķĻêµIJ":111911,"å°ıå¿ĥç¿¼ç¿¼åľ°":111912,"ĠÙĤدرت":111913,"Rick":111914,"nip":111915,"ĠLua":111916,"大åĪĢ":111917,"æľ¬çº§":111918,"éĺ²çģ¾":111919,"çϾä½Ļ":111920,"åIJ«æ°´éĩı":111921,"Ñĺал":111922,"è¿Ļä¹Īå¤ļçļĦ":111923,"è¸ŀ":111924,"ĠBarrel":111925,"ĠRecher":111926,"Ġreformed":111927,"æĦĽçļĦ":111928,"Everybody":111929,"åħ¬çĽĬæĢ§":111930,"طرØŃ":111931,"ĠReciprocal":111932,"viz":111933,"ä¿ĿåŃĺåľ¨":111934,"ä¼łç»Ļ":111935,"ĠAsync":111936,"Uniform":111937,"ĠVolk":111938,"éĩİæĪĺ":111939,"çŃĶæ¡Īè§£æŀIJ":111940,"å°ĸ端":111941,"æľīä»Ģä¹Īç͍":111942,"à¥ģम":111943,"Ġà¨ħ":111944,"Ġhydrate":111945,"Ġintersecting":111946,"æĩĴæĥ°":111947,"ä¼łè¾¾äºĨ":111948,"_Name":111949,"çļĦåį°è±¡":111950,"ĠAin":111951,"à¦Ļà§įà¦ķ":111952,"å¤ļä¸ĢçĤ¹":111953,"ĠиноÑģÑĤÑĢан":111954,"转å½ķ":111955,"èIJ½å¹ķ":111956,"ĠColombo":111957,"iddy":111958,"èĭıæł¼åħ°":111959,"ĠTransc":111960,"åħ·ä½ĵè¦ģæ±Ĥ":111961,"Ġberd":111962,"åıĤåĬłè¿ĩ":111963,"Ġsatisfactor":111964,"Ġknelt":111965,"æĺ¯ä¸įä¸Ģæł·çļĦ":111966,"éĹ²èģĬ":111967,"èĢģ头åŃIJ":111968,"ovas":111969,"她被":111970,"ç³»ç»Łå·¥ç¨ĭ":111971,"kaan":111972,"×ķת×ķ":111973,"èĪĴå±ķ":111974,"å·¥èīºåĵģ":111975,"traditional":111976,"é«ĺè´¨éĩıçļĦ":111977,"ykle":111978,"ĠÕ°Õ¥Õ¿":111979,"æĦŁè¦ºåΰ":111980,"Ġescalate":111981,"Ġpoblació":111982,"缴è§Ĵä¸īè§Ĵå½¢":111983,"ç«Ļ起身æĿ¥":111984,"Mak":111985,"çļĦå¾®ç¬ij":111986,"ĠCage":111987,"ĠFargo":111988,"Ġrempl":111989,"ĠزÙĨاÙĨ":111990,"Ġancillary":111991,"æĸĩæľ¬æ¡Ĩ":111992,"ç¯ĦåĽ²":111993,"ĠSlavic":111994,"algebra":111995,"ĠÙĪÙĤاÙĦ":111996,"Ġmuster":111997,"Ġvoort":111998,"Preferred":111999,"æĿ¥åΰè¿ĻéĩĮ":112000,"èĢģæĿ¿å¨ĺ":112001,"Ġklar":112002,"Ġë³´ê³ł":112003,"åľ°ä¸ĭ室":112004,"æİłè¿ĩ":112005,"Ġcholera":112006,".')Ċ":112007,"/media":112008,"Ġearl":112009,"ĠMura":112010,"ĠNij":112011,"éĥ½çĿ£":112012,"åĽĽæĿ¡":112013,"ĠXOR":112014,"IDER":112015,"è¯Ħæµĭ":112016,"Ġbiographies":112017,"Äįuje":112018,"æ¼ĶçļĦ":112019,"Ġmicrobiology":112020,"çĽĺæĹĭ":112021,"è¡Įä¸ļä¸Ń":112022,"åĸĿçĿĢ":112023,"å¿«éĢŁå¢ŀéķ¿":112024,"Ġspokeswoman":112025,"ĠÕĢÕ¡Õµ":112026,"ĠBalkans":112027,"Pars":112028,"Ġternary":112029,"çļĦæĸĹäºī":112030,"ĠEO":112031,"itya":112032,"ĠJays":112033,"åĽ½ç¨İ":112034,"å¼Ģæŀª":112035,"éĹ®åΰ":112036,"Ġequid":112037,"é¢ĦæĦŁ":112038,"åħħè£ķ":112039,"Ġcausative":112040,"Ġев":112041,"_call":112042,"(mat":112043,"Ġpropane":112044,".Ref":112045,"æģ©æĸ¯":112046,"æķĮæĸ¹":112047,"å¡«æĸĻ":112048,"æŁĶæĥħ":112049,"Ġoccupant":112050,"-East":112051,"ĠTrending":112052,"ĠTaiwanese":112053,"Ġfaçade":112054,"游åĩ»éĺŁ":112055,"åĶłåı¨":112056,"enade":112057,"entious":112058,"åľ¨ç½ij绾":112059,"åĩºåħ¶":112060,"æĮĩæ¨Ļ":112061,"Ġgrinning":112062,"Ġantar":112063,"åı³è¾¹çļĦ":112064,"ÙİØŃ":112065,"沿ç͍":112066,"ĠNOTES":112067,"Ġà¸Ļาย":112068,"ĠGregor":112069,"finding":112070,"Ġtigers":112071,"çļĦä½ĵ积":112072,"以éĻį":112073,"Ġposibilidad":112074,"æ·±åij¼åIJ¸":112075,"骨çĽĨ":112076,"çŃijåŁº":112077,"ĠPalo":112078,"Ġbirthdays":112079,"DPE":112080,"æĹĹè¢į":112081,"ÙĤطة":112082,"ĠسبتÙħبر":112083,"Customers":112084,"Ġnourishment":112085,"ĠokoÅĤo":112086,"èĩªè¨Ģèĩªè¯Ń":112087,"ĠTreasurer":112088,"ĠLSU":112089,"ĠLankan":112090,"ocarp":112091,"ubishi":112092,"è§ģæĪij":112093,"ัà¸Ĺ":112094,"社ä¼ļæ²»å®ī":112095,"èIJ½å¯¦":112096,"æĸ¹åIJijä¸Ĭ":112097,"åĬ³åĬ¨çĶŁäº§çİĩ":112098,"æĪ°åł´":112099,"踪影":112100,"åľ¨ä»ĸçľĭæĿ¥":112101,"寡å¦ĩ":112102,"奥æŀĹåĮ¹åħĭ":112103,"ĠstoletÃŃ":112104,"?a":112105,"cab":112106,"olut":112107,"ĠCaj":112108,"orto":112109,"ĠGrac":112110,"Ġunmarried":112111,"ä»ĸä¸įä¼ļ":112112,"Ġclown":112113,"Ġprecondition":112114,"éĥ½åĸľæ¬¢":112115,"æ°ĶäºĨ":112116,"å¤Ħäºĭ":112117,"åijĬè¾ŀ":112118,"incs":112119,"æĹ©äºĽ":112120,"Ġterutama":112121,"Ġdistribución":112122,"ĠØŃاÙĦت":112123,"è·ijéģĵ":112124,"ĠÙħرØŃ":112125,"æĺ¯å¯¹çļĦ":112126,"_COMM":112127,"hancing":112128,"Ġburs":112129,"ĠJOURNAL":112130,"æľĢåŁºæľ¬":112131,"åı¯ä»¥æıIJä¾Ľ":112132,"ullende":112133,"è§ĤçľĭäºĨ":112134,"æĬĢæľ¯ä¸Ĭ":112135,"å¾Įãģ«":112136,"Ñģин":112137,"-hospital":112138,"稳éĩį":112139,"ĠBoone":112140,"åIJ¯è¶ħ":112141,"Ġnoses":112142,"/widget":112143,"Ġrefrigerant":112144,"Ġপরà§įযনà§įত":112145,"adto":112146,"æīĢæĥ³":112147,"Storm":112148,"æ£Ł":112149,"Ġoptically":112150,"马è¹Ħ":112151,"å·²ç»ıä¸įæĺ¯":112152,"-cig":112153,"ĠBeans":112154,"ĠHistoire":112155,"иÑģал":112156,"çĶ³è¯·è¡¨":112157,"ä¸į好äºĨ":112158,"}=-\\":112159,"åı¯èĥ½ä¼ļ导èĩ´":112160,"ä¸ijéĻĭ":112161,"两ä½įæķ°":112162,"×ķ×ŀ×ķת":112163,"ĠVicente":112164,"ĠÑĦоÑĢмиÑĢованиÑı":112165,"奢ä¾Īåĵģ":112166,"-network":112167,"\"As":112168,"eva":112169,"xu":112170,"Ġfred":112171,"çļĦå°ijå¹´":112172,"æĺ¯åħ·æľī":112173,"åľ¨åįİ":112174,"ĠGTP":112175,"交ç»ĻäºĨ":112176,"ĠÑĩÑĢез":112177,"ุร":112178,"å®īè£ħäºĨ":112179,"Highlight":112180,"Ġà¦Ĺà§įরহ":112181,"\\xi":112182,"ĉName":112183,"Ġhá»ĩ":112184,"igten":112185,"orty":112186,"Ġuska":112187,"è¿ĺ为":112188,"ĠProbe":112189,"Ġinsults":112190,"attend":112191,"ĠÙĦÙģ":112192,"Ġcollage":112193,"ĠÐļÑĥÑĢ":112194,"cznego":112195,"Ġsnatched":112196,"Ġricord":112197,"à¸Ĺัà¹īà¸ĩหมà¸Ķ":112198,"ĠâľĶ":112199,"ĠSaddam":112200,"éͦæłĩèµĽ":112201,"ĠÑģÑĤоÑı":112202,"actorial":112203,"å¾ĹéĿŀ常":112204,"Ġzgod":112205,"×ķס×ĺ":112206,"ÑĢее":112207,"Ġpotencia":112208,"boBox":112209,"æ©Łåζ":112210,"ĠExpense":112211,"ç¬¬åĽĽæĿ¡":112212,"å¯ĨåĪĩåħ³æ³¨":112213,"大ãģįãģı":112214,"ĠBewegung":112215,"CER":112216,"moral":112217,"çļĦæĿĥåĬĽ":112218,"Ġrei":112219,"åľ¨çłĶç©¶":112220,"ĠrÄĻ":112221,"ĠStarr":112222,"å®ļ罪":112223,"Ġfeito":112224,"Ġcurator":112225,"Ġboils":112226,"ä¸Ģå®ļæľĥ":112227,"åħĪçĶŁè¯´":112228,"мона":112229,"Ġramach":112230,"æĭĮåĮĢ":112231,"Ġllamado":112232,"-butyl":112233,"itore":112234,"Ġbn":112235,"##Ċ":112236,"以西":112237,"çĶŁè®¡":112238,"æĿ¥çĿĢ":112239,"achs":112240,"Ġentw":112241,"ĠZab":112242,"æĸ½ç͍":112243,"人çļĦçĶŁæ´»":112244,"åįĬæŃ¥":112245,"ĠGrö":112246,"Ġsticker":112247,"Ġmoderated":112248,"ãĤ«ãĥ¼":112249,"á±ļá±":112250,"ногие":112251,"Ġurn":112252,"Ġtame":112253,"ĠIEP":112254,"ĠPren":112255,"ĠPCM":112256,"ĠDodd":112257,"Ġpractising":112258,"raciones":112259,"红åħī":112260,"éĻ©äºĽ":112261,"ĠPolly":112262,"Ġberasal":112263,"ĠTomatoes":112264,"ذÙĩب":112265,"Boost":112266,"ängt":112267,"Ġë²ł":112268,"åįĹåĮĹæľĿ":112269,"-playing":112270,"ĠÙĬؤدÙĬ":112271,"à¸Ħวà¸ļà¸Ħุม":112272,"åĴĮå®¶éķ¿":112273,"å¦ĤéľĢ":112274,"æĢ»éĩıçļĦ":112275,"_samples":112276,"æī¬å£°":112277,"éĽĦä¼Ł":112278,"æİ¨è¿Ľä¼ļ":112279,"èĤ¥æ²ĥ":112280,"unicode":112281,"è¾ħèѦ":112282,"ĠHenrik":112283,"ä¼ļ计æĬ¥è¡¨":112284,"ĠÑĢабоÑĩиÑħ":112285,"ĠCites":112286,"åľ¨ä¸ĸ":112287,"Ġsait":112288,"æľ¬åŃ¦æľŁ":112289,"强壮":112290,"ütt":112291,"ç½Ĺå¾·":112292,"Ġseme":112293,"Ġfavorably":112294,"Ġpowst":112295,"Ġwrongdoing":112296,"çļĦäºĭæĥħäºĨ":112297,"ĠJudas":112298,"ĠìĭľìĬ¤íħľ":112299,"ĠLinden":112300,"Ġinterprets":112301,":nil":112302,"Ġsulphate":112303,"Ġcardiomyopathy":112304,"åľ¨ä»ĸ们çļĦ":112305,"好åIJ¬":112306,"Ú©ÙĪ":112307,"ĠPlumbing":112308,"ACM":112309,"ĠErfol":112310,"ĠاÙĦÙĥرÙĬÙħ":112311,"Ġnephews":112312,"ĠÔµÖĢÖĩÕ¡Õ¶":112313,"{},":112314,"}R":112315,"ĠBEGIN":112316,"ä¸įèĤ²":112317,"ogels":112318,"ĠUUID":112319,"æĬĬåŃ©åŃIJ":112320,"তম":112321,"irlo":112322,"æł¹æľ¬æ²¡":112323,"Ġtagging":112324,"åĮºåĪ«äºİ":112325,"ĠMcCoy":112326,"à¹Ģà¸Īà¸Ļ":112327,"Ġì¹ľ":112328,"Ġ[-]":112329,"ĠGlobes":112330,"Ġdécouvrir":112331,"otically":112332,"ä¸įçĶļ":112333,"è¦ģé«ĺ":112334,"æľ¬åIJĪåIJĮ":112335,"社ä¼ļå·¥ä½ľ":112336,"ç»Ŀä¸ĸ":112337,"å·¨æĺŁ":112338,"à§Ģà¦ķà§įষ":112339,"Ġstocking":112340,"èIJ½å®ŀæĥħåĨµ":112341,"ĠMaver":112342,"Ġroyalties":112343,"Basically":112344,"Ġдвижение":112345,"Ġreassure":112346,"ĠSerializable":112347,"Caption":112348,"-equipped":112349,"Ġsymbiotic":112350,"ĠSOM":112351,"duizend":112352,"Ġparten":112353,"Ġroam":112354,"observer":112355,"æĪij们ä»Ĭ天":112356,"Ġdefiant":112357,"ĠبÙĬÙĥÙĪÙĨ":112358,"西游记":112359,"Ġsuccessively":112360,"Ġphotore":112361,"å°įæĪij":112362,"Ġخاک":112363,"åį·äºĮ":112364,"ĠMilli":112365,"Ġknitted":112366,"ëĤĺëĬĶ":112367,"æľµæľµ":112368,"篮åŃIJ":112369,"ĠSomali":112370,"ĠðĿij¦":112371,"è½°åĬ¨":112372,"æī¿åĮħ人":112373,"ĠMedicina":112374,"Ġmencari":112375,"sage":112376,"Ġpai":112377,"Ġgóc":112378,"ĠLek":112379,"Ġnearing":112380,"ĠVass":112381,"åIJįåī¯":112382,"Chord":112383,".jackson":112384,"æŀ¶çļĦ":112385,"-Friendly":112386,"Ġliquidation":112387,"Ġvacations":112388,"íļ¨":112389,"ĠMiracle":112390,"Ġ\"@/":112391,"liwoÅĽÄĩ":112392,"urethane":112393,"(Name":112394,"Ġcine":112395,"ivin":112396,"Ġimágenes":112397,"éĤ£é¢Ĺ":112398,"Ġ.----":112399,"ÑĢиÑģÑĤа":112400,"æł¡çº§":112401,"éĻĦåŃIJ":112402,"domin":112403,"ĠVerfü":112404,"ĠDemographic":112405,"缼å¤ı":112406,"æ¯ı天éĥ½åľ¨":112407,"lemish":112408,"绿èī²åıijå±ķ":112409,"Ġgelden":112410,"Weekly":112411,"ЦÐĺ":112412,"Ġcombinatorial":112413,"Ġaches":112414,"çļĦåIJ¸æĶ¶":112415,"igations":112416,"ÑĤнаÑı":112417,"ä¹Łç͍":112418,"Ġagg":112419,"æĽ´åºĶ该":112420,"Ġlonged":112421,"åIJ¬æĩĤ":112422,"Ġlograr":112423,"Ġbitmap":112424,"ĠÙħÛĮÙĦ":112425,"èĮĥåĽ´ä¸º":112426,"áŀ»":112427,"è¯Ńè¨ĢåѦ":112428,"Ġsalesman":112429,"ĠÄijo":112430,"ĠONLINE":112431,"ĠMelan":112432,"Ġintimidation":112433,"ĠSubstanti":112434,"ĠÑĢегÑĥлÑıÑĢ":112435,"Ġae":112436,"Ġtha":112437,"stalk":112438,"unod":112439,"å¹´æĺ¥":112440,"óch":112441,"ĠΨ":112442,"ĠConnie":112443,"Ġavan":112444,"Ġeros":112445,"Ġguise":112446,"ä¸Ģå®ļæľī":112447,"Ġ×IJ×Ļ׳×ķ":112448,"ÑģкаÑĤÑĮ":112449,"åIJİæĿ¥åıĪ":112450,"åIJIJåĩº":112451,"Histoire":112452,"Ġpomp":112453,"ноÑģÑĤÑıми":112454,"ਾਰ":112455,"à§ĩষà§įà¦Ł":112456,"ĠSlovak":112457,"Ġeuropéenne":112458,"Carb":112459,"]ãĢģ":112460,"repeat":112461,"Ġnello":112462,"Ġgarb":112463,"Ġabit":112464,"æīĢçŁ¥":112465,"جرة":112466,"å§Ķå©ī":112467,"çĶ·åŃIJçļĦ":112468,"ListNode":112469,"éĻĪæĹ§":112470,"aturik":112471,"æķ£åıijåĩº":112472,"> ĊĊ","æ¸ħåįİ å¤§åѦ","åĮĸ æĪIJ","ograf ie","ĠHum ph","g il","j us","ning ar","ç»Ń èĪª","Ġоб наÑĢÑĥ","çģµ åĬĽ","ĠTom orrow","ĠSat isf","æ· ¬","åŁº æķ°","ĠMar itime","Ġà¦ħ à¦Ń","宿 主","i é","Ġh ust","åľ §","产 å¦ĩ","è´¯ éĢļ","ä»İ严 æ²»","Ġcal f","ä¹IJ äºİ","Ġsw ings","Ġfell ows","Ġwork book","è¯Ń çļĦ","è¨Ģ çļĦ","读 äºĨ",": {","大 å¸Ŀ","Ġcraw l","T alk","çľ¼ 羸","çļĦæ°Ķ æ°Ľ","b ill","c ulture","ä¼ļ ç»Ļ","åħ¨ 社ä¼ļ","Ġant ique","Ġspecial ization","ĠÑĢаз нÑĭÑħ","ĠÑĦоÑĢ Ð¼Ñĭ","b ility","ot y","ĠP iano","人 社","ĠDe ck","Ġsum m","عÙĦ ÙĪÙħات","subscript ðĿIJ","Ġm ươi","ä¹Ł åºĶ该","Sc anner","Ġrob bery","éĩĩåıĸ äºĨ","èĥĥ èĤł","ĠÄį i","- row","åħ¶ çī¹å¾ģ","éķ¿ çĽ¸","缴æİ¥ å½±åĵį","Ġhypothes ized","ĠRee ves","Ġad orable","é²ľ æĺİçļĦ","Ġnu anced","身 åīį","ĠE cho","ä¾Ľ éľĢ","æī¿ ç§Ł","游æĪı çļĦ","Ġclar ified","c aster","pe ace","ä¸ĭ åĨĮ","ä½ł å®¶","Ġcons ciously","æ²ī çļĦ","Ġfem me","ä¸į论 æĺ¯",". btn","ĠB iz","ĠH K","à¸Ĭ à¹Īวà¸ĩ","è¯ģæĺİ äºĨ","Ġlug gage","Ġcytok ine","olog ue","Al ways","ĠPier ce","- word","Ġse bag","Pat ients","伪 éĢł","ì¦ Ī","b os","ĠRom antic","Ġlegisl ators","ĠSubt ract","ĠF lying","cy j","erg er","æ¤į 被","open ia","Ġmonaster y","æ· ¼","Ġ\\( (\\","Ġبد ÙĪÙĨ","ç¨İåĬ¡ æľºåħ³","åİŁ ä»¶","åı£ åı·","Ġsom atic","å½ķ åζ","Ġ×IJ× ļ","Ġbra kes","Ġso fa","Ġev al","ĠEnt om","ä»ĩ æģ¨","æ· µ","æĶ¾ è¿Ľ","sequ ent","ĠAdvent ures","æ¶Ī æķ£","à®ķ à¯į","-in fo","ĠÑĢе ÑģÑĥÑĢ","âĸ ª","åĸĿ èĮ¶","çĽIJ æ°´","P si","Ġt rench","Ġun in","åѦ åīį","Ġmind er","do ctor","ges ter",".IO Exception","A j","Ġun res","æĿ¥ 访","und i","nah me","D one","und efined","Ġsupp orter","声 ç§°","æı¡ ä½ı","è¿Ķ è¿ĺ","UL D","al ms","主 æĿĥ","ä¸įåIJĮ çļĦæĺ¯","çIJĨè§£ 为","èĥ½ éĩıçļĦ","Ġbear ings","รั à¸IJ","ĠByz antine","P HP","ç±³ åħ°","App endix","ÑģÑĤÑĥ пи","R u",".R eg","Ġd ances","èĩª 强","æķ° åįĥ","èħ ±","Ġcond iciones","Ġbul lets","File Name","Z T","Ġcapac idad","æĵ ¾","amb il","ĠÎŃ Î½Î±","Ġh och","Ġpart i","Ġpersever ance","Ġn ont","ĠT ac","Ġ} ),Ċ","æŃ£ èĥ½éĩı","ä¿¡ èªī","åįģ æĿ¡","é»ij å¤ľ","ĠعÙĨد Ùħا","ĠC ALL","ĠпÑĢе п","ĠاÙĦØ´ ÙĬ","ç·Ĭ å¼µ","æ¾Ħ æ¸ħ","à¶± à·Ĭ","åħ¬ åŃĻ","×ķ ×Ļ×ķת","åĨĻ åľ¨","Ġr ansom","Ġtour naments","RA W","ĉ data","èĮ Ĺ","Ġmen us","ç¼ĸ ç»ĩ","ç§ijæĬĢ å¤§åѦ","ĠControl s","çļĦ人 ç±»","ãĤ¹ ãģ®","Part y","; ãĢĬ","} .\\]ĊĊ","Ġl bf","Ġi j","æĪij们 è¿ĺ","Ġsocial ism","ĠMag yar","bas ic","Ġdream ed","Ġ׼ ת","ĠAssess ing","= =\"","Ġn ud","pec ified","ä¸ī 个人","é ·","ion o","ï¼ģ ï¼Ī","Ġpa is","Aut hentication","ĠCos m","ĠTib etan","Ġprophe cy","ä¹Ł åı«","ĠÑĢе зи","abil idade","d if","Ġ\"\" \"ĊĊ","æ¶Ĥ æĬ¹","< stdio","åľ Ń","åıį åĬ¨","stit ial","ĠPar agraph","ä½ł æĬĬ","被 æĪij","èIJ¥ è¿IJ","ĠSte f","ĠÑįлем енÑĤов","Ġíķ¨ ìĪĺ","-f lex","ç·´ ç¿Ĵ","s led","Ġh lav","Ġj Query","Ġele phants","ä¸ĵ 人","ç³»ç»Ł åĴĮ","æĺ¯ä¸Ģ 次","ĠاÙĦس ÙĥاÙĨ","از Ùħ","ĠPh armacy","Rel ations","形容 è¯į","Ġgri pping","\\ \",","Ġfor fe","è² ¿","åģı åIJij","ĠOpt imal","ĠIF N","ĠBoy d","Ġresemb ling","ä¸Ģ 棵","åѦ åΰ","æŁ¥ æĺİ","ĠMir ror","Ġt iga","ĠAcc om","ĠN ah","Ġun g","æ°´ 管","ĠEr geb","Ġlaws uits","ĠU tt","ĠCo operative","æĥĬ èī³","ä¹ı åĬĽ","ĠCell ular","Ġphon ics","åįķ纯 çļĦ","iv ement","Ġ= (","Ġall ied","Ġ' )Ċ","第ä¸Ģ åIJį","éķ· çļĦ","ä¹¾ åĿ¤","g ran","p q","åĩ ±","Ġav ail","Ġgl am","åľĭ åħ§","ĠRE QU","Ġথ াà¦ķà§ĩ","aras htra","Ġm ener","iz in","Ġper oxide","iel en","Ġpartic olare","Cl uster","L F","Ġe cl","ĠD addy","ik l","Ġshort ened","çIJī çĴĥ","j avascript","Ġb ahan","ass ing","æĹł è¾ľ","app ings","ç½® æį¢","åĤ ¢","IV ES","ĠÏĥ ÏĦη","ĠEsc her","tern a","ĠWe alth","åħĭ åζ","Ġinitial ization","ĠSan ct","Ġ: ãĢĬ","Ġcorner stone","ä»ĸ åİ»","ĠV ista","ĠÐŁ оп","à¯įà® ©","Ġretro spect","Ġt asting","çļĦ 综åIJĪ","ase ous","ov ý","amp ed","ö ll","Ġsw ollen","ãĥª ãĥ³","ĠболÑĮ ÑĪин","ä¸įå¾Ĺä¸į 说","群 ä½ĵçļĦ","浪 æ½®","ĠNS String","Ġapre nder","诸èijĽ 亮","Ġ{ ...","è·¯ ç¨ĭ","Ġпод об","_ ind","Ġprodu ção","ĠFl oyd","Ġbal let","ä½ĵç³» 建设","ĠÑĢÑĥб лей","Ġaccus ations","Ġan no","Ñģ ем","aw ning","çļĦä¸Ģ 份","ĠÑĢо ÑĴ","_ char","y cin","麻 çĹ¹","ç¬¬åĽĽ èĬĤ","est ead","Ġrad iotherapy","ĠReg istered","åŁºç¡Ģ çŁ¥è¯Ĩ","ĠPerson nel","ĠPlay ing","Ġv Å¡e","Ġmov imento","Ġvol um","Ġinhab ited","Ġto ile","ä¸į åħį","Ġamph ib","ĠاÙĥت ÙĪØ¨Ø±","- interest",". You","= &","Ġcal idad",".p assword","aa a","è£Ĥ 纹","æĿ° åĩº","acion ais","-hydro xy",": a","h urst","åľ° ä»İ","è¨Ģ èijī","ET TER","Par se","ĠëĤ´ ìļ©","ĠRenew able","n il","Ġs unt","æĺİ çŁ¥","ĠAN AL","ĠH ull","ÏĮ ÏĦε","ги Ñı","å¥ĭ æĪĺ","ĠAd vertising","Ġvan ished","èĩ´ è¾ŀ","Ġthreat ens","ĠJud gment","Ġban anas",". Property","Ġv rou","Ġv ég","æ¸ħ åĩī","ĠठĶ","ér ature","Ġreceipt s","ov el","æľī èĥ½åĬĽ","åĽŀ è¿ĩ","اد ÙĬØ©","ĠIN VENTION","Ġrev olt","åħ¬ åħ¬","è¾ĵ åĩºçļĦ","bit o","Log ic","Ġdebt or","Ġmarginal ized","ighb ors","Ġambul ance","ĠG UID","Ġreson ates","= A","le c","æĪij è®°å¾Ĺ","Ġmil ioni","à¸Ĺ าà¸Ļ","ÑģÑĤвен ное","éĹª åħī","ĠBill ion","bro ken","ĠNg uyen","X ml","Ġb ishops","ind y","ina fter","ä¸ IJ","ru z","Ġelement al","ĠEduc ação","Ġacadem ia","pro perties","ย าà¸ģ","ãģ¾ ãģļ","æĹ¢ èĥ½","Ġsupplément aires",". floor","æĪij们 åħĪ","Ġprodu its","çŁ³ å®¶åºĦ","ĠÑĢе алÑĮ","Ġundergo es","ĠинÑĦоÑĢма ÑĨии","Ġ} ,ĊĊ","ä¸ī æĹ¥","Ġut ilit","ĠGreen land","Z S","iew icz","/s ervices","ä»Ļ 女","Ġsesu ai","Ġsp iders","è·Ł æĪij说","ĠON LY","çļĦ çİĭ","ĠP regnancy","èĦļ è¸ı","γ α","ç¯ĩ å°ı说","çĴ Ģ","}$ ,","ãģ§ãģ¯ ãģªãģı","stud y","Refer ring","ĠавÑĤом оби",". If","per haps","M Hz","å¹¶ èģĶ","ĠCon rad","à¹ģ à¸ľ","Trans ition","èĥ¸ åīį","Ġpitch ing","ighbor hood","ĠPh on","Ġprev ail","- we","B ind","ies a","ÙĪ Ø¶","ook ed","ĠCl ock","ç§ijåѦ ä¸İ","ç¥ĸ æ¯į","Ġâĸ ²","X i","çļĦ å·¥åħ·","ĠJ ika","ick i","Ġmon ot","å¨ ±","Col lege","èľ Ĺ","/ ph","åıĪ ä¸Ģ个","ĠاÙĦÙħ ÙĤ","ĠAm azing","-n ight","æ¤ħ åŃIJä¸Ĭ","ĠTrip advisor","[ start","ĠC ST","ли на","rid o","ĠMon ument","ĠÑĨе лÑĮ","åı® åĺ±","ĠÑĥд об","C ash","ro j","èĩª åįij","ä»Ģä¹Ī æł·","D EC","Ġm oll","Ġdem ise","ĠاÛĮÙĨ Ú©Ùĩ","Ġc idade","as u","ack ages","æīĢ æ¬²","pro cedure","Ġautom ate","ÑĨион нÑĭÑħ","Ġoso b","å®ŀäºĭæ±Ĥ æĺ¯","ER A","Ġmys ql","ĠCP I","C os","以 æıIJé«ĺ","Ġwork outs","Ġб ал","ëĬĶ ëį°","ĠEth nic","à¸Ńยูà¹Ī à¹ĥà¸Ļ","Ġhered itary","- ended","Ġe agle","ĠW ah","åĬł å·ŀ","ĠEr d","Ġex quisite","Ġhist ória","åĬ ¹","ĠH IGH","åħ¨ åľĭ","з ем","Ġagree ing","Ġverd ad","çļĦ çĪ¶äº²","Ġz ÅĤ","éĩij åħī","िठĤ","Ġpow in","ç»Ŀ对 ä¸įä¼ļ","qual ified","çīĪæĿĥ æīĢæľī","t alk","str öm","æĻ® æ³ķ",".re nder","ap ons","ĠV ander","Ġsp acious","æ°´ ä¸Ĭ","St ates","Ùģ ÙĤ","ู à¸Ķ","\" Yes","Ġtur moil","ut ar","ĠB hat","ov is","她 ä¼ļ","Ġder en","Ġmult idisciplinary","ست ر","ĠThe odore","Ġcustom ization","å¥ĸ 项","éĹľ 注","Ġch ili","load er","æĺ¯æĢİä¹Ī åĽŀäºĭ","ĠاÙĦØŃد ÙĬØ«","ä¼ ¶","Ġinv as","-D ay","w b","at aka","è¿ĺæĺ¯ æ¯Ķè¾ĥ","-p o","Ġexist ential","=\" \";Ċ","H ad","以 太","Ġdi ra","è§£ æ³ķ","åIJĦ æĹı","Ġsm arter","view port","è² «","ĠAsp ects","K orean","ĠM d","Ġk ph","åħ¶ äºĮ","两 ç»Ħ","çį Ħ","æģ¢ 復","áĥIJáĥ łáĥ","деÑĤ ÑĮ","P ART","Ġn x","ĠS ung","ĠF ax","åı¯ å°±","ĠÑģ ÑĢеди","æ¹ĸ 人","Ġneces ario",". const","éĹ» åIJį","Ġ×¢ ×ij","Ġpoison ous","Ġp og","ĠC ara","Ġan ton","ĠD ates","ĠAl to","ÑĤа ÑĨии","约 åįł","fe atures","è³ĩ æľ¬","Ġp onder","æĪij çľĭåΰ","å°Ĩ å®ĥ","æŀģ åħ·","CL C","ĠD U","Ġcorrect ive","Ġindu cing","Ġتع اÙĦÙī",". Inter","éľ ¾","Ġм Ñĥж","çī©ä¸ļ 管çIJĨ","Ġn oc","Ġqu ota","ç¤ ģ","ä»Ģä¹Ī åı«","åķĨ è´¸","ĠInt ra","-e ast","ĠC ake","ĠN ão","è¿Ļ ç¬Ķ","ĠShe ffield","v ig","äºĨ æĪij们","Ġwe ary","åĩº æ¼Ķ","η μο","Ñļ Ñĥ","Ġmeny ebabkan","å±ĢéĻIJ æĢ§","p iration","ï¼ī ãĢĭ","Se q","ĠDef endants","à«įઠ¯","Ġl äs","pl ac","ÙĪØ¯ Ùĩ","Ñĸ н","æķij åij½","Ġcateg orical","Ġancest ry","D al","çļĦ åįķ","å¦Ĥ åľ¨","Ġam usement","çϽ çİī","å¹» çģ¯","æľī ä¸įåIJĮçļĦ","ĠÑģ обÑĭ","æ¥ ŀ","ç¿ Ł","ĠEven ing","ĠSU MMARY","K W","åĴĮ åŃ©åŃIJ","ÙĪ ÛĮد","ĠCent imeter","hel f","Ġsu ed","è¿Ľ çIJĥ","ä¸Ģ 书","大 èĤł","çŃī éĥ¨éŨ","åħħè¶³ çļĦ","/ U","D it","æıIJ çĤ¼","Ġprof und","çĻ» å±±","à¸ĵ à¸ij","Ġmira cul","Û Ĩ","åħĥ 宵","çłĶç©¶ ä¼ļ","ä½įç½® çļĦ","Ġпол ноÑģÑĤÑĮÑİ","à¹ģร à¸ģ","çIJĨå·¥ 大åѦ","çŁŃ æĹ¶éĹ´åĨħ","ĠPre cision","ĠÙ¾ رÙĪ","Ġv ÄĽt","ign er","ts ch","çı Ĥ","à´ Ł","Ġsuper conduct","è°ģ çŁ¥","ĠâĨĴ ĊĊ","Ġpopul asyon","ĠG PA","æĪij们 æĬĬ","å½ĵäºĭ 人çļĦ","Ġhemorrh age","Ġ ili","ä¸Ĭ åı¤","ĠпÑĢо Ñģ","asion ally","ag l","Ġj ets","åĵģ æł¼","ĠSett lement","Recomm ended","æĪĺ åIJİ","Ġpresent a","çłĶç©¶ äºĨ","åħŃ çϾ","ĠпÑĢи ваÑĤ","æ¡Īä»¶ çļĦ","Ġreef s","Ġ ï¼İ","Ġg g","Ġobs ession","Ġp als","åĩº ä¹İ","Ġet ching","èµŀ èµı","åĪĽä¸ļ æĿ¿",". ad","S OL","head ers","Т ак","Ġorganis ational","_ delete","Ġb ude","Ġb awah","ç» ®","åįĬ æľĪ","Ġaccess ory","é̲ ä¸ĢæŃ¥","Ġa rab","ãĢĤ ï¼īĊĊ","ä¸į ä½ľ","æĪIJ 績","åĽŀ è°ĥ","اÙĦ ع","ί ν","UN G","èµĭ èĥ½","D rug","qu ick","Ġres iding","oy l","ä¸Ģèά 人","è°ģ çŁ¥éģĵ","Ġbow ls","ĠKa plan","Ġc aves","çļĦ æĢ§æł¼","çģ¯ æ³¡","Ġпом ога",".Control s","äºļ马 éĢĬ","Ġt asted","ĠC af","ä¸Ģ æľµ","ç»ĵ èĤł","亲 çľ¼","ĠHar vest","ĠSal em","{c ases","R outes","ĠD io","åľ° åĪ©","Ġes crib","ĠÏĦ ε",". route","Ġin ferences","ĠP AC","Ġdream ing","access ible","F n","-t aking","Ġ×ķ ׾×IJ","å¥ł å®ļ","g ado","ĠA ircraft","æĺ¯ å®Įåħ¨","ä¹ĭ æ¯Ķ","æĸĩ ç§ij","计 åħ¥","Ġgra f","Ġrep ayment","ĠÑĦ ай","OT E","Ġper tain","念 念","Ġব à¦Ľ","Ġviol ating","åıij表 äºĨ","çłĶç©¶ 人åijĺ","ĠпÑĢед о","Ġreimburs ement","in ig","ĠSc out","ĠPer l","çŃij çī¢","particular ly","ä¸Ģ åij³","ĠG ST","Ġshe lters","Ġfun ção","Ġep igen","ĠопÑĢеделÑı еÑĤÑģÑı","Ġfich iers","á ¾","Ġtoler ated","çϽ éĵ¶","ĠDig its","ĠBang kok","Ġnest ing","çļĦ çŁ³","ĠT G","ĠQ ur","Ġfire place","Ġrug ged","am ientos","ĠR ash","ÙĪ Ø§ÙħÙĦ","Ġpot encial","Sh ared","éĴĪ çģ¸","ĠVer bs","Ġcu ad","m ie","Ġam orphous","Ġob ras","ĠÐŁ ÑĢед","åİĨåı² æĸĩåĮĸ","Ġmosquito es","à¹Ģลืà¸Ń à¸ģ","ĠE VER","éĥ½ å¾Ĺ","Ġop ener","ĠDon na","Ġion ization","浸 润","em as","ĠF rage","æĶ¾ åѦ","< double","ĠB rowse","ob iles","Ġgr ind",".R ep","ĠResp iratory","ot ations","æĪij 个人","ĠV otes","Ġfin ns","ĠвÑĭ з","æĪIJç«ĭ çļĦ","Ġwaste ful","ĠSD K","Ġwithd rew","Ġ' Ċ","åĽ¾ ä¸Ĭ","åıijå±ķ ä¸ŃåĽ½å®¶","ma al","ãĤī ãģªãģĦ","è½´ 线","Ġser otonin","Ġant if","åįĢ åŁŁ","Aut om","åľ¨ åħ¶ä»ĸ","ä¹ĭ æĹ¥","Ġparallel s","L G","ĠS ind","ĠM emb","Ġby lo","ĠIns ight","ĠAmer ika","Y M","ĠC ache","ĠG leich","ient ial",".S h","ç»Ŀ对 æĺ¯","ĠAdapt ive","óst ico","l ift","çļĦ 交","Ġl ava","Ġexp osition","åĩĨå¤ĩ çļĦ","ĠMad ame","ĠÑĩи Ñģел","Ġl aps","end ered","Ġbet s","建 åħļ","Ġcol abor","Ġgl itter","bur st","Ġasp iration","Ġincom patible","A us","ä¼Ĺ å¤ļçļĦ","ä¸įå¾Ĺ å·²","Ġrent ed","Instance State","Ġl ol","æĢ§ éĹ®é¢ĺ","èĭı è½¼","ĠBur k","Õ Ģ","è´¨ æĦŁ","èµ° åĩºåİ»","ĠAdv ice","å°ı 女åŃ©","Ġno i","ĠHash Set","çĭĢ æ³ģ","à¹ģ สà¸Ķà¸ĩ","Ġre lic","åĩł åįģå¹´","Ġm ár","/ android","好 çİ©","Ùĥ رÙĬ","Ġtw ists","æĮ½ åĽŀ","ĠMund ial","è»ĭ çĹħ","Ġj erk","ĠÑĤ ÑĢав","èĥĮ éĥ¨","Ġìŀ Ħ","E I","T x","ÙĪ ÙĬÙĦ","п ÑĢ","Ġair borne","-E urope","an ese","Ġsal iva","ุ à¹Īà¸Ļ","樣 çļĦ","Ġinstance of","åĴ½ åĸī","n ap","Ġw icht","Ġ' $","èµ° èµ°","ĠMuse o","ä¸ĸçķĮä¸Ĭ æľĢ","Ġupp ercase","Ġd h","ik ian","Ġab lation","Ġstat t","Im per","Ġpossess ing","ĠLock e","ä¸į æĶ¾","Ġcl ocks","容 å¿į","åįİ åįĹ","åĪ» çĶ»","ĠCardi ovascular",") _ĊĊ","Ġs angu","æķĪ ç͍","Ġrep ression","æĻ® æ´±","ä¸įåľ¨ ä¹İ","Ġassim ilation","æĦļ èł¢","u ks","He at","Ġر ÙģØª","Ġpod czas","ä¿¡åı· çļĦ","c op","Ġse aw","Ġlaw ful","缺 è¡Ģ","Ġos ób","ĠпÑĢед ÑģÑĤави","èĢĮè¨Ģ ä¹ĭ","ĠFisher ies","H s","Ġm ów","ä¸Ģ å¹¶","æĸ ¬","ok ia","ä¹ĭ åIJį","uc her","Ġsupp er","åı£ 岸","ĠWar riors","_m ethod","Rec ogn","িন à§įন","ĠO phthalm","ov id","注 éĶĢ","Ġmach ining","幸ç¦ı æĦŁ","Ġmetaph ors","Ġnat ür","ĠM ish","ìľ ¨","تب اط","Ī à¸°",".d jvu","ras i","æĭ¥ æĮ¤","ĠгÑĢа ÑĦи","Ġcan ned","Ġconc aten","èĹı çĿĢ","温æļĸ çļĦ","ĠاÙĦتع ÙĦÙĬÙħ","` ;Ċ","Ġel k","æĪij们 æīĢ","chn itt","Ġage ing","ĠPri est","éĬ ĺ","g old","ĠD art","ĠD uty","ĠU IC","ood le","带 éĺŁ","æ»ĭ 润","ĠS out","å¸ ¥","Ġsh aded","å¨ ´","æĺ¯ä¸Ģ æĿ¡","é¢Ħ æľŁçļĦ","ĠPsych ol","ä¿Ĺ ç§°","འĸ","$ .ĊĊ","w ife","ĠS urely","Ġv eto","Ġpro getto","Ġel Åij","ĠApp ellant","Ġk ern","ris ing","기 를","Ġgly cer","ar re","ä¼ĺ äºİ","èĥĮ éĿ¢","æĿĤ 交","( msg","c yst","ust e","Ġtra che","Ġsk illet","åĢŁ ç͍","ĉ List","Ġf uss","Ġg estion","ot rop","ä¼ļ åľº","å¼Ģ æľº","Ġer kl",".s upport","-d istance","ĠAtt empt","{ U","Ġj ó","å°± åĴĮ","è¾ Ĭ","å®¶ åĴĮ","ĠÙĥ بÙĬر","ç©¿ æIJŃ","ĠInc ident","ĠSy ll","ĠRoll ing","ĠT J","ä¸į æģ¯","æķij æĬ¤","Ġd yes","ĠV ascular","å¢ŀ å¹ħ","Ġident ifiable","ç©´ ä½į","轩 è¾ķ","ĠG os","Ac cepted","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","j b","ä¹Ł åºĶ","Ġbal cony","æŃ»äº¡ çİĩ","( Http","Ġs ph","ĠL ob","æĬ¥ ä»ĩ","Ġconform ity","Ġhook s","al most","eng lish",".de v","Ġপর িব","Ġbios ynthesis","Ġfran çaise","c ant","大 æĺİ","ze ÅĦ","æ»ļ æ»ļ","举è¡Į äºĨ","Ġsoll ten","Factor ization","ild er","s un","游 离","Ġped ig","Ġcurtain s","âĢĿ ï¼ī","æĮĩ æľĽ","顺 åºĶ","ĠJud y","éĴ¢ 管","å̼å¾Ĺä¸ĢæıIJ çļĦæĺ¯","umm ies","Ġanál isis","Y A","{ o","ĠD OC","éĤ ĥ","ric anes","ÑĪ ÐµÐ½Ð¸Ñİ","éĽĨåĽ¢ çļĦ","ä»Ĭå¹´ 以æĿ¥","re ply","ä¸Ģ æ»´","th ren","æĬĵ æīĭ","/ Z","ĠT odo","ö st","é»Ħ çĵľ","ĠEst imation","Ġregister ing","Ġka žd","- anal","а в","л Ñĭе","åī§ ä¸Ń","Ġespecial mente","ÑģÑĤана вли","y ellow","ĠIn vent","Ġا ض","Ø· ب","çļĦåľ° æŃ¥","V II","Ġо кон","太 大çļĦ","å¼ķ èµĦ","ĠL H","ä½ł è¿ĺæĺ¯","çݰ å°Ĩ","ä¸ĩåħĥ çļĦ","å¯Ħ æīĺ","ĠB BB","н нÑĭй","ik ka","èµ· éĩį","-f ashion","æĴ¤ éĢĢ","åħ¨ æĹ¥","Ġnon zero","F ine","Ġm olding","Ġ} {","ли в","Ġmerg ing","ä¾® è¾±","ĠCh oi","ĠAll ison","ĠPre v","ĠWars aw","Ġdamp ing","Ġextern ally","ĠEpidem iology",", âĢĶ","Ġbl iss","å¤į è¯ķ",".a uth","Ġdisreg ard","Ġm osaic","op old","å¾Ī é«ĺåħ´","两 项","çī¹ å°Ķ","abil a","è¶ĭ åIJij","ĠCON ST","Techn ical","omencl ature","Ġch op","alt ed","Ġuph old","f ib","æĿ¥ è§£åĨ³","çļĦ人 æķ°","Ġве ÑīеÑģÑĤва","ighth ouse","ophys iology","B arb","c ourt","k ul","对 æĸ°","åľ° 为","åΰåºķ æĺ¯ä»Ģä¹Ī","Ġencompass ing","im ag","ér ale","ETA IL","P ick","Ġg ens","Ġpen elitian","ĠIndian apolis","(f inal","é¡¶ 端","Ġtransport ing","åľ¨è¿Ļ åĦ¿","د ÙĨ","æºIJ æ³ī","åİ¿ çļĦ","ĠCollect ive","ĠW erner","Ġdiagn ostics","ä¹ĭ åĬ¿","ä¾Ŀ èĪĬ","Author ity","羸 åŃIJ","éĩĿ å°į","ĠP eb","ben z","Ġcater ing","ĠLe ipzig","Ġnuest ros","ĠRes cue","åĨ² åĪº","à· Ļ","' _","åľ¨ è¿Ļæł·çļĦ","Ġpr isons","åĤ Ģ","åij¼ åͤ","ê² ł","åļ´ éĩį","c zenie","æĢ§ åŃIJ","Ġ/ =","-g rand","èĸ ij","[] (","Ġplot ting","Ġcompound ed","Ġp ane","åĪĬ çī©","ба ÑĤÑĭ","ĠNaz is","иÑģа ние","Ġpatri ot","Ġвме ÑģÑĤе","ĠM Ps","为 缮æłĩ","大 æīĭ","第äºĮ æŃ¥","- US","Ġin verter","Ġpers isted","par agraph","ä¼ł åĩº","æĺ¯ä¸Ģ çīĩ","Ġnational ist","御 åı²","Ġ×Ĺ ×Ļ","Ġbund les","Ġe ben","ع ÙĪØ¯","åıį è¿ĩæĿ¥","oh ist","ä½İ åİĭ","ż Äħ","f ocus","äºİ ä¸Ģ","æµģ æ·Į","ĠجÙĨ ÙĪØ¨","为 åķ¥","ε ÏĦαι","çĶŁäº§ çİĩ","zi om","Ġíķ Ń","Ġsket ches","- Ad","èĩ´ çĹħ","}{ |","Ġস াথà§ĩ","ีย ม","ĠF letcher","æ¼ ª","丰 çͰ","umin um","天津 å¸Ĥ","ip yo","Ġac uerdo","Ġrespect ing","ĠÑĤÑĢан Ñģп","ĠGRO UP","çľĭäºĨ çľ¼","éļİ æ®µ","Ġdiscre p","Res pond","Ġpack aged","Ġci Äħ","ĠOk tober","ĠتÙĪØ§ÙĨ د","ĠWikip édia","ess ing","Ġsp esso","Ġëĭ ¬","ĠSS L","åĪ ģ","ĠAP A","Ġré du","ĠL ots","æīĭ ä¸ĬçļĦ","ÑĢи ÑģÑĤи","æĿ° åħĭ","- forming","Ġd um","åĴĮ æİ§åζ","eth nic","Ġо но","Ġserv i","Ġ×Ķ×ŀ× ĵ","ĠCer ro","Ġs d","人 ãģĮ","çĸ ±","Õ¡Õ¶ Õ«","Ġpenny weights","ä¸į åĩ¡","Ġag ility","let a","æĺ¯ä¸Ģ 款","ç»Ī æŀģ","åħ¶æ¬¡ æĺ¯","> ,Ċ","ÑĢи ÑĨа","Ġdon ner","ר ×Ļ×ļ","Ġпо ÑģколÑĮкÑĥ","ĠMal ay","in ence","Ġb ate","и мÑĥ","ĠB US","ab ella","erm is","æ°Ķ å¾Ĺ","Ġб ÑĢо","cem ic","Ġà¹ĢภĤ","åľ¨ éĩĮéĿ¢","ram os","Ġrel apse","Ġcol s","-d eterm","åħŃ å¹´çº§","-st ory","ĠBo at","åѸ éĻ¢","ĠÑģо един","स à¥įत","Di agn","车 è½®","μα ÏĦα","ĠMong ol","ĢáĢ» á̱á̏áĢ","A bb","ĠG aming","éĹ µ","Ġdet ained","Ġза ÑĤÑĢа","Ġsem inars","ĠChe f","Ġsuperfic ie","Ġs ä","ĠE QU","ди Ñı","nic os","(l ambda","Ï Ĭ","Ġra ils","ĠRet irement","è¸ ¹","Trans lation","ÑĦоÑĢ Ð¼Ð¸","ĠSho pping","o os","ent hal","ĠD ynam","Ġcons om","客 车","èΰ éĺŁ","ĠÑĥÑĩи ÑĤÑĭ","ĠP ly","ox ygen","AT S","ĠMeg an","ĠTow ard","Arab ic","Portug uese","Ġb ritt","Ġth ym","qu arter","åĩº åįĸ","â̲ ,","- _","çļĦ åIJĮ","iat rist","Ġпов ед","ĠComment ary","ĠHV AC","P U","c hens","em ing","å·¥ä½ľ æĹ¶","اÙĩ Ùħ","Ġpal av","äºĮåįģ å¹´","æĪ° 鬥","he tti","说 èĩªå·±","é¦ Ĵ","áŀ Ģ","Ġdownload s","æĿľ çĶ«","èIJ¥ä¸ļ æĶ¶åħ¥","w orms","Ġh ose","å¼ł æŁIJ","ĠÑģÑĤа в","ä¸įå¾Ĺ è¶ħè¿ĩ","Ġrival ry","Ġпомо Ñīи","èĦijæµ· ä¸Ń","ĠS cul","pe ated","๠Ĭ","AT O","Lab els","ount y","éķ¿ æĸ¹å½¢","建 äºİ","æŃ¤ 项","åIJĥ èĭ¦","ĠEd win","Åij s","Ġm ijn","Ġl atch","ener ate","Ġdist ractions","λ ικά","Ag ric","ĠÑģооÑĤвеÑĤ ÑģÑĤвии","Ġth rom","åľ¨ 被","cy t","med ian","å¢ŀåĬł åΰ","æ¡£ 次","ĠWell ness","d istance","ĠC ars","her ty","ä¹Ł å·²ç»ı","Ġsl ipping","ĠDis abilities","Ġinform ing","ëIJľ ëĭ¤","åģļ 大","еÑĤ о","ĠEn lightenment","ĠпÑĢи бÑĭ","ĠÐĴ ели","acc uracy","Pop ular","ol tre","Ùİ Ø©","ĠMet all","ĠMal ta","åīĩ æĺ¯","ä¸Ń åıijçݰ","ä½ł è¦ģæĺ¯","Ñģи й","ĠHouse hold","ĠCH ECK","ò n","çļĦ åįķä½į","Ġst unned","Ġal ley","å¹¶ æł¹æį®","IN F","ç»Ĩ å¾®","ä¸įçŁ¥ æīĢ","Ġentertain ed","å°ı å¼Ł","ж де","带 宽","cal c","Ġmort ar","ĠÑĤÑĢе ÑĥголÑĮ","L iving","å¥ ļ",".sub string","ĠKil ometers","æħ· æħ¨","åĨĽ åĮº","ย าย","Ġкак ой","ĠØ· رÙĬÙĤ","ï¬ ĥ","Ġupgrad ing","ĠD ul","Ġcomput ations","ĠTw elve","íİ ĺìĿ´ì§Ģ","Y ellow","Ġas hes","ĠD ATE","ĠN G","æĽ² éĿ¢","Ġconcent rating","ĠVer d",".n umber","åį±éĻ© çļĦ","Ġbranch ing","ĠAlb any","nom bre","åı ½","Ġmat éri","emb rance","Ġž ivot","ĠMoh ammad","à¹Ģà¸ģีà¹Īยว à¸ģัà¸ļ","ĠH U","Ġcong rat","ĠV est","空 æł¼","à¸Ĭ ืà¹Īà¸Ń","ĠK O","Ġ} ).","ĠÙģ ÙĦ","Qu ote","]( /","ëĿ¼ ê³ł","obacter ium","( iii","ĠW rong","åѦ æ´¾","Ġtemper ament","Er rors","á̝áĢķáĢºáĢ ħ","åİ» è¿ĩ","æŃ» äºİ","éĺ³ æĺİ","å®¶éķ¿ ä»¬","ĠB uilder","ç¥ º","æ°Ķ æµģ","Ġa quest","ĠA udi","Ġsp ikes","åħ· é«Ķ","is lation","om bo","ä¼ļ æĬĬ","Ġcost ru","Con ference","éĵ¶è¡Į åį¡","к лон","ĠY as","äºĮ 年级","è¿Ľè¡Į æ¯Ķè¾ĥ","ĠFort une","Ġtempt ing","Ġs ack","åĽ ²","åIJĪ ãĤıãģĽ","å¼ķ æµģ","梦 å¹»","麻 çħ©","Ġcourty ard","ic amente","oc arcinoma","ĠRe y","Ġph ương","änd e","ĠHoff man","ä½ł åĨį","好 åIJĥçļĦ","å¹¶ åĪĹ","年代 åĪĿ","ĠÎŃ Ïĩ","ä¸Ń åѦçĶŁ","Ġinv o","ĠAg osto","Ġmyst ical","辨 åĪ«","Ġannoy ed","Ġa per","Ġm ots","Ġl ions","人 æĿĥ","ت ÙĤ","Ġdist ancing","Ġkun st","ĠGC SE","p ared","od b","èᝠåīĤ","çͰ éĩİ","å¦Īå¦Ī çļĦ","Ġfuel ed","Ġgran ite","Ġroyal ty","ent ies","ĠL t","æ³¢ éķ¿","OT AL","æµĩ æ°´","Ġhyp oxia","Perm ission","ĠSh apes","ĠMy c","Ġtan pa","Ġbon ne","Ġdisc overs","HE AD","ĠاÙĦØ£ ع","Ġfre q","ĠA min","ĠاÙĦØ£ د","tan ler","o arthritis","Ġk b","ap en","ĠV OL","åı¯ä»¥ å¾Ĺåΰ","ä¸ĩ åĨĨ","ระ หวà¹Īาà¸ĩ","Tra ining","im ps","æľ¬ éĩij","ĠD iane","rib e","她 ä¸į","ç«Ļ äºĨèµ·æĿ¥","åĩĨç¡® æĢ§","-min us","æĢ» æľī","elen ium","Ġspont aneously","çŁ¥åIJį 度","ĠÅĽw iat","em oc","Ġac ordo","Ġma id","ĠAntar ctica","ĠfÃŃs ica","roll ment","ĠInvest ors","ĠPass ion","j ala","an imal","ĠM ilit","å¤ļ éĩį","eb ack","åªĴ é«Ķ","fin ite","éĺĢ éŨ","J M","ĠP PT","ĠHe gel","çĤ¸ å¼¹","/ get","Ġp ies","ä¸Ĭ åįĥ","å¦Ĥ å®ŀ","å¤ĸ 壳","çıł åŃIJ","éĢī åĩº","ny der","Ġ? >","Ġadapt able","Ġà° ħ","ĠArchae ology","\" <<","ans hip","å¦ ĵ","èĩ´ åij½","çͳ è¯ī","èį· èĬ±","Ġt ors","ĠA BS","è¡Į èĢħ","ĠAn imation","Ġver z","Ġarbit r","; -","V a","ĠTh ir","主 åĭķ","åįĹ å®ĭ","Ġeth ic","à¸ķ à¸Ńà¸Ļ","æĬµ 御","Ġattend ant","R EC","Ġи ÑĤ","Ġded uctions","ĠRespond ent","_ stdio","Ġwitness ing","m ars","åıĤ ä¿Ŀ","Ġter b","ste hen","ĠPen ny","Ġst ellen","ĠRet ro","ĠPa ula","Ġpip elines","ĠConc ord","ĠB ü","ok ol","å¤ļ è°¢","Ġtr out","Ġterm asuk","æĢ§è´¨ çļĦ","æĺ¯æĮĩ åľ¨","ĠCL ASS","In ject","åĪĩ åı£","ç²ĺ è´´","Ġwarr ants","Dig it","æ¾İ æ¹ĥ","Ġo stat","ĠCan ter","ĠÑįÑĤи м","Ġmelan ch","æ¯Ķ åĪ©","çĪĨ çł´","Õ¸ÖĤÕ©Õµ Õ¡Õ¶","ĠÑĥÑĢав нениÑı","Ġbov ine","c za","Ġle pt","Ġmon archy","Ġten emos","мен ÑĤÑĭ","ĠÙħد ÛĮر","Ġmour ning","ĠJ W","Ġarr iv","ìŀIJ ê°Ģ","ĠOper ational","Ġrend ers","Ġdetect able","ĠPL AN","Ġë² ķ","ĢáĢ»á̱á̏áĢ Ľá̽","Ġqu in","ER IC","ĠTi O","ĠP rentice","ĠW I","Ġrespect o","Ġclean up","ô m","ĠAnne x","å°± ä¸įè¦ģ","åŃIJ æłij","漫 éķ¿","人æīį çļĦ","åı¯éĿł çļĦ","ç¶Ń æĮģ","éģĵ 人","çͱ æĿ¥","Ġwarn s","ĠLingu istics","le ave","çľ¼ çļ®","cer al","åĵª ä¸Ģ个","å¾IJ å·ŀ","Ġprosper ous","´ ī","Ġsuper market","_f unc","çĿ¡ äºĨ","ĠSing ular","= device","ĠM atching","ĠIn valid","Ġpr atic","åĢĴ éľī","çĸij ä¼¼","Ġmol ten","Ġstra ined","×ķר ×ķת","}}\\ ),","ĠCompan ion","ĠHab itat","r ath","ant wort","å¿ĥ äºĭ","Ġnew ton","åĢĴ åľ¨","Ġutil izar","od end","Ġ< >","ren o","åıįæĺł åĩº","................................................................ ................................................................","ç²¾ éĢļ","åĨĻ å¾Ĺ","çͰ éĹ´","é̲ ä¾Ĩ","Ġobs essed","I ron","æĪ Ł","-st op","å½ĵåīį çļĦ","漫 éķ¿çļĦ","Ġdegrad ed","Ġби бли","åͤ éĨĴ","ĠE ck","ĠL al","æĪij å¿ĥéĩĮ","éĤ£ 份","æ·± åIJ¸","è¿« 使","Ġa par","æĹ¶ ä¸įæĹ¶","f etch","ar it","Ġm Ã¥","å¿ĥ ç¥ŀ","اÙĨ س","uck le","èĮ« çĦ¶","av ir","Ġbus hes","à´ ¨","Sh ipping","Ġoccup ies","Ġdere chos","åı¯ åı£","á» Ń","Ġcommand ing","æķ² éŨ","ç¯Ħ åľį","ĠAnaly ze","Ġsos ial","b uffer","çī¹ å¼ĤæĢ§","Ġdetail ing","Ġspl ash","á̬áĢ¡ á̝áĢķáĢºáĢħ","ĠI vy","ä¸Ĭ éĥ½","Ġtr ud","è¨ Ĺ","Ġد اخÙĦ","äºĨä¸Ģ è·³","ech a","га ни","Ġcapt ion","Ġtag ged","\" ])Ċ","K i","-s w","åĺ Ĩ","Ġwis ely","ĠGy ne","è¾ Ļ","Ġz oning","Ġsl it","ĠاÙĦØ£ رض","-re ported","è¾Ĩ 车","Ġlou der","e ce","an ity","使 åĬ²","Ñģк ÑĥÑİ","ĠRes on","Ġtrust worthy","è¿Ł çĸij","t urn","¯ ¯","ĠNin ety","_ RO","Ġর াà¦ĸ","Ġwheel chair","顯 çĦ¶","(@ \"","Ñıв лениÑı","v w","Ä ķ","太 好äºĨ","Ġdoc s","ĢáĢ»á̱á̏áĢĽá̽ á̬áĢ¡á̝áĢķáĢºáĢħ","çļĦ æľĢåIJİ","ä¸į 符","ield ing","+ H","åħļ æĢ»æĶ¯","AC TER","çŃĽ æŁ¥","ĠConvers ation","ap un","Ġfe br","ĠEst her","_ email","k iego","Ġd ang","Ġb ÄĽ","ÙĬ اء","che v","æĸ¯ å¡Ķ","ĠÙĤ ÙĬÙħØ©","Ġcompens ated","ĠRefer anser","ĠMeasure ments","è¾¾ ä¸įåΰ","ĠпÑĢи води","/A IDS","inds ay","éĸ¢ æķ°","ĠSter ling","g ene","g ling","ĠT ruck","è¿Ļ ä¸Ģ个","Ġ×Ļ ×¦","åºĨ 幸","Ġcyt oplasm","Ġstraw berries","divid ed","ĠC FR","Th an","lig t","ĠÑģиÑģÑĤем е","æĮĩ çĤ¹","AT ES","col ors","ä¸ī个 æĸ¹éĿ¢","ĠÚĨ ÛĮ","åĩºå¸Ń ä¼ļè®®","ä¸Ģ åĨį","éĹ® äºĨ","ĠLam bert","Ġbr ushed","ĠкоÑįÑĦÑĦиÑĨи енÑĤ","Ġc ál","Ġst aged","è¿Ļ éĥ½æĺ¯","ĠØ¢ زÙħ","à§Ĥ প","ĠBrig ade","åºĶ 符åIJĪ","Ġк ÑĢеди","ĠAt om","` ,Ċ","ĠF IT","act ivated","åİĤ çļĦ","Ġinfer t","Output Stream","Çİ n",".m icrosoft","оп ÑĢиÑı","çļĦç¥ŀ èī²","ìĪ ľ","Ġartif act","c ine","Ì Ħ","Ġn hi","Ġgar ments","ä¸įèī¯ åıįåºĶ",", u","is ance","个 大","hed ron","Äģ r","= âĢĿ","åı¯ è¡ĮçļĦ","Ùħ اÙħ","Ġday time","ื à¸Ļ","èĴ¸ é¦ı","ر Ùĥ","å°ij åħĪ","Ġtext iles","Ġesc aping","Ġê´Ģ 볨","AM L","ç§Ł æĪ¿","ĠRest oration","Ġk ok","Ġster oids","! Ċ","çľĭ ä¸įåĩº","缸 ä¼´","ĠHe aling","æĹł è§Ĩ","ί δ","éĶĢåĶ® æĶ¶åħ¥","ä¸Ģ çŀ¬éĹ´","ĠвÑĭ Ñħод","Ġexec utable","ĠRef lection","æ»ŀ åIJİ","ĠRug by","Ġyour selves","æľ¬ å±Ĭ","åIJ¦ åīĩ","è¿Ļä¹Ī 大çļĦ","éģĵè·¯ ä¸Ĭ","ĠNut rients","ĠAutom otive","ĠCham bers","åı° çļĦ","ικ ÎŃÏĤ","ĠLaure nt","F lex","Ġan k","ĠL ance","Ġdr ills","Ġconn ective","æľĭåıĭ çļĦ","M IT","w and","ĠD OS","ä¸ĭ åİ»äºĨ","ä½ł æĺ¯åIJ¦","-b o","ĠاÙĦØ£ رشÙĬÙģ","å®ŀéĻħä¸Ĭ æĺ¯","ë¯ Ģë¡ľ","Ġcommence ment","æ©Ħ æ¦Ħ","ç͍ å·¥","Ġд иÑģ","arch ing","ĠÐł аÑģ","Ġscr ub","ĠÑĥни веÑĢÑģиÑĤеÑĤ","ozyg ous","Ġ( «","ĠW P","è¿Ļ å°Ĩ","ee ks","çħ§ 亮","Al ready","éģ¿ åŃķ","Ġpet ite","Ġuter ine","ol ina","ãĤĭ ãģĵãģ¨ãģĮ","à± Ĭ","unning ham","çŁ¢ éĩı","f actor","ĠP erc","** .:","ĠMan ifest","Ġcheck out","ĠRom ance","ut as","Ġj oka","Ġdis connected","Ġche wing","Ġsk up","ั ม","éģį å¸ĥ","ĠB ool","ih ar","Ġ Ó©","ĠF ees","æĪij è¿Ļ个","åıĺ æĢģ","åѦçĶŁ 对","è³ĩ éĩij","综ä¸Ĭ æīĢè¿°","Ñĥ Ñĩи","Ġexper i","au ge","Ġexpl ode","Õ¥ Öģ","Ġor ally","all on","å¹³ å¹³","èĩªçĦ¶ èĢĮ","di agn","ĠFundament als","é¢Ħ æĸĻ","ÙĪÙĨ ت","è°ĵ ä¹ĭ","ocument ed",".value Of","Z hang","åIJİ å°±","å¾Īæľī åı¯èĥ½","ĠогÑĢани Ñĩе","ul ia","б ÑĢе","Ġconvenient ly","Öī ĊĊ","Ġskept ical","åIJİ å¤©","Ġer ase","_P RO","ÛĮÙħ ÛĮ","ĠSac ramento","ar ithms","Ġb ells","ĠSt rait","Ġ% }ĊĊ","æĪIJåĬŁ åľ°","èĪª è¡Į","å¼Ģåı£ éģĵ","Ġlap ar","çŁ¥ æĥħ","ism atic","ada an","Ex change","Ġcat hedral","æľīæīĢ å¸®åĬ©","ĠBal tic","Õ¡Õµ Õ«Õ¶","Ġin ici","çļĦ å¹´","ĠN IH","-h u","ĠÑħ оÑĤÑı","Ġdin osaur","à¸ķà¹īà¸Ńà¸ĩ à¸ģาร",": `","æĮĩ å°ĸ","ĠÐļ аÑĢ","\" But","çļĦ äºĶ","Ġtrans gender","æīĢ以 æīį","Ġpoll ing","æijĩ æĻĥ","ĠâĻ ¦","æĺ¯ 缮åīį","Ġд ене","éĿĴ éĿĴ","V ALUES","çļĦ 计åĪĴ","Ġexpon entially","å®ī ä¿Ŀ","اÙĦ Ø«","ার à§ĩ","Ġذ ات","ISS ION",". select","æĹł æķ°çļĦ","Ġdel inqu","-b uilt","Ġser pent","Ġbow ling","çļĦæľĢ æĸ°","Ident ify","le kt","ĠD anger","æĪij å½ĵæĹ¶","ÛĮ Ø·","ĠG N","Ġun paid","Ġspec ulative","Th row","Ġsl ammed","åĬ¿ å¿ħ","Ġneuro degener","on ica","re duce","ber ty","ik us","å« ¡","D EN","çļĦ ç±»åŀĭ","ä¸Ģ 竳","Ġme est","两 åľ°","Ġhel ium","Ġuns ere","ĠMov ies","\" fmt","Ñĩ Ñĭ","åĨį æľī","ateg oria","': '","åı¯ä»¥ 对","æł¹æį® åľ°","缮æłĩ åĴĮ","G CF","[ C","å°Ĩ è¿ĻäºĽ","çıŃ ç»Ħ","æ°¸ éģł","ĠJul i","E asy","åĮĸ 身","å®Į å¤ĩ","-c arbon","Ġза па","ĠSynt ax","Ġо Ñħ","Ġdou bling","åĵį äºĨ","Ġnational e","Ġساز ÙħاÙĨ","_ up","ĠAk adem","_ J","çļĦ å±ĢéĿ¢","éģ ·","Ġë Ŀ¼","Ġdé p","è¿IJèIJ¥ åķĨ","åŃĺ éĩı","Ġfright ening","Ġn enÃŃ","ad ia","æ³ķ 令","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ","( expected","w m","çĤ¹ æĺ¯","Ġsing ers","μ ÏĮ","/j query","las se","Ġmouth s","ĠÑĢÑĭ н","Ġadminister ing","Ġreg iment","ĠاÙĦÙħ ÙĦ","ĠÚ© ÙĦÛĮ","ĉt emp","S eb","W ellington","缸 符","Ġimp atient","lev ard","Pol ice",", ##","Ġprodu z","ĠChar acters","àµįà´ Ł","t b","Ġc asing","Ġsl udge","×¨× ¡","ĠÙĪØ§ÙĦ د","ĠEli ot","ĠÑĦинан Ñģов","Ġth ức","à¸ļ à¸Ħ","ows ka","æ¯ı个 æľĪ","Ġyouth s","Ġm ente","大 ç¥ŀ","åIJį çīĩ","ä½ľç͍ äºİ","Ġfasc ination","ĠL amp","col on","Ġalarm ing","ĠاÙĦاجتÙħاع Ùī","é£ Ħ","ä¿Ŀ æļĸ","-y ou","ĠÑģÑĥм ма","H op","c py","Ġt ÃŃnh","度 éĩı","ç»Ħ åĪĨ","Ġgovern o","Ġfore ground","ĠRE VIEW","å¹´ ä¸Ń","æĹ¥ ç¨ĭ","ร à¹Īาà¸ĩ","ç®Ĺ æ³ķçļĦ","è¿Ļ æĶ¯","ell ants","ib ia","æĭį æĭį","ĠÐij и","èIJ¥ä¸ļ æī§çħ§","Ġsymmet rical","Ġpist ol","ĠFilip ino","R ules","Ġl est","Ġwild ly","ĠCalcul ating","ot us","ĠB ey","ĠE arlier","per formance","åºķ èķ´","å®ŀéĻħ éĹ®é¢ĺ","ĠاÙĦت ج","EG F","第åįģ äºĶ","Ġab c","×ķ× ¥","çľ¼ çľ¶","èĭ± åĭĩ","帮 ä»ĸ","EL S","cl ed","å¤ļ æĺ¯","ä¸Ģ åĽ¢","cy cles","Õ¡Õ º","çļĦ大 éŨ","ĠاÙĦاع تد","Ġm úsica","EL F","Ġstack s","åı¯ çͱ",".S earch","ĠMar l","Ġfel ony","en ched","ers et","iz ados","éĹ «","æĸ° 课","çł ¥","ĠÑĥ лÑĥÑĩ","Ġho og","Ġnic otine","x o","~ /","е ви","她 对","ÅĻ ej","ĠÐĶ Ð¶","ä¸įç¡®å®ļ æĢ§","ä¸Ģ 模","Ġcl am","ä¹ĭ æĦŁ","ĠNe o","åľ£ ç»ı","Ġrook ie","åħħåĪĨ èĤ¯å®ļ","ĠÑıн ва","æľī ç͍çļĦ","为 å¥ijæľº","ec zy","inn itus","Ġexperiment ing","Ùij ÙIJ","Ġprosec utors","kor zyst",". os","Ġver de","* }","ĠT ing","ä»İ æł¹æľ¬ä¸Ĭ","æĺ¾ åį¡","Ġcorrect ing","Ġcav alry","Ġch ords","Ġmism atch","Ġجد ÛĮد","g ap","ĠN SC","л Ñĭй","åij ¦","åĩºçĶŁ äºİ","Sche dule","is ers","ul monary","ah aran","(\" ./","Ġಠ¬","ĠHand ling","ĠSqu ares","ĠобÑĥÑĩ ениÑı","ram ient","äºĮ æŀģ管","é¦ĸ éĢī","åħ´ èĩ´","H ydro","S port","Ġde que","Ġclass ics","åħħ æĸ¥","jo ining","Ġib n","Ġtilt ed","Ġw izard","Ġz ie","ç͵ æĬ¥","åįĹ çĵľ","æĽ´å¤ļ åľ°","å¤ĸåĽ½ 人","Princ ipal","ĢáĢ»á̱á̏áĢĽá̽á̬áĢ¡á̝áĢķáĢºáĢħ á̝","ĠCons olid","IL Y","Gener ally","Ġceleb rities","-reg ulated","ac jÄĻ","Ġtrans genic","Ġsam t","ĠEl ena","uj u","ĠGener ic","åıªè¦ģ æľī","çļĦ è¶ĭåĬ¿","Ġsur tout","æĢ» å·¥ä¼ļ","Ġج اÛĮ","el te","×Ļ ×Ļף","Al ign",".get Name","Ġà¦ķ ার","èįī æľ¨","ÑĤÑĭ е","ĠConsult ado","URR ENT","pr inc","æĦŁ å®ĺ","æİ¨ ç§»","ذ ÙĬ","çͲ éĨĽ","èģĬ èģĬ","ä»ĸ ä¸İ","åŁİ åİ¿","æ³¢ çļĦ","å°ĩ è»į","config uration","ĠAra bs","st ag","ĠC erv","Ġdet ox","×Ļ׾ ×ķת","ĠP IN","ĠV ale","ج ات","ĠM ast","çī ½","Ġ× ĺ×","Ġdeb it","Ġeth ylene","Ġdiss ipation","ĠÑģп и","ÑĩиÑĤа ÑĤÑĮ","K ap","举 äºļ","ÙĴ ر","L oss","et as","ĠS PE","å®¶ 常","åıĹ è¿ĩ","Ġrest o","RE M","ĠBas el","ĠEs sex","寺 åºĻ","ench ymal","Ġcom erc","ĠK uwait","è¿Ļ次 çļĦ","< iostream","z ego","â ļ","ĠF CC","Ġr m","C arbon","h á","æµ· åı£","gan o","èī² è°±","æĢ» 线","Ġur ges","\"ĊĊ Ċ","ç쵿´» æĢ§","åīįæīĢæľª æľīçļĦ","leuk in","w ah","ont a","èĭ¥ ä¸įæĺ¯","Ġconsult ations","Ġdesc ub","F lor","Ġd io","ĠNic arag","ĠNUM BER","ĠCont est","je va","त à¥įय","Ġbrief ing","+ z","it ra","The ta","è¾ĥ å°ıçļĦ","mm as","Ġestud ios","åĵ® åĸĺ","ç¿ İ","Ġsw ords","ÑĢаз д","Ġtijd ens","alb um","à¹Ĥ à¸ķ","ĠGib bs","f rames","åĪ¶åº¦ åĴĮ","æ¡Ĩ ä¸Ń","éĽĨæĪIJ çĶµè·¯","p aces","od em","ÑĤ ного","éĽĨåĽ¢ æľīéĻIJåħ¬åı¸","ĠProv idence",". ge","ĠH ide","Ġblock ade","-n umber","Ġtherm odynamic","Ġtrib ut","-dig its","Ġexplor atory","Ġ à¸ľà¸¹à¹ī","an ine","åĮ ¡","å¤į 产","rid ges","Ġann ée","éĺ ij","å®ŀ æķ°","论 è¯Ń","ĠÑĢа н","åĨĻ ä¿¡","çĸĹ ç¨ĭ","ĠBlack well","Ġко оÑĢдина","Ġस म","P u","ol ite","ĠB H","ä¸Ģ缴 没æľī","ĠCit izen","ĠÑĥÑĩ ÑĢежд","ĠN SA","ign y","Ġ) =","Ġheart beat","èŃī æĺİ","st roke","åĨħ èĨľ","-b al","éĩİ å¿ĥ","CD C","Ġpled ged","ens ely","oss ary","é±¼ ç±»","Ġrecon oc","Ġrhyth mic","s j","Ġp si","ัà¸ĩ à¸ģ","ĠKid ney","ĠSabb ath","çݰ å·²","çª ĺ","ä¸ĵä¸ļ æĬĢæľ¯","Ġre e","ĠC ogn","ĠD jango","ist ica","éħĴ æĿ¯","ĠPrep ared","Ġstate wide","Ġimm erse","ĠÙħج اÙĦ","ĠG le","大 为","sp oken","ĠQ uran","ç²ĺ èĨľ","ĠK art","(g rid",", )","_ client","Ġwh ip","erm o","Ġο ÏĢο","å¾Ģ è¿Ķ","第ä¸Ģ æĿ¡","æİı åĩº","éĢī è´Ń","-c irc","Ġbi ome","Ġsem ana","Ġborrow ers","_ next","or iasis","Ġh p","ig ua","Ġд об","Ġmet iculous","åIJ¬ ä»İ","Ġpast ry","ĠPur ple","pp led","è¿Ļ æľ¬","ĠK A","ãĥ §","rad ing","Ġhabit ual","åĬłå¿« æİ¨è¿Ľ","ãĥ³ãĥ Ĺ","çĴĢ çĴ¨","çĪ Ľ","ĠSc ots","æĿĢ å®³","ãģĭãĤī ãģ®","è´µå·ŀ çľģ","Ġìŀħ ëł¥","Ġpess oa","w is","ĠJ B","æ¶² çļĦ","éĥģ éĹ·","çµĦ ãģ¿","åĤ· 害","Ġмн ог","S ent","Ġhal ves","Art ikkeli","åıį åĢĴ","ĠQu in","ĠÙĩ ÙħاÙĨ","Ġce il","ĠMil ano","` s","ĠH ast","Ġver wend","çģ« çĥ§",".C ore","Ġauthor itarian","çļĦ 个人","åľ¨ ä»»ä½ķ","ä½ł åı¯èĥ½","uc us","Ġpal ate","ĠاÙĦÙĥ تاب","Ġg ems","ber ra","Ġme zi","ph rine","åıĪ å°Ĩ","Ġste eds","èįī åĿª","Ġcort isol","Ġfilos of","Ġd Ã¥","Ġple ad","ĠاÙĦس ÙĬاس","Ġdistingu ishes","Ġde comp","ĠW heat","åĽ½ ç«ĭ","éķ¿ éĢĶ","æ¸ħ æ·¡","åĶ §","Ġб аз","ĠGovern ments","ĠXV II","ÑĦ он","Ġgly cos","Ġburst s","çĭłçĭł åľ°","Ġun paralleled","rit able","ER VER","æ±ī 书","Sim on","çŁŃæľŁ åĨħ","å®Į åIJİ","å·® é¢Ŀ","Ġ×ŀ× Ľ","Last ly","Ġ[... ]ĊĊ","C AR","åĬ ´","è¯ģ çĽijä¼ļ","ĠSc roll","éĿŀ常 éĩįè¦ģçļĦ","Ġsv ÄĽt","Ġembarrass ment","ä»İ ä¸Ĭ","ices ter","Ex cept","unt as","ĠAcc uracy","åĵ¡ å·¥","Ġquel que","å¾ĭå¸Ī äºĭåĬ¡æīĢ","W ire","r gb","ĠS ole","å±± æ²³","/h ome","Ġδ ι","Ġabol ished","è¿ĩ å¤ļçļĦ","Ġmill iliter","Ġgot ta","èĥĮ çĿĢ","Ġsen ator","Sl ide","Ġvort ex","çĤ¹ å¤ļ","åįİ ä¸½","Ñīи м","Ġê· ¼","ä¸Ģ åĽŀ","æĺ¯ 羣","åĨħ ç½®","ES E","æĪªèĩ³ 缮åīį","z na","èĢģ çļĦ","Ġbu il","-sh aring","ĠCook ing","g ro","Ġso aked","ØŃ ÙĦØ©","cho ose","å°ĸ åı«","ĠEc ological","n in","ign al","åģľ åľ¨","icon duct","Simpl ifying","_ request","j ad","Ġt icks","ĠA IR","ĠD ict","ó k","Ġent rada","ذ Ùĥر","ĠвÑĭ ÑĩиÑģ","à¯ģத à¯įத","Ġdern ier","ĠT umor","æĹ¶ éķ¿","ä¸Ń èı¯","æĢ§ æĥħ","Ġmon k","å®īåħ¨ åĴĮ","Ġspecial ize","ĠEr rors","Ġdirection al","ä¿ĿæĮģ åľ¨","Ġmant en","ĠREF ERENCES","Ġin nych","are as","ock ing","è¶Ĭ è¿ĩ","éĨ ļ","æı¡ çĿĢ","Ġë°ľ ìĥĿ","Ġprobabil istic","Ġfor ged","aut iful","Ġconsequ ent","ÑĤоÑĢ Ð¾Ð¼","å¼Ģ å¿ĥçļĦ","被 她","æķĻåѦ åĨħ容","Ġgrass es","Ġsul le","h ak","m obile","æĺ¯ å¤ļä¹Ī","Ġhigh s","Ġген еÑĢа","Ġprofic ient","å®ŀ å¹²","ĠRep orter","ĠMon ster","Ġluc rative","Ġnécess aire","Ġ ï¼Ľ","éĺ» æĬĹ","Ġmon et","帮åĬ© ä¸ĭ","Ġk Wh","inn ov","valu ate","åĨĴ çĿĢ","Ġë§İ ìĿĢ",". br","Ġp its","Ġh ình","大 åı£","Ġм оз","äºĨä¸Ģ å¼ł","æīĢæľī 人çļĦ","å½Ĵ äºİ","Ġin verte","æĪĺ éĺŁ","ç¼ĵ åĴĮ",": i","P ag","P ush","Ġrest ricting","ાઠ°","ÑģÑģи в","qu é","èᝠçļĦ","Ġ×ŀ× ij×","soft ware",". pe","ven es","the l","å¢ŀ æķĪ","play ed","ç´§ç´§ åĽ´ç»ķ","Ġcruel ty","Ġindict ment","R AM","Û İ","est roy","ĠH IP","Ġro da","... \"ĊĊ","ĠÙĬ ج","èĨ ½","æıĴ ä»¶","Lou is","ĠTelesc ope","Ġg ag","Ġpart itions","Ġins urers","ĠSaf ari","éĽ£ 以","Ġ) }Ċ","æĹł åıĮ","ĠAl a","Ġdok ument","ĠÑģледÑĥÑİÑīи е","Ġl iner","带 äºĨ","æİĮ æŁľ","Ġsandwic hes","éĩįè¦ģ æĦıä¹ī","EM BER","تر ÙĨت","âľ Ķ","l iquer","åľ¨ è¿ĩåİ»","Ġr ôle","å» ļ","ż s","ä¼ļè®® ç²¾ç¥ŀ","_ LOG","ĠÙĪ Ø°ÙĦÙĥ","Ġcou pon","ĠDun ay","S G","Ġl icha","äºĮ æľŁ","Ġtre as","Ġни же","William s","ĠAtmosp heric","ĠSe vere","æĶ¶ 款","声 èªī","ä¹° æĸ¹","Ġker ja","ON OM","Ġconf use","Ġinj unction","人åı£ çļĦ","- u","â ĩ","说 ä½ł","ĠGeorg es","举åĬŀ äºĨ","ĠGujar at","k d","çļĦ çĥŃæĥħ","åľ¨ æľªæĿ¥","åĴ ¦","×Ļ ×ķ×Ŀ","ĠPol ite","èĦ± æ°´","Ġম ত","ĠSun set","è®ĵ 她","Ġped al","è¿Ļ åIJį","ĠRe creation","é¦ĸ åıij","Ġbow ed","ĠпÑĢави ло","im ony","æµģ åIJij","ั à¹Īà¸ĩ","ĠLe aves","带 ä½ł","Ġup held","缸 æĢĿ","Ġsl ipp","èªŀ è¨Ģ","ĠB om","ä¸į æīĵ","ĠN ielsen","Ġph p","æį¢ åı¥è¯Ŀ说","ÑĢÑĭ в","ĠMill iseconds","沿 éĢĶ","ல à¯įல","Ġfebru ari","ä¹Ł æľīäºĽ","Ġi os","é¾Ļ çļĦ","举 æŃ¢","为äºĨ 使","Ġdeb ido","-st atus","Ġmiss es","Ġì° ¾","Ġth ro","ä¸Ń 没æľī","çݰ éĺ¶æ®µ","å³ Ļ","organ isation","Ġpac ientes","ĠKrish na","Ġà¤Ķ र","f ur","Ġна ÑĢ","= .","g p","ä¸į åĸĦ","æĥħ ä¾£","âĢĶâĢĶ âĢĿĊĊ","วิ à¸Ī","ĠT ipo","ĠRe actions","ning er","æ·± çŁ¥","åºķ çļĦ","po que","ĠÑĢе д","огÑĢаÑĦи Ñı","osex uality","pro c","Ġpol ity","-d emand","è¡ĮæĶ¿ æī§æ³ķ","ä¹Łæ²¡ ä»Ģä¹Ī","Ġconvey ing","Ġpriorit izing","_ con","Ö ±","Ġmeaning less",". Open","Ġre op","ĠP arl",".... Ċ","×Ļ× ¨×","Ġmed idas","Å¡ k","Ġprz ew","åΰ èĩªå·±","é£İ æ°Ķ","æĶ¯ 票","omy ces","麻 麻","ứ ng","ol ated","Ġas ynchronous","Ġar k","Ġpre ach","ete enth","æİ¨åĬ¨ äºĨ","ĠMad ag","ĠÐŀп ÑĢеде","ä¸į ä¼ij","IN ESS","Ñĩа ÑģÑĤ","ĠPe pper","Des cribe","m ier","al nya","æĢ§ çĬ¶","Ġgr á","åĮ» ç§ij","èij¡èIJĦ çīĻ","M Q","S her","if act","yst y","å¤į æŁ¥","ÑĤо н","Ġci ò","d ob","Ġquest e","ĠEm erson","ĠQual itative","unis ipyo","[ MAX","Ġcan ine","Ġra ft","ied z","на м","Ġstri pes","Ġmuc osa","ĠRect angle","Ġmicrom eters","Ġa chter","Ġpre defined","Ġco ined","æł¼ éĩĮ","åŁİ 主","ĠCar negie","ä¸Ģ è¿ŀ","rom an","Ġag gi","Ġpe g","没 å¿ħè¦ģ","æĺ¼ å¤ľ","ch apter","Ø´ اÙģ","ĠØ£ Ùħر","Ġê·¸ ë¦¬ê³ł","ĠTechn ological","CE LL","Ġин дивидÑĥ","à¹Ģà¸Ĥ à¹īาà¸","ĠIU CN","f ection","Ġb idding","ä¸Ģ æľĪ","Ġcl aw","Ġcomp osing","ĠChrist ina","Pl ot","ĠбÑĥ к","å±ıå¹ķ ä¸Ĭ","l ain","ÃŃ do","ب ÙĬرة","çĶŁäº§ èĢħ","ĠпÑĢед ÑģÑĤавлÑı","< input","è¦ģ åģļåΰ","ĠÑĢа м",".E xt","plet ely","Ġп Ñĥнк","ek ak","Ġdown stairs","ла м","OT O","éĵ¾ æĿ¡","Ġdis place","Ġbro ch","âĶ ģ","Occ up","l ior","p ick","Ġn ests","æĿ¥ å¾Ģ","ili h","ãģį ãģ¾ãģĹãģŁ","Ġiron ic","chedul ing","åĭĺ æİ¢","deg ree","ÏĦ οÏħ","æİĴ æ°Ķ","ograf i","ĠRain bow","ĠاÙĦÙĤر Ø¢ÙĨ","L abor","ve hicle","èĩª è¯Ń","Ġ/ ><","Ġter ug","Ġ×ķ× ©×","æİ¨èįIJ çļĦ","ĠQué bec","é«ĺ å°Ķ","ĠRe x","ax on","å®ĥ èĥ½","ĠAd vertisement","社ä¼ļ åѦ","/m atch","Ġprofessional ism","æµ® åĬ¨","饥 饿","/ equivalent","ĠM ys","ä¸Ģ æĭ³","ult ats","ĠGe ology","åı« 人","éĴ» çłĶ","ĠвоÑģ пиÑĤа","ĠLoren zo","Ġs ibling","ik ir","æ¤į åħ¥","ĠSem inar","ĠS itu","æıIJ åIJį","ç®Ģ 约","é£ŀ éĢŁ","æľ¨ æĿ¿","ĠЧ а","ĠS UR","Ġun sett","' eau","_ var","ĠST ART","Ġpump ed","ĠOpp osition","? ...ĊĊ","end u","è£ħ æľī","ĉĉĉĉ Ċ","Ġmm Hg","Ġdifférent es","âte au","ĠÑĥ ÑĤвеÑĢ","Ġge ology","å²Ĺä½į ä¸Ĭ","may be","'=> '","ãĢ ĸ","ĠT rag","ĠM ongo","çİ ĸ","ĠK udos","à° Ł","à¸Ľ ัà¸Īà¸Ī","âĶ ľ","éĸĢ åı£","Ġpúblic a","ä¸İ ä¼Ĺ","Ġpres cribe","utt gart","Ġrough ness","Ġpolymorph ism","-count ry","ĠRw anda","Ġm A","å¿ĥ æĦı","Ġev ol","ç͵ 线","ĠEng aging","ĠÚ¯ رÙĪÙĩ","ĠKey nes","Fe atures","ĠAN OVA","ĠW itness","é ge","b ung","¼ áĢ","çļĦ åı¦ä¸Ģ","ä½ µ","为 她","æĿij å§Ķä¼ļ","éĻIJ é¢Ŀ","ĉt ry","Ġgrat uit","Õ¥ÖĢÕ ¨","/ img","> :","Ġb iting","ies en","Ġun ilateral","Ġlas ers","å®Ī æ³ķ","ä¿ĿéĻ© 人","Ġredund ancy","ĠÑģовеÑĢ ÑĪен","çļĦ éŁ³ä¹IJ","ĠD airy","ik ers","æĹł çŁ¥","ç͵ å¹³","Ġpers ists","Ġequ iv","åħĭ éļĨ","رÛĮ ÙĤ","иÑģа ÑĤÑĮ","F it","Ġc rossover","Ġin compet","а лов","Ġcon te","Ġacqu ainted","ĠاÙĦس ÙĦاÙħ","Ġresist ed","a on","çļĦ æŃ£ç¡®","ich é","éĩį 度","ĠCom fort","èģĶ æīĭ","ĠAm ber","ĠCal gary","çĤº ä½ķ","UR AL","æľºæŀĦ åĴĮ","agram s","èľľ èľĤ","Ġsmok ers","çļĦ è§£éĩĬ","^{ +","Ġtop ography","ода ÑĢÑı","ĠQual ifications","R ON","j ian","çļĦ æĻĤéĸĵ","åģļ æ¢¦","èĭ± åľĭ","Ġlen ker","Ġdivers as","Ġinf atti","çĮ Ŀ","è²» ç͍","ĠH apit","äºĭ äºĭ","éĢı è§Ĩ","éĴ¢ æĿIJ","Ġroof s","Ġl umbar","Ġpract ise",".C ross","ç´¢ æĢ§","ĠAustral ians","Ġвз ÑĢоÑģ","ĠM ole","ĠL iqu","ó rm","æµĭ ç®Ĺ","Ġni em","å®Įæķ´ æĢ§","iv it","Ġform ative","-s um","丧 å°¸","ICAgICAg ICAgICAg","L em","ä¸Ģ æĹı","() ))","æķ°æį® éĽĨ","éĩijèŀį æľįåĬ¡","ĠAlber to","ĠWARRANT IES","t ool","çİ º","åħ¨ è¦ĨçĽĸ","çī¹ æķĪ","ä¼ģä¸ļ åıijå±ķ","ĠFlex ible","Lower Case","/bl ob","Ġmeningkat kan","ä¸į å¤į","Ġeffic iencies","Ġ át","cc ió","Ġple thora","Blue print","Ġrept iles","Ġac claimed","Step hen","λο ÏĤ","opl ankton","ĠAcknowledg ments","Ġجز Ø¡","åĩı æİĴ","Ġgraph ite","вед ение","au en","Ġlife cycle","ÑĢÑĥ ÑİÑĤ","Ġphot ographers","mod ified","Ġblog ger","æł¹æľ¬ å°±","Ġnost ra","Ġqu ir","æŃ£ 缴","Ġgall eries","ĠInfant ry","\" \\","Ġd ung","æĺ ĩ","ĠO ok","ĠK uh","çݯ çIJĥ","æ¦ ķ","æ½ĩ æ´Ĵ","帷 å¹ķ","ĠK ell","ä¾ĭ é¢ĺ","Ġembarrass ing","Ġgebru ikt","ik in","Ġprincip io","Tw enty","ĠwiÄĻ c","h aving","ĠS ain","est amps","åĴ Ĩ","ä¸ĭ 乡","ka ar",".e u","ظ ÙħØ©","m ajor","ĠÑģ меÑĢ","Ġठĸ","åĵª 裡","Ġপ াত","×ķ× Ľ×ľ","amp u","df s","ĠÐij Ñĥ","eder b","åIJĪæł¼ çļĦ","ĠRab bi","ĠFitz gerald","å°± çľĭåΰ","ec ip","Ġка пиÑĤа","Ġин ÑĦек","iform es","ĠCorre ction","{ h","ä»· éĴ±","æİ¨ è¿Ł","AL TER","RO SS","ä¹Łä¸į 好","ÙĬÙģ Ø©","第ä¸ĥ 竳","Ò ±","us ch","Ġal right","res ident","Ġcontin ual","ãģĹ ãģ¦ãģĦãģŁ","ĠZe us","ĠMut ual","ĠH ä","Ġok res","ĠMcK in","(type of","åİ» çľĭçľĭ","à¸Ķ ิà¸Ļ","el as","åĨ Ĺ","æĪij们 ä»İ","An im","Ġà¦ķ ি","_f ilter","sl ug","C as","F air","× £","ed ere","met adata","Ġcross word","ĠاÙĦÙĤ د","éĹª éĹª","Ġcareg iver","Ġt earing","æĺ¯ ä»ĸçļĦ","为 åĽ½å®¶","ä¹Ł 許","Ġbu ys","Al ice","é¥Ń åIJİ","ĠBre xit","æĽ¾ç»ı çļĦ","åѦçĶŁçļĦ åŃ¦ä¹ł","Ġpare ce","æīĢ å¸¦æĿ¥çļĦ","åĨĽ è®Ń","èĢģå¸Ī åĴĮ","last ing","Ġaqu arium","nah men","èĩ³ å°Ĭ","Ġw ary","Ġr ond","ä½ł 说çļĦ","æµ· 峡","Ġcut off","èİ« éĿŀ","Ġexhaust ive","à°¿à° ¨","ĠSel bst","ter o","ĠR AD","ore g","ph ysical","çľĭ åľ¨","ho pping","Ġ×IJ× ©×¨","ù ng","background Color","ĠокÑĢÑĥ жа","ĠTrig onometric","pro gress","温 室","éĢīæĭ© æĢ§","ĠIsrael ites","Ġwarr anted","ĠRO I","on ation","ãĤĴ ãģ¤","ĠاÙĦÙħ Ø®","nÄĽ jÅ¡ÃŃ","жд ениÑı","Ġdiverg ent","Ġfor s","åĽĽ 级","ار ت","å·®ä¸į å¤ļäºĨ","ziÄĻ ki","Ġinform s","¶ ĊĊ","Ġlors que","D G","pp les","为 çͱ","ठħ","çĶŁäº§ ä¼ģä¸ļ","丼 书","åѦ éķ¿","è¿ĩ åī©","çŃī å¤ļ个","åı¯ä»¥ 被","Ġdisc s","ਠķ","Ġoccup ancy","Ġhyd rated","Ġdict ators","yy yy","éĺIJ éĩĬ","Ġpharmac ological","ĠðĿIJ ´","-bre aking","w l","Ġsl ack","Ġdat i","ĠÙĤ سÙħ","Ġма ÑĪи","ĠباÙĦ Ùħ","ë© Ķ","ìĺ ¨","ĠMort on","ĠCher ry","V EN","ĠاÙĦ ÙĴ","cons ciously","ë©´ ìĦľ","Ġpy ro","ĠD ud","é ly","Ġpr ů","约 ä¼ļ","ĠкÑĥлÑĮ ÑĤÑĥÑĢÑĭ","ĠBib code","çļĦ èĦ¸ä¸Ĭ","ĠM ight","ob ody","Ġب Ø·","ç§» åΰ","æĿ¾ å¼Ģ","æł¹æľ¬ ä¸į","ĠBreak fast","ĠD ivers","Ġhe mod","ä»ĸ ãģ®","ĠK IND","ien cias","åĽĽ æµ·","Ch oice","ÉĻ s","ĠÑģа й","nd an","ĠN ina","ĠDem o","สั ม","ä½ĵ åŀĭ","Ġlong itud","书 å±Ģ","åħĭ éĩĮ","åĨľä¸ļ åĨľæĿij","Ġfav ors","}$ .","sa id","ĠNorm ally","ĠSuz uki","_ once","Ġin ductive","ĠH b","大 æłij","åºĦ åŃIJ","] ));Ċ","ol iber","ĠM int","éķ¿ å¤§äºĨ","Ġgr ids","æĪ¿ éĩĮ","Ġcere bell","= F","ĠP aste","ay ah","Ġdep ois","rid ing","rad y","Ġس ÙĦاÙħ","_point s","Ġvast ly","Ġdict ate","ĠопÑĢеде лиÑĤÑĮ","å²Ĥ ä¸įæĺ¯","Ġinve ce","ĠS ight","Th ai","ĠNot ification","ĠSol o","سب اب","ĠConvers ions","Ġchuck led","ĠB olog","åĨĻ çľŁ","κ η","å°½ æĹ©","={ '","à¤ķ à¥įत","æĵ¦ æĭŃ","Ġwie ku","lic hes","Ġless en","Con c","æĺŁ åħī","伺 åĢĻ",". ref","ĠF ILE","ci us","gl ut","æĨ §","ĠvÅ¡ ak","Ġes k","æİ¨ ä»ĭ","æķ°æį® åĪĨæŀIJ","ĠÑĤо н","Ġком ан","Ġfro gs","Ġcohort s","Enc oder","е ÑģÑĤи","ÑĤ нÑĭе","ä¸Ń å°Ĩ","fer ably","åIJij 举","Ġer halten","Ġrepresent a","ĠChief s","ÑĨион ной","_ Y","Ġw an","ot rophic","ĠM aker","çϾ è´§","人ä¸İ 人","纪å½ķ çīĩ",". default","æŃ ©","ass i","天 çİĭ","ĠIs le","ä¹Łæĺ¯ æľī","èĦ¸ é¢Ĭ","Act ual","ÑĢ Ð¶Ð°","ĠN ab","äºĴ éĢļ","ĠRat ings","- er","ĠL emon","ĠSp ell","\\in fty","Ġepidem iology","åĩº åĬĽ","ous ed","è¡Į æ¥Ń","form a","Ġret in","Ġinf ra","éļı 身","å±ŀ æĢ§çļĦ","Ġdeliver ies","çݲ çıij","ĠMAN AG","_ U","Ġrespons iveness","Ġinsp ector","Ġ] ;ĊĊ","Ġrenov ation","Ġ{ (","æ²ī éĩįçļĦ","æľīæķĪ æĢ§","Ġcorrespond ent","åIJĮæĹ¶ è¿ĺ","ĠBenef it","VEL OP","o C","çī¹ è´¨","æĨ ¬",".string ify","R ain","ĠP OP","ie gel","Ġver ge","給 ä»ĸ","ĠEight y","ĠاÙĦØŃÙĬ اÙĩ","D ynamic","r ather","оÑĢ Ð¾Ð¶","ĠÚ© ÛĴ","ãĢį ãĢĤĊĊ","è®ĵ ä½ł","bour g","عر اض","ĠEk sterne","ĠF ract","å°ı çģ«","å°½ äºĨ","å¿ħé¡» æľī","ĠApplic ant","/ log","W a","_ html","en ig","red ient","ock ed","è®® é¢ĺ",".H ash","è¤ Ĵ","çļĦ ç͍æĪ·","ä¹Ł ç§°","ä½Ĩ ä¸įèĥ½","Ġbus ca","าล ัย","Ġd ictionaries","Ġcheer ful","Ġch ac","в ÑĪиÑħ","Ġass ort","IN ST","ul te","ĠH ubble","ĠPro to","Ġmill s","ĠProv ided","_ rec","æĥ³ 念","åıĺ è´¨","æµģ 产","转 åŃIJ","Ġsum a","æIJŀ å¾Ĺ","is pr","Ġand ers","Ġqu ed","Ġshe ath","Ġм ÑĥÑĪ","çļĦ人 æł¼","Ġcheck point","骨 è´¨","é¤IJ é¦Ĩ","ĠÑħ озÑıй","Ġmanip ulating","ĠMan it","c us","Ġworks pace","Ġorganiz er","ĠоÑĢгани за","èĩª 驾","çĤ ¬","================ ========","Ġcorro bor","r atory","it re","ä¸Ń æłĩ","ÑĢа к","çĸ µ","åİĨ æĹ¶","åĿļ åĽº","çīĽ é¡¿","ĠÐłÐ¾ÑģÑģи йÑģкой","ĠwÅĤ as","ent ries","åľ¨ çľĭ","åĪĨéĴŁ åIJİ","Ġmand ated","al ary","Ġv ÉĻ","Ġм не","设å¤ĩ åĴĮ","-reg ulation","åIJį çīĮ","樱 æ¡ĥ","Ġspat ially","代表 æĢ§","ĠBrit annica","k amp","è³ ¦","ÙĦÙħ Ø©","ĠУ кÑĥпно","éĭ ª","åĩı éĢĢ","ש ×Ļ×Ŀ","Ġconson ant","好 æ¶Īæģ¯","è¿IJ éĢģ","ĠWat ts","W inter","ĠM iz","ĠE CM","se par","失 æİ§","ĠÙħÛĮ اÙĨ","circ le",". ne","P ok","\\ Delta","Ġr t","Ġob solete","áĥ ľ","ĠX L","她çļĦ æīĭ","(p age","Ġdif ÃŃ","æ¯Ķ ä»ĸ","ä»ĸ们 ä¹Ł","ought on","æ´ģ åĩĢ","ĠCounsel ing","Y esterday","Ġad tong","м он","ĠVer de","Ġì¤ Ħ","o il","ath am","Ùģ Ø§Øª",".s ource","åĩĨå¤ĩ äºĨ","غ ÙĨ","Ġdial ysis","ĠMalays ian","æľ¬ èĬĤ","Ġন à¦¿à¦ľ","åĽ½æľī èµĦ产","Ġgior no","usah aan","s ic","çļĦ 第äºĮ","ĠH än","ĠÑģÑĤÑĢа нÑĭ","@ section","ib id","lic ts","ä¸ĵ å±ŀ","æŃ¦ 士","à¸ģาร à¸ĵà¹Į","Ġacid ity","çļĦ åıij","çļĦ æľīåħ³","çļĦ åĽ½éĻħ","Ġinform áció","ĠSoph ia","om rÃ¥","Ġmov imiento","à±įà° ¨","Ġfest ive","çļĦ 游æĪı","ĠT ay","ĠG ym","å°± ä»İ","表 åĨ³","æĹł æľº","äºĶ 年级","ç»Ŀ ä¸į","顺 çķħ","Ġmol ti","Ġkole j","U DE","t ube","Ġg ere","ĠD ixon","ant z","Ġintern s","é¢Ī æ¤İ","Ġto re","Ġen cephal","Ġdur ant","Ing redients","ĠM oy","ĠF old","æĻĵ å¾Ĺ","Ġmater n","otechn ol","èĢĮ çİ°åľ¨","å°ij äºİ","Est a","Ġsurviv or","å¼ ©","åİŁ åīĩ","ran a","m eth","Ġب ÙĬت","Ġvari os","b io","Ġع بار","Se ason","Ġo at","ĠÙĦ Ø¥","äºī åIJµ","Ġspecific s","éĵ¶ è¡Įä¸ļ","ĠPo ems","Ġtur bo","æĺ¯ åħ¶","-st ore","ðĿij ij","rypt ed","Ġcher che","æĴķ è£Ĥ","Ġpro cent","Ġun im","Ġд ÑĢев","Ġprogram mers","Ġat yp","Ġroad map","Ġperm utation","èIJ¬ åħĥ","in ux","Ġre leg","ĠM ID","å°ı 說","Ġо ÑĪиб","åIJij ä½ł","Ġmed iate","amb igu","çĿ¡ çĿĢäºĨ","FF ECT","Oper ations","- result","Ġw anna","ÑĤ нÑĭй","æĸ° å¨ĺ","ĠCook ie","ĠAnthrop ology","ci ences","ï¼ī =","çĭł æĬĵ","Ġ à¹ĥหà¹ī","Ġchar coal","лÑĮ зÑı","ĠâĪ ©","ãģĭ ãģ«","×ŀ× ĵ","Ġghost s","ĠA val","è¿Ľ åĨĽ","Ġneg li","Se conds","å°į èijĹ","_l oss","çŃī æķĪ","Ġrh s","R am","åĩŃ ä»Ģä¹Ī","Ġwie le","Ġproduct o","олÑĮ но","-qu arter","Ġbol ts",") T","å¤į ä½į","ĠÕ¸ ÖĢ","æĪij ä»Ĭ天","éľĢè¦ģ è¿Ľè¡Į","ĠÙĨ دار","Ġস à¦Ļà§įà¦Ĺ","建ç«ĭ ä¸Ģ个","С Ðļ","มาภķ","ratt utto","ĠاÙĦاعتد اÙĦ","s aurus","ent on","ow ell","op lan","åĮĸ èĤ¥","她 èĩªå·±","ĠAl ess","work er","ĠRE AL","Ġmedi ator","ĠEl astic","Class es","èµŀ åĬ©","ĠJose f","ú a","èģĶç³» æĸ¹å¼ı","że j","ãĤŃ ãĥ£","K al","v ate","ĠT ours","à¥įठ²","}} }{","ĠMap le","( un","re iche","uc ceed","åIJĥ åĸĿ","ाठ£","åħ¬æľī åζ","Ä ¯","Ġal f","ĠL U","ä¸Ĭ åŃ¦æľŁ","ä¸ĩ 个","ç§ģ åĭŁ","Ġpéri ode","Ñģ ол","Ġcl ones","æ°ij çļĦ","áĢ ¾","竣çĦ¶ æĺ¯","äl le","åIJį é¢Ŀ","à¯ģà® ±","èľ¡ çĥĽ","åij Ĥ","Äį ek","Ġré alis","Ġlé Äį","- area","Ñĩ ении","ĠÙĤ ابÙĦ","ĠCalcul us","Ġfuer za","Ġinaug ural","u ze","å¹³ åĪĨ","Ġest ekak","ÑĢи Ñĺе","Ġgrand son","ĠU L","Ġpr id","ian za","é© ¯","ĠÐļ ом","ĠPed iatrics","C ivil","ĠM og","ä¸ļ æĢģ","èĢĥ åľº","×¨× ¦","å¥ĩæĢª çļĦ","Ġst itch","åľ¨ 人","æĹ¥ è¶ĭ","æĺ¯ å¤ļ","æĶ¶ åī²","ðĿij ł","交æĺĵ çļĦ","ĠBrun swick","ĠB ek","Ġdo br","Ġcont ractions","Ġé én","Ġà¦Ĩম াদà§ĩর","ĠاÙĦ رÙĪ","交 æīĢ","ิ ส","è ce","Ġcomment ing","ĠWend y","ĠоÑĩе ÑĢед","ub in","á i","åĽł åľ°","æ¶ Ł","ID TH","(p arent","Ġreject ing","ĠAur ora","Com pleted","ais se","éĻĦ çĿĢ","Ġfrag mented","ĠAg ile","ĠFran çais","Ġhyp othalam","Ġvolunte ering","Ġszcz eg","p ain","un ched","oll er","Ġbel ts","air d","ł× Ĵ","è´µ éĺ³","ĠìĿĺ 미","'aut res","ĠÑģвоб од","ag y","çŃ IJ","Ġthem ed","Ġanal ogue","li us","Ġinvent or","示èĮĥ åĮº","Ġзада Ñĩ","Ġf ountain","๠ij","å² ¡","Ïĥ ία","Ạ³","ĠÑģе годнÑı","E ARCH","å¹´ äºĨ","Ġpre natal","cur l","æĤ² åĵĢ","Ġresemb lance","ĠR if","å±Ĥ éĿ¢çļĦ","ĠAccess ibility","িত à§įর","Download s","Stre et","analy se",") P","Ñĩ нÑĭм","erd ings","Ġà¦Ń ার","Ġì ±ħ","ari amente","ä¸Ģ个 éĹ®é¢ĺ","è§£ èĦ±","Ġtransl ator","î n","Ġw ilt","ä»ĸ å®¶","Ġform ación","è·¯ æĺĵ","Ġinform ations","æĨ İ","æ©¡ çļ®","æĸ° 西åħ°","é£ ½","Ġо знаÑĩа","Ġda erah","çĹĽ å¿«","Ġpet als","æĬµ æĮ¡","MO OCs","广æĴŃ ç͵è§Ĩ","c ong","Ġim itation","Ġnovel ty","ĠÐŁÑĢи ÑģÑĤÑĥп","ĠComb ine","Ġtranqu il","ĠBec ome","å±± ä¸ĭ","Ðł Ðŀ","Ġreact ors","Ġp ly","Ġst rap","ont rol","ef it","arg on","ĠÙĨ س","Ġоб озна","ar ÃŃa","ust o","are mos","æµģ éĢĿ","Ġinf ancy","å¡ ¾","моÑĤ ÑĢ","ĠNeu rology","Ġh ues","Ġan ys","Ġab ide","Ġlif ts","Ġbright ly","ĠAppro ximately","Ġsar Ãł","im oto","ra x","eth oven","é£İ æīĩ","è§īå¾Ĺ å¾Ī","Click Listener","Ġস াম","ĠD OWN","äºĨä¸Ģ ä½į","çĨ ¹","اء Ø©","åĨį æĬĬ","åįĬ æĻĮ","æĨ ¤","Ġfreed oms","b x","æĹ¶ å°±","д ви","çļ® èĨļ","ÃĹ ĊĊ","âĸ ¶","âĸ ĵ","ĠBa um","Ġinstrument ation","Ġperpet ual","ĠP AN","ĠW ien","Ġad ecu","Ġri ot","r ero","Ġrem nants","ĠProt ect","Ġsoc iedade","临åºĬ ä¸Ĭ","ĠاÙĦØ· ÙģÙĦ","Ġp ans","çļĦ åı¤","çļĦ åħĥç´ł","лÑı ÑİÑĤ","Ġgot o","ĠEd itors","ĠDen is","Ġreact ing","ĠKer ry","w omen","ĠT ennis","ä¹ĭ å¤ļ","åĮĸ 管çIJĨ","Ġmark ings","ãĥ« ãģ®","Ġdiscrim inate","åĪ» 度","ĠðŁ Į","ĠÐĿа пÑĢимеÑĢ","Ġbreat hed","g aben","k ary","stit uting","å°½ æĥħ","ĠNot ably","Ġdam s","çŁ¿ ä¸ļ","æĸ°åĨł çĹħæ¯Ĵ","为 å®ľ","Ġdist ract","ç»ıèIJ¥ çļĦ","кÑĥ лÑĮ","åĬłå¤§ 对","æĪIJ å½¢","rap ie","鼶 çĤ¹","é¤IJ æ¡Į","Ass essment","Ġalign ing","èŁ Ĵ","é¢ł è¦Ĩ","Ġpam ph","ick e","ç½® 身","Ġsum ber","ĠCN C","éĥ½ åı¯","ĠRoman ian","æĥ³è±¡ çļĦ","ĠÙĩÙħ ÛĮÙĨ","Ġtroubles hooting","al ach","Ġnot ch","à¸Ń าà¸ģาร","Ġactiv ates","Ġter k","Ġess ent","Ġbrain storm","Ġré pond","ĠDeg rees","Ġà ĵ","çģ« çĪĨ","Ġdivor ced","-go vernment","åħļç»Ħ 书记","' clock","@ {","à Ī","Ġк ÑĢÑĭ","ç¡® åĪĩ","ĠØ´ ÙħاÙĦ","çŁŃ è§Ĩé¢ij","ĠDevelop ments","Ġfur ious","ujÄħ ce","èĦij åŃIJéĩĮ","à±įà° ¤","Ġíĥ ľ","ãģ«éĸ¢ ãģĻãĤĭ","Ġs are","), \\","åıª çŁ¥éģĵ","Ġsol ute","Ġhand ing","空 æ´ŀ","AD O","Ġspl its","Str ateg","Ġviel en","ĠExamin er","M K","N at","[ left","ut ex","ĠB ess","ome z","æĪij们 ä¸įèĥ½","emb ang","vol g","ĠGes und","à¸ŀืà¹īà¸Ļ à¸Ĺีà¹Ī","红楼 梦","g enden","åѦ åłĤ","æĹł äºĭ","Ġnos so","Ġelectron ically","Ġling ering","ĠB row","车 åİ¢","app lic","Ġsom eday","æIJ IJ","rand o","æī¹ 次","åĪĺ éĤ¦","Ġsz ko","اط ÙĤ","Ġpess im","ĠH ess","ä½ł åıĪ","缸 å°į","æ® ´","оп а","ĠList ing","æ¸IJ è¿Ľ","tw itter","ĠRab bit","-function al","Ġl ace","é rt","éĻį è§£","æĬĹ è®®","Ġcontext o","å¾Ģå¾Ģ ä¼ļ","è¿Ļ æĸ¹éĿ¢çļĦ","Ġmod ulated","åħ¬åı¸ åĴĮ","ina ção","ĠHer b","Ġdiss ent","an ça","Ġsw orn","ç£ ĭ","代表 äºĨ","Ġà¦Ĩ à¦Ľà§ĩ","Act ually","Ġcomm end","use ppe","ASS WORD","T re","æĸ Ł","ä¸ī ç±»","ĠпÑĢи ем","éĢIJ 漸","or ch","æľī åĩł","reib en","Crit ical","Y X","ĠExper iences","Ġве ÑģÑĮ","åĨ¶ éĩij","ä½ł ä¸įèĥ½","é»İ æĺİ","ðŁĮŁ ðŁĮŁ","= ['","en ance","çļĦ åĬŁæķĪ","æĺİ æĻº","Ġе дин","AAAA AAAA","åħĥ æ°Ķ","An notation","éĺ¶ æ¢¯","ìĦ¸ ìļĶ","Ġunp ublished",") ](","Ġf idelity","Ġب Ø¥","ĠZ ap","é»Ħ å¸Ŀ","àµįà´ °","Ġmetast ases","Ġpedag ogy","- rank","z io","åħ¥ çĿ¡","她 è¦ģ","Ġsur geries","åıijçĹħ çİĩ","os as","åħŃ å¤§","ĠNe utral","ত ার","ĠMagn us","Second ary","ĠÑģлÑĥÑĩа ÑıÑħ","หม à¸Ķ","Ġn iew","Ġdet achment","çĹħ åı²","Ġpast ure","Ġhes itated","} <","ch r","reg ist","à¸ŀ วà¸ģ","ĠاÙĦج ز",". \\)","ĠC ec","身 躯","ĠLe ib","à¸Ķ ัà¸ĩ","æĢ¥ è¯Ĭ","è§£åĨ³ çļĦ","éĢı æĺİçļĦ","Ġcart ridge","СС Ðł","å±± æŀĹ","bor ah","åıĥ èĢĥ","Ġgerm ination",". Arrays","è¿Ļ å¹ħ","æ° ĵ","åħ¨ å¿ĥ","èĢĥ é¢ĺ","å¦ĩ ç§ij","Ġmig raine","ĠR andy","çĹ ¢","à· Ħ","ĠANS W","ĠBris bane",". ar","© ×Ķ","æ°´ æ³µ","èħ «","æ®ĭ å¿į","end region","Ġlong time","çŁ³ 墨","ĠVal le","Ġmur ders","Ġzn ac","ĠV augh","æĩ ¼","åīª åĪĩ","/ u","é¦Ļ æ°´","èį¯çī© æ²»çĸĹ","in ally","ĠB ates","Ġal iens","Ġpres upp","Ġgra bbing","ĠD ahl","Ġdo ivent","au h","Ġser ait","Con vers","Ġextra vag","Ġdetermin istic","opath ic","is able","ç¤ ¦","ado op",". es","s peed","Ġ icy","ĠF asc","ĠL iam","Ġam plit","Ġel ites","ç»Ļ çļĦ","Ġminim ized","è¡Ľ çĶŁ","v ii","Ġp add","æľī æĿ¡","ÃŃ os","Ġprincip ally","Ġméd ia","Ġconoc er","Ġsummon ed",") C","Ġapp la","Å¡ i","Typ ography","â̦ ..","à¹ģ à¸Ķ","Ġein ige","Ġinform atie","Ġswo je","Ġaten ción","代 è¨Ģ","羣 èıĮ","Ġsl ider","AR DS","Ġlist ings","åĮ»çĸĹ åį«çĶŁ","Ġnumber Of","ĠØ£ Ø«","Ġfing ert","( img","act ors","å¹´ åįİ","ĠMost ly","ాఠ¨","Ġdispar ity","ê ´ij","ĠPro sec","Ùĥ ار","å¾· å°Ķ","Ġpool ed","Ġassign s","αν δÏģικÏĮ","ப à¯į","ä¸Ģ éĹ´","/h r","æĿ¾ å¼Ľ","æļĹ èĩª","æĺİç¡® è§Ħå®ļ","ÃŃt ÄĽ","ĠBerg er","çŃĶåºĶ äºĨ","ĠD ai","ä½ĵ åĴĮ","è¾¾ å°Ķ","çĶŁæ´» åĴĮ","åıįåºĶ çļĦ","å§ij å§ij","éļ» æĺ¯","Ġкла ÑģÑģи","Ġves icles","ĠÑįнеÑĢ Ð³Ð¸Ð¸","éĩį é»ŀ","æĢ¥ äºİ","_p art","Add r","(size of","esz cze","çļĦ æĪIJ绩","ĠH LA","ĠSe crets","ج ÙĬÙĦ","ĠAm ph","âĦĥ ï¼Į","Syn onyms","B rian","æ¯ İ","und ert","å¨ Ħ","Con cept","æĻļ æĬ¥","æģį æĥļ","pt o","ire t","cul as","åIJį æł¡","è¯Ħ åΤ","post a","ĠSem in","ĠCru ise","ĠCoron avirus","ĠDoll ars","Ġremodel ing","ĠEscher ichia","Ġsu icidal","å¹¶ æĬĬ","å²Ľ 屿","Ġdisapp ears","Ġprol ific","ç¼ħ ç͏","m ale","б ок","åĨħ容 åĮħæĭ¬","éĢı äºĨ","K ar","Ġa while","Ġwh ipped","èĩªå·± åĴĮ","ĠAr bor","Ġroz p","ĠвеÑĢ Ñħ","ĠÏĢα Ïģα","Ġus ability","ĠExp ected","Ä Ĥ","é«ĺ ãģĦ","容 è²Į","Ġplant ations","éĤª æģ¶",". â̦ĊĊ","ard ia","ĠY in","de en","æŃ£ æ°Ķ","sl ow","reb bero","f acts","Ġl ied","ä¸ī èĢħ","éª ¸","ä¸ĩ è¾Ĩ","红 åĪ©","à¸Ī ึà¸ĩ","Ġcatast rophe","S leep","Ġk ier","大 åŁİå¸Ĥ","Ġproject ing","_c ost","éļIJ 约","åĬ± å¿Ĺ","à¸Ľà¸£à¸° à¹Ĥย","ĠG rat","ä¹Ł åIJĮæł·","Ġer ro","å¼ķ åĩº","åĢŁ æŃ¤","Ġprincip als","op ausal","å°Ĩ 该","çļ® ä¸ĭ","é±¼ çļĦ","ĠاÙĦب ØŃر","decl are","? \\","ä¸ī 项","æĸ¯ 大","ING TON","ì¶ Ķ","od ied","主 åŃIJ","Ġem anc","æĽ´åĬł çļĦ","ë§ Ŀ","ĠRout es","èģĮèĥ½ éĥ¨éŨ","h k","om ination","pt ides","åĬł å¼·","æ½ľ èĥ½","æī« çłģ","ĠHE ALTH","Ġp ituitary","ĠB ax","à¸Ĺั à¹Īว","ĠG li","Ġme z","ä½ł å·²ç»ı","è¿ĺ 说","离 线","Ġconc ave","éĽª å±±","ĠÑĤе ÑĢа","Ġ×¢ ×","ĠV ER","两 ä¼ļ","Ġج ا","ĠExec ution","çĹĽèĭ¦ çļĦ","çĭłçĭł çļĦ","g ov","Ġside walk","Ġtax onomy","ĠDer by","Ġcon osc","ï¼ģ ï¼ģĊĊ","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ","el ic","åľ° åĴĮ","æĶ¾ çļĦ","Ġrev ital","ĠнаÑĩа ла","éħµ æ¯į","ĠP U","çİĭ å®¶","ÏĮ ÏĦη","ãĤı ãĤĮãĤĭ","åĢ Ĩ","æīĵ ä»Ĺ","Ġcult ivating","è³ĩ æł¼","ĠOB JECT","Ġl umber","ĠE sk","met ics","uest as","æ½ľ ä¼ı","Ġgoss ip","ĠW izard","Ġimpact ful","åĨ· æ±Ĺ","â ng","ÐŁ ÑĢе","ĠBusiness es","ĠSens ing","he ts","Ġre ins","Ġen vy","æ¸ħ é¦Ļ","OR N","Ġbusiness man","à¯ģà® Ł",". ui","çļĦ ä¿ĿæĬ¤","ob ra","ج اÙĩ","Ar c","Ġмож е","اØŃ Ø«","Ġbuil dup","and ung","pl ays","Ġsh uff","Ġв оÑĤ","itt al","èĨ ł","åģľ é¡¿","ĠÑĤа ким","w x","Ľ ×Ķ","ä¸Ģ æīĢ","к оле","che in","æĥ³ èµ·æĿ¥","ĠØ® طر","æĭĸ æĭī","ĠÑģлÑĥ ж","Ġmater i","ĠìĻ Ħ","\" =>","ĠF X","ä½İ ä»·","type of","è¶ĬæĿ¥è¶Ĭ 大","ãĤ³ ãĥ³ãĥ","製 ä½ľ","ĠÐŁÑĢиÑģÑĤÑĥп ÑĻено","_ format","f et","çļĦ 她","ÑĪ Ð»Ð¸","æľĽ åIJij","纹 çIJĨ","\\ User","Ġд огов","Ġanim ations","Ġfunctional ities","I i","æĿ¥ 人","ĠCh r","ĠSh ane","éĸ Ĵ","={ (","-A ss","Ġfont s","- ra","C K","] ãĢĤĊĊ","çĶŁ åĩº","ÙĪØ± Ø´","Ġachie vable","å±Ĭ æĹ¶","o of","èĥ½ ç͍","è¡Į ä¹ĭ","we e","æį® ç»Łè®¡","Ġع ÙĦÛĮ","por ate","Ġens l","æĺ¯ åIJ§","æĺ¯ åįģåĪĨ","å½ İ","Ġcond ens","ĠÙĤ اÙĨÙĪÙĨ","ederb örd","S and","] ][","st elling","ä¸İ ä¼ģä¸ļ","Ġо казÑĭва","åĿļ 飧","Ġseg reg","å²Ľ ä¸Ĭ","éĮ¯ 誤","Ġpartic iple","à´ ª","r ö","Ġob last","ØŃ ÙĬØ©","á» ķ","ĠпÑĢед ÑģÑĤавлÑıеÑĤ","Alex ander","ĠN orge","æīĵ 磨","ĠLand es","Ġne v","ĠO PT","-s erver","uff ix","En joy","ä¸Ŀ 毫ä¸į","åįģäºĮ 竳","-W est","æ¡Ĥ èĬ±",": ',","b j","Ġcom ún","æĸ° ä¸Ģè½®","ĠCom pletion","ey n","Ġਠ¹","' +","ĠA SE","ĠL ut","Ġarr anging","èģĶç³» çļĦ","Ġ׾×Ķ ×Ļ×ķת","Ġpars ley","Ġsten osis","_ amount","æķĻ ä½ł","è§ī æĤŁ","ĠÐľ Ñĭ","é¦Ĵ 头","us ic","è¿Ļ个 åIJįåŃĹ","éĺ¿ æł¹","Ġih nen","Ġতার িà¦ĸ","an nt","æĺ¯ ä»Ģ麼","ж но","Ġ×ij× ©","à¹Ģà¸Ľ ิà¸Ķ","âĹı ĊĊ","[ max","ĠB ali","å¹´ åĴĮ","èĢģ çΏ","Ġmet iculously","Ġgre ase","ĠSc ales","äºĭæĥħ çļĦ","ĠÑģоÑģÑĤоÑı ние","ĠN orte","ĉĉ ĠĠĠ","ĠÙģ ÛĮ","æij© å°Ķ","Ġguard ians","/ go","/ Comment","Y e","ig ate","åı¯èĥ½ 导èĩ´","Ġles bian","åĵĪ ä½Ľ","Ġcritic isms","çī¢è®° 使åij½","ĠÙħرد Ùħ","t ails","Ġt udo","ĠM uss","ĠIn hib","æĿ¡ çļĦè§Ħå®ļ","Ġد ÙĦÛĮÙĦ","ĠDr um","ĠScript ures","çĹī æĮĽ","ĠC rop","åŃ °","å°ij å¹´çļĦ","িঠŃ","IS PR",".P oint","Ġpod rÃŃa","å¼± çĤ¹","Ġли ÑĨ","Ġpl anners","Ġput ative","api ro","cip itation","Ġk de","ula ire","áŀ ĵ","W ORK","{ [","Ġi ç","åĿ Ĥ","Ġস à§ĩ","ĠØ® ÙĦ","UG H","Ġhes itation","L java","è¦ģ èµ°","Ġra k","Ġgr âce","à¤ķ à¥ĭ","ĠФ и","Ö¸ Ö¼","ĠExpress ions","ĠÐŀÑģ нов","at itis","ĠG ad","å¤Ħ éķ¿","请 æĤ¨","ĠPres ence","éĢŁåº¦ å¿«","Ġpolic ing","Ign ore","转è¿ĩ 身","é¡ «","Ġind ifference","éķ· æľŁ","å®£ä¼ł æķĻèĤ²","f ass","ĠF iscal","Ġhe ra","ĠN iem","ä¼ļ æĽ´","ĠZ ahl","è¾ĵ äºĨ","缮åīį 为æŃ¢","çķ¶ åĪĿ","Ġин ÑģÑĤиÑĤÑĥ","Ġíļ ¨","Ġc rap","ĠUn ve","æŀ¶ ä¸Ĭ","ĠObs erver","Ġnot withstanding","ĠIn i","á ticos","åĬ¡ å·¥","ator ia","ĠWill is","Ġasym metry","l ord","æľī éĴĪ对","Ġprint ers","sh ots","ĠRES P","Ġj ov","é¢ ĵ","Ġz de","Ġfl ashing","主é¢ĺ æķĻèĤ²","p ak","èĩª ç«ĭ","äºĶ 彩","J R","ud ing","ä½ł éĥ½","åĨĻ æ³ķ","Ant i","Ġresent ment","ud der","Õ ·","el im","Ġ ¥","uk aan","Æ¡ n","Ġanten nas","×ķ×¤× Ł","ĠFerr ari","åĪĩ å¼Ģ","ĠRob otics","Ġtheor ists","Ġseek ers","Ġtask ed","æīŃ å¤´","Ġmonument al","ĠH ole","æĪij 被","åĪĨ æµģ","æµ· ä¸Ń","ĠCS V","Menu Item","f requency","sp ects","ĠAr row","Ġpas o","inf ection","Profess ional","Ġg dzie","ow att","res ist","ãĥļ ãĥ¼ãĤ¸","y et","te ger","Ġins omnia","Ġpor osity","å®ģ æĦ¿","Ġ×ij ×Ļת","-bl ack","Ġtrait ement","Bet ter","为 ä¸Ģä½ĵ","ç§ ½","ip art","Ġab uses","çī¹ åĮº","Ġple asures","æĸ° æĿIJæĸĻ","çϽ çĻľ","aut re","éd ias","ĠC ly","ä¸ĭ åĽ¾","æ¸ ¾","ä¿¡ èµĸ","Ġpsych osocial","ĠM obi","ç¥ Ĥ","=\" \">Ċ","ĠPro ve","åĸ ª","åij½ åIJį为","éħį ä¸Ĭ","arb ij","à¹Ģล à¹ĩà¸ģ","Cle an","Applic ations","A gg","Ġt rough","ĠN un","å°± åľ°","Ġpres erves","Ġindividual ized","à«įઠ°","ĠRevel ation","xt ap","ĠY uk","çĤ¹ åΰ","Ġimport ancia","Ġstat i","讲 å¸Ī","设置 äºĨ","ĠLabor atories","U U","che my","×Ļ× £","ĠLe u","积 æ°´","ĠسÛĮ ستÙħ","Ġscu ola","æĺ¯ ä½ķ","র ি","Ġpat io","åķĨ æĪ·","á» ı","ĠGu ides","ĠRem oval","ä¾į åį«","Õ ©","æľĢ 主è¦ģ","ĠCon v","Ph ilipp","æĢĴ åIJ¼","mus ic","åĴĮ æĶ¿æ²»","Ġresp uesta","Ġimp ending","è¶Ĭ å°ı","oph obia","Ġмноги Ñħ",". As","ent lich","åĽ½ æĹĹ","è ces","详 è§ģ","æĺ¯ä¸į åı¯èĥ½","åħ±äº§ åħļçļĦ","Ġtwe ets","capt ion","Ġs Äĥ","ĠN ä","èĩ Ł","å°ı ç¨ĭåºı","æİĴ æ³Ħ","æĥĬ åı¹","ĠA be","èĩª å¦Ĥ","Ġair flow","ĠMac beth","åł¡ åŀĴ","Ġg aseous","ĠY ong","ä¸ĢçĤ¹ éĥ½ä¸į","ĠÄij ó","big g","Ġmobil ization","Ġíĥ Ģ","ousse au","ä¹łè¿ijå¹³æĸ°æĹ¶ä»£ä¸ŃåĽ½çī¹èī²ç¤¾ä¼ļ主ä¹ī æĢĿæĥ³","Turk ish","大 å®Ĺ","ä¸ĵ项 æķ´æ²»","à²Ĥ ದ","Ġla quelle","Ġorder ly","then ing","Ġprobl ème","ĠS ell","ĠW oj","ĠAn c","åĽĽ ä½į","ç®Ĺ åĩº","æij ¹","æĭ· è´Ŀ","Ġcon cret","çĪ »","æŀģ åĬĽ","Ïģ οÏħ","Ġà¦ħন à§ģ","ĠProte ins","E u","ĠA o","ç» ¯","ĠÑģ еÑĤ","bt ed","ĠÐĺ Ñģп","ĠÙĦÙĦ ت","ãĥ»ãĥ» ãĥ»","à¹ģà¸Ļ ว","integ ration","Ġher m","èħ ĭ","æĭī åĬ¨","ðŁ ļ","oll s","Ġget All","æĬ¥ åΰ","ĠX en","éĺ² èħIJ","Ġé lect","Cont rib","è³ º","åIJī å°Ķ","åŁºç¡Ģ设æĸ½ 建设","ĠÑģкоÑĢо ÑģÑĤÑĮ","Ġn ossa","Ġpro pre","ec er","CP I","ulière ment","comm ittee","Ġcamp uses","ĠpÅĻÃŃ pad","\" Oh","ν Ïī","ĠÐĵ е","ĠакÑĤив но","ĠLanc aster","-work ers","j ana","çļĦ æľĢé«ĺ","л ка","Ġ×ľ× ©",".de gree","åĨį ä¹Łæ²¡æľī","â nd","ĠÑģÑĤа ÑĤиÑģÑĤи","Ġdrive way","诧 å¼Ĥ","ón ica","åįģäºĮ æľĪ","ĠÙħص ر","Ġpe qu","æĹł åģ¿","ÄĽ st","unction al","user Id","det ail","Ġparas itic","ĠWolf gang","Ġп окÑĥ","ĠFl ora","Ľ× ĸ","W G","äº Ł","Ġor a","ä¹Łæĺ¯ å¾Ī",". 'Ċ","Ġn ég","leg t","Ġ×ľ× ª","å¥ĩ å¦Ļ","ĠGood man","ow ler","å¹³ ç§»","æİĪ æ¥Ń","è´¢åĬ¡ 管çIJĨ","Ø·ÙĦ ÙĤ","ĠBiomed ical","ĠAzerba ijan","N ic","è¿Ļ åĩłå¹´","cl ic","ж ноÑģÑĤи","伤 å¯Ĵ","æĦŁè§ī èĩªå·±","Äĥ ng","çĶŁ çĮª","Ġsp re","é¢ĺ 为","èIJ½ åħ¥","ĠоÑĢи ги","ĠM UST","ĠG ou","ener ated","ST ER","Ġspecial izes","_f irst","æ»ij éĽª","ucc i","m ine","Ġw ol","ad ay","Ġhand book","大å¤ļæķ° 人","ĠBol ivia","çļĦ åIJ§","ĠT WO","æĪij æľĥ","æĹł 常","ãģı ãĤĭ","ĠUse ful","Õ¥Õ ´","Ġsyst olic","ë ĥ","Ġ Æ","ig rant","åĽŀ å®¶çļĦ","Ġsim plement","à¦ķ ল","ä½Ľ å±±","ĠMat th","æ£Ģå¯Ł æľºåħ³","ĠاطÙĦ اعات","_ th","Ġc iel","Ġn ama","æĪij å¿ĥ","az es","çĭ Ļ","è¿ľ äºĨ","ĠPol ym","Data Source","ĠÙ¾ رد","Ġ×Ĺ ×ĵ","ĠB ST","Ġj eder","å¸ĥ æĭī","çļĦ åİ»","com position","èĭ ŀ","ãĢĭ ï¼ļâĢľ","t g","èĢģ 天","ĠValue Error","Ġcuk up","Ġre el","un ken","ĠK ah","管çIJĨ å±Ĥ","ĠÐŁ ÑĢ","Ġcual es","éĺŁåijĺ 们","Ġa plik","iv ol","åĶ ł","åī¯ éĥ¨éķ¿","ู à¸Ļ","ĠHam mer",": ]","Ġsu nd","çŁ¥ è§ī","ä¸ĩ ä¸ĩ","æķħ 宫","ÑģÑĤи ÑĤÑĮ","Ġ×ľ× ª×","ĠاÙĦت ÙĤ","åĮ¿ åIJį","Tex as","T X","Ġp ů","اÙĦ ÙĤ","çŁŃ 线","ĠباÙĦ Ø¥","itate a","M aria","çļĦ è¯Ħä»·","em t","æĪij 好","Ġmy c","Ġب Ùħا","Ġfun nel","åĻ ľ","éĿŀ éģĹ","åįĥ åı¤","ĠAl ready","å·¥ç¨ĭ åѦéĻ¢","åī¯ å¸Ĥéķ¿","ĠÙĪØ§ÙĦ ÙĨ","èµŀ æī¬","ĠÑģ ло","att ie","Ġdesign ate","å¯Ĩ éĴ¥","èϽçĦ¶ åľ¨","ç§ijæĬĢ æĪIJæŀľ","Ġalt ura","འ¢","Ġcer amics","Ob viously","i ÅĤ","Ġ ðĿĴ","è® ļ","ĠÑģи лÑĭ","ĠÑįлем енÑĤа","Ġпо и","Ġprec ursors","gl ise","ĠSur f","udd le","人为 æľ¬","Ġt ion","ĠL AB","land ers","çľ¼ è§Ĵ","uck ing",".h ash","Ġש ׾×IJ","ÑĤÑĥ ÑĢÑĥ","æĬ¥åijĬ ä¸Ń","ÑĤив нÑĭÑħ","ниÑĨи па","í ĶĮ","æĿĥ åĬĽçļĦ","Ult imately","ç§ijåѦåıijå±ķ è§Ĥ","Ġ Äĩ","Ġde ity","ÙĪ ÙĬÙĥ","Ġhack ers","ĠÑĢаÑģÑĤ ениÑı","æĪij ä¸İ","对 è§Ĵ","Ġsub urbs","Ġج سÙħ","æĮĩ导 æĢĿæĥ³","Ġpolar ized","Ġض د","ĠNatur ally","åĮ»åĬ¡ 人åijĺ","ÑĤ ого","主 页","åĽº æľī","âĸ ij","Ġay uda","les ia","åıijå¸ĥ æĹ¥æľŁ","ĠIh re","fight ers","_ api","ĠD ON",".S ervices","Chem ical","ĠF ot","Ġinter ruption","ки н","Works heets","mem bers","Ġcon es","Ġا ثر","åĪĨ éļĶ","ла кÑĤи","Ġillust rative","Ġquot id","åıijæĶ¹ å§Ķ","z p","izz iness","Ġprzy k","j ut","ĠD rain","Ġnot a","ĠSt ick","Ġ ¬","Ch ief","Ġinde bted","ĠÐĺ ÑģÑĤо","M H","d aughter","æ¿ ķ","ĠС ШÐIJ","ีย à¸ļ","ç»ķ ç»Ħ","Ġultr asonic","é nt","ÙĪ ØŃ","ĠLand s","Ġbench marks","' inter","ik ai","ew s","ĠAf rika","èĤī çľ¼","Ġpin point","Never theless","K as","ĠC ao","Ġwh ichever","pt ive","Ġsp ac","Ġsim ulator","ĠDe borah","Ġbest imm","åľĨ å¿ĥ","ĠEth n","ĠобоÑĢ Ñĥд","åĽ½å®¶ æłĩåĩĨ","ĠStr ange","öl ker","è¾½å®ģ çľģ","æĸ°åįİ ç¤¾",".tw itter",". exp","l ittle","Ġb aj","ĠB alk","Ġdi ber","Ġsix teenth","> ()","ÃŃ culos","ĠÙħ ÙĦÙĬ","AR P","é»Ħ èī²çļĦ","ĠLI KE","Ġসাল à§ĩ","ĠZ am","åĨį å°Ĩ","æ¿ Ĵ","ĠÚ¯ ÛĮ","ĠVis itors","ĠEgypt ians","Ġsvilupp o","é«ĺ æ¡£","Ġmarket ers","Ġconduct s","ĠпÑĢоизвод ÑģÑĤва","ĠмеÑĢ Ð¾Ð¿ÑĢиÑı","åΰ ä½ł","ĠCh ung","å®ŀ å¤Ħ","Ġdisc ord","tr zym","é»ĺé»ĺ åľ°","rv ats","ĠPret ty","w agen","è¿ĺ ä¸įèĥ½","åħĪ åİ»","Ġна де","Ġdep iction","转 è´¦","ĠMan uscript","Act ivities","ĠSom mer","Ġpalab ras","ĠCOUR T","C ette","ĠB erm","ĠD ru","æ²¹ èıľ","bet ter","Ġcom eback","ĠK ick","交 ç»ĩ","éĽĨ ä¸ŃçļĦ","Ġexec utes","Ġimpair ments","Ġveg gies","again st","ẳ ng","åIJĮ çIJĨ","ied ades","åĽŀ é¦ĸ","Ġconst ipation","Ġmon ol","ĠWilliam son","ãģ§ãģĻ ãģŃ","ä½ĵçݰ åĩº","ãģķãĤī ãģ«","il or","ĠTh in","åħī äºĨ","Ġhom ogen","ĠBrit t","çļĦç¥ŀ æĥħ","ç®Ģ缴 å°±æĺ¯","Ġb ids","ĠW itch","ĠU CLA","Ġbud dy","áĥĺáĥ ľ","ĠDream s","æĭĽåķĨ å¼ķèµĦ","C ulture","Ġ ****************************************************************","com put","éħį ç͵","ĠJun i","Ġdoctr ines","Ġde hydrogen","av at","éĥ½ æ²Ĵ","(\" [","æĶ¶ äºĨ","au coma","_D ATA","ĠLuther an","ĠNiet zsche","- aff","Ġcont ours","Ġcre ar","áĥIJáĥ ł","Ġstere o","ÙĤÙĬ ÙĤØ©","ĠK rak","Ġhab er","æĢĢ æĬ±","mo oth","Eng land","×Ļ׾ ×Ļ×Ŀ","åĴĨ åĵ®",". Key","çļĦ 温度","ĠD IV","LO AT","ĠlÃŃ nea","et ra","æĺ¯ ä»ĸ们","ĠO v","ä¸Ĭ å²Ĺ","ĠIn structor","åĽ¾ çĶ»","à¸Ĭà¸Ļ ิà¸Ķ","ĠгÑĢаж дан","è¿ĩ éĩı","å¿ĥ äºĨ","ĠBe h","play ers","Ġmais on","ë§ IJ","an ch","ĠE igen","Ġtra der","Ġб ол","éĻª çĿĢ","Ġn ave","ra um","ä¹Łæĺ¯ åľ¨","Res olver","ĠCur ve","éĿ¢ç§¯ 为","éĥ½ä¼ļ æľī","ìŀIJ ìĿĺ","ிà®ķ à¯įà®ķ","( it","ĠW erk","ign ement","å¿ĥ 室","æĥ³ ä¸Ģæĥ³","/s ub","Ġcal ming","æľĢåIJİ ä¸Ģ次","åĺ´ ä¸Ĭ","TP L","Ġbibli ography","ĠHerm ann","ãĤĦãģĻ ãģĦ","Ġp ä","çļĦ æİªæĸ½","缸 è¾ĥ","Âł ĠÂł","è¯Ń å¢ĥ","work ers","ĠDo ctors","Ġutil ise","Ġদ িন","èĬĻ èĵī",".sw ift","éĤ£ èά","Ġchar s","èĮ §","да Ñĩа","ĠÐĴ оз","HS V","Ġжи дко","ĠMah arashtra","ĠÑĦи лÑĮ","l á","Ġun affected","åı¯ 为","çī© ä»¶","åıª è§īå¾Ĺ","ĠGra b","åĨ° åĨ°","ĠTre vor","Ġsoy bean","_ ;Ċ","f ielder","ĠB IG","ä½į å±ħ","æľĿ èijĹ","æ²ī éĻį","Ġtack les","Ġper missible","å¦Ĥæŀľ ä»ĸ","-h ow","Ġми ÑĢ","æĪij çŃī","对 åĩĨ","de ad","æ¸ħ æī«","ĠMac ro","ĠGold man","èµĮ åįļ","ĠPain ting","Ġadorn ed","M oving","h og","çļĦ çĹĩçĬ¶","Ġpr udent","ĠSus p","å§¥ å§¥","以ä¸ĭåĩłä¸ª æĸ¹éĿ¢","Ġted ious","ĠT rop","ä¸Ģ è´¯","if ie","в ла","Ġrel oad","ĠJer emiah","G as","ĠB J","Ġstr ides","ãĢĤãĢĤ ãĢĤãĢĤ","ĠDick ens","以 å¾ħ","Ġam using","Ġser ene","æŃ¤æ¬¡ æ´»åĬ¨","F N","ĠM EN","uk un","ĠMar athon","ç§ģ ä¸ĭ","Ġlang ue","zÄħ t","p ell","ĠE arn","èĢĮ å¾Ĺ","ва ний","客 æłĪ","Ġburn out","Ġju ices","èĪŀåı° ä¸Ĭ","оÑĢ Ñĥ","Ġcompet ed","èīºæľ¯ åĵģ","çģŃ äº¡","(L ong","- mentioned","Ġa com","Ġcont ests","Ġcar ga","uit able","sim ilar","纲 é¢Ĩ","丫 鬣","Ġdere cho","I z","am ino","Ġfil ming","Ġpen insula","ĠVict ory","( app","ons on","Ġwid ened","ĠInvest ing","à¸ģ วà¹Īาà¸","æ¡Ī åŃIJ","sk ich","æ§ĭ æĪIJ","Ġì¹ ´","Ġquar antine","Ġth rott","ul kan","Ġill icit","={ `","ĠST D","าย ุ","驱 éĢIJ","Ġoverlook ing","hid upan","Q B","p ang","æ¸ħ åģ¿","åıijå±ķ è¶ĭåĬ¿","ĠPer cy","ç´§ åĩij","éĿ¢å¯¹ éĿ¢","ĠSens ors","( |",") ==","å½ĵ äºĨ","便 æ°ij","åľŁ æľ¨",".p age","èĿ ł","- ever","A qu","ult z","-M ar","itar ia","æĻºèĥ½ æīĭæľº","ĠObs ervation","Ġни м","Ġexplo iting","Ġbureauc racy","C ole","x sl","大 åĶIJ","è¿Ļ åĽŀ","Ġattach ments","# {",": layout","Ġg cd","Ġwh ist","ĠCl aus","Ġbre wing","IJ× ª","( {\\","ĠG ore","ठı","Ġí Ĩł","Ġvo iced","çij ļ","纸 å¼ł","Ġoste oporosis","ĠR ak","æ·± æĢĿ","æĹ© æľŁçļĦ","ĠвÑĭ боÑĢ","追 éļı","糯 ç±³","M utable","ĠÑģ ÑĢок","Ġsub types","ĠCon ven","çĦ¡ æķ¸","-a uthor","ĠAB OUT","D EF","i ram","t gn","ĠÑĢа за","Ñģа д","éĺ¿éĩĮ å·´å·´",": H","ch rom","äºĨä¸Ģ éģĵ","æĺ¯ä¸Ģ 種","ĠÑį ÑĤÑĥ","ç§ĭ åĨ¬","= false","Ġc DNA","ĠM add","ä¸Ĭ æīĭ","éĿ¢ ä¸ĬçļĦ","hed en","ĠPUR POSE","Ġc ie","и й","åĹ ·","Ġsp indle","}ĊĊ ĊĊ","pon ential","Ġge ared","Ġmagn ets","åİĤ éķ¿","æ±ł å¡ĺ","Ġcardi omy","Ġvamp ire","ĠC rew","ur z","为äºĨ éģ¿åħį","hus us","åĤ¬ ä¿ĥ","åıĹ害 èĢħ","- ret","\\ .ĊĊ","ä¼ł éĹ»","isc opal","о Ñı","ä¸į èĩ³äºİ","Ġв клÑİ","Ġpol yp","uls ions","åľ¨è¿Ļ æĸ¹éĿ¢","ĠоÑĢгани зма","Ġzd rav","Ġense ñ","ä¸ ¡","res ources","æľī å¾ħ","æ¯Ķ ä½ł","äºĨä¸Ģ èµ·","é¦ĸ 缸",". Type","æ¸ ¤","ĠÏĢ Î»Î·Î¸","Ġconnect ors","gra ce","Ġm k","çļĦ 模å¼ı","Ġqu atro","ry an","×Ļ ×Ļ×Ķ","åĬŁ åĬ³","unn ers","诸 ä½į","Ġisot opes","ĠT omas","os ide","ap ar","ä¸ŃåĽ½ ç»ıæµİ","Ġdé part","Ġmid point","- vers","Ġd ó","Ġre yn","è° į","çī¹ åľ°","ĠBy ron","åī¯ æķĻæİĪ","ĠCle aning","[ string","Ġk ins","åĬ¨ èį¡","лÑĮ нÑĥÑİ","ĠAb el","å¦Ī çļĦ","iat iva","Des ktop","Ġdiss ociation","ĠMur der","Ġannounce ments","ãģ¹ ãģį","åIJīæŀĹ çľģ","çļĦ éĻIJåζ","Ġre plicated","Pol ish","Ġacry lic","å·² æľīçļĦ","æĽ´å¤ļ çļĦ人","رÙĬ ÙĦ","SU M","imm ers","ĠнеÑģк олÑĮ","ĠT akes","ĠV y","ĠÙĪ Ùħا","ĠDe z","çªģ å¦Ĥåħ¶","çħ¤ æ°Ķ","Ġfruit ful","iar ism","C zech","N ear","ÙĨ Ú¯ÛĮ","à´ ²","Ġrespect able","_ default","Ġc uring","н оп","å½ Ĩ","п ÑĢа","Ġaus ge","Ġa venue","ĠS é","Ġloc ating","失 æķĹ","åį° ç«ł","ĠY ing","ĠBl ut","ĠComp ounds","Ġalb umin","ĠVari ation","Ġدار اÛĮ","ĠEmploy er","Ġhomeless ness","å½¢åĬ¿ ä¸ĭ","Ġпо Ñħ","' ):Ċ","ĠM üller","ä¸Ń æŃ¢","被 认为æĺ¯","éĿŀ 线æĢ§","ĠCol leges","Ġhab il","áz ÃŃ","re ira","al ie","Ġl odge","ĠI Enumerable","Se ven","èµŀ æĪIJ","ç͵è§Ĩ æľº","ĠEvalu ating","轻轻 çļĦ","äch st","ĠBew eg","éľĢè¦ģ注æĦı çļĦæĺ¯","Ġstagger ing","ãĢ Ĺ","å½¢ ä½ĵ","æºIJ çļĦ","ai ra","pan ies","-P CR","Ġreb uilding","CN N","ĠD enn","å®¶ ä¼ģä¸ļ","åħį å¾Ĺ","è¨Ń ç½®","Ġscrut in","Ġ×IJ×ķת ×ķ","ĠÙħÙĨØ· ÙĤÙĩ","ĠMorm on","Ġsu f","ä¸Ń æĸ¹","Ġint ram","åºĶ å°Ĩ","Ġë ¸","è·¯ åĨĽ","Ġplan o","Ġpeel ed","r án","Ġm oc","Ġh ir","ĠL ug","ĠG ri","Ġsa usage","Ġest ates","æĴ ²","math scr","ä¸ĢçĤ¹ ä¹Łä¸į","ĠΤ ο","Ġl än","åľ° ä¸ĬçļĦ","å°±æĺ¯ è¿Ļ个","éł ĥ","çļĦæĥħ æĻ¯","ĠIngl ês","ong an","æī¿ æİ¥","ä¹İ ä¹İ","Ġhor r","實 é©Ĺ","El izabeth","ĠUN IVERS","Ġanalys ing","Ġilleg ally","} else","Ġb inder","éĥ½ åºĶ该","åħ¶ 为","æĹ¥ æ´»åĬ¨","Ġgre p","ENC Y","หว ัà¸Ķ","Ġlingu istics","åĩĿèģļ åĬĽ","Ł ģ","Ġt á","Ġt rophy","il and","ä½ Ł","å§ Ŀ","åĥµ 硬","顽 强","vel ocity","Ġг ÑĢи","c ube","æľī ä½ł","å¤ļ 大çļĦ","head ed","ĠBlock chain","ĠпеÑĢв Ñĭй","Ġc og","ight ed","we it","Ġâ ĩ","亲 身","Ġsuper hero","åģľ æ»ŀ","ĠØ® ر","ju ven","ĠNord ic","åĭĺ å¯Ł","G it","æ³ ¸","对 åŃ©åŃIJ","å¼Ģ å±Ģ","ج ÙĨ","è¦ģæ±Ĥ åĴĮ","Ġgro ÃŁe","Ġenzym atic","ç·¨ 輯","èı© æıIJ","ĠPar am","Ġiter ate","Ġmurm ured","F ish","l k","ĠPa olo","ãĤ ¼","ਠ¦","Ġinspir ational","ä¹Ĵä¹ĵ çIJĥ","ĠIn cluding","ĠRes idential","ĠAut hent","ÃŃ da","Ġsub merged","ÏĦ Ïī","åĬŀ çļĦ","ем ой","CL UD","o ze","ch urch","Ġha unted","ãģij ãģŁ","å¦ĸ åħ½","ifer ous","ĠKy oto","ĠczÅĤ owie","Ġch iam","ind ung","åħĥ å¸ħ","ĠLe one","Re ceive","çµ µ","Ġbar red","mm mm","åΏ åķĨ","Sch olar","R ose","iv ert","Ġemerg ent","áĥĶáĥ ľ","åľ¨ å½ĵæĹ¶","ap r","sub a","ä¼° è¨Ī","ĠW rest","Ġac ronym","Ġbo ast","ilit ating","ëł ĩ","ãĤ¯ ãĥª","Ġyouth ful","S ym","u ž","头 æĿ¡","Ġت Ú©","ze pt","-p resent","-a fter","Ġdar auf","Mult iply","+ s","M X","ĠS iem","Ġj eszcze","éĥ½ ç͍","âĢĶ âĢĿĊĊ","ĠCom un","unt za","t in","çª ®","èĬ± èįī","éĢĻ æīį","è¸ Ĭ","phant om","ĠInvest ments","ĠاÙĦÙģ ÙĦÙĥ",". age","ä¹Ł å°±ä¸į","çĿ ¾","Ġfl are","Ġest amos","æİĴ 污","à¥įठ¸","_ items","Ġsc op","Ġaut our","æĭħ è´Ł","Ġপà§įর থম","Organ ization","á»± c","( query","Ì Ĥ","åĮ ®","èª ķ",".d to","ĠOb esity","ĠHum idity","ĠConcept ual","s ent","Ġp iss","社ä¼ļ ä¸Ń","æĥ¯ ä¾ĭ","çļĦæĹ¶éĹ´ åĨħ","Ġwy korzyst","Ġbij voorbeeld","Ġconting ency","T rend","oc ortic","ub ahan","Ġres olver","ob ox","缸 æ¯Ķè¾ĥ","Õ¡Õ ·","Ġeffort lessly","à§ĭ à¦ľà¦¨","Ġliv ro","ĠC YP","ne al","Ġra ced","æĤ ħ","åį° å°¼","Ġthin ner","bed a","éļ¨ å¾Į","ĠV L","éĥ½ æ¯Ķè¾ĥ","Ġfl ashed","æ¯ı ç§į","Ġens ino","Ùİ ÙĪ","Ġtrust ees","Ġinterfer ing","Ġobt ener","ĠG arn","éĿĴ äºij","enc ers","ä¸įæĸŃ åıijå±ķ","ĠM ali","ĠD ress","ĠF alk","æĥ ®","åıĮ èħ¿","Ġtour ing","Ġкол лек","Æ°á»Ľ c","= /","å°¼ åħĭ","ĠвÑģÑĤÑĢе Ñĩа","ä¸į å°ıçļĦ","Ġun biased","åĩº çı¾åľ¨","TR Y","ãģ«ãģª ãģ£ãģ¦","Ġfare well","èĦijæµ· éĩĮ","ĠS HE","主 æĿ¿","Ġem pat","æľĢ çα","Ġ\\(\\ {","ĠEm manuel","p our","is ierung","çļĦ è´¹ç͍","et ings","Ġr uth","sh aw",".D ef","ĠÑģÑĤа ли","ück en","_ op","as in","га л","Ġprop ensity","Ġow l","人 éģĵ","åѦçĶŁ åŃ¦ä¹ł","Ñīа еÑĤ","注åĨĮ ä¼ļ计å¸Ī","èĬ³ é¦Ļ","Åĵ ur","lak ang","Ġamyl oid","èİ«åIJįåħ¶ å¦Ļ","v all","ĠL ópez","cl ub","amp al","ÑĤи на","ogen es","ĠRed e","exec ute","ĠÙĨسب ت","S r","j av","ä¹ĭ é£İ","éĿ¢ 容","Ġdef lection","не ÑĢа",":h over","ĠTeh ran","éĤ ¸","-A meric","åł± å°İ","Ġjs em","ve k","为 人æ°ij","èĩª 带","Ġreg roup","Ġд ок","æį¢ ç®Ĺ","ç®Ģåįķ åľ°","æŃ£ç¡® åľ°","ĠÄij ưá»Ŀng","çłĤ æµĨ","opath ology","g uez","è¿Ľè¡Į æ£ĢæŁ¥","oir s","éĽĩ 主","de b","ç͵ åİĤ","-S tep","Ġd ubbed","ank ind","åĩĨ æĹ¶","ĠUS C","ĠIN R","-S aharan","åºĶç͍ çļĦ","å°±ä¼ļ 被","æ©Ł 械","èĺ ¸","Ġd ues","Ġen rol","ä½ł 羣çļĦ","å®¶ åħ¬åı¸","äºij 计ç®Ĺ","æı¡ æīĭ","Ġвой нÑĭ","Ġpar an","Ġest rat","osc ale","ĠFra u","ĠB ien","Ġcur ry","Ġchar ities","Ġس اخت","ĠNot tingham","-inf ected","è¾ľ è´Ł","å¤ļ ä½į","Ġent ender",".Are Equal","ĠC afe","ĠRe ceived","社ä¼ļ 责任","åĽ½ æĥħ","ä¹ĭ çİĭ","ix in","son ian","çĶļèĩ³ è¿ĺ","éŃĶ çİĭ","प à¥įर","ÑİÑīиÑħ ÑģÑı","- volume","ĠW irtschaft","åĨħ èĦı","Co ord","ĠKil ogram","ĠjÄĻ zy","< m","Ġc ron","æĩ ¸","ĠArch bishop","æĤĦ çĦ¶","åı¯ä»¥ åĪ©ç͍","Ġsl ag","Ġsequ entially","-fund ed","ĠM its","ç»ıæµİ åѦ家","à¸ŀ าะ","ĠLo ans","E Q","ಠŁ","ĠCons ortium","éĺ¶æ®µ æĢ§","Ġì¤ij ìļĶ","ĠE vel","åĽ½ ç±į","part icip","ç³»åĪĹ æ´»åĬ¨","åijµ æĬ¤","% ).ĊĊ","F emale","Ġa vere","ug i","iqu ette","íķĺ ë©°","اخ تÙĩ","×ķ×¨× ļ","ĠW ick","çŃī ä¿¡æģ¯","Ġhigh ways","Ġaspect os","å·¥ä¸ļ ä¼ģä¸ļ","ĠÑģе лÑĮ","çѹ éĽĨ","éħ° èĥº","Ġweit ere","[ ];Ċ","è¦ģ åģļ好","Ġbl inked","_y ear","Ġ×ŀ×IJ ×ķת","_ _","Ġconcess ions","ĠH ellen","个人 ä¿¡æģ¯","lor o","åħ³éĶ® æĺ¯","ĠиÑģполÑĮзÑĥ еÑĤÑģÑı","Ġclass ifications","æŃ¦ åĬĽ","Ġfem oral","ĠLog istics","浩 çĦ¶","K il","åľ¨ æŁIJ","Ġз вÑĥ","æľįåĬ¡ è´¨éĩı","Ġamount ed","bl ad","Ġtravel er","Ġhil arious","ĠگرÙģ ØªÙĩ","-fashion ed","G roups","if ers","Ġen forcing","Ùĩا Ùħ","ĠNot ebook","èĥĮ 诵","è¡£ 裳","Ġfer ro","Ġp q","ĠV ote","ĠTh row","ем ого","acc um","ä¹ĭå¤ĸ çļĦ","ĠPok émon","Ġsubsid y","Ġdiscrep ancies","\\ <","大 éĿ©åij½","Ġform ul","漫 æŃ¥","Ġpr istine","-c arb","æĮ¯ å¥ĭ","å®Ł éļĽ","Ġin scription","๠Ĵ","Ġher pes","æĥĬ æħĮ","]( ../../","ĠопÑĢеде лениÑı","éĢĢå½¹ åĨĽäºº","æ²® 丧","åıij è´§","é¢Ħ åijĬ","ĠоÑģнов нÑĭÑħ","Commun ication","ver ify","Ġad hering","Ġra ggi","Ġed uk","à§įঠŃ","èģĶç³» ç͵è¯Ŀ","ĠId i","bu ilt","ĠA e","Ġ\" *","åīį 线","åĪĻ åı¯","Ġw aving","et ect","ĠS isters","ĠاÙĦÙĥ Ø«ÙĬر","åį³ä½¿ åľ¨","èªį 羣","ord ial","Ġind ent","ĠSh ield","ĠLe hr","Ġsuper l","éģĹ è¿¹","Ġby ÅĤa","导 åĩº","лÑĮ ном","-t rack","æŃ» åľ¨","ÑĢов и","çļĦéĤ£ ä¸ĢåĪ»","F ail","el ho","read s","Ñĩе ÑģÑĤва","Ġobtain s","Ġt unes","è² ¢","Ġstabil ized","F at","ĉ string","ĠP PP","ä¹ĭ ä¸ĬçļĦ","ä¾Ľ éĶĢ","溶 æĢ§","Ġmin er","Ġcur led","اÙĦ د","ĠÑĢаÑģ ÑĪи","Ġm ates","Ġ* (","ĠQ in","æł¼ æĭī","èIJ¨ åħĭ","óg ico","Ġd eng","çļĦ èµĦéĩij","ĠK ern","ÙĤ Ùħ","ĠQu ando","ê° Ĵ","Ġpict ured","ì n","е Ñģ","å·¥ä½ľ ç»Ħ","ĠнекоÑĤоÑĢ Ñĭе","is as","ĠW iel","åŃĹ æķ°","èĭ¥ æĹł","ä¸įä»ħ å¦ĤæŃ¤","izz are","ĠФ оÑĢ","Ġconstitu ency","st orage","ci i","Ġz ah","ano i","åĩºçīĪ çļĦ","ĠConf uci","ĠVe hicles","m eg","Ġl g","Ġeffect ed","ÄĽ k","ĠÙħØŃ ÛĮØ·","Ġenfermed ad","Ġd azz","ĠS ulf","ĠUnivers ität","- et","D og","ĠM OT","主 é¡Į","Ġeld est","ĠÑĢекомен дÑĥ","T ony","T unes","人æ°ij æĹ¥æĬ¥","zn ik","Ġsacrific ed","ĠпÑĢоÑĦеÑģÑģи оналÑĮ","Ġmorn ings","d ro","y un","ĠG OD","Ġla ure","èĢģ é¾Ħ","Ġop aque",".L oad","ĠجÙĩ اÙĨ","社ä¼ļ ç»Ħç»ĩ","缸åħ³ æĢ§","Ġpsych ic","y en","ic in","ĠT art","对 æĪijçļĦ","å¹´ åħ¨åĽ½","Ġchron ological","èĥ° èħº","_ al","ĠT enn","Ġpe at","åIJĥ 饱","è¿ŀ éĢļ","Ġge ven","={ \"","ku uta","Ġsimpl ifies","è¿Ŀæ³ķ çĬ¯ç½ª","gester one","Ġd ancer","Ġج ÙĪ","æĭĴ çµķ","\" ãĢĤ","ç¾İ åĮĸ","IC U","Ġaction able","èĦij è¡Ģ管","ĠHel ena","Ġont st","ĠÐĹ ÐµÐ¼","çļĦ åĽŀçŃĶ","Ġh ut","Ġ? >Ċ","LE Y","æ¶ī æ¡Ī","ĠHel m","Ġinvent ions","試 é¨ĵ","åľ¨ åħ¬åı¸","Ġen vol","ich o","erv ille","çĤ¹ ä¸Ĭ","Ġi ets","ç³ ľ","è¿Ļæł· å°±","第äºĮ å±Ĭ","atal ytic","Ġweb page","ums i",".index Of","exper ience","ãĤĤãģĹ ãĤĮ","ĠK op","éĥ½ ä¸İ","æĬĹ éľĩ","Ö¸ ×Ķ","à¸Ńะ à¹Ħร","Ġb aff","Ġse hen","غ Ø·","Ġblog gers","ĠпоÑĩ ÑĤи","Ġh ither","ĠT icket","å¤ĸ åĮħ","ç»§ ç͵åύ","ĠCook ies","Descript ors","çļĦ æ¯į亲","åĪ ¨","ม ืà¸Ńà¸Ļ","iment ary","ĠAdv antage","ĠÐĹ Ð½Ð°","ĠINT EGER","Ġf iss","å¹´ æľŁ","Ġam ort","Ġmain s","Ġbo ek","åĪĨæŀIJ æ³ķ","ĠIN ST","ĠÐľ Ñĥ","åıªè¦ģ æĺ¯","à¹Ģส ริม","Ġdebug ging","Å ¿","è¦ģ 使","æīĢ åģļçļĦ","Ġmod ific","åIJį è¨Ģ","aw ai","ĠìŀĪ ìĸ´","åįĥä¸ĩ åĪ«","×Ļ×¢ ×Ķ",": text","T rain","ä¸į èµ°","ÃŃ cios","Ġpo ignant","п ен","Ġà¦ħ রà§įথ","Ġfill er","Ġpes quisa","Ġintens ified","åľ¨ ä¸įåIJĮçļĦ","ip ada","ord inary","æľĪ çIJĥ","(t itle","éģĹ åĺ±","ĠFar mer","Ġkiss ing","est ing","åı¯ çĸij","åIJİ å¤ĩ","Ġsp onge","å¼ķ åĬĽ","åķĨåĵģ æĪ¿","Ġsucceed ing","ĠвнÑĥ ÑĤÑĢи","çĶ» çĶ»","åįķä½į åĴĮ","æĽ² 线çļĦ","ãģĹãģ¦ ãĤĤ","粪 便","ç¤ Ļ","ä¸ī 缸","ĠConn or","åĩ¶ æīĭ","å«ģ ç»Ļ","纪念 é¦Ĩ","Ġscaff old","ä¸į æŃ£","ra pped","Ġvol te","ä¹Łä¸į çŁ¥","Or Default","Ġhem os","ĠUnderg round","ÃŃ na","Ġmin utos","Ġgl omer","-p ost","å¸Ĥåľº 份é¢Ŀ","ĠPart age","ĠF ishing","æ± ¾","æľ¬ æĺ¯","Ġel kaar","It alia","ĠSa úde","à¸Ĥà¸Ńà¸ĩ à¸ģาร","ĠФедеÑĢа ÑĨии","ĠS oy","Ġbl onde","-b tn","å¢ŀ çĽĬ","-p ath","ĠÑĤ оже","Ġlocal es","га ÑĢ","ĠÑģоб ÑģÑĤвен","Ġh é","ĠпÑĢе кÑĢа","ĠKel vin","ĠHass an","人 å±ħ","太 好","m A","Ġn ik","ĠP izza","ĠB ark","ä¸į 失","ĠCh al","è¿ĺ æĺ¯ä¸Ģ","Ġno ve","Ġع بر","crib es","ç®Ģ è¿°","Mod ified","å°ıæĹ¶ åIJİ","ĠPi per","ĠÑģÑĤанови ÑĤÑģÑı","Ġm iesz","Ġг о","èŀº ä¸Ŀ","Process ing","s ers","à IJ","× ļ","çļĦ æĪĺçķ¥","pe es","çľĭ çĹħ","-s uccess","ç§» 交","تر ÛĮ","c ause","om ány","ä¹ Ĺ","æĮī éĶ®","Ġdé m","ĠÑįк Ñģпе","à§įল াহ","Ġgouvern ement","ar ic","ĠJ ab","Ġequ ipo","ನà³įನ à³ģ","b uilder","c ra","ë ¹","Ġv ested","æľī èijĹ","ŀ× ĵ",".S ave","rec ated","ĠBul ld","p olar","ĠC Y","æĢĿ æ½®","Ġantic o","all back","ä¹ĭ 说","eth ical","æ°Ķ åĴĮ","Ġprepared ness","ÃŃt ás","Ġtet ap","Ġzd row","etz ung","E H","Ġth ief","Ġk ini","天 åĨħ","Ġhoriz ons","Ġt int","å°Ħ æīĭ","ĠRob b","Ġconoc imiento","Author ization","k ach","ĉ C","åıij åĩºäºĨ","ä¸ī 代","Ùĥ ثر","Ġtw itter","è¿ĻéĩĮ éĿ¢","åįģäºĮ æĿ¡","çĶĺèĤĥ çľģ","- '","Ġc ose","ä¸į åĬł","Ġag itation","æĹł å¿Į","_ img","æ±Ł å¸Ĥ","IT LE","ãĥ¬ ãĤ¹","Any one","忽çķ¥ äºĨ","ä»ĸ ä¸įæĺ¯","èι åıª","Ġtur f","Ġkdy ž","ĠCarp enter","r ne","Ġsp ores","éľ ĵ","ç͵ åĽ¾","Ġdoubt ful","欺 è¯Ī","ĠB orough","äºĨ ä¸įèµ·","act ivate","åĪĨ 段","主 线","Ġaut oc","Ġge o","Ġsec und","ál is","Rel ative","E sc","Å ½","ģ áĢ","Ġp ours","Ġte g","Ġtrans ducer","Con struct","Ġinsp ected","å¼Ģåıij èĢħ","Ġbelong ings","ëĮĢ ë¡ľ","Ġincl u","ĠCoven ant","is el","ем ÑĮ","=' /","æĬ½ æŁ¥","ozo ic","æŁ ¬","å·¥ä½ľ 计åĪĴ","Ġindividual ity","Ġrevolution ized","Ġpestic ide","çļĦ ç³»ç»Ł","est im","çĶŁ åīį","èĬ± æ¤Ĵ","׾ ×Ļת","িঠĵ","Ġmar ca","ĠاÙĦØŃ ر","âĻ ¥","T rade","ĠE uras","æľ¬ æĽ¸","åħ¬ åľĴ","ç¯ ±","Ġpred ic","ĠÑĢаз ÑĢе","é§ IJ","v ous","ĉ de","ãģķãĤĮ ãģ¦ãģĦãģ¾ãģĻ","ĠFire fox","} e","Ġp addle","åĨ¬ 奥","ynth ia","am ation","é£İéĻ© 管çIJĨ","Ġú nico","ĠMot ivation","-x s","Ġpremi ere","Ġc ops","ĠT ir","ne v","ä¿Ĺ è¯Ŀ说","æŃ¡ è¿İ","èģ¯ åIJĪ","Ġкомп он","ĠÏī ÏĤ","E ric","{ }Ċ","Ġin cont","ra ins","ĠPath ol","æĬĴ æĥħ","ä¹Ł è·ŁçĿĢ","é«ĺ æ°´å¹³","社 å·¥","æł¼ æŀĹ","Ġfam ÃŃlia","ç»ĻäºĨ æĪij","çļĦ çī©è´¨","çĸ ¸","Ġsun k","_ AD","ĠAd mission","ö ld","çŁ³ èĭ±","ĠMan ning","æĪª åĽ¾","ä¸įç͍ æĭħå¿ĥ","Ġliv ello","+ h","ĠK od","ĠUn certain","æľª è§ģ","åij³ ç²¾","ex tern","çϽ çϽ","-e lect","Ġкомп ÑĮÑİ","ĠÑĢаÑģÑĩеÑĤ а","$ .Ċ","T ap","åij¼ åij¼","js ce","Ġperf usion","prof essional","å¼Ģå¹ķ å¼ı","T ot","Ġна зна","Ġμ M","åħ§ çļĦ","Cap acity","Ġíı¬ íķ¨","A part","ĉ list","ĠG ale","æľ¬ èįī","å¹³ åĿ¦","æľį çļĦ","åĨ· æ·¡","çļĦ大 éĩı","Ġtow els","es per","ä¼ļ å±ķ","card ia","Ġintens ely","Ġdifer encia","P ain","Ġcomp ressive","çī¹ åĭĴ","iter ation","à§ĩঠ¡","ĠJack ie","ä¸įäºĨ çļĦ","λλ ά","è´ª 污","Ġ\" ...","ĠRel ation","Ġdigit ally","åĪĽä½ľ çļĦ","Ġlifest yles","Ġske ptic","è´ª 婪","ĠÑĤÑĢÑĥ д","Ġbust ling","in ous","ĠR ough","ord a","Ġ$ (\"#","Go ing","Ġfire wall","ĠاÙĦرب ÙĬع","ä¸Ń æĹ¥","-in spired","Ġarrest s","ipher al","Ġ×¨× IJש","é ben","Ġint elig","fer ential","ink y","Ġcomple teness","ĠJu venile","çļĦ åIJĪä½ľ","é«ĺ æłĩåĩĨ","éĩij èī²çļĦ","ij u","-t rained","Ġcapital ize","ĠCirc uits","s an","ino a","Ġsex es","ç¶ĵ æŃ·","abilit Ãł","( let","_ update","ĠR oles","Ġ구 ìĦ±","ın ı","ầ u","ĠпопÑĥ ла","L ew","为 代表çļĦ","åıij èĬ½","cy m","اÙĦ ج","æĢ» çĿ£","Term s","% D","aus end","ç¬ij èĦ¸","æľīä¸Ģ æĿ¡","ĠìŀIJ ìĭł","| =","人 å°±","Ġk N","Ġcont a","ç¥ ¯","amp ing","alle ts","uv res","\\({}^{ +}\\)","Ġdocument ing","误 åĮº","Ġrh iz","æŀ¯ çĩ¥","Ġprat ique","Ġcic lo","Ġl umen","åıĪ æĬĬ","ĠYour self","Ġdownload ing","Ġt ierra","Ġse ks","ĠSevent h","Ġf im","人 ãģ¯","ú ng","åį¡ è½¦","åĮ»çĸĹ åĻ¨æ¢°","ĠاÙĦØ´ خص","etti in","ĠvÃł o","en able","Ġy uan","CT V","ĠGeoff rey","Ġk ró","缮 çĿ¹","ĠEl ite","ĠTrans it","ç½ij绾 ä¸Ĭ","Ġê² Į","×Ļר ×Ļ×Ŀ","Ġalumn os","v irtual","âĢ °","对 éĿ¢çļĦ","Ġnear er","å¥Ĺ é¤IJ","æĶ¾å¿ĥ åIJ§","zá lez","z nych","Ġreal ms","AT URE","é«Ķ é©Ĺ","Ġsubstit uting",".con current","çĭ¼çĭ Ī","Ġwh it","so far","ü hl","è¶³ äºĨ","Ġmot ivating","Ġimm ensely","W ir","å¹´ åĿĩ","ä¸ī 峡","Ġval ore","Ġintens ities","åĥµ å°¸","á»ĭ nh","æĺ¾å¾® éķľ","åı¯ ç¬ij","ж нÑĭÑħ","(s ource","æľŁéĹ´ çļĦ","ï¹ IJ","ÑĨенÑĤ ÑĢа","ĠJ ia","åı¯ æİ§","ian y","ãĢĤâĢĿ ãĢĬ","ĠCont rast","ĠNurs es","×ķפ ×Ķ","ĠMob ility","' r","N V","çº ¶","Ġdev oid","ç»ıæµİ çļĦåıijå±ķ","æĭĽ æĶ¶","çī¹å¾ģ çļĦ","ĠLis bon","ac ción","Ġrel ativity","çϽ ç»Ĩèĥŀ","ãģĦ ãģ¾ãģĹãģŁ","ãģĮ ãģĤãĤĬ","设计 ä¸İ","ä¹Łä¸į æľĥ","bal ances","ĠÙĦÙĦ ØŃ","ĠпÑĢоÑĨе ÑģÑģе","éĢĻ åħ©","Ġinc ision","غ ÙĦ","Ġtrain ers","ĠMagn et","Ġmaj estic","orient ation","' ]ĊĊ","iz zo","æĿ¥ è¿ĻéĩĮ","Ñģк омÑĥ","US H","æĶ¿åºľ éĥ¨éŨ","Ġà¦ķ ল","Ġp aternal","å®ļ 为","á ÅĻ","ä½Ĩ ä»į","éĩij åŃĹ","оÑĤ пÑĥ","ãģĭ ãĤĭ","çķ¶ ä¸Ń","Ġfol klore","缸 çα","ç»ıæµİ 建设","ĠInt ers","Ġplant as","Ġdiss ection","ĠJer ome","ÙİÙĨ ÙĴ","J s","è¿ ´","we bsite","Ġfam ine","åħ¸ èĮĥ","ĠÑĤа м","Ġinstall ment","Ġneutral ity","ĠاÙĨت Ø®",".Cont ains","ik awa","å·¥ 人çļĦ","çħ ²","sch ule","Ġfung si","[ label","Ġdam ned","pat rick","满满 çļĦ","- cycle","Ġpar sing","ä»ĸçļĦ æīĭ","र à¥Ģ","å®īæİĴ éĥ¨ç½²","ä¸ĵ项 è¡ĮåĬ¨","Ġsop rattutto","ĠL ös","Ġris que","åĪĽæĸ° èĥ½åĬĽ","ĠìŬ 룬","Ġt c","ĠJ ain","åĺĢ åĴķ","Ġd ici","Ġm oth","ou k","×Ļר ת","Ġreconc ile","ä¸ī å±Ĥ","没æľī 被","ĠÑĤ омÑĥ","nam en","Ġпло Ñģко","ר׼ ת","ç¶ľ åIJĪ","ot ec","å§ Ĺ","Ġind ifferent","èģĶç³» 人","ĠاÙĦج اÙħ","ĠобÑĬ ÑıÑģ","Ġdiaphrag m","Ġa ún","çļĦ æķ°åŃĹ","èĩ³ ä¸Ĭ","اد ات","æĪIJåĬŁ äºĨ","Ò »","al us","Ġت Ùı","æĸŃ å¼Ģ","Ġ×Ķ× ŀ","оп ÑĢи","IM O","cz nych","Ġcalibr ated","ĠBiod iversity","( pos","ĠD ash","åľ¨ 建","Ġor t","ï¼Ī ï¼īĊĊ","Ġmon oc","Ġع ÙĤ","Ġden ies","åĢĭ æľĪ","å°Ķ çī¹","çļ® çļĦ","Ġ\" )","ĠX OF","å¤įæĿĤ æĢ§","ĠMembers hip","éĻ º","ty wn","ä»Ģä¹Ī æĦıæĢĿ","Ġbad an","çĥŁ éĽ¾","樱 èĬ±","os est","ĠN ish","éĢ ŀ","çĽĺ çĤ¹",".b uild","ĠPRO GRAM","YS IS","£ p","ĠH aj","Ñĩ ноÑģÑĤÑĮ","åħ¬ åħ³","ог Ñĥ","ä¼ł 羣","ĠâĪ Ģ","åľ°æĸ¹ çļĦ","Med ium","Ġíİ ¸","Ġsp ins","西 åŁŁ","å±Ģ å±Ģéķ¿","ĠÑĪе ÑģÑĤÑĮ","ĠP ricing","ak ra","åıij æĺİçļĦ","æ· Ħ","æĿİ ä¸ĸ","Ġ×ŀ× ĸ","ĠتØŃ ÙĤÛĮ","à¸Ĭà¸Ļ à¹Į","ĠÙĨÙĤ Ø´","B road","j ing","he ten","Ġde ï¬ģ","ĠH SL","Ġpre ocup","äºĮ åĵ¥","çϽ éĽª","ä¸Ĭä¸Ģ ç¯ĩ","Ġква ÑĢ","B ACKGROUND","åıij è¡ĮçļĦ","Ġz ug","Ġtra va","Ùĥ اÙģ","Ġbook let","å¼Ĥ æŃ¥","è§ĦåĪĻ çļĦ","ĠLight ning","Ġà¸Ħ à¸Ļ","Ġt ents","ant ar","æŀģ å°ij","ĠComb ining","l r","à ij","el is","å¹´ 以ä¸Ĭ","æ°´ çħİ","è¿ĺæĺ¯ ä¼ļ","ĠاÙĦØ£ÙĪÙĦ Ùī","ĠH os","é«ĺ åľ°","Ġbed rooms","éĥ½æľī ä¸Ģ个","èµ¶ ä¸Ĭ","Ġsubstit utes","Con clusions","Leg al","or get","ES C","Ġexperien cia","ĠEst imate","ç¹ģ å¿Ļ","Ġa ire","Ġ) {Ċ","ĠEr in","Ġnou v","} &\\","................ ..","org es",".b oot","Ġdisapp ro","Ġfort ress","é̼ è¿ij","ç¶² è·¯","Ġthromb osis","绣 é¢Ĩ","åĦ ¡","Ġà¦ı à¦Łà¦¿","Ġborrow er",", W","ĠE lections","Ġk y","Cl ub","ĠEl ijah","ÛĮد ÙĨ","æĤ¬ å´ĸ","Ġembark ed","ĠDipl oma","ĠF AC","ie kt","ä½Ĩæĺ¯ å¦Ĥæŀľ","Ġinter course","ĠSe eds","Ïĥ Ïĥ",".m ake","æŀ¶ åŃIJ","Ġдо ÑģÑĤÑĥп","èĤ¿ èĥĢ","åݨ å¸Ī","ĠLoc ated","Ġelic it","Ù ¢","Ġm ég","ä¸Ń åĩºçݰ","Ġcl oning","åľ° æŃ¥","ep och","åĽ¾ å½¢çļĦ","Ġت Ùĩ","Ġseg uridad","礼 åĵģ","اØŃ ÛĮ","Ġgro cer","ç°¡ 缴","èµĶåģ¿ è´£ä»»","æİĴè¡Į æ¦ľ","Ġf action","×ķ× Ķ×","Ġinitial ized",".st ack","éĻª ä½ł","à ĭ","ou m","Ġcat ar","ĠVul ner","çľĭ ä¸Ģçľĭ","ĠAng l","综åIJĪ ç´łè´¨","Pr ivacy","Ġpad re","N Z","Ġcon ceive","åľ¨ æĥ³","Ġпо Ñıви","ét és","Ġê°ľ ë°ľ","ĠRegular ly","Ġd afür","ĠB amb","ans ka","åĽ½ èIJ¥","umber land","缺 æ°§","Ġma upun","龸 éģĵ","Ġкомп лекÑģ","[ pos","Ġa ft","à¸Ļ ะ","Ġeigen en","æĪIJ交 éĩı","ĠØŃÙĤ ÙĪÙĤ",". ind","ĠD ere","大 åįĬ","×Ļ× §×","Ġty ph","导 ç͵","Ġmill ing","ati u","é̲ äºĨ","Ġvent ral","ĠBright on","ĠëIJľ ëĭ¤","on ana","ll es","äºĮ åįĥ","ĠIr ving","Ġclim ax",". {","V ers","æĸ° å¾ģç¨ĭ","ĠRes et","ìĹ ¼","Ġsw ell","Ġpsych otherapy","ĠDIS C","Ġprere quisite","Ġnostalg ia","Ġprocess us","arg ent","Äį ka","Ġдо Ñħод","Det ailed","mont on","Ġrecom end","ĠPART IC","M ais","Ġd ah","æĺ¯ åIJĹ","Ġna am","_n odes","Ġmeng alami","Ġয à¦¾à§Ł","_e vent","Ġmoh ou","Q UE","éļ¾ çļĦ","(t oken","ĠRed ucing","ĠÑģоÑģÑĤоÑı ниÑı","Ġw omb","Ġl ounge","ĠPl ane","Ġil ust","ä¿¡ç͍ è¯ģ","\\t au","Ġsummar ies","éŃĶ æľ¯","Other s","ç´Ľ ç´Ľ","D ra","R ear","ov ir","åŁ¹åħ» åѦçĶŁçļĦ","দ à§įর","-pl ane","Ġczy li","棺 æĿIJ","ĠP est","ĠR ita","åĩº éĶĻ","åİŁ çĤ¹","For ward","ìł ij","Ġdé termin","åľĺ éļĬ","? >Ċ","ur ved","åľ¨ çĶŁäº§","ĠDis pon","Prior ity","Ġclo ak","ie b","æĹ¥ åħī","èµĦ æ·±","yd d","ç²¾ç¥ŀ çĹħ","Ġlock er","Ġgru nd",". Image","K P","ĠH DL","å¿ĥ çİĩ","ĠÑĢаз ÑĢа","à¸ľ ิà¸Ķ","åĽŀåΰ å®¶","Ġ×Ĺ ×ijר","Ġжи ве","Ġremind ers","-act ivated","m ul","æľī 空","å±± èį¯","Ġnovel ist","ĠTurn ing","Ġaug mentation","ĠS is","åĴĮ å¤ĸ","æľ¬ çĹħ","æĬķèµĦ 人","软 骨","Ġlie utenant","ĠConn ections","ĠHem isphere","Ġked ua","æ°ijèIJ¥ ä¼ģä¸ļ","ĠA xis","ĠO UR","ĠK ru","èĢģ å¹²éĥ¨","iet e","Ġap nea","å¿ĥçIJĨ åĴ¨è¯¢","ĠWhe eler","Ġst ains","å¼Ģ å°ģ","ĠQu ite","ĠÕ ¬","ĠFin ish","è¡° åĩı",": self","c itation","n pm","Ġи меÑĤÑĮ","Ġshe ds","çݯå¢ĥ 污æŁĵ","Ġho je","å¹´åīį çļĦ","ĠwiÄĻ cej","а на","çŃī äºĨ","å¸ĥ èݱ","ĠÙĨ ÙĪØ´","ĠÐľ ак","篮 æĿ¿",".Data Frame","p ok","ĠM ush","Ġà į","â̦ .ĊĊ","ĠEx isting","çݯ åį«","ล à¹Į","ĠPath ology","Ġaer ospace","Ġrod ents","p ole","æľ¬ èģĮ","ä½ĵ å¾ģ","æ·»åĬł åīĤ","Ġimpart ial","Ñļи ма","Ġl ượ","å¾Ī æ¸ħæ¥ļ","åħļ 课","å¤ľ èī²","án chez","å°±åľ¨ è¿ĻæĹ¶","Ġslee ves","æĪĸ å°ij","iqu é","ĠLear ners","ï¼ŁãĢį Ċ","el ier","Ñħ аÑĢ","bel ow","为 åѦçĶŁ","C itations","çļĦ è·¯ä¸Ĭ","Ġv enge","Ġdan ced","ĠìĦ¸ ê³Ħ","g rown","Ê ¿","ĠSt raw","д енÑĤ","Ġcor az","ç«ĭ 项","æľį èį¯","çij ¤","ĠMod elling","]( ../","ĠоÑģ новним","³³³³³³³³ ³³³³³³³³","ĠSomal ia","p olit","in ers","ĠN PR","Ġr asa","ĠK C","éģĵ æķĻ","Ġsc am","约 æľī","çIJĨæĥ³ 信念","Ġse b","ä»ĸ æĽ¾","ĠY uta","ĠUn i","리 를","Ä« n","ä¸Ĭåįĩ åΰ","Ġverte bra","f w","f ax","l x","æĹ¶ æīĢ","we is","æİ¥ 觸","Ġrem ission","els er","彩 票","Ġfør ste","V III","ï ľ","Ġa ustral","ĠG ink","Ġpar ab","ãĢį ï¼Ī","缴æİ¥ çļĦ","Tr uth","Ġuniform ity","é»ijé¾Ļæ±Ł çľģ","ᱣ á±","e at","ĠN amed","åĩº éĿ¢","Ġdown side","Ġv uel","ĠF ighting","å¡ Ĺ","ias i","æľ± åħĥ","Ġfloor ing","valid ator","Ġintrig ued",". not","Y o","æĥ ¬","æĮī ä½ı","iet ic","表示 为","mark ets","Ġин ÑģÑĤÑĢÑĥ","ĠInsp ection","Coll ins","Ġk ale","çĹ Ĭ","rict ions","èµĦæºIJ åĴĮ","Ġuniform s","Ġcontrad ictions","éģİç¨ĭ ä¸Ń","Ġw i","Ġgen o","åij¼ åı«","ep en","Ġsur real","æľī ä»Ģ麼","æĪij åģļ","对 è§Ĩ","çī¹ éķ¿","æµģ éĢŁ","text area","Ġconver ges","èĥĨ åŃIJ","Ġpit ches","à§İ স","Î ¥","Ġm ound","Ġstr utt","è¿IJè¡Į çļĦ","ĠÎľ α","ĠExhib ition","p ring","Ġthe aters","an uts","Ġjoy ful","اÙĪ Øª","çĸ¯ äºĨ","ĠdifÃŃ cil","¢ ×Ķ","è§Ħ éģ¿","ĠBl o","ç¼ĸ åī§","Ġbed time","ä¹³ 头","ç¥ŀç»ı åħĥ","æĭĴç»Ŀ äºĨ","Ġinformáció k","ĠL ists","к ог","ار ات","ä¼ĺ ç¾İçļĦ","æĺ¯ä¸Ģ éĥ¨","ĠÙĤ اعدÙĩ","-re port","ÑĪа еÑĤÑģÑı","æ¼Ĥ æµ®","Ġmultim édias","Ġsp leen","ä¹Ŀ å·ŀ","Ġviol ates","CM YK","áģ ĭ","ĠëĶ Ķ",". row","Ġpre ached","Ġwork ings","Ġkon nte","ĠIn her","çϼ åĩº","åıĤä¸İ åΰ","Ġorient ations","Ġdeploy ing","ĠDim ension","ĠEnh ancing","M esh","ö nt","اÛĮ ر","Ġà¦ľ াত","èģª æĺİçļĦ","ĠC ITY","Ñĩ нÑĥÑİ","åħ¨ 天","respond ing","åįĸ å®¶","Pe er","éģķ ãģĦ","ĠTr uman","ç¨İ è´¹","æľīå¤ļ 大","Ġaspir in","äºĨ çľ¼","åı¤ ç±į","æ´ŀ å¯Ł","Ġchrom atin","Ġlapt ops","å¯Ŀ 室","- going","ĠS AF","ĠM är","ä¹Ł æ¯Ķ","æŀľ æłij","åĩĿ è¡Ģ","Ġبع دÙĩ","Ġíķ¨ ê»ĺ","çļĦ å®ŀæĸ½","ess ori","Ġdis so","........ ..","Ġx xx","ĠChrist ie","ol on","ve ctors","Ġor anges","wo f","ä¸Ģèά éĥ½æĺ¯","-com ponent","Ġtact ic","Ġattent ive","Ġcleans ing","Ġmú lt","d v","Ġc obalt","ĠPre ston","Or den","å¦ĤæŃ¤ çļĦ","иÑģ Ñĭ","ç¥Ŀ ä½ł","ĠÑģлÑĥÑĩа й","ĠBos nia","m ui","u ção","åĵĪ åĪ©","ÙĬÙij Ø©",".am azon","l ampi","{ Y","ist y","ĠE as","éĢĻ ä¹Ī","-l im",".d ll","åŃķ æľŁ","寶 寶","à«ģ àªĤ","T PS","ÅĤ ych","Ġhal o","Ġdump ed","im etry","è¿ ¸","ä¸ĩ æĪ·","çŁ³ çļĦ","comm ission","Ġvo ork","ui ção","Column s","Ġì² Ń","F riends","Ġh amb","åı¯ åıĺ","Ġso fter","ĠSim mons","Ph ylum","ĠEt im","ĠShel ley","à¹Ģà¸ķ à¸Ńรà¹Į","g iv","åĨ ½","次 åºı","çłĶ ä¿®","Ġadv ising","Ġbro ccoli","ਠĤ","\\n u","ÑĢов ой","Some one","çĶŁéķ¿ åıijèĤ²","et u","ĠC ork","Ġbe ad","ap oda","cc c","ä»» çͱ","ç²¾ åŃIJ","Ġimm ature",".t otal","ĠCons ent","Ġf j","å°ı åŃ¦æł¡","ä»· 款","ãĢij ï¼Į","ĠBar ber","Ġwel comes","Request Mapping","æĻĭ 级","Ġ×§ ר","à¹Ģà¸ī à¸ŀาะ","大 ä¸Ģ","ни н","ÏĦε ÏĤ","Ġবà§įযব হার","\" -","ä¸Ń 举","æ¶ Ŀ","Ġд еÑĢ","åĽĽ äºĶ","ĠEn cyclop","Ġfire works","Äģ t","ellig ence","Ġmic rop","ĠÄiji á»ĩn","ophy ta","ĠHypot hesis","รà¹Īาà¸ĩ à¸ģาย","ĠRe b","ec ost","å¾ ´","å°ı é±¼","ย ม","é¡¹çĽ® 管çIJĨ","é¢Ŀ å¤ĸçļĦ","-h op","ĠاÙĦØ´ Ùħس","Ġkomun ik","çα å°Ķ","Ġbro kers","å½Ĵ è¿ĺ","éĴ¢ æĿ¿","organ ized","' alt","ĠU M","Ġ/ ĊĊ","çĹħ çĹĩ","读 éŁ³","Ġste hen","Un ix","ĠPres ervation","Ġmoder ne","ĠCoun c","大 å¥ĸ","æ´» äºĨ","æĶ¾ æĺł","ĠÑĤ оп",".g rid","è¸ı ä¸Ĭ","or ce","åĨħ æł¸","Ġspect ators","اض ÙĬ","Ġ.. .,","ad rat","åύ ä¸Ń","绿 åľ°","Ġ×©× Ŀ","Ġul cers","ĠÑģк олÑĮко","ĠBart on","ĠÑģоÑģÑĤо иÑĤ","as ional","fer n","åĿ ·","åĽŀ åij³","äºĨä¸Ģ ä»¶","Ġartic ulation","à§įয à§ĩর","ĠMet als","æ··åIJĪ çī©","Ġtent ative","ĠпоÑĤ ен","Ġsign ify","åģ¥ èĦ¾","ä»Ļ åŃIJ","ĠëĮĢ íķ´","ĠKan po","ĠÑĥÑĢав нение","W s","om n","ĠT end","åΰ å®¶","на ÑĤа","æĢ» åĴĮ","-t reatment","Ent re","ĠF ritz","Ġsp äter","ãĢĤâĢĿ (","Ġpress es","ĠÙĥ Ùĩ","éļĶ çĿĢ","ĠÑģи н","Ġassass ination","le h","ag ara","ill age","çϽ è¡Ģ","ส à¹Į","æ²¹ çĶ»","ĠBo ost","Ne il","ãģ§ãģį ãģªãģĦ","Ġpig ments","为 èĩªå·±çļĦ","Ġ- (","ĠSe al","éĿŀ çī©è´¨","Ġë° ķ","ĠاÙĦØ® Ø·","Ġjs me","Ġbatt ling","Ġmund ane","é rio","åĬ¨ æijĩ","è§ģ 她","over flow","ĠоÑĤ меÑĤ","é¢ij é¢ij","à¹Ĥ à¸Ľà¸£","éĴ¢ ä¸Ŀ","IF ICATION","Ġε ξ","................................................................ ..","à¦ŀ à§įà¦ļ","çļĦ 马","Ġcl oves","æĢ¥ éĢŁ","Ġred sh","à¹ĩ à¸Ī","æĥ³è±¡ åĬĽ","Ġjav ascript","Ġباش ÙĨد","à¸ģิà¸Ī à¸ģรรม","Ġnouv elles","< img","Ġd rie","çĦ¶åIJİ å°±","å®ĮæĪIJ ä»»åĬ¡","Ġви дов","Ġ×Ķ ×Ļ×ķ","æĥł æ°ij","Ġadvert ised","ĠEle anor","th inking","aut é","à§ĭ প","بر د","æ°£ æģ¯","trans late","Ġлег ко","ĠFert il","ĠN eph","Ġr á","æ¯ Ĺ","ĠY ield","att ack","িঠ¡","à¹ĥ à¸ļ","ĠPre viously","k ä","Ġ( )Ċ","æĥħ ç·Ĵ","åĬł æĿĥ","Ġprov isional","éĵ µ","å±± è·¯","Ġarr h","umn i","Û± Û²","Ġpartition ing","Ġsne ak",".\"\" \"Ċ","W y","Ġde pleted","大 殿","表 æĢģ","ise ase","tern ess","ann en","ç½ij çļĦ","ĠNorth western","åĩ¡ äºĭ","áĥIJáĥ ĵ","ĠÑĢ Ð¾Ð»ÑĮ","ãģ¡ ãĤī","Ġacon te","à¯ģà® ®","å®ŀåľ¨ 太",". ï¼Ī","åľ¨ ä¼ģä¸ļ","ĠL ia","æł Ħ","两 级","çŁŃ çļĦ","ðĿij ļ","gl omer","è§£éĩĬ éģĵ","- expression","ĉ add","é cole","ĠComb at","éĩİçĶŁ åĬ¨çī©","å͝çī© ä¸»ä¹ī","ost atic","ĠK ara","éķ¿ å®ĺ","ı m","Ġspr inkle","Ġlandl ords","ĠплоÑīа дÑĮ","-conf idence","ow ana","ĠW ass","Ġk omb","ĠO vers","ĠCl erk","è¿Ļæł· åı¯ä»¥","Ġfi ery",". price","Y a","r ive","w elling","Ġ ï½","vel d","çľĭ åĩºæĿ¥","åĨĻ ä¸ĭ","ĠÙĨ Ø®",".P ar","æĵįä½ľ çļĦ","âĢĻ ;","èĬ± é¦Ļ","å¢ĥ åľ°","建设 å·¥ä½ľ","à´ £","ĠEm brace","ĠVal encia","ĠÑģп оÑĢ","chedul er","ãĤµ ãĥ¼ãĥ","M ot","ial is","é ct","å®ĥ ä¼ļ","æĿij 级","è¿Ľåħ¥ åΰ","~~~~~~~~ ~~~~~~~~","Ġcompart ments","Ġ'+ '","Ġo mission","çĤ Ĭ","é«ĺ 空","ier an","Ġë ł","calcul ator","Ġhass le","l ungs","Ġo cular","è¶³ 迹","Ġwer ken","æģIJæĢĸ çļĦ","Ġfreel ance","ĠCanter bury","Ġantib acterial","ĉ protected","ĠPro d","ни ке","ĠSa unders","æĬ¤ çħ§","ãĤĴ ãĤĤ",": )","yn y","çª ĸ","å¦Ĥæŀľ ä¸Ģ个","è¿Ļä¹Ī ä¸Ģ个","Ġinterview ing","flow ers","Ġelim in","Ġrég ion","åĹĵ éŁ³","-aware ness","Ġrheumat oid","- auto","å·¥ åĨľ","Ġmas se","áĥIJáĥ ļ","Ġbath ing","ĠL F","Ġall ure","åĸ ½","ä¸įèĥ½ ç͍","æĿĢ èıĮ",".n l","ĠKind ern","æĪij ç»Ļä½ł","âĢĶâĢĶ Ċ","Ġlow ers","éĸĭ çĻº","ĠMet aph","çļĦå¿ĥ çģµ","è¿Ļ ä¸ľè¥¿","Ġper ch","Ġdivers ified","Ġcabin ets",". abs","K evin","Ġav id","uest os","çŁŃæļĤ çļĦ","èģĮä¸ļæĬĢæľ¯ åѦéĻ¢","> ().","K y","Ġbe asts","** .ĊĊ","æĮĩ æİ§","åIJĥ çĿĢ","Ġze igt","ĠConf idence","Ġphosph olip","åħ¬å¸ĥ çļĦ","ĠKos ovo","_ the","âĢ ĥ","Ġat mos","Ġmar c","}} (\\","ĠC rom","çĶŁ åŃIJ","çĦ¶ åľ°","° .","ä½Ĩæĺ¯ åį´","β α","ĠGeorg ian","á½´ ν","ul ants","Ġâ Ł","交 éģĵ","è§ģ åΰäºĨ","Ġpop e","Ġdivers i","Ġfur ry","Ġw od","ĠE y","cont rollers","é£ŀ èι","رÙĬ Ùħ","- olds","{ W","Ġk j","大 ä¸ĵ","åĴĮ å®īåħ¨","ĠRe formation","èĤī 身","Ġма лÑĭ","Ġde j","um u","th ings","Ġsk ÅĤad","åħ« æĸ¹","بد Ø£","Ġ= -","Ġmuc us","Ġasympt otic","Ġanch ored","Ġman ier","Ġatt r","äºĨä¸Ģ éĺµ","Le an","-le ading","ãĥģ ãĥ£","æĸ°ä¸ŃåĽ½ æĪIJç«ĭ","ay aan","ĠاÙĦت ارÙĬØ®","Ġmal adie","Ġন ির","n k","ass emb","Ġstart led","ium s","Ġsal ient","رب ÛĮ","æ¹¾ åĮº","ĠM ey","Ġent ail","Ġд ем","(\" {","RE EN","æ³¢ 浪","ä½ľåĵģ ä¸Ń","Ġfl own",".s qrt","Ġع اÙĦÙħ","ĠHar riet","-re aching","Ġmes ma","Ġnod s","Ġž iv","Ġnarrow ly","Ġintertw ined","Ġz est","Ġcons ortium","Ġت ÙĨاÙĪÙĦ","AM ES","ĠChe ss","æ³ķå¾ĭ 责任","Ġmig lior","ne as","ç¼ĸ æİĴ","设置 çļĦ","èµ¶ å¿Ļ","Ġbra king","ĠÕ¥ Õ¶","d le","ent en","èĢĮ 使","Äħ t","_d b","Ġid iot","ÙĴ ÙĦ","æľĢ好 æĺ¯","wend ung",".Ass ert","ĠC ret","ĠM AG","ay as","å¦Ĥ æľŁ","Ġbl anc","oci ate","Ġview points","ĠдеÑĢе в","Ġun pack","Ġrem un","ĠðŁ ĩ","èķ ©","æľĢ大éĻIJ度 åľ°","- rest","en ity","ä¸į èµ·æĿ¥","Ġlast Name","ĠØ¥ ÙĦÙĬ","à© ģ","å®Ŀå®Ŀ çļĦ","Ġúlt imos","C orn","Ġqu od","åĩº æ°´","åħ¨ çıŃ","ع اÙħ","æĪ´ ä¸Ĭ","-he arted","ĠназÑĭва еÑĤÑģÑı","L IN","ç® į","æİ¨ ç¿»","èĥĥ åı£","гÑĥ ÑĢа","Ġwand ered","ाल à¥ĩ","Ġfisher men","Austral ian","- ID","C er","at zen","ĠSt ones","åįķ åIJij","æķĻåѦ 缮æłĩ","ológ ica","ĠMoz art","( end","H ier","n esty","Ñģ ен","ï¼Ł ï¼ģâĢĿĊĊ","æ·± éĤĥ","yt ics","Sh arp","ĠAD C","ÏĢο ι","ĠA ux","Ġcomp lements","ÙĬ اÙĩ","å¦ĸ æĢª","Ġfran ces","æĺŁæľŁ åħŃ","สิ à¸Ĺà¸ĺิ","éĵħ ç¬Ķ","con de","ĠCent imeters",".st rip","è§Ĥå¯Ł åΰ","Ġsophistic ation","Ġcom orbid","åĪĨ é¡ŀ","举 æ±ī","ÑĽ е",".sc ss","ol g","Ġex ogenous","Ġcl aws","az ol","rac ia","Ind ependent","éĹ² ç½®","Ġdrag ons","Ġunreal istic","ĠProfession als","N ight","Ġhe uristic","åĪĨ 红","å½ĵ æĻļ","æĤ ¶","Ġì§Ģ ìĹŃ","ặ t","ì Łģ","Ġ ãĢijĊĊ","or re","Ġse ptic","Ġind iv","der abad","Re ason","bre vi","ĠЧ е","Ġalg umas","è¾² æ¥Ń","ĠI TS","é« ¦","å®ŀ å®ŀåľ¨","å¾® ç¬ijçĿĢ","T ips","s ce","ä¸į æĸ¹ä¾¿","ä½ł 没æľī","ন ি","人åijĺ åľ¨","Ġtro op","çĽ¯ èijĹ","Ġযঠ¦",". username","ëĬĶ ëĭ¤","ĠSpring field","ĠKl aus","ĠManufact urers","Ġнеп оÑĤпÑĥ","Ġشر کت","è¿Ļ å°ıåŃIJ","Ġbl anks","Ġcho res","ga ard","ä»ĺ åĩºçļĦ","Ġinform asi","åĸĬ çĿĢ","Ġдиа меÑĤ","ĠF ault","ç¾ ¹","Ġrad iant","ĠPer kins","Ġpra v","Ġθ α","à¹Ģà¸Ĭ ืà¹īà¸Ń","ĠDiet ary","å¯ µ","Ġpartic ulier","ĠÙģ ÙĪ","ĠÙĨ ÙĪØ±","ĠWill ie","ÑĤив нÑĭе","æĶ¿åįı å§Ķåijĺ","a illes","ĠE lla","ĠG ong","Ġqu ais","Ġпо ÑıвлÑı","Ġplant a","ĠW M","Ġk onse","ĠG amma","Ġsh aken","Ġdel im","med icine","Ġorig en","æĺ¾ç¤º äºĨ","ĠDev on","é£İæł¼ çļĦ","мож но","ĠGab ri","ĠÑĤеÑĢÑĢи ÑĤоÑĢии","J am","t ok","ĠS AN","ĠCoord inate","åĤĢ åĦ¡","á Ĩ","im etric","è§ ij","Ġapp elle","ä¸ŃçļĦ ä½ľç͍","Ġпо ÑģÑĤ","Ġorigin ates","å¹¾ 天","ìĨ ¡","æĹłçº¿ ç͵","çĦļ çĥ§","ĠP ang","ç͍ ä»Ģä¹Ī","ĠX avier","ма Ñı","à¸Ħร ู","Ġдома ÑĪ","ĠÒ »","ĠC alled","ห à¹Į","umer ator","ĠMart ÃŃ","Ġcoast line","à³Ĩ ಯ","Ġw att","ä¸Ģ çŃī","å®ī åİ¿","-f inal","ì§Ģ ëĬĶ","ç»ıåħ¸ çļĦ","Ġreag ents","f ixed","ĠV iolet","第åįģ åħŃ","gener ate","Obs erver","ĠWinds or","æį ħ","In ventory","æĤ ĸ","Ġlist e","Ġnumer ically","è¹ ¤","ĠMain taining","Ġexcess ively","he ly","åĴĮ çłĶç©¶","Ġpl anner","ä¹Ł èĥ½å¤Ł","è¿Ľ ä¿®","Ġ' \"","ĠRe ver","ä v","lex ia","Ġak hir","Ġqual ité","= r","ĠR aff","Ġ\" ))Ċ","Ġpol ish","iet ies","Ġwonder fully","Ġdry er","appro ach","ÑĢемен но","åĿļå®ļ ä¸įç§»","Ġindef inite","D X","Ġon board","åĩº åĬ¨","æºIJ æĢ§","åij¨ åħŃ","ĠاÙĦسر عÙĩ","ĠCOND ITIONS","å°± è§īå¾Ĺ","è° ´","اÙĦ ÙĬد","éĢģ ä½ł","Ġvel mi","Ġdiff us","Spring er","tanler ia","il ine","ĠL IFE","ĠMin im","ä¸įåı¯ æĪĸ缺","ĠKen ny","à¹Ģà¸Ĥ า","Ġcultiv ars","ĠKN OW","Ġap ós","}} }\\","Ġpie z","åĪĽéĢł åĬĽ","ĠCS R","ĠML B","Ġسر عÙĩ","Ġupl ift","fl utter","å¿« æ¨Ĥ","Ġaprendiz aje",". cloud","] ):Ċ","m ak","注 缮","æ²§ æ¡ij","Ġm asc","ĠIn k","com ments","æłij ä¸ĭ","Ġtut ors","-k ind","Const raint","ĠA O","ig ms","Ġг Ñĥ","é¹ Ĭ","Ġmaster ed","è®¤çľŁ åŃ¦ä¹ł","åįģä¸Ģ 竳","Ġbetray ed","Ġzitu en","G EN","å¹¶ 举","ĠاÙĦÙħ دار","Ġpret ending","ĠHomes chool","H indi","Q t","æĥ Ń","Ġz ir","åıĪ å¼Ģå§ĭ","Ú© ز","ار ض","å·¥ç¨ĭ éĩı","çļĦäºĭ åĦ¿","ĠBook marks","×ķ׾ ×Ļ","Ġdil ute","Ġadvis ers","å®° 缸","飧 带","Ġparal ysis","Ġaggress ively","im il","åľ° 毯","主 å¹²","Ġext rac","åĨľ ä½ľçī©","æ±Ł æ²³","Ġدر ÛĮ","å¡« è¡¥","çĤ« èĢĢ","Imp act","er View","ĠT x","to chrome","ĠRec ording","æĪij åį´","æľĢ éĢĤåIJĪ","Ġexpl ica","çļĦä¸Ģ æł·","×ij ×Ļ×Ŀ","Ġearn s","åħ¨æĹ¥ åζ","ä¸Ģ å±Ĭ","Ġbel le","Ġlo in","ĠMer cy","çľĭåIJij äºĨ","ĠEC G","ç»Łæ²» èĢħ","N am","æĻ Ĺ","缴 è¨Ģ","éĢĻ å°±æĺ¯","ament al","Ġglac iers","h w","ĠB onds","ĠG ert","æŃ» 人","å¿§ èĻij","Ġkont s",") ...","Z Y","Ê Į","ort une","æľĢ ç¾İçļĦ","ĠEnter prises","ĠWhit ney","ĠREP ORT","l od","Ġv ere","com par","åįı åĬĽ","Ġyoung sters","æĶ¿åºľ åĴĮ","ĠDec isions","at ok","Ġc orso","Ġf ollower","ĠC umm","ĠL icht","ort ality","Ġsh ipment","ident e","-f rom","Ġcr ashing","ĠС к","æ³¢ æ¾ľ","iot ensin","çļĦåĨħ æ¶µ","ĠAbd ullah","Ġbip art","Ġاص ÙĦÛĮ","ĠT us","ĠH ume","erm a","å¹¶ çͱ","æķĻåѦ 设计","Ġбли з","коном Ñģки","\\ neq","Ġس Ù¾","产åĵģ åĴĮ","æľ¨ 头","âĹ ¦","à¹ģละ à¸ģาร","u é","Ċ ĠĊ","ĠE f","., ĊĊ","Ġpres criptions","ĠÑģп ÑĢа","Ġposit ives","ĠGro ÃŁ","N y","çļĦ 产çĶŁ","ĠRe levant","\\) _","ä¿¡æģ¯ åĴĮ","Ġìł ij","Work er","Ġto en","ĠR ender","å°ı 鼨","ĠExt rem","ç¶² ç«Ļ","advant ages","æłĩ æĿĨ","ĠOr b","inc are","ĠBe v","ãĤ¸ ãĤ§","Ġmask ed","ĠÙĦج رÙħ","Ġf ringe","ĠD rosophila","Ġind ist","Ġcolon ists","åħij çݰ","æİ¡ ç͍","ĠNatal ie","Å ģ","Ġj aren","ĠU ma","æİ °","Ġsk ate","æŃ¥ æŃ¥","ĠPre valence","Ġforg iven",", ...ĊĊ","j c","éĿ¢ ç©į","ĠQu ote","ara an","æįŁ çĽĬ","æ©Ł åύ","OB JECT","人 å®¶çļĦ","Ġha w","ĠاÙĦت Ø´","िठ®","鸡 èĤī","ĠкÑĢа й","ĠÑģек Ñĥн","prob ably","ĠÑĺÑĥ нÑĥ","Q M","س ات","äºĨä¸Ģ å¥Ĺ","模 çī¹","-d ess","Ġsocial ization","Ġка ÑĤего","IV ER",".L abel","Ġnos otros","Ġbiom arker","ä¸Ģ è¾Ī","äºĮ 代","äºĨä¸Ģ 段","η μ","å°¿ éģĵ","ä¸Ģ åħ±æľī","erv a","ç¢İ äºĨ","ĠSubl unar","G MT","ve e","ĠV intage","æ³ķ å®Ŀ","اÙĨ ا","ĠÔ µ","stre et","/ sec","A round","[ _","Ù Ķ","åı¯èĥ½ åľ¨","åİĭ æł¹","Ġevent o","Ġâ̦ ,","Ġoccup ying","主ä½ĵ 责任","Ġ×ĸ ×IJת","éĨ« çĻĤ","ĠBroadcast ing","\\ gamma","f ait","m oney","Ġp ardon","Ġforest ry","ÈĽ ie","ĠCarm en","w ir","Ġm anganese","ĠG ear","Ġr er","ĠPro posal","az or","æľįåĬ¡ äºİ","ĠIm mediately","Ġgym n","æĪIJ åĵ¡","æīĢ ä¸º","Ġت ÙĪØ³","(s ys","ĠIN V","Ġalt ijd","å®ī å¸Ĥ","ĠÏĦ Ïį","åĢŁ çĿĢ","个æľĪ çļĦ","æīĢè¿° çļĦ","ĠколиÑĩе ÑģÑĤва","alde hyde","Ġindef initely","J osh","\\ Component","ĠD oyle","ĠÙĪ Ø§","Ġmodern ity","äºĮåįģ åħ«","Ġmes mer","Ur ls","ĠVo IP","ë²Ī íĺ¸","ãģĦ ãģĨ","ĠÑĦ Ñĥн","éļľ å®³","表çݰ å¾Ĺ","æĸ½å·¥ çİ°åľº","被害 人","¥ 幸","Ġd iver","ĠL er","л об","ĠSt yles","ä½ł åĸľæ¬¢","éĩį æ¸©","ĠTR ANS","äng er","Ġunre liable","éĿĻéĿĻ åľ°","ĠS alam","ri ere","åĬ¨ 人","uj emy","åļ£ å¼ł","H IP","T emperature","Ġper plex","ĠUr b","ĠKil ograms","ç·© ç·©","d bo","Ġcom enz","Ġat rib","rou se","Ġro pes","马 çļĦ","Ġgre edy","ĠиндивидÑĥ алÑĮ","ix els","ĠAss oc","ett ed","为äºĨ ä¿Ŀè¯ģ","Ġnu ovo","à¸ķร à¸ĩ","ન à«ĩ","/ blog","åĩĿ ç»ĵ","ĠLen in","Ġp uff","ch ap","ä¸į è¿ľå¤Ħ","èµ° è¿ĩåİ»","Ġrecurs ion","Ġts unami","Ġwen iger","ĠHern andez","Ġا Ùħر","ĠWh ilst","è¡Ģ èĤī","é£Ł çī©çļĦ","å®ı ä¼Ł","å£ ĩ","Ġsc issors","ç¼ ī","ĠEng el","Ïĥ αν","ĠÙĩ ÙĦ","èIJĮ èĬ½","Ġsour cing","' }","imes ter","éħį åζ","çł´ æįŁ",".F ore","Fig ures","hand lung","ĠAR M","åŁİéķĩ åĮĸ","Ġг одÑĭ","áĢ ¸áĢ","åİĭ åĬĽçļĦ","ë¡ľ ìļ´","know ledge","Ġreper c","ë Ī","ĠF inger","为 æĮĩ导","å®ļ å±ħ","èĪ µ","æ· ©","ä¿Ŀ 湿","å¿« æŃ¥","ä¼ģä¸ļ æĸĩåĮĸ","ĠPer th","æ±ī åŃIJ","åĩ¹ éĻ·","Ġn ib","Ġcon ferred","ĠB N","人 éĢł","Ġsl ate","ĠVisual Fractions","g ray","ż a","ĠMult imedia","ãģĬ ãĤĪ","å½ĵ çĿĢ","çļĦä¸Ģ åı¥è¯Ŀ","é§ Ľ","Ġtrat amiento",". controller","Ġty rosine","Ġми нÑĥÑĤ","ĠÙħجÙħÙĪ Ø¹Ùĩ","re ten","Ġs ings","Ġinvestig ative","ãĥĭ ãĥ¥","( <","Ġd ared","Ġth á»ĥ","Ġle uc","Ùģ ÙĪ","ä¾Ľ æ±Ĥ","Ġsem if","Ġtem as","ä¿® è¡¥","ĠEduc ación","ĠQuestion naire","ç§ī æī¿","Ġdeut schen","ert as","æĹ¥ è¯Ń","ĠÑ ļ","åĢį çļĦ","imb ing","å£ģ åŀĴ","å®īè£ħ åľ¨","á¿Ĩ ÏĤ","为 大","åī ĥ","Ġtri ang","اب ÙĤ","é½ Ĵ","æĢĢ ä¸Ń","èµĦæľ¬ çļĦ","總 æĺ¯","Ġlicha am","วิà¸Ī ัย","Ġd ucks","Ġan h","管 å§Ķä¼ļ","Ġelect rom","ĠпоÑĤ ом","Ġzá klad","Ġcél ulas","æľ¯ åīį","Ġcertain es","ĠAct ing","ൠ½","ĠÑĤо ÑĤ","Ġphenomen al","Ġcum pl","åĴĮ å¤ĦçIJĨ","++ ]","ĠChe cks","Ġinternational e","ĠSam pling","Ġpubl ik","è³¼ è²·","ĠAlger ia","ĉ name","Ġl ute","çŃ ł","Ġع ب","+ t","Ġcan non","éĤ£ 樣","è de","Ġemb odies","Ġ×ķ× Ĵ","Ġpag an","çε 士","_ as","c opyright","Ġd á","le an","åĴĮ ç»Ħç»ĩ","Ġint olerance","åīį ä¸ĸ","ĠÙĪ Ùħع","ism an","Ġwa its","Ġar id","ç¸ «","ä¸ĸ纪 åĪĿ","ĠT GF","Ġv yt","åı¯ æĥ³","她 å·²ç»ı","Ġx en","án ak","kl ich","ĠBuild ings","Az ure","ĠвопÑĢоÑģ Ñĭ","åĽ½ ä¹ĭ","à¹Ģà¸Ĥ ียà¸Ļ","æ°´ åŁŁ","ä»Ģä¹Ī éĥ½ä¸į","à¸ķ à¹ī","åı¦ è¡Į","Le on","ĠC atch","ver o","ä½ł 說","Ġна ÑĪей","çļĦä¸Ģ å¹ķ","Ġexc uses","æīİ æł¹","æĺĨ ä»ij","Ġcalm ly","-Europe an","- cur","/ react","m ad","åħ¨ éķ¿","æĦı 象","å¤ĩ 注","Ġsuper st","ĠMet ab","Dec ision","ĠNeg ot","Ġthigh s","ç͵ èĥ½","æ¸ħ äºĨ","è¡Ģ 红","ห à¸į","èĻļ æĹł","ĠAdd ressing","Ġkn it","ç¼ĺ åĪĨ","Hist orical","ĠD aisy","th anks","æ°´ è§£","ĠBer uf","ĠкÑĥлÑĮ ÑĤÑĥÑĢ","m ethyl","r ÄĻ","} .Ċ","两 éĿ¢","èĢģ æĺ¯","Ġfunction ally","ĠMe ch","ĠPeriod ic","Ġprzed staw","ĠLuxemb ourg","u ation","ĠB its","ise x","ER ENCE","æ³¢ æĸ¯","ĠìĥĿ ìĦ±","Ġglac ier","O Y","Ġdisc ourses","ĠPack et","ĠCarb ohyd","ĠT U","ĠB RA","åĴĮ æĹ¶éĹ´","ĠJ est","de w","ë¡ Ģ","èµĦæĸĻ çļĦ","à®° à¯įà®ķ","ĠÄĮ esk","æĨ§ æĨ¬","R oyal","åľ° åIJį","æķĻ åĬ¡","è¦ı åīĩ","ĠпÑĢодÑĥк ÑĨии","çļĦ çĶŁ","Ġtr ên","-f in","_c ase","Ĺ× ©×ij","ĠÅ¡ kol","Ġpredecess ors","ä¸Ń éĸĵ","Ġ\\ {","è§£ 説","é£ŀ å¿«","Ġpoly meric","Ġenhance ments","ா஠¯","Ġreject s","ĠмеÑĤ оди","F ilm","Ġinstit uted","unc her","رÙĬÙĥ ا","ur ized","å¾Ĺ ä¸Ģ","èĭ ĩ","_s ource","èĤ¾ åĬŁèĥ½","-Ver lag","S pecific","] $","Ġhad e","æī¾ ä½ł","ĠVer ify","Ġdifer ente","ä»ĸ è§īå¾Ĺ","ĠK E","éĥ½ å±ŀäºİ","ä¸ĩ 亿","å¾· åįİ","ĠÐŁ Ñĥ","è¿Ľä¸ĢæŃ¥ æıIJåįĩ","Ġsel enium","èį¡ èį¡","ÛĮØ´ Ùĩ","伪 è£ħ","èħ¦ è¢ĭ","ĠC andidates","Ġle k","åı¯ éĢī","Ġwork flows","åĨľ çī§","è´¢ åĬĽ","ĠDep th","ĠØ® دÙħ","æī© åħħ","éĺĪ å̼","Ġsh akes","Ġcr ater","强è°ĥ äºĨ","×ŀ ×Ļ×Ŀ","Bin omial","u che","ç³ §","æİĴ 骨","æŃ» æŃ»","åĸĿ éģĵ","ĠArab ian","Ġextraord in","Ġativid ades","Ġs itt","æĺ¾çĦ¶ æĺ¯","Ġannot ated","ĠìĦ¤ ëªħ","Ġfor ge","Ġ# [","åĬŀ 好","Ġ×Ķ× ĸ×Ķ","æ¤į æłij","×ķצ ×IJ","æĬĹæĹ¥ æĪĺäºī","W ie","v p","ag li","人 æķĻçīĪ","ĠQ B","ز ا","G reek","Ġd st","Ġdes e","宫 å»·","IR S","-ind ust","ĠÑĤÑĭ Ñģ","å°ı èĬ±","Ġ×ŀ× ĺ","place holder","å®Ī åį«","OT H","çij ª","ĠST EP","ç³ĸ æŀľ",".de bug","åħ³èĬĤ çĤİ","Ġpresum ption","Ġlingk ungan","å¤ļ æĥ³","æ¯Ķ ä¸Ĭå¹´","Ġed o","é£İéĻ© çļĦ","Ġsed ent","Ġкла ÑģÑģа","ĠVern on","ä¸İ éĿŀ","Ġcommun ion","Ġchlor ophyll","T ables","Ġw s","if iques","olog ne","ĠAd emás","à¸Ī ัà¸ģ",".set State","æŃ»äº¡ çļĦ","Ġobstruct ive","} .ĊĊ","Ġdis connect","æµ· æ£ł","çĬ¶ çļĦ","责任 人","akt ion","è¿Ł è¿Ł","ĠSid ney","ĠpÅĻ es","çªģåıij äºĭä»¶","éŁŃ èıľ","æĺ¯ 天","ĠR W","æ¶² æĻ¶","çĮ® è¡Ģ","Ġúlt ima","i hat","ro tic","ĠD ram","ĠCom edy","å¤į æĹ¦","Ġgi ov","Gu ide","' ag","/ ad","æ¼ ©","å¤į åı¤","à¹ģ ม","åħĴ ç«¥","çѹ èµĦ","od zi","In clude","(\" -","Ġele tt","éĿĻ ç͵","æĢĿæĥ³ ä¸Ĭ","ĠÐŀ на","Ġadm itting","辨 è¯ģ","jar ah","ä¹Ł åĽłæŃ¤","ach ie","æ°´ æ±ł","AL K","Ġа пÑĢе","Ġsection al","Ġwa ard","Ġepidem iological","ä¸Ģ åĩ»","sh arp","éĥ½æĺ¯ çͱ","ĠпÑĢи ме","åįĸ æĸ¹","IR A","å·§ åIJĪ","帶 ä¾Ĩ","ĠHaw k","è±IJ å¯Į","æĶ¹ æĪIJ","Ġб Ñİ","Ġkom on","ĠF IL","cl iffe","代表 æĢ§çļĦ","Ġminim izes","è½ī 身","çĽ¸ä¼¼ çļĦ","Ġprzypad ku","v ell","åıĪ å¦Ĥä½ķ","à´ ķ",")= >","Ġuns atisf","Ġtrad icional","à§Ĥ রà§įণ","å¼Ĭ 端","ĉ bool","ĠC EST","ĠV ij","æ°ij çľ¾","Ġпа м","ĠInfect ious","Ġingl és","J ane","ĠS aving","åľ° é»Ħ","èĢĮ 没æľī","æķĻ å®ĺ","计 çĶŁ","è¿IJ åĬ¿","åīª è¾ij","(T reeNode","èł »","Const ruction","ĠÑĪкол Ñĭ","-St ar","Ġcomm its","ਠ¼","æĿĤ èįī","žÃŃ vá","çļĦ åĬ¨åĬĽ","ĠB enn","лÑı Ñħ","éħ¸ çĽIJ","ĠÐĺ ва","ãģĪ ãģŁ","ĠShir ley","C ra","ĠK atz","ز اÙħ","ĠEd iting","ਠ¹","Ġlect urer","æ»ĭ åħ»","Ġসম à§Ł","ĠF us","ç»´ å°Ķ","اب د","åĪº åı²","Ġ×ij ×Ļ×ķתר","å®ļä¹ī çļĦ","Ġmand ates","æĶ¾å¤§ åύ","v f","çľĭ å®Ī","ĠMay er","Ġblood stream","Tr ump","ĠExt ract","Ġbetray al","b ots","k ot","Ġp ensions","ä¸į åħ·å¤ĩ","æĿ¥ å®ĮæĪIJ","ord re","å°ı é»ij","她 æīį","æĺ¯ä¸Ģ 座","enc oded","ĠInter val","åĬ£ åĬ¿","Ġremed iation","ĠMull er","w g","ĉĉ Ġ","iel i","ç²¾ ç¾İ","æĶ¯ è¡Į","Ġtal ags","çļĦ主è¦ģ åİŁåĽł","Ġmotiv ational","Ġmund ial","or gen","ï¼ £","ĠS nyder","è¯į çļĦ","ĠConf igure","ä¸ĵåĪ© æĿĥ","ĠbÄĻd Äħ","åĴĮ åŃ¦ä¹ł","åĽ½ èµĦ","Ġte e","Ġtw isting","ni u","Ġê²ĥ ìľ¼ë¡ľ","ĠTalk ing","b ear","ĠC yp","说 èµ·æĿ¥","rac use","æľĽ è¿ľ","éł Ĺ","ç´§ äºĨ","Ġestab a","Ġpas ado","ĠìĿ´ íķ´","ä¸ĭä¸Ģ åĪ»","ีà¹ī ย","æĺŁæľŁ äºĶ","Ġcurs ed","å¤ī åĮĸ","d ies","åľ¨ æĪij们çļĦ","éĤ£ 次","æľª æĪIJå¹´","Ġ! Ċ","Un its","ç¯ĩ å¹ħ",".B ase","æ·±åħ¥ çļĦ","ĠMah m","Prom ise","ag her","ر Ø®","æĹ¥ æ¸IJ","å±ķ å¼ĢäºĨ","Ġlong ue","æ¶² ä¸Ń","ظ ÙĬÙħ","ĠÚ¯ ÛĮر","å¯ĨåĪĩ 缸åħ³","ĠпÑĢогÑĢам мÑĭ","J ones","Ġre inst","Ġun inter","çŃī è¿Ľè¡Į","åĬł æĮģ","æģ º","Ġche ating","ĠÙĤ ÙĦ","å°ıæľĭåıĭ 们","[H entet","_ if","ĠB ai","å¤ĸ 伤","Ġmat riz","âĪ Ģ","ema akt","Ġtan ah","aps ing","Ġبر Ú¯","Ġafford ability","alm az","ic l","人 人éĥ½","pr iv","å±ķ éĸĭ","Ġíķ ©","Ġmisunder stand",": I","åľ¨ 第ä¸Ģ","åĬł çĤ¹","åIJĦ æĿij","é¢Ħ è¨Ģ","Ġbapt ized","pr és","åŁİå¸Ĥ 建设","èģļ åIJĪçī©","éī Ħ","ĠлÑİ Ð±Ð¸","Ġoutwe igh","ä»ĸ 表示","å¤ļ æľī","az en","Ġsoft ened","kov á","Ġíħ Į","s id","Ġd är","ĠL IN","Ġا غ","ov ina","èĩªå·± æīĢ","ran i","Ġmem oria","ä¸ĩ å¤ļ","Ġground ing","Ġstreng thens","Ġinsp ires","大å°ı å§IJ","st ates","Ġem ph","ih a","ÙĨد ا","Ġtend erness","aten ess","人éĻħ åħ³ç³»","ĠPly mouth","Ġtalags aon","R untime","æīĭ 游","æµģ 泪","ÑĤе л","æĶ¾ æ£Ħ","社ä¼ļ åĮĸ","ĠPer ception","ĠØ´ Ú©ÙĦ","Ġmů ž","Ġcôt é","Ġl uk","Ġper ish","ãģ® ãģł","bo a","ur se","å¹´ ãģ«","ĠUn ified","Ġcost itu","èĭ¦ æģ¼","Ġdro its","Ġign ores","Ġrational ity","ĠÙĪÙĩ ذا","ĠÑĦÑĥнк ÑĨиÑı","Ġsid lakan","ĠRah men","Ġseaw ater","- rated","; a","Ġf ury","Ġcommon place","Ñĩи Ñģли","ĠCir culation","ae us","⣠¨","ä¼ļ å°Ĩ","ary l","ĠÑģ веÑĢ","å¸Ĥ ä¸Ńå¿ĥ","ãĢĭ âĢľ","Ġmon oxide","CH A","С и","ĠBi as","Úĺ ÙĪÙĩ","Ġ×Ļ׼ ×ķ׾","ĠÑĢоÑĴ ено","/ image","h ya","Ġm ansion","Ġhyper bolic","Ġà´ µ","Ġhurd les","ĠC yr","èİ· èĥľ","ĠìĿ´ ë¦Ħ","-res ponse","ĠвоÑģ па","A le","F H","] ];Ċ","ĉ j","Ġv ont","ĠØ Ł","-g radient","Ġswe ating","Ġmuit as","Ġpent ru","Ġваж но","ĠHe ide","å¤ĸ åĬł","Ġcar otid","âĪ ©","çĥŃ çĥĪçļĦ","ä¹Ł æľī人","ĠÑģ мÑĭÑģ","两 çľ¼","-s eries","ä½İ ä½į","红 èĬ±","ди ÑĤÑĮ","ĠPost er","à¹Ģà¸Ĭ ืà¹Īà¸Ń","T ow","г Ñĸ","讲 äºĨ","æĶ» æīĵ","Ġpurs uits","Ġnob ility",") )ĊĊĊ","Ġse colo","Ġcan als","ĠDes ktop","å½ķ ç͍","åī§ åľº","Ġphen otypic","check box","Fe ed","è°¦ èĻļ","Evalu ation",": P","l bs","Ġth ì","ĠR ide","太 å®Ĺ","Ġhum ming","è nes","mark t","çıį è´µçļĦ","μα ÏĦοÏĤ","él ior","Ġtrav ellers","å®´ ä¼ļ","{( }\\","ĠvÃŃ ce","ĠP CA","س Ø·",".F irst","ĠпÑĢе обÑĢаз","-bl ind","ĠCarm ichael","ĠÑĢели ги","Ġin sc","ct l","um bo","åľ¨ è·¯ä¸Ĭ","éª ·","çĽij å§Ķ","å°½ åħ¨åĬĽ","Ġface book","åįģä¸Ģ æľĪ","æ£ķ èī²","k J","ĠW aves","б ÑĢÑĮ","çģ¯ çģ«","ĠTim er","Ġaffid avit","} u","Ġc reek","声 ä¸Ń","ล าย","ĠUS S","ĠSm ooth","EP T","asm us","Ġdis cret","ãģ« ãģ¨","v oll","Î ¨","åĮ £","âĪ ĺ","ĠFront ier","çµĮ æ¸Ī","Ġt ighter","on ate","åľ¨ åĽ½å®¶","èĢĥ éĩı","æ·± å±Ĥ","æľ¨ è´¨","/ users","è¿ĺ ä¸įçŁ¥éģĵ","Ġam el","åħ¨éĿ¢ æİ¨è¿Ľ","Ġt ête","çļĦ æĹ¶åĪ»","Ġr ins","åĴĮ ç²¾ç¥ŀ","åºĶ 交","è¢ĭ åŃIJ","L atin","if l","ä½ł å¿ħé¡»","Ġret our","çļĦä¸Ģ å®¶","æ½ į","æĬ½ æIJIJ","Ġbomb ard","äºĶ å®ĺ","Ġok re","ç»Łè®¡ åѦ","Ġdesert ed","ow anych","äºĨ æĮĩ","Ġim print","åħ¨ éķĩ","[ node","Ġh ic","ä¸į äºĨè§£","éĹ Ĩ","Ġcl iffs","ç©¿ çļĦ","Ġsecret ed","Ġtamb é","à´¤ àµįà´¤","+ D","Ġd essa","çļĦ è¯Ńæ°Ķ","ĠB ram","Ġhas ht","ä½Ĩ å®ŀéĻħä¸Ĭ","ĠEng els","Ġbi olog","Ġsa x","å¿ĥ éĩĮçļĦ","åºĶ ä¸İ","åĨį ä¸ī","ÑĤи м","ĠOr din","ĠRa um","W ARE","m our","çļĦ è¡ĮåĬ¨","Ġas phalt","Ġinst ru","æĶ¾ çĿĢ","ĠRep ública","_s plit","å¸ĮæľĽ èĥ½å¤Ł","Ġmel odies","ä¸į太 好","ŀצ ×IJ","n ova","he mer","åŃĹ å½¢","Ġد ÙĦ","Com pat","åıijæĮ¥ ä½ľç͍","åºĶæĢ¥ é¢Ħæ¡Ī","c rum","Ġre claim","Ġse ab","Ġré fé","åħ³ å¿ĥçļĦ","ä hl","ä¾Ĩ çľĭ","ĠPlan ck","Ġgeb en","èµ· é£ŀ","Ġcal cular","Ġref eree","æĭ¿ åΰäºĨ","èĤī ç±»","Ġα á½IJ","硬 å¸ģ",".R un","æĭĸ åĬ¨","ĠStaff ord","ĠPok emon","/ Al","Ô ±","ç͍ è¯Ń","ĠCan berra","çĿ¡ åīį","Act s","è¡Ģæ¶² 循çݯ","åīµ æĸ°","_ words","æŁ¥ çľĭäºĨ","apt ure","IS P","æĹħ éģĬ","Ġwra ps","é o","å¼Ģ æĮĸ","æ¨ Ł","Ġgl are","èŀį åĮĸ","Ġmass acre","ĠKing ston","ç¼ł ç»ķ","æĶ ¥","èĩª çŁ¥","å¾Ĺ 失","Ġfin an","ä¸įæĺ¯ 说","éĢĴ ç»Ļ","ãĤı ãģij","F Y","Ġgra cious","缼 ä¸ĸ","æij¸ æij¸","ubb ing","çµ± è¨Ī","ĠNumer ous","ÙĨت اج","Ġcater pill","as ch","å°± è¿ij","æĹł é¡»","书 åĮħ","åįĥ çĵ¦","OT A","Ġesc ort","çݰå®ŀ ä¸Ń","ิà¸ļ ัà¸ķิ","åIJ Ń","rom pt","对 åIJ§","罪 åIJį","åĪĬ çĻ»","ä¸į对 åĬ²","[ f","åıij æĬĸ","Ġapp ellate","以ä¸ĭ åĩłçĤ¹","âij ¥","ĠUN IX","ĠMess enger","F DA","åĩº ä¸į","Ġche at","Ġ×ķ ×ij","ãĤ¸ ãĥ£","= S","п ом","表çݰ çļĦ","ĠAff ordable","ode a","׾ ×ij","ä¿® çħī","Ġrecept ive","\" Is","i ab","Ġqu arts","Ġsub string","Ġheart felt","äºĮåıī æłij","ĠT un","am ong","éĩ ľ","æľ¬ æĢ§","湿 çĥŃ","×¢ ×ķת","Ġb akter","ow iÄħ","ĠâĢ »","对 æĪij说","ĠZ ip","Ġelect ive","åħ« 大","Ġsound track","Ġhybrid s","Ġmad re","ĠPhill ip","Ġconced ed","Ġcorp se","h ay","Ġপ à§ģর","ĠDay ton","æ³ī å·ŀ","Ġëĭ¤ ìĸij","溢 åĩº","Const raints","Ġméd ico","ĠÑĢиÑģ Ñĥн","Ġlia ison","ĠResil ience","ĠW almart","åı· ç§°","Man ufact","åĽ½åĨħ çļĦ","ĠУ кÑĢа","æįķ èİ·","æĦ§ çĸļ","Sil ver","qu iv","ok al","ĠPro z","ET F","omy cin","éķ· èĢģ","( color","f ed","è¦ģ 好","Ġstr ata","Ġreal t","ä¸ĥ çϾ","âī ¡","ou les","ĠC unningham","н ож","ĠZ eb","åįİ ä¸Ń","è¿Ļæĺ¯ 个","Ġcapac itors","Ùħا Ùĭ","è¦ĭ éģİ",".F ont","å¥ĭ åıij","Ñij ÑĢ","ĠÙħت ÙĨ","ĠProdu cer","çļĦ 樣åŃIJ","è¿Ļ åı¯","ĠQu otes","à¸ŀ à¸Ń","æĺ ±","è°ĥ åīĤ","Ġboot strap","P Q","l ion","çļĦ åĮºåŁŁ","è¦ģ 让","è£ħ åħ¥","Ġphen yl","ä¸į 带","Ġex its","ĠØ£ بÙĪ","-M e","èĢ ĺ","Ġch ia","ert os","åħī æłĩ","ĠÙħÙĨ ذ",".A b","æµĵ åİļçļĦ","Ġoxid ized","Ġz org","é£Ł çĽIJ","æī¾ ä¸Ģ个","çĦ¶åIJİ æĬĬ","N u","ĠT rem","åľ¨ ä¸ĬéĿ¢","éĢļ åijĬ","ä½Ĩ æľī","еÑĢ ÑĤ","æĸĹ å¿Ĺ","Ġmemb res","ç¼Ķ 约","ĠHosp itals","Ġunder lined","áĢ ·","arl ow","_d im","çĶŁåij½ åĬĽ","Ġsmooth ing","ĠArab idopsis","s olution","Ġout lining","æıIJé«ĺ åΰ","é² ¨","罪 æģ¶","Ġphon etic","Ġure a","åıij åŀĭ","ual i","éĤ£ 段","Ġpos ing","St ruct","è¯Ĺ åı¥","Reg istry","ibil idade","ĠP VC","ib it","Ġacc ents","æŃ£ å¤Ħäºİ","ç¦ı çī¹","åĢĴ åľ¨åľ°","urg ence","och t","ç»ı常 ä¼ļ","inher it","W ik","Ġ* .","éĤ£ åıĮ","ax i","Ġvol leyball","Ġen amel","åłĤ åłĤ","Ġcommunic ates","Ġveloc idad","-d ark","Ġfront s","ĠStarb ucks","åįģ ä¸ĢæĿ¡","è·Ł è¿Ľ","æ²³ è¾¹","ĠÑģÑĤ ÑĢок","ĠEmb assy","Ġhippoc ampus","U i","in em","ub ation","Ġpos itivity","-h idden","Ġmemor ize","Ġtodd lers","ĠO sw","ಠ¬","è¿ŀ åIJĮ","éĢĤ ç͍çļĦ","室 温","lev ance","_p arent","è¦ı åĬĥ","ãĥĹ ãĥª","ãģ«å¯¾ ãģĹãģ¦","em arks","Ġar be","åĮĹ æŀģ","Ġconv ict",".n ih","çģĮ æľ¨","缸 çŃīçļĦ","Ġpo ziom","fl age","å±± åĿ¡","å¢ŀ æ®ĸ","ĠÙĬ ص","æŃ İ","èĬĤ æ°Ķ","ĠCas ino","Ġstead fast","Ġرس ÙĪÙĦ","Ġsout heastern","F etch","ĠC ement","ĠP ension","ĠF G","Ġgu ild","å®Ŀ èĹı","log ram","hav en","Ġs inks","ä¸Ģ è¯ķ","ĠB ytes","æĺ¥ 天çļĦ","æĢ¥ äºĨ","Ġpet ty","ĠоÑĤно ÑĪениÑı","Ġarsen ic","st im","Ġst roll","qu ares","å¹¶ å°Ĩåħ¶","urs ions","æī¹ å¤į","ĠTra cy","ĠRub in","elect ronic","Ġfor ts","Pro jects","ĠBe ethoven","ç¿ Į","}{ *","Ġexplo its","微微 ä¸Ģ","æ£Ģå¯Ł å®ĺ","} A","Ġh inter","ä½ ļ","ĠP W","å·¥ä½ľ æĹ¥","æł¡ å¤ĸ","ĠÑĥ Ñĩен","sk u","С о","à¹Ģà¸Ķ ิà¸Ļ","(* )","ĠAnders en","- api","ï ¸°","Ġre cycle","åŁºæľ¬ åİŁåĪĻ","Ġহ তà§ĩ","Ġf ruct","æĺ¯ åŁºäºİ","Ġev it","amb o","顺 åĬ¿","rab ble","æĥ³åΰ è¿ĻéĩĮ","GR ect","Ġenlight enment","ت Ùı","è° Ľ","Ġpat ag","Ġplay wright","àµģà´ Ĥ","ir á","Ġdis lik","é¢Ħ åIJİ","Ġsuff ice","Ġett ä","Ġê· ľ","Ġeukary otic","- string","] ])","Ġun answered","×ķ× ij×ĵ","ем Ñĭй","AB S","sub section","Disc ussion","ĠKazakh stan","- add","c é","al ta","ĠÑģ ÑĢазÑĥ","Ġtrans national","Ġincre ments","Ġbast ante","ĠتارÛĮ Ø®","- position","el p","ĠK athy","ä¹Ł ä»İ","ĠAs per","å¸Ĥåľº ä»·æł¼","; \"><","Ġ ËĨ","Ġret iring","Ġм он","_c ategory","æľ¬ çļĦ","åįķ åįķ","It aly","模 樣","åIJ¬ éĹ»","Ġauthor ize","ĠEffect iveness","l auf","ch as","-t oggle","å¾· æĭī","struct ured","ĠABC D","ç¾İæľ¯ é¦Ĩ","Ġef ekt","J en","el ope","è¿Ļ ä¼ļåĦ¿","æĹ¶ éĻIJ","Ġint rus","çIJĨ çļĦ","Anal y","Ġdispers al","c Äħ","ĠW B","ä¹Ł æĮº","æĹł ä»İ","Ġâ ŀ","ãģ® ãģĬ","-st re","æīŃ çŁ©","Ġдан ной","Ġenf rent","Ġstraw berry","cart es","ĠPatri ots","j ury","() `","社ä¼ļ å®ŀè·µ","é»Ħ åľŁ","-S A","ĠMag ist","Ġdop ing","Ġmul ai","b und","é£Ł æĮĩ","æ²¹ èħ»","å®Ĺ éŨ","à¦Ĥ শ","Ġescol a","å¹»çģ¯ çīĩ","设 为","Ġме д","驾 é©Ń","Hash Map","Ġplac enta","b ys","Ġl ords","ĠS essions","ĠD inner","Ġj ars","ĠK oz","æľĢ å¿«çļĦ","-d omain","åĽłä¸º è¿Ļ个","客 æĪ¶","Ġmicro structure","rot ate","Ġm au","Ġком мÑĥ","å°±ç®Ĺ äºĨ","sf c","ĠÙħجÙħÙĪ Ø¹Ø©","v io","ä¸į éķ¿","ure t","ĠJ PL","ست ÛĮ","éĩĩ访 æĹ¶","C AS","Ġon emoc","Ġk emb","éĥ½ å·²","An th","综 è¿°","Sl ot","ĠScot ia","çķ° å¸¸","Dist rict","Ġtá» «","æķ£åıij çĿĢ",".rand int","Ġconject ure","( other","ur in","Ġint angible","åζ æĪIJçļĦ","Ġcar amel","Ġgovern ors","éĥ½æĺ¯ æľī","è¯ļ æĦı","Ġdiscipl ined","é£ĺ é£ĺ","ĠÑĤеп ло","Ġcomprend re","Ġcontag ious","Ġte il","次 ä¼ļè®®","è¿Ļç§į çݰ象","Ġpour rait","Ġurban ization","ĠClay ton","} ))","ig ator","ä¸Ģ æĹ©","Ġdo omed","غ ÙĬ","ijn en","}/ \\","æĭ¨ 款","è¯ģ 人","çĶŁäº§ æĢ»å̼","çĴ Ł","Ġcz yn","ĠPartic le","滿 è¶³","' {","ĠB ür","éĥ½ è§īå¾Ĺ","ps in","Ġent hal","æĺ¯åIJ¦ 符åIJĪ","ĠEns uring","é«ĺ äºĨ","ven cy","ĠÐļ ÑĢа","лен ной","æĭŁ åIJĪ","è½´ çļĦ","nym i","æĬijéĥģ çĹĩ","s chema","res p","_{ -","éŁ ¬","åĮĹ ä¸Ĭ","è¿Ļä¹Ī å¿«","রà§įঠ¶","ĠVik ings","¤ ×Ļ×Ŀ","Ġas i","éĢļ è¯Ŀ","Ġtrans porter","åģľ äºĨ","å°¼ å°Ķ","åŃĶ éĽĢ","Ġfu era","ä¹³èħº çĻĮ","Ġasse z","Ġarbitr arily","å°ı å··","è°ĥ éħį","大家 åľ¨","_t op","åľ°ä¸ĭ æ°´","çļĦ åħ´è¶£","ç¦ ¦","Ġprogram as","Ġlim ite","-p ound","(b ase","åijĬè¯ī 她","Ġত বà§ĩ","èĮħ åı°","åı¯ æĮī","æĶ¶ èµ·","çĬ¶ åħĥ","Ġein z","ÙĦÙĬ ات","ษ à¸IJ","Ġ×ij×ŀ× §","Ġh obbies","ä¸Ģ è§Ī","ãĢģ ãĢģ","ĠJ ian","ĠK err","Ġfin anced","ĠÐŀ ÑĢ","Ùħر اÙĩ","/w p","Ġverschied enen","Ġfl ashes","æ°ij æĦı","æĤ ¯","sk o","Ġinform ações","ĠÄij á»ĥ","Ġà®ħ வ","< >(","ant ib","ĠSt okes","æľįåĬ¡ å¹³åı°","ض Ùħ","-st im","骨 æŀ¶","Ġкажд ой","æľī åħ´è¶£","代 åı·","åIJĦ æĸ¹éĿ¢çļĦ","èĬ± æł·","ĠPe ck","ÏĮ γ","ko a","èĥ¶ åĽĬ","Ġdivers ion","Ġë¯ ¼","ĠKath leen","_ ad","pt us","ese z","Ġtherm ometer","UM BER","Ġplain ly","éĽĻ æīĭ","ĠRap ids","ĠPresbyter ian","\" Well","iv orous","ĠM oor","ri am","社ä¼ļ åıijå±ķ","ott est",".l ocal","Ġil mu","Int ent","éĺ» åĩ»","Ġsen ators","Ġoc clusion","Ġpemb elajaran","M ade","ç»Ļ å®ĥ","Ġplan eta","ĠÑģÑĤ ÑĢан","web kit","ĠTECH N",") //","Ġt aux","Ġn emat","ä»ĸ æĮĩåĩº","Ġunderstand ings","ÅĽ cia","Ġimpl anted","Ġy en","est ar","大 é»Ħ","èĬĤ åģĩæĹ¥","éĻIJ æľŁ","oph osph","String s","å¤ľ çļĦ","ĠкоÑĤоÑĢ ÑĥÑİ","-v irtual","ĠMoz amb","- One","ĠW ahl","ĠL IB","ä¸Ń 人",".g z","Ġcab o","cap ital","ĠCorn wall","Ġflux es","culos keletal","ĠпиÑĤа ниÑı","- ness","R V","Ġ ern","éĥ¨ éĥ¨éķ¿","èĤ¡ åĪ©","宣 ç§°","Ġalt ers","ä¸ĭä¸Ģ ç¯ĩ","好çľĭ çļĦ","t as","åĵģ 質","era ção","èĸ °","ador as","èµŀ åı¹","Jack son","O US","Ġn autical","Ġg eld","Ġ* ,","æķ´ é«Ķ","Ġdirect ives","è¡Į为 人","Ġдиаг ноÑģÑĤи","ł ×ķ×¢","le urs","ä¸ĭ è°ĥ","è¿ĺ 以为","æŀľ åŃIJ","ĠSh u","æĭī ä½ı","raft s","ĠDis cipline","çªĹ å¸ĺ","Ġpron unci","Ġни Ñĺе","èĩªè§ī åľ°","Ġê¸ Ģ","ĠW ish","-se lect","ĠEvery body","Ġcyt os","Middle ware","Lect ure","ä¸İ æĪij们","æĥ³ æĬĬ","ex ternal","Ġben ar","áŀ Ħ","Ġju xtap","ĠPap ua","Ġmengen ai","es ley","åĩº æ±Ĺ","Ġdi agon","Ġbacter ium","æĴ¤ 离","reib ung","ultat ua","ĠThe ories","ä¸ĭ ä¸Ģ次","AP S","Ġweb inar","angel o","Ġgam ers","Ġkonts ultatua","ä¸Ģ 审","ä¸į 代表","æ± ¶","Ġalk ali","à¸Ńยà¹Īาà¸ĩ à¹Ħร","Ġмол од","åĩº çĶŁçļĦ","ence phal","×ķ ×ķ×Ķ","ĠSe v","на ÑĢ","Ġblue print","Ġminim ally","åĪĽä¸ļ èĢħ","Ġrect angles","Ġà¸ŀ ระ","对åħ¶ è¿Ľè¡Į","ĠStra ight","ĠO mar","ĠTo ast","ä¸įæĸŃ å®ĮåĸĦ","å¤ļå°ij 人","è¨ĺ éĮĦ","Ġmarch ing","Ġcar c","çģŃ äºĨ","ĠAutom ated","Ġsuck ed","çĤ¹ 亮","Ġbi otechnology","æķĻåѦ æĸ¹æ³ķ","ĠогÑĢаниÑĩе Ñļима","大 èĴľ","ä¿Ŀ å§Ĩ","èĥľ ä»»","åģı è§ģ","---------------------------------------------------------------- --------","Ġto pping","ÏĦ ηÏĤ","è¶Ĭ éĩİ","No iz","} y","Ġt arde","ĠI ris","ual a","ãģĨ ãģ¡","éŃĶ åĬĽ","ê² ¬","æ©Ł éĹľ","/ ac","/ uploads","m il","z os","ĠØ£ رب","ص ابة","Ġdiagn ost","çģĮ 注","Ġchampions hips","çİĭ å°ı","Sp ain","Ġsoci ological","Ðĵ Ðŀ","หà¸Ļ à¹īา","- condition","ĠS ail","ĠF amiliar","好 æĦŁ","eng age","Ġsim p","à¥įठľ","Ġann um","æ®ĸæ°ij åľ°","ĠпÑĢедпÑĢиÑı ÑĤиÑı","ĠB EL","à§ĩঠ¹","éĽĨä½ĵ ç»ıæµİ","à¸Ħร ัà¸ļ","ĠPrinc ip","érie ure","ĠEthiop ian","B BC","\\ quad","Ġdem ean","åIJĥ ä¸į","į ¼","é ress","Ġgo ose","Ġgr ated","æŃ¦ æŀĹ","ç»§ èĢĮ","sm anship","ä¸įåıĺ çļĦ","ĠFle ming","oblast oma","( col","en al","Ġk asar","ip ro","éĥ½ æ¯Ķ","å®ŀ åĬĽçļĦ","Ñĩа й","ุ ษ","Õ«Õ ¯","Ġflavor ful","Ġreplic a","è¶´ åľ¨","\\ usepackage","u ins","è¿Ļ çķª","м б","æĶ¿ å§Ķ","åĬŁ è¯¾","Ġprot ested","rack et","Ġве ÑīеÑģÑĤв","Ġà´ ķ","ãĥ¡ ãĥ³ãĥĪ","ĠвоÑģ ÑģÑĤанов","Ġflags hip","' ][","æ°Ķ çIJĥ","d uring","ĠP uzzle","被 è§Ĩ为","ĠBe ast","Ġens uing","igraph ic","Ġjealous y","å®¶ åįıä¼ļ","åıĹ äºº","è¯Ħ æ¯Ķ","Ñĩа ÑĤÑĮ","楼 å¸Ĥ","åĪĽéĢł åĩº","ĠRic ardo","Ġempir ically","Ġà¦ķথ া","E PS","è¶ ¨","Ġch oses","ов Ñĭе","à´ ¸","ĠF amous","éļ ħ","ese orang","à¥ĩ श","ĠDet ective","моÑĤ ÑĢеÑĤÑĮ","éĬ· åĶ®","( all","M oh","Ġap o","/d ist","ĠGO OD","Ġornament al","åΰ åĵªéĩĮ","Ġz iek","ĠAr cher","ĠAss y","ä»»åĬ¡ æĺ¯","æĬ½ çĥŁ","æĸ°éĹ» ç½ij","p ag","Ġn ós","Ġer ano","Ġflu ent","Text Field","社ä¼ļ主ä¹ī å¸Ĥåľºç»ıæµİ","འ´","Ġnombre uses","Ġì° ½","- ent","- che","天 èī²","æŃ£ ä¸Ń","æĽ¾ ä»»","çļĦ大 åĬĽ","Ġrot ations","ĠPent agon","ко ÑģÑĤÑĮ","à¹Ģà¸Ļ ิà¸Ļ","ĠFal con","åı£ å¾Ħ","æķij äºĨ","ĠÑĦоÑĢ Ð¼Ðµ","ÑĨион нÑĭе","Ġre aff","ä¸Ģ åŃ£åº¦","ĠD SM","ang ements","Ġad verb","Ġparticip atory","Ġsegment ed","Ġpenet rating",". Update","** )","åIJĮ æĢ§","éĢļ 车","ä½Ĩ è¦ģ","äºĶ æĺ¯","Ġpost partum","Int rodu","L ET","Ġfil aments","æł¹æľ¬ å°±ä¸į","ĠFull er","åĴĮ è´¨éĩı","è¾ «","iss an","ĠÙħÙĪ Ø§ÙĦÙĬد","ĠCoch rane","ĠCard iac","ĠTrust ees","ĠRaj as","( sc",". me","ow ment","ç¥ŀ æĿ¥","ĠSc al","μ ÏĨ","user content","Ġdak ong","L OC","[ @","m alloc","Ġb ằng","ä¸į 强","ĠV B","ог е","ĠEn able","ba ik","é»ĥ éĩij","Ġмноги е","ĠspoÅĤ ecz","-respons ive","Ġat rophy","Ġle vy","çĥŁ çģ«","Ġhorm on","ç»ı纪 人","Ġmou vement","Ġbe gging","åIJĮ ä»ģ","Ġem blem","ĠSp aces","ãģ¨ ãģĭ","Ġnews letters","Ġанг лий","r ill","ä¸Ĭ è·¯","ä¹ĭ äºĮ","羣 æľī","ĠAll ergy","Ġpod s",".E vent","Ġbreath s","æģ¢å¤į æŃ£å¸¸","Ġле каÑĢ","饿 äºĨ","Ġê¸ ¸","à¸ķว à¹Į","- standard","ĠTh ou","èµ° è¿ĽäºĨ","unn able","ä¹ĺ 车","Ġreb uilt","य ा","Ġlan tern","q ing","et et","Ġre usable","æ²»çĸĹ çļĦ","æ´Ľ æĿī","ĠÚ©ÙĨ ÛĮÙħ","Ġski ing","\" --","Ġan arch","ĠD ex","ÙĪØ± ت","Un less","è§£åĨ³ çļĦéĹ®é¢ĺ","unn an","ĠNC ERT","est yle","åĴĮ åºĶç͍","ass ed","ind ers","ĠPro posed","æĦŁ è§¦","Ġdev ise","Ġà¦ķ à§ĭ","Supp lementary","ĠLiber ation","饼 å¹²","ar riage","Ġm V","Ġke hidupan","ival ence",".f ill","Ġbackground Color","交éĢļ å·¥åħ·","ãĤı ãĤĬ","á̽ áĢ","åĩº äºĭ","ile e","ĠCon centration","én ergie","기 ìĹIJ","र à¥įव","Ġwa ż","ĠSuper visor","åı¯è°ĵ æĺ¯","Õ Ń","Ġm ango","ĠV ish",".C urrent","×ŀ ×ķ","ĠH CC","äºĶ ç§į","ĠPh ar","Cl osed","ž enÃŃ","éĻį åΰ","Ġconcept ions","æľºæ¢° åĮĸ","J K","IJ ×ķת","ä½ł 没","西 æ±ī","Ġrest less","è¿ŀ 线","æĥĬ å¥ĩ","ÑĢан ениÑı","åĭ¤ åĬ³","ä»ķ äºĭ","m aps","w idget","× ł×ķ","åıĸ çļĦ","ÑĤи ва","è´§ çī©çļĦ","S anta","åĪĨ æ¯į","éĥ¨ 份","æĸĻ éħĴ","Ïĥ Ïī","Ġknock out","ĠاÙĦÙħج تÙħع","Ġgobier no","ĠC oh","æĹł 误","åĪ© 害","-d iv","çϾ å®¶","èϽ æľī","Ġде ÑģÑı","ĠíĺĦ ìŀ¬","Ġre ap","å°± å¦Ĥ","æľ¬ èµĽåŃ£","è¿Ļ个 è¿ĩç¨ĭ","ĠPer forming","ĠAlex andra","ĠاÙĦز اÙĪÙĬÙĩ","é¾ IJ","Ġarch ived","Ġcas inos","èħ° æ¤İ","dat etime","Ġconsolid ate","Ġl le","st orms","ĠF ü","æĶ¶ åħ»","ĠС ан","æ°¸ ä¸į","è®¤çľŁ åľ°","Can adian","ник ом","ĠProm pt","ĠMes opot","Ġsynthes ize","Ġsediment ary","n od","Ġev olves","åħ¥ èģĮ","Ġdef orestation","kt f","Ġing in","碳 æ°´","ç͵åĬ¨ 汽车","Ġunser er","Ġfor n","Ġst ature","åĴ İ","Ġsk ulle","åħ± èµ¢","Ø· ÙĬÙĨ","é£ŀ è·ĥ","Ġing estion","ĠSym fony","Ġay ant","áĢĶ áĢºáĢ","-tal let","S ie","nt own","åħ³ éŨ","éĩĮ è¾¹","ä¿® è¾ŀ","èµĽ éģĵ","Ġkin adul","Ġdict ated","Ġnue vas","Ġl Ỽ","ĠM ega","ĠU EFA","æĬĢæľ¯ ä¸İ","ĠRec ipes","æ¼Ĩ é»ij",". per","ĠA ST","Ġst ent","Ġfirst Name","cent os","æĢĿ æĶ¿","ä¸Ģ次 次","اع ت","Ġstar vation","Ġвоз вÑĢа","ãģŀ ãĤĮ","O ffic","ib u","åIJij 社ä¼ļ","ank ing","Ġsum med","Ġut ama","å°±ä¼ļ æľī","zer w","ĠJud ges","ĠMes a","为 æŃ£","åĢ Ķ","åIJį å®¶","â̦â̦ ãĢįĊĊ","uit en","à§Ł ার","cel ain","Ġав гÑĥ","ĠBild ung","Ġreluct ance","C ou","ĠH ick","/m od","ĠGu ill","ĠØ£ÙĨ Ùĩا","åĸ· å°Ħ","Ġpropag ate","s ense","Ġp he","æµģ æĺŁ","åħ¨ä½ĵ åħļåijĺ","åįģä¸ī 竳","Ġspark ling","r k","Ġ ãĦ","æĢ» èĥ½","è°ĥ ä¾ĥ",".N ext","ĠCard inals","ĠLouis ville","å±Ī æľį","Ġo ats","Ġr èg","ivid ade","å¢ŀ éķ·","ç u","ĠBo oth","et able","ol us","å¤Ħ 以","çĭ ŀ","åĮĹ å¹³","Am b","appro ximately","ĠÑģам ÑĭÑħ","ĠÑģÑĥÑīе ÑģÑĤвÑĥеÑĤ",". Start","> `","æĺ¯ æĹłæ³ķ","Ġmin ut","ĠLe icester","èĽ ¤","è·³ åĬ¨","åıĮæĸ¹ çļĦ","ĠEmp irical","Ġrepair ing","ová bbi","Ġw inters","ic er","çĽ §","åįģ éĩĮ","Ġdist illation","Ġword ing","çŁ³ 榴","μ ÏĮÏĤ","ãĤģ ãģŁ","Ġdar ipada","à¹Ħมà¹Ī มี","das arkan","B h","le ben","of i","é¦Ļ çĥŁ","å¢Ļ ä½ĵ","ĠPC s","ีà¹Īย à¸ĩ","ĠBatt alion","Ġcortic oster","W enn","è® ¥","ĠSt uttgart","ĠPsych iatric","Ġsel uruh","éĩį åIJ¯","ann otation","ĠباÙĦ ا","ç¾Ĭ æ¯Ľ","dig ital","= models","ĊĊ ĊĊĊ","Ġit andi","ĠAd olf","ಠ£","çļĦ人 æĿ¥è¯´","ha el","Ġà¦ı স","ĠmiÄĻd zy","ĠMadag ascar","æĪij å¾Ĺ","Ġmod ality","è§£ å¼Ģ","att ention","èѦ æĪĴ","Ù¾ ÛĮ","à¦¾à¦Ľ à§ĩ","çı¾ 實","ĠT uc","ĠP ens","Ġwater proof","å¼ł æī¬","Ġpot ency","大家 åı¯ä»¥","Ġconcom itant","¢ ת","æµģ è¡Ģ","æĭī åĬĽ","æ¯į åŃIJ","Ġκ ά","ĠKim ber","ĠPom pe","Ġstair case","Ġ×Ķ× ¡×","ðŁ Ļ","off ice","æĥĬ åij¼","¤× ¡","满足 äºĨ","v ÄĽt","ĠS co","Ġattack ers","Ġà° Ĺ","Ġfib rous","})\\ ),","Ġpodcast s","æľ±åħĥ çĴĭ","ë ģ","Ġd ato","ĠS CC","Ġal ph","人 æīĭ","ä¹Ł éļıä¹ĭ","ew ay","Ġê ¶Į","Ġcom a","åİ¿ åŁŁ","Ġgil ay","Serialize Field",". Command","_ root","ial a","å°ı 说çļĦ","çŃī åĬŁèĥ½","æĪĸ ç͍","ãģĮ å¿ħè¦ģ","( right","b oss","á Å¡","pos able","å±ŀ åľ°","çŃĶ æĩī","åĪĨæŀIJ å¸Ī","Ġperm utations","Ġsv é","p ure","é»Ħ çĸ¸","å¸ĥ æĸ¯","çķĻ çĿĢ","Or ders","eling en","Ġantiv iral","N orthern","Ġo sp","ĠAng les","ãĤ¤ ãĥ³ãĥ","ãģĿãĤĮ ãģŀãĤĮ","Ġmilit ia","ĠUrugu ay","ĠT ig","ill or","Ġj ong","ĠCh urches","Ġshort cut","åĢĴ æķ°","Ġintellectual s","Ġlu ar","Ġshield ing","Ġh olog","est ra","Ġо жи","头 çļ®","ç»Ļ ä»ĺ","Ġе ÑģÑĤе","Ġà¦ħ ধ","夹 æĿĤ","ĠVacc ine","\" .\"","re pository","ĠM itch","é¤ ħ","aren ce","è¿Ŀ 竳","åıĤä¸İ äºĨ","ĠMart y","ĠSn ake","ĠвоздÑĥ Ñħа",". connect","Ġo or","ol azione","åľ¨ çݰ代","Ġ\" :","ع ÙĬ","ĠÙħÙĪ Ø³","Ġabandon ment","ĠCrypt o","ĠRou ge","-ha ired","åĮ»ç§ij 大åѦ",". ####","åı¯ ä»ĸ","Ġfin er","List Item","ĠÙĥ Ø«ÙĬر","/d L","θ ή","æĦĪ åıij","å¤ļ åĬŁèĥ½","å®ŀ æĥł","è½® æµģ","å¼¹ åĩºçļĦ","}= (","ĠStevens on","B H","ĠT ensor","è¦ģ è¿Ľè¡Į","å±± åºĦ","åŁ¹ é¤Ĭ","Ðł иÑģ","ĠØ· ب","×ķש ×Ķ","B rowser","re in","at rice","ĠM p","转 弯","Ġdownt o","ĠRol le","Ġhá» £p","ä¾ ¥å¹¸","Ġmay o","Ġdet te","è¡Ģ èĦĤ","æ²³æµģ åŁŁ","填空 é¢ĺ","A O","åĩº åĩ»","Out let","éĽķ åĥı","ĠEspa ñ","Z H","} A","Ġr icon","æĪIJ äºĨä¸Ģ","é© ħ","ä¸ĭä¸Ģ 代","Ġtouchdown s","Ġf rem","å¹´ éĩij","ĠSt ella","èĦ ħ","çĹħ çģ¶","åĽĬ èĤ¿","ĠاÙĦÙĦ غة","Dire ctions","[ e","ĠS word","Ġ= .","大 è¡£","è£ ĺ","å°±æĺ¯ æĬĬ","女 å©¿","As ian","ĠÙĩ دÙģ","æĢİä¹Ī çľĭ","ĠGl ac","Ġpod le","ô te","ò ng","ä¼Ĭ æĭīåħĭ","Ġìłľ ê³µ","Ġпок ÑĢÑĭ","ĠAer ospace","cl uster","èµ° åĩºæĿ¥","âĢĶâĢĶ âĢĿ","楼 å±Ĥ","Ġaggreg ated","ä¾ĿæĹ§ æĺ¯","Ġ모 ëijIJ","Ġh uv","Ġv zd","ily n","代表 人","Ġcircul ated","Ġdust y","! \")Ċ","z ier","åľ¨ åĽ¾","大 æŃ¥","天 çļĩ","å³ ª","pat ients","Ġple ading","æľ´ ç´ł","Ġrepent ance","åľ¨ ä½łçļĦ","åĸ ļ","ĠZ um","Ġgr as","ãģª ãģŁ","éĸĭ å¿ĥ","Ġtrig lycer","Math Step","ĠÙħÛĮÚ©ÙĨ ÙĨد","à¸Ľà¸£à¸°à¹Ĥย à¸Ĭà¸Ļà¹Į","S ESSION","åıĮ èĦļ","Ġsek itar","Ġbuck ets","ä»İ严治 åħļ","d ream","ĠTr im","ĠDef ining","zi ak","w ives","Ġs x","èĩªå·± 对","اÙĦ Ùģ","çļĦ大 éĥ¨åĪĨ","ĠÐľ еÑĤ","Grid View","Fi xture","æ¯Ķäºļ 迪","F G","k N","z hen","Ù ł","åľ ĥ","ä¸ĭ åıij","Ġmat ière","eb ug","Ġlo af","ĠPay ne","ĠNap ier","à¸Īัà¸Ķ à¸ģาร","Ġmoy en","ĉ super","Ġ ../","ĠW ings","æķ´ å½¢","Ġspeed ing","Ġdiss imilar","μα ν","ĠWW II","Ġgeop olitical","Ġбибли оÑĤе","I UM","v ote","en ça","ĠM ia","em erg","ĠB ene","å°±æĺ¯ ä½ł","ľ× ļ","ĠØ® صÙĪØµ","ĠÑģÑĤа л","Ġont ology","ĠCross Ref","ĠRo of","Ġko ÅĦ","/ an",": +","Ġt m","åĴĮ èĢģ","all enges","ip olar","ä»İ æĪij","Ġgrow ers","ST OR","缸åħ³ è´Łè´£äºº","Ġbur ger","Ġpeace fully","æĶ¾åľ¨ äºĨ","ĠTele phone","Ġpreschool ers","B uk","Ġpres cribing","ìĿ ij","ĠBere ich","_ rows","ĠS DS","Ġal arms","ore l","à¸ļ รร","ä¸įä»ħ èĥ½","ĠFound er","åı¬å¼Ģ çļĦ","Ġrecon cil","Ġdun ay","æķ° ãģ®","ode ficiency","Ġeas ing","大家 åºŃ","Ġcounsel ors","' ent","åľ¨ åIJĮ","åĵģ ç±»","St rip","ÏĦ ί","Ġge le","ĠSw ing","çĿ¡ çĿĢ","ĠM LA","åŁºç¡Ģ çļĦ","秸 ç§Ĩ","J ay","ub u","å°ı çĭĹ","ier no","Ġsuggest ive","Ab ove","Ġglut amate","C ómo","l ost","ch ars","æľī åIJįçļĦ","ĠThe o","éĹ® è´£","ãģ§ ãģĤ","ĠGod dess","ĠÐļ ÑĢоме","ç¹ ª","æ½ľ æ°´","ĠÑĩа Ñīе","ät ze","on an","ĠB old","Ġk Pa","è¦ģ éĢļè¿ĩ","Ġé lé","Ġне лÑĮзÑı","Ne ither","污æŁĵ éĺ²æ²»","æ°¸è¿ľ ä¸įä¼ļ","Ġвла ÑģÑĤи","ĠHero es","ĠWikis ource","ser ve","åĪ¶åº¦ æĶ¹éĿ©","à¥Ĥ न","ĠczÄĻ ÅĽci","Ġ à¸ľ","çı ©","ãģ« è¡Į","ploy ed","Ġrecord er","Ġdro plet","ĠJon as","ह à¥ĩ","à§ģর à§ģ","Ġwart ime","[ right","Ġw ickets","Ġin scribed","ĠL ucky","åIJİ èĥĮ","ï¼Ł ï¼ģĊĊ","å°Ĩ ä»İ","åĪĻ ä¼ļ","夫 æĸ¯åŁº","En v","-w ritten","c ou","çļĦ ç͵影","le ader","Ġmod ulate","ĠLe an","Ú¯ ÙĪÙĨÙĩ","令 æĪij","è² ©","èĤ© è´Ł","Ġdipl omat","æµıè§Ī 次æķ°","Job s","ĠY ao","Ġmechan ically","ĠAut onomous",".Aut owired","Ġatyp ical","g at","ary ing","Ġrec ruits","乡 亲","Ġnormal ize","å£ģ çĶ»","顺åĪ© è¿Ľè¡Į","ĠPlace ment","Nor wegian","Ġl ance","Ġg ö","ä¸ĭ èIJ½","اÙĨ زÙĬاØŃ","min i","Ġill uminate","Ġbit terness","Ġspons orship","িষ à§įà¦ł","Ġb s","ĠF ond","Ġob raz","ale igh","AC P","éĻį ä»·","(t otal","Ġnov ice","éĢĴ åĩı","Ġкон кÑĥÑĢ","æİ© æĬ¤","ç¥ŀç§ĺ çļĦ","hd ad","m V","ĠS ikh","好 å¿ĥ","å·¥ä½ľ æĢ»ç»ĵ","ĠSh ows","åıĺå¾Ĺ æĽ´","= B","åĴĮ çĿ¦","ĠHaw kins","Ġm aks","ol ulu","ä¸ī äºĶ","Ġدر صد","追 寻",". facebook","Ġte kn","æ·± æ²ī","Ġcamb ios","çľ¯ çľ¯","Ġenvision ed","Ġt ad","Ġev oked","IN O","ìĬ ¹",".next Token","ĠDE VELOP","ìĿ´ëĿ¼ ê³ł","ĠB ET","Ñī Ñij","éĩĩåıĸ æİªæĸ½","Ġsynd romes","Ġk ec","ys ql","çļĦä¸Ģ åIJį","æī¾ åĩĨ","åĪĿ ä¸Ģ","å·ŀ åĮº","Ġsn el","дÑĥ ÑĤ","æĸ½å·¥ åįķä½į","ĠBolog na","C op","iv ar","out ed","å°± å¦ĤåIJĮ","æĹ¥ èIJ½","ç»ıæµİ æįŁå¤±","IC EF","æ²¹ æ¼Ĩ","Ġein zel","åIJī ä»ĸ","Ġgosp od","C AN","atur ity","çĺ «çĹ","ĠÅ º","à¦ļ à§įà¦ļ","âķIJâķIJ âķIJâķIJ","pon de","ĠCons umers","çļĦ èĤ©èĨĢ","ys et","æį į","Ġد ÙĪÙħ","Ġhuman ities","ä¹° å®¶","â̲ (","çľī éłŃ","Ġmask ing","ÑĴ Ñĥ","let ons","åıijçĶŁ è¿ĩ","за д","Ġfort unes","ĠL N","к ÑĤа","Ġdec ipher","è´¨ æĬ¼","åĥı æĪij","æµĭ éĩıçļĦ","ĠCons cious","ൠ¼","Ġkin ahabogang","Ġcourage ous","h c","Ġd ès","ĠT oul","ĠV ä","èIJ ¤","å¢ŀ 产","é¡¹çĽ® ä¸Ń","Ġbit coin","ĠRan ch","ĠB uffer","oc ellular","书 ä¸Ĭ","å¤į ä»ĩ","}} ^{\\",") _{","Ġan ion","ε ÏĢ","ан ÑĤи","Ġwyst ÄĻp","æĺ¯ 说","好 åIJİ","éĤ£ éĩĮçļĦ","æ¡ Ģ","ни ми","Ġreview er","ãĤ¢ ãĥ¡ãĥª","Ġcaps ules","Ḡį",") e","L ists","_ day","w riters","ĠR iemann","åĴĮ å¼ł","Ġsp it","çī¹ è®¸","书 é¦Ĩ","æ±ī 代","ĠEvolution ary","Ġun ittest","红 æŀ£","æĹ© èµ·","宣 èªĵ","ĠWork place","ĠMult ic","ĠDaniel s","Ġsuprem acy","ig ar","be ans","ĠCan vas","Ġস াহ","åħ¬å¸ĥ äºĨ","C d","ĠS oup","ut sch","ĠC ove","Ġ\\ $","×Ļ× ŀ×Ķ","群 å²Ľ","Ġ×ij× ł×Ļ","ĠÑıзÑĭ ка","Ġcens orship","ĠVolunte ers","ató rio","çļĦ è¶ħ","ru ch","Ġflow ed","第åįģ ä¸ī","ĠاÙĦاÙĨ زÙĬاØŃ","Ġbask ets","j ung","Ġa vez","æĽ Ĩ","Ġexp ire","Ġsub units","Ġrun way","æķĻèĤ² åİħ","лÑı еÑĤ","æ¶ĪåĮĸ éģĵ","b inary","и Ñģп","ia ux","Ġqu an","æľª æľī","å®Įåħ¨ ä¸į","ĠDi ary","Ġà¦ı মন","(l ength","çĺ«çĹ ª","G uest","Ġd itch","-m illion","Ġне из","Ġر شد","ÅĽ wiad","ĠEst ad","Ġcam el","ĠSU V","ĠManit oba","Ġ à¹ģลà¹īว","Ġ à·Ģ","ri ak","æ²¹ ä»·","讲 课","лÑĥ б","emp el","GF loat","Ġorbit als","Indones ian","ĠT ür","ĠT ovábbi","ä¹ĭ 主","Ġpe ÅĤ","Ġdecor ate","+ q","о ÑĨи","æīĵ åΰ","Ġident ifiers","Ġиз гоÑĤов","ãģ£ ãģ¦ãģĦãģŁ","对è¯Ŀæ¡Ĩ ä¸Ń","ĠоÑĤноÑģи ÑĤелÑĮно","à§įà¦ŀ ান","B MI","ĠIs olation","à¹Ģวล า","ĠI hr","è¿ĺ å¾Ī","请 示","Ġintegr als","ĠL PS","åı¯ åIJ¦","Ġexp elled","åĩĢ å̼","Ġze al","Ġastron omers","Ġwhis key","Ġoverd ose","Ġm ama","åıij 声","ĠCon cent","温 æ°´","第äºĮ æĿ¡","amer ican","_ context","` )Ċ","e conom","ad ge","ĠP ose","ain an","èĩ³ æŀģ","Ġide ologies","Ġпла ÑģÑĤи","Ġhang s","Ġure th","Ġreck less","éĿ¢ åĽ¢","Ġми ÑĢе","سÙħ Ùī","Ġbuff alo","Ġharb our","al at","et in","ĠM ere","åľ¨ æľĢ","æĪij 覺å¾Ĺ","é«ĺ æĢ§èĥ½","çĤ¹ ä»Ģä¹Ī","Ġty ing","ר ×Ļת","Ġni ño","å½»åºķ çļĦ","Ġpall iative","æĢ ħ","çłĶ åΤ","ĠRep rint","T U","l st","Ġ ________________________________","Ġв одÑĥ","Ġdi pped","å¦Ĥæŀľ ä½łçļĦ","-m ass","To List","ä¸ĸçķĮ ä¸Ń","æķ£ äºĨ","Ġprogress ing","æ·¡ å®ļ","Ġcup c","Ġbag gage","ĠS ear","ĠT ense","表 åįķ","à§įঠ§","Ġsk ins",".d ir","à¯įà® ®","ÙĪÛĮ ÛĮ","Ġshr inking","ãĤ¢ãĥ¡ãĥª ãĤ«","ĠØ ¦","Ġpo orest","-in formed","ĠProduct ivity","Ġfigur ative","] \"","ĠA BA","ä¸Ń ä¸ĵ","æĹ¶ ä¼ļ","å» ¿","uc ional","Ġfact ores",".l ower","丽 èİİ","лек ÑĤÑĢи","Ġmetaph ysical","ĠJes ús","Ġunint ended","/ file","াঠł","ëĭ ´","ĠÐŁ е","çĹĽ çļĦ","çĪĨ 竹","Ġeyeb row","çļĦ éĿ¢ç§¯","ĠRe ef","æķĻ åħ»","ĠÑĦ ев","年代 çļĦ","non atomic","éªļ æī°","Ġinter m","az ionale","åį´ ä¹Ł","èĥ¡ åIJĮ","Ġrid ges","ĠDal ton","Ġczas ie","+ N","N ATIONAL","Ġins er","Ġspecial izing","è§ĦåĪĴ åĴĮ","Exper imental","ĠعÙħÙĦ ÙĬØ©","Ġcomunic ación","urb ed","Ġchrom ium","& E","/ un","i O","çļĦ åIJĦ","æľ¬ çİĭ","ĠSch u","Ġstory line","-st ructured","èģ½ èªª","ĠëĺIJ íķľ","ĠSau ce","Ġì¶ľ ëł¥","ัà¸ĩà¸ģ ฤษ","ä½ľ æģ¯","Ġet iology","Ġconf isc","æıIJä¾Ľ ä¸Ģ个","è¯ģåΏ 交æĺĵæīĢ","Ġtert entu","é£ŀ ç¿Ķ","å¯Į 士",".B ack","Ġfingert ips","Ġu v","æĪij们 å®¶","Ġtot als","ĠâĪ ª","çĶŁåij½ åij¨æľŁ","ĠìĿ¼ 본","B rad","Z O","at uring","çļĦ éĩı","äºĭ ä¾ĭ","åħ¥ åºĵ","ĠSch uster","ÄĽ r","Õ¥ÖĢ Õ«","éģ© ç͍","Ġê³¼ ìłķ","M iller","_ msg","Ġf ü","ĠÑĥ ÑģÑĤойÑĩи","Ġgen au","_n ull","ĠTim eline","ĠкиÑģ лоÑĤ","ann es","å¸Ī å¼Ł","åħ¬åı¸ æ³ķ","Ġcomment ators","第åįģ åħ«","奴 å©¢","oglob ulin","Ġ .....","ঠĺ","èµ ĥ","ä¸Ĭä¸ĭ æĸĩ","龸 çİĭ","ĠвоÑģ емÑĮ","- help","\\ rho","i in","Ġs yl","ad ura","Ġcommun ism","ĠMed ien","åİ¿ åħ¬å®īå±Ģ","æŁIJ ä¸Ģ个","Ġпи Ñīе","r ases","ĉ float","ĠE ig","Ġthere on","æĬĬ å®ĥ们","Ġsal ads","æĹ¥æľ¬ ãģ®","Ġresist ors","Small est","å¤įå·¥ å¤į产","{ |","al iation","am eth","äºĽ 许","(\" \");Ċ","åķĨ æ¥Ń","Ġconc ord","ĠPar se","nÃŃ k","ĠNumer ology","æ« ĥ","f ried","便 èĥ½","缮åīį å·²","以ä¸ĭ ãģ®","па ÑĢ","ĠSund ays","宾 è¯Ń","Vir gin","Ġsl ogan","ĠGen re","o ji","ĠC LE","èĩª å·²","èģ °","ĠÎ Ī","æľĪ 饼","æ°Ķ åİĭ","Ġbel um","管çIJĨ ä½ĵåζ","èĬĴ æŀľ","åįģä¸ī æĿ¡","Ġenrich ing","( de","[ T","pp el","ĠK ons","é£İ çŃĿ","è¢ĸ åŃIJ","ĠBed ford","Ġla ut","ä½Ĩ åħ¶å®ŀ","ÑĤа л","ÅĤ ÄĻ","Ġbi ologically","ĠÙħÛĮ Úº","ש ר","Path s","l ug","åīį å¤ķ","Ġfl akes","ĠLe ah","æĺ¯åIJ¦ èĥ½","Ġfoot er","২০ ০","ĠGust av","bring ing","In fl","太 ä¹ħ","æĸĩåĮĸ 产ä¸ļ","amm ers","à¥ģ à¤","ĠPass age","Ġле Ñĩение","Ġà¦ļ ল","ĠмаÑĤеÑĢи ал","Ġìĺģ íĸ¥","ĠبØŃ Ø«","Ġacquaint ance","ĠS idd","Ġv inden","ÛĮ ÙĦÛĮ","erg ies","èIJ¥ åĪ©","Ġaccess ion","By Name","顺 æīĭ","æIJŀ å®ļ","Ġδ á½²","ÙĪ Ø£","å±± ä¸Ń","ĠG ould","æį ¨","Ġsk ut","åŁİ åĨħ","åĪĿ è¡·","Ġspirit ually","èµĦæĸĻ æĿ¥æºIJ","ĠStre pt","om aterials","ĠR ost","è¿Ļ éŨ","èĩª æķij","èĢĮ 导èĩ´","æĸĩ æŃ¦","UR A","æ±ĩ éĽĨ","ĠFe eling","ĠMet rics","Per fect","Ġdrift ed",") âĢĵ","Ġm alloc","ร à¹īà¸Ńย","éĢĤ åºĶçļĦ","ĠRel iability","Ġric hest","ĠпÑĢоÑĨе дÑĥ","d orf","Ġ ศ","æľ¬ éĻ¢","ick le","Ġsim bol","è¿IJ ä¼ļ","ĠCol our","éĢĢ äºĨ","function al","åIJĮå¿Ĺ 们","Ġgal van","ĠB enson","ĠÑĥ дов","Ġnon fiction","ĠÙħÛĮ زاÙĨ","ĠLi ouville","Ġdepart ing","Ġurg ently","м оÑģ","åħį éϤ","Ġpowder ed","еÑĤе лÑĮ","Ġwen ig","æĿ¥ ä¿¡","è§Ĥ æľĽ","æī¿ è¿IJ","è¿ħ çĮĽ","Indust ry","ĠBless ed","ĠÙĪØµ ÙĦØ©","Poss ible","ĠLithuan ia","N an","éĢĤ éĩıçļĦ","èĨ Ī","Ġber l","è§ĦèĮĥ çļĦ","æī¾åΰ ä¸Ģ个","ĠLim itations","Ġmemor andum","; [","I de","_ port","ä¸ĵ åζ","涨 ä»·","ĠкаÑĩе ÑģÑĤва","j p","ĠH MS","åľ¨ æīĭ","çłĶç©¶ åıijçݰ","ãģª ãĤĬ",".p ayload","Event Args","ĠÙħØŃ د","ĠAccount ability","ãģ®ãģ§ãģĻ ãģĮ","respons ible","G irl","ä¸Ģ çĵ¶","ay ana","ill ian","åĩº 头","ä¸Ģå®ļ ç¨ĭ度ä¸Ĭ","ĠSens itivity","æĩ· çĸij","åĴĮ çĶŁäº§","çĽĬ æ°Ķ","Ġunf avorable","è¸ı åħ¥","Ġimmunos upp","Ġb anc","åıij æ³Ħ","oh ol","Ġcho ix","ĠGu ided","Ã¥ k","Ġë² Ħ","Ġk ry","çŁ¥ å·±","à¹Ģภĺ","ĠÐļ огда","اذ ا","rÃŃ guez",". aut","Ġs plic","éĿ¢ 缸","å¤į æķ°","æĺĵ æĩĤ","对äºİ ä¸Ģ个","Ġp pt","æľº åĴĮ","Ġfil s","-con fig","ืà¹Īà¸Ń à¸Ļ","Don nell","лож ениÑı","IFI ED","M ON","d g","æĹ¶ åı¯","åıª æīĭ","ec o","Ġmin ors","ä¾Ľ åħ»","è®® äºĭ","ç»´ äºļ","ç±³ å°Ķ","Ġpropri a","B rowse","з еÑĢ","æĺİ ç¢º","Ġspec ulate","ĠAugust us","Ġreass uring","Elect ronic","åĿİ åĿ·","n ad","åĩºäºĨ ä¸Ģ个","hu is","ÑĤелÑı м","ĠCoord ination","ç©© å®ļ","Ġflatten ed","- State","å°Ĩ 为","å°±æĺ¯ 个","Ġaut istic","çļĦä¸Ģ çīĩ","é´ »","Ġbekan nt","' },Ċ","D ifference","çļĦ æĶ¶åħ¥","Ġfor aging","ell an","Ġi x","æĢİ æ¨£","à¸Ĺย à¹Į","à§ĩত à§įর","Ġà®İ ன","å·¥ä½ľ ä»»åĬ¡","Ġpolit iques","opt ers","ãģĹãģ¦ ãģ¿","Log ged","iaz za","Ġad ept","红 èĸ¯","ho z","éĻį åİĭ","pat cher","Ġли ней","ĠÑıзÑĭ к","аÑħ аÑĢ","Ġinh aled","çļĦ æĿ±è¥¿","Ġj as","ĠZ ug","ä»· ä½į","ä¼ģ äºĭä¸ļåįķä½į","ĠØ´ Ú©","_m atch","Ġmodern ization","æĺ¾ç¤º å±ı","ĠChand ler","é»ĦèĬ ª","Ġm ason","Ġv ive","é«ĺ åĪĨ","ĠInd oor","Re ports","è¿Ļç§į äºĭæĥħ","ãģı ãĤĬ","Ùı ÙĪØ§","Ġalleg iance","W iki","Ġo de","Ġl ij","å½ĵ ä¸Ģ个","åı¯ä»¥ èĢĥèĻij","èĪª æ¯į","ีย à¸Ķ","Ġju in","浦 举","åīĸ éĿ¢","P oor","UR CE","å³ ¥","ä¿ĿæĬ¤ çļĦ","New ton","ĠSem ester","Ġcuc umber","Ġt ÃŃch","ĠR UB","res ión","end re","身 å¾Į","à¥įठ¹","åı« ä»Ģä¹Ī","Ġ-* -Ċ","( ll","r ude","an onymous","ĠR ocket","æŀ Ń","min a","ั à¸ķร","ĠÙĩ ÙħراÙĩ","æķ°æį® ç±»åŀĭ","( default","w issenschaft","Ġb az","ma res","AR GET","ä½Ļ åľ°","ĠComp act","åľĨ å¼§","æĹģ çļĦ","×ķ×ij ר","ĠTec n","Ê Ķ","Ġf um","åѦ 好","èĻ §","å®Ŀ çī©","omer ic","Ġlung o","- Level","it ution","ä¸į è¦ĭ","çĤ ¯","iss ä","èĢĥ ä¸Ĭ","ä½İ äºĨ","ĠGl ory","Ġeth os","Text Box","ĠSi O","第åįģ ä¸ĥ","å¾Ģåīį èµ°","Ġfibrobl asts","ĠA very","âĢľ â̦â̦","ĠN im","Ġpre term","Ġz onder","Ġgu err","æĬĵ èİ·","mat ched","Ġaan v","Ġâľ ħ","J I","åı¯ åı¯","Ġund e","Ġtrans ports","áĥIJáĥ ¡","大 æ±ī","åģļ 强","ç»´ å¥ĩ","à¸Ħ à¸ĩ","à¹ĥ à¸ķà¹ī","ĠRep ro","Ġlogarithm ic","ĠÑĪ ÑĤо","gi ore","าวิ à¸Ĺย","Ġha u","ice ps","åı¯ä»¥ åIJij","æĿij éķ¿","ç»Ħç»ĩ å®ŀæĸ½","ĠWorld s","zen i","Ġstress ors","åŁºéĩij 管çIJĨ","you ng","ĠпÑĢакÑĤи ÑĩеÑģки",") }{\\","以 è¾¾åΰ","che id","çϽ åıij","ĠUS ER","Ġtw or","ĠобÑĢаз Ñĥ",".App lication","( br","çļĦ ç̧","el ius","ãĢĤ >>","Ġre union","ĠA FP","Ġval eurs","Ġо ÑīÑĥ","æŃ£ 弦","Ġadv ises","ĠاÙĦت Ùģ","å¿į çĿĢ","Ġorth odox","Ġsol ves","-b orne","Ġfr ü","ุ ล","Ġplate lets","f uer","at ism","éĢĻ è©±","åĨĽ æ°ij","ĠBe am","Ġvo ed","Ġaf ric","çļ± çº¹","ĠAdapt ation","ĠM ek","å¼ł 大","Ġbed ding","ĠElect oral","Ġsel ves","Ġatheros clerosis","ä¸Ģ 转","åĬł æģ¯","Ġra ff","° ï¼Į","åħħ æ²Ľ",".H as","ĠÎij ÏģÏĩ","Ġf d","Ġб ога","ĠSch ro","Ġrad ios","ÙĬÙħ ÙĥÙĨ","à¹īาภ«","ä¸Ģ åģļ","æ·¡ çĦ¶","àµįà´ ¯","æĩī該 æĺ¯","Ġprofes ional","in ander","Ġ× ¡×¤×¨","æ¸ħ èĦĨ","Ġpa wn","sk ie","è¡Įä¸ļ åįıä¼ļ","ĠPlaintiff s","à¹Ģลืà¸Ń à¸Ķ","/ Second","Ġt abel","ä¸ī åħĥ","çݰéĩij æµģ","çļĦ æĬ¥åijĬ","ĠP ixel","ĠE ph","æĸĩ åѸ","æŀĹ æľ¨","Ġleft over","κ B","(n umbers","追 èµ¶","ĠاÙĦØ£ Ø®","ĠÐķ го","Å« n","iconduct ors","人 ç§°","Ġsu fic","åĴĮ æķĻèĤ²","å®ŀ ç͍çļĦ","irc hen","ĠSo zial","ðĿij Ł","é½IJ å¿ĥ","Ne uro","' ass","ĠN ora","åħī 度","ç½ij æ°ij","Ġà¦Ń াল",": T","F lu","ĠF ans",".... .ĊĊ","Ġdis continued","Ġpart isan","amp uan",":: $","ä¿® ä»Ļ","Ïĥ ί","Ġuns ur","Conf irm","-val ued","Ġp inned","åľ¨ æİ¥åıĹ","è¿Ľ åľº","Ġdi astolic","num ero","ãĤ·ãĥ ¥","Ġch ond","ĠвÑĭ бÑĢа","Ġtrim med","ĠÃŃ nd","ang ka","ä»ĸ ä¸Ģ缴","å°ı é¼ł","Ġam alg","== ĊĊ","æµ· è¾¹","Ġconf essed","èģĶ ç»ĵ","ĠاÙĦÙħ Ùģ","-in vasive","ĠBo om","åĮĸåѦ åıįåºĶ","ĠSav annah","Ġsag t","Ġzosta ÅĤ","Ġro ar","æĥ³ 说","ĠX CT","æĢ¥ çĿĢ","诺 è´Ŀå°Ķ","འĦ","Ġaffili ates","ĠÑĥг ол","educ ated","Ġp ueblo","Ġex cretion","æĿİ å°ı","è¿Ļç§į äºĭ","EF T","æĦŁæŁĵ èĢħ","Ġquad ril","Ġmuj er","ĠÏĢÏģ Ïī","Ġмик ÑĢо","K F","á ln","Ġ) )","Ġbat ches","åĩºåıij çĤ¹","ĠاÙĦÙħس ÙĦÙħ","z al","çļĦ 女åĦ¿","ĠM IS","æĶ¶ 纳","str ateg","å¸Į å°Ķ","Ġscript ures","ç«¶ çĪŃ","Ġ'* '","are lla","Ġpart ecip","ç»Ļ èį¯","ĠZ imm","ли и","å¸Ī å¾·","ĠFor g","äºĨä¸Ģ ä¸Ŀ","Ġlim p","ĠâĨ ĵ","dr ive","Ġpad res","åįģä¸ĥ 竳","çī¢åĽº æłijç«ĭ","Ġb umps","åľ¨ å·¥ä½ľä¸Ń","à¹ī าร","Ġworld ly","èĥ½åĬĽ 强","ÐĴ Ñģе","åĽŀçŃĶ éģĵ","Ġmix es","ĠTrin idad","Ġ ï¼Ŀ","Ġund en","ĠQ t","åij¨ ä¸ī","Ġsum mation","ĠCur ry","ĠØŃد ÙĪØ¯","ĠDest roy","Ġk s","çŃī ä»·","è§Ħå¾ĭ çļĦ","Ġdend ritic","Ò ³","Ġh ati","li as","Ġmagn ification","Ġimag ining","Ġgi á","åĦĦ åħĥ","en viron","åįĹ éĢļ","yp se","Ġseem ing","ĠExpl ained","ĠWeek end","ĠпеÑĢв ого","Import ant","is és","=\" ../","èĬĤ æ°´","è¿ŀ 带","ĠPr z","è´§ 款","ä»ĺ åĩºäºĨ","çĽĺ çļĦ","ল à§įপ","ĠMill ions","ков и","Ġëħ ¼","L orem","ä¸ļ çķĮ","ĠIm ag","ĠpÅĻ ip","H Q","d emo","人 æĹı","Ñĩ ном","Ġfirst ly","ö ss","LL E","Ġweight ing","ĠÄ ¯","oral y","辨 认","ĠRF ID","; }","ĠT ina","ĠT aste","ĠM ild","大 åĵŃ","ï¼ļ [","Ġapp rend","è¿ĺ åŃĺåľ¨","æĹł å°½","æµĭ ç»ĺ","λ Ïį","æ·· èĽĭ","ĠÐĽ Ñİ","調 æŁ»","ĠAT T","Ġbol ster","ĠاÙĦØ« اÙĨÙĬ","ĠEndocr inol","ĠT rophy","ĠJ UST","æĦŁ æĥ³","Ġber at","æľ« å°¾","åĴ¬ çĿĢ","Ġouts ourcing","t ant","ĠM ih","æĶ ĺ","éĢł åı¥","åij¨ åĽĽ","Ġcop olymer","Descript or","E k","ra iser","Ġhe ures","ов ого","Ġvari as","éľĢè¦ģ 对","ris is","ĠCL I","hund rede","ä¸į æİī","两 çϾ","æİ¨ åIJij","Äį né","Ġsymbol izes","Ġweaken ing",". order","_ button","Ġb h","èµ· åĬ¨","Ġimpact o","ĠEV s","ë¨ ¸","S alt","d ump","un en","ĠR ousseau","ĠH omo","ä½İ ä¼°","}{ (","äºĴ æį¢","é¹ ĥ","ĠSil k","Ġstrat ified","itt el","Ġgener als","Ġdevast ated","Ġan z","Ġk husus","æĺ¯ä¸į åı¯èĥ½çļĦ","Consider ing","Ġì ĵ°","伸 å±ķ","Ïĩ ή","èĥ¸ èĨĽ","çϽçĻľ é£İ","d epth","åİĨ å¹´","Ġsqu amous","äºī åħĪ","åŁİå¸Ĥ åĮĸ","V G","Ġs inter","ãĢĤ ï¼ī","å®¶ éŨ","iff any","OT S","Ġsex y","ĠÙ¾ زش","Ġfashion able","_V ERSION","Ġconhec imento","Ġverwend et","缸 éĢļ","---- ---","å¾Ī åĥı","åij¨ æĺĵ","å¸ĮæľĽ 对","ĠEL ECT","Ġà¦¹à§Ł à§ĩ","моÑĤÑĢи м","[ ...","Ġm c","ch oline","ĠPro spect","ìĹIJ ëıĦ","å¸ĮæľĽ éĢļè¿ĩ","len ÃŃ","Ġáĥ Ļ","com be","ull ing","åĽłä¸º æľī","ĠÙħÙĪ Ø§Ø±Ø¯","åѤ åĦ¿","ĠëĤ ł","總 çµ±","ifik asi","è¿ij æĿ¥","Äģ s","å±ĭ åŃIJéĩĮ","ÑĬ л","Ġtid y","Sur vey","ĠContin uing","ĠZamb ia","ĠSt ad","Ġ' )","umb a","Ġflav on","ĠRu iz","ĠRud olf","Ġgez ond","ĠIn verse","ãģĦ ãĤį","ĠReview ed","æ°ijæĹı åĽ¢ç»ĵ","Ġlleg ar","ĠAnglic an","E g","ĠL adies","Ġcom pt","int es","Ġر ÙĬ","Ġsil hou","åįĪ åIJİ","æ§ Ł","宽 æķŀ","ë§ ī","æĭ¨ æīĵ","Ùħج رÙĩ","ĠÒ» ÓĻм","- La","Ġf aktor","Ġre par","Ġant iquity","اÛĮ Ø·","çĦ¶åIJİ åıĪ","ëĤ ł","Ġcris py","ë ķĮ","æİ¨ åĭķ","Ġvis cous","ĠImm une","ĠES G","Ġexacerb ated","ĠP ou","å¹¶ ä¸įçŁ¥éģĵ","াঠ«","iam ond","ĠпÑĢо ÑĨ","èİ« åIJįçļĦ","è¿Ķ 乡","Ġfunc iones","Ġchat ting","ĠSME s","为 导åIJij","ethod s","Ġhom me","×Ļש ר×IJ׾","Ġpopula ção","B razil","j at","ĠP ST","ĠH older","Ġz iem","åıª ç͍","æĭ¿ åĩºä¸Ģ","_m ain","vol ent","Ġo mit","Ġal erg","Ġhe ed","Ġbl ond","åįģ å¤ļ","ran king","Ġmen opause","à¶ ½","Ġquad r","éĢıæĺİ åº¦","Ġanne aling","Î §","åŃIJ åľ¨","eth ane","Ġind ign","æıIJ è´¨","Ġatt ire","åĨį èĢħ","Ġvis ceral","åĪĿ ä¸ī","ç§ijæĬĢ è¿ĽæŃ¥","ø n","ä¸ĸ纪 æľ«","B atch","Å ĮÄĨ","or ange","Ġper ts","Ġside ways","Cl ock","Log o","éĢĤåºĶ æĢ§","Ġfle eing","Ġprecip itate","åĽłåľ° åĪ¶å®ľ",") Skip","åĩº åİĤ","ph rase","Ġد اÙĬرÙĩ","ĠاÙĦشع اعÙĬÙĩ","ä¸į åĭķ","è¾ Ĺ","ĠÙĤ طع","ائ ÙĤ","ĠIre ne","Ġdescript or","Ġvag u","ãĥĹãĥŃ ãĤ°ãĥ©",". math","c ÃŃ","Ġre pos","æ° °","ite z","اÙĦ Ùĩ","-s oluble","Ġmen cion","Ġprec isa","åĶIJ è¯Ĺ","å§ĵ æ°ı","Ġcontro le","ĠвÑĭпол нÑı","Ġê¸ Ī","keep ers","Ġoversee ing","F resh","ë Ĩ","Ġwh ims","Ġche fs","ĠठĽ","ana o","æ²³ 西","åĿIJ ä¸ĭæĿ¥","Ġprote ase","æĸĩä»¶ åIJį","éĹª èĢĢ","ÓĻ Ð½","Ġkl ass","ĠسÙĨ Ú¯","×ķ×ŀ ×Ļ","Ġt ester","Ġv ant","åºĶ å±Ĭ","Ġconver gent","ĠU R","к лоп","ps um","çİ°åľ¨ æĪij们","ĠAnn als","éĢĥ çĶŁ","ĠìĹŃ ìĤ¬","Ġkond isi","l ant","à Ĭ","åĴĮ ä¸ĢäºĽ","æıIJ çĿĢ","ann ie","车 祸","Ġgro oves","Ġstrat ification","Ġìŀij ìĦ±","ĠC VD","广 ç͵","Ġëı Į","[ len","ask ell","ĠDes igned","stit uto","CO DE","æ·¡ æ°´","ÑĻ Ðµ","Ùĥت ÙĪØ±","Ġin patient","est ination","以 身","Ġag r","Ùİ Ùĥ","Ġnational s","ĠCreat ivity","夹 è§Ĵ","_ child","z g","ĠMün chen","ac ock","og t","asc a","ĠOut standing","éĤ® 票","åĬ² åĦ¿","ĠاÙĦربÙĬع Ùī","à¸ĩ à¹Īาย","Ġredu z","оÑģ ÑĢед","ĠÙ¾ ÚĺÙĪÙĩ","ä¹Łåı¯ä»¥ æĺ¯","æķ¸ éĩı","ĠGrand ma","åĤ³ ä¾Ĩ","ëIJĺ ìĹĪ","å¿ħä¸įåı¯ å°ijçļĦ","ãĤĴ ãģĻãĤĭ","çĭ¬ å®¶","Ġgrasp ing","æ°ijäºĭ è¯ī讼","Ġrejo ice","Ġstrang ely","ĠM OV","æľ¬ å¸Ĥ","ĠLe ist","åĽłä¸º æĺ¯","éĢĥ èĦ±","çѹ åĪĴ","ĠBang alore","ĠìĿ¼ ë°ĺ","åħ¶çī¹å¾ģ åľ¨äºİ","b ok","Ġqu oting","éĢļ æ°Ķ","å°±æĺ¯ äºĨ","失 è¡¡","ĠDr ivers","çĿ« æ¯Ľ","+ R","Ġt ÃŃm","ÑĢ Ñİ","op at","大 åĪĩ","া à§°","Ġpar sed","Ġsm ugg","ank en","ĠQu arters","ĠCo at","çĶļèĩ³ åľ¨","_n umbers","åħ¨åĽ½ åIJĦåľ°","æĮij è¡ħ","Ġmuit os","Ġambient al","омеÑĤ ÑĢи","Ġwür de","J ason","Ġd ÄĽt","éĥ½ æľīäºĽ","oy e","Ġopp ressed","itu ary","ĠС ÑĤ","Ġtor ment","æĺ¾èijĹ çļĦ","対 çŃĸ","Ġphysic ist","Ġsulph ur","ĠH Y","ĠL NG","Ġsh rine","没 éĤ£ä¹Ī","Ġprov oke","Ġdec ks","åģı ä½İ","Ref resh","ĠÑģооÑĤвеÑĤ ÑģÑĤвÑĥÑİÑīи","Ġsecre cy","Ġ ÖĦ","es on","å¼Ģ æºIJ","ish ly","çł ¾","Ġgl acial","ĠSc r","åĩı åİĭ","новни ка","ĠHaw ks","ëIJĺ ìĹĪëĭ¤","Ļà§įঠķ","-gu ided","ĠHunting ton","Ġmalf unction","- ear",".C ode","ذ ر","ĠAppro ved","ĠاÙĦØ« ÙĤ","Ġunders core","Ġ(+ )","ĠAnaly zing","\\ delta","c ov","é¢Ħ è§Ī","col es","åĮ»çĸĹ æľįåĬ¡","Ġon click","æĪIJ è´¥","ĠاÙĦ اÙĤتص","Ġpur pos","Ġinvol untary","æī§ åĭ¤","ĠÕ ·","é¢Ŀ çļĦ","è±Ĩ çĵ£","Ġprev ailed","ä¸ĭä¸Ģ ç§Ĵ","Ġmisunder stood","æĸ¯å¤§ æŀĹ","} ={","Ġ ðĿ","è¿ĩ çĿĢ","è¨ Ŀ","ĠID s","ĠاÙĦÙĨ س","ĠTH C","Mc C","Miss ing","Ġpelle ts","Ġte oria","æīĢ åıĹ","主 æīĵ","Ġag ony","Ġع رض","Pro du","两个 åŃĹ","ĠÑĤак ого","zia ÅĤa","Ġro be","oph ysics","èĩªçĦ¶ çģ¾å®³","ÑĨион ного","測 試","Ġcan oe","åľ° 段","åħļ 代ä¼ļ","Ġpatient ly","ĠLi ability","-R el","ĠBur ma","ĠвÑģ ей","è°£ è¨Ģ","á ī","åIJĦ è¡Į","ĠHar lem","æ´ĭ èij±","ĠGDP R","管 线","oss ing","软 å¼±","Ġobl ique","M U","ĠM err","qu ake","ĠThe rapeutic","á val","ç± ł","éļı é£İ","Ġlat in","abs orb","um ont","iz k","ठ½","缴æİ¥ å°Ĩ","æĢª ä¸įå¾Ĺ","Ġ미 êµŃ","ĠRand all","Ġexh ilar","C ards","a ution","Ġe chter","Ġ{ },Ċ","æĪIJ æīį","é«ĺ 涨","åıĺ 大","íļ į","ĠPhilosoph ical","èĻIJ å¾ħ","w aters","ĉ get","ä¸Ĭ è¿Ľè¡Į","Ġsp oj","ĠRe con","Ġform ulae","Ġsub scriptions","åįĹ ä¸ĭ","ĠBel ief","à¹Ģà¸ģ à¹ĩà¸ļ","Ġdispar ate","ĠSubst ance","Ġש×Ķ ×ķ×IJ","Wil son","æĹł å°½çļĦ","arg uments","èµ° ç§ģ","SS L","ĠRES EARCH","éĢļ äºĨ","ล ำ","çģ« äºĨ","Ġsal ty","Ġدر بار","ĠÑĢезÑĥлÑĮÑĤа ÑĤ","Ġвозмож но","et ik","èIJ½ äºĨ","è¶³ å¤ł","éķ¿åº¦ 为","/ man","×ķ× ©×IJ","Ġserv icios","ç»´ åŁĥ","ĠPol sce","ét at","Ġvirt u","æĪIJåijĺ åĽ½","_ FAIL","And erson","æ³¢ ç½Ĺ","ி஠µ","Ġré p","çļĦæľĢ 好","_g raph","åīĬ åĩı","æľĢæĹ© çļĦ","ĠCB SE","} .\\]","ãĢĤ )","ot as","äºİ å¿ĥ","çľĭ æľĽ","che on","Ġdissatisf action","w irk","ĠB arker","éĵ ł","è» Į","éĩijèŀį å¸Ĥåľº","Ġwood land","ĠHebre ws","r ily","Ġk hi","Ġup front","Ġठ«","大家 ä¸Ģèµ·","èĭ¥ ä¸į","Ġmor als","åı³ ä¸Ĭ","æķĻåѦ è´¨éĩı","éĩİ åħ½","Uk rainian","ĠBench mark","ri ps","åĨ· èĹı","_f rame","ĠPort rait","çį µ","她们 çļĦ","à¸ģล ัà¸ļ","el den","ĠG eg","被 æī§è¡Į","åĨĽ éĺĢ","åıijçĶŁ åIJİ","Ev idence","develop ed","è¯ ħ","ä¼ģä¸ļ ç»ıèIJ¥","é¢Ħ çķĻ","ĠинÑĤе ÑĢе","ĠпÑĢом ÑĭÑĪ","اÙĦÙħÙĬÙĦ اد","rom a","Ġover haul","ни ÑĨе","-d ollar","ĠCo aching","ç¨ĭåºı åijĺ","ĠMill imeters","çļĦå¿ĥ æĢĿ","à¥ĥ ष","f ors","çŃī å¼ı","ç²¾ èĩ´çļĦ","ü b","æķĻèĤ² åŁ¹è®Ń","Ùİ Ùī","å®Ĺ 主","Ġwid ening","ĠCOL OR","Ġper ten","ت Ø´","ĠTr ich","Ġbehav es","-h ard","Ġfa ctions","End point","è´ Ī","Ġbre thren","ext ends","Ġviol ently","ຠ²","Ġprá ctica","ç»Ļ人 ä¸Ģç§į","ĠSpot ify","T ar","Ġa isle","Ġdifferent ially","åįĩ 温","ĠÙħÙĨ اسب","ĠCons istent",".log in","Ġscr atching","ĠгÑĢÑĥ н","ĠParticip ant","Ġf ak","ç͍ æĦı","ern o","导 读","æ¯ı æ¯ı","Ġcapt ivated","èĪª è¿IJ","-F ree","ĠLeg ends","äh lt","æĸ°åĨłèĤºçĤİ çĸ«æĥħ","ĠSerge ant","w indows","ĠC ain","å¹´ å°ij","该 æĸ¹æ³ķ","ç»Ŀ ä¸įä¼ļ","Ġpan jang","èĥĨ åĽĬ","ĠFOR M","' }Ċ","çĶŁ éķ¿çļĦ",".C OM","ç¨İ éĩij","ph thal","Ġdem ost","Ġка Ñģа","Ġrefer rals","_l ocal","འĵ","Ðľ е","ãĤ³ ãĥ³","K at","e as","Ġn c","ãĢĤ ...ĊĊ","ĠP ris","pl ash","Ġso zial","ij ks","åĬ© åѦ","cover ing","ÙĦÙĬ س","ç¼Ŀ åIJĪ","ĠAub urn","ãĢģ ãĢIJ","ĠCon sequences","èĢĥ ãģĪãĤĭ","æłĩåĩĨ åĴĮ","- covered","t iny","am atan","ĠF ris","车 éŨ","å©ļ åIJİ","×ijר ×Ļ×Ŀ","ĠFra gen","大家éĥ½ çŁ¥éģĵ","ĠMong olia",". Al","ç ĥ½","Ġb rim","ï¼Į \"","Ġfam ously","åŃĺ åħ¥","åĦ¿ç«¥ çļĦ",": <","ĠP ip","ĠH ouses","ÙĦ غ","Ġte h","ÃŃ du","Ġsm irk","é»Ħ çļĦ","æł¹æį® èĩªå·±çļĦ","Ġtax onomic","Ġprem iers","ãĥ© ãĥ³ãĤ¹","Ġpel vis","Ġclar o","-sm all","< bool","ĠB oden","ĠF ay","ĠG ü","Ġat roc","Ġsh udder","å°± åı¯èĥ½","ĠBro ken","(u int","Ġproces os","åħ¨å¿ĥ åħ¨","S orted","ä¸į 稳","Ġj ab","ĠBe en","_t arget","Ġpsych iatrist","ĠTor re","ĠVari ety",".App end","ä¸İ æĹ¶","Ġfl ange","æĮģ æľīçļĦ","Ġgu inea","ÐIJ ÐĿ","Ġri pple","Ġih rem","G Hz","ë Ŀ","av erse","ç¥ŀ åύ","çģ¯ ç¬¼","ë¶ ģ","çĤĴ ä½ľ","å¾ĹçĽĬ äºİ","ed ad","اÙħ ÛĮ","اÙħ ÛĮÙĨ","Ġdest e","Ġmig rated","ĠInst ance","% \"","n ation","æº ´","ards hips","Ġಠ¤","w c","Ġb orough","æĸĩ 稿","æļ §","ĠPol ynomial","ĉĉĉĉ ĉĉĉĉĉĉ","ĠاÙĦغ ذ","ĠاÙĦعÙħ ÙĦÙĬÙĩ","a ar","å¤ļ ç͍","å¤ļ 说","æ°´ 温","Ġdev em","æıIJä¾Ľ äºĨä¸Ģ个","æĶ» åĬ¿","Ġtight ening","n ear","ÄĽ jÅ¡ÃŃ","åĪĨæŀIJ çļĦ","çĸij éļ¾","页 çłģ","ĠMod erate","/p ost","Ġnarrow ing","åIJĦ个 æĸ¹éĿ¢","D ream","ĠW inner","æĸĩ åı²","åİ» ä¹°","纸 è´¨","Foot er","Ġpier cing","Ġp uck","ĠI CE","her tz","ĠH CV","åĽŀ æµģ","ä¿® 羣","Õ¸ Öģ","åĤ¨ èĥ½","åį«çĶŁ éĻ¢","ë° ķ","åĦª åĭ¢","ĠPlate au","f ond","n ith","Ġn ectar","un wrap","èĤ ħ","æ°ij åħµ","ica ção","Ïĩ α","ĠCha os","P ret","S cientists","t opic","çļĦ åij½ä»¤","ĠN ate","çݰ åŃĺ","æ¡ ¦","ä¹ĭéĹ´çļĦ è·Ŀ离","}` );Ċ","ĠêµŃ ê°Ģ","ĠZur ich","D anish","ĠCl im","Ġmot ivo","Ġred dish","Ġét udi","Ġmie jsce","ãģĦ ãģ¤","Ge ografia","ĠAlb ums","ãģ¾ãģŁ ãģ¯","ĠPars ons","z ion","Ġ Ú","ä¸į å°±","Ñĭ л","é« ĭ","ĠÙħ ÙIJÙĨ","Ġant ih","读 æķ°","åºĶ该 åľ¨","åŃ©åŃIJ们 çļĦ","ĠForest ry","ả ng","à¹Ģส ียà¸ĩ","ÄĹ s","\\ Schema","Ġy ak","次 è¦ģ","åĽŀ è¿ĩ头","Ġbr ushes","à°¿à° ķ","æĭ¦ æĪª","æĸ°é²ľ çļĦ","Ġt élé","ĠN ike","ĠN ex","Ġpo ch","èĬ± èĬ±","ĠEm il","Ġmom s","Ġstre pt","工人 éĺ¶çº§","-Cent ury","S weet","r usion","ĠG rams","ib b","çŁ ľ","Ġв Ñĸд","ĠÑģ еÑĢе","Ġstand out","èŀ ĥ","Ïģο ν","溫 度","- ob","n io","ç on","Ġdeterm inar","Ġë° ±","ĠбÑĥ ма","x FF","è¾ Ħ","ĠSt amp","ç»ı 绾","å·® éĶĻ","丹 麦","æĽ¼ èģĶ","Ġbuff ers","ಿಠķ","Ġ``` ĊĊ","ÙIJÙij Ùģ","ĠAcadem ia","- default","P ressed","op ort","å®īåħ¨ äºĭæķħ","æĻ¯ çī©","most ly","è³ Ĭ","æİ§åζ åĴĮ","Ġindex ing","åĩĿ è§Ĩ","ĠÏĦο ῦ","æij©æĵ¦ åĬĽ","m ai","Ġf órm","ä¸į æĶ¹","å°ij è§ģ","å±Ĥ 级","Ġпол ови","Ġadj unct","å¥Ķ æ³¢","×Ļ׳ ת","è¿Ļ两 天","ледова ÑĤелÑĮно","Ġdisgu ise","v änd","Ġsim il","åij¼ åºĶ","ĠSing leton","Ġnan ost","ĠHam pton","DR ESS","ĠBou levard","Db Context","es cence","Ġh ak","Ġk re","ci er","ĠÙĪ Ø¯","ĠHigh light","æ¯ĶèµĽ çļĦ","ಲà³įಲ ಿ","Ġr ifer","ä¸Ń éĥ½","奥 迪","à¸ł ัย","Ġни ка","ĠGraph s","ĠÚ©ÙĪØ¯ Ú©","- values","Ġpick le","åįģåħŃ ç«ł","ĠNumer ator","_ Z","Ġw ollen","il ai","Ġso aking","Ġfl ats","åı¯ä»¥ æľī","Ġobject ed","osp heric","ĠTest Bed","ĠHuman os","ĠMer ge","纤维 ç´ł","åįģåħ« 竳",". %","çļĦ éģĵçIJĨ","Ġan hyd","è¢ ħ","à¸ķ ะ","Ġhem isp","Pack et","Ġgroom ing","Ġtoile ts","Ġp aw","Ġг г","ĠAss urance","_t ask","اÙģ ÙĬØ©","Ġhandic ap","Ġpo zn","Âł Ġ","ĠSy nd","/M in","s weise","ä½ł ä¸įä¼ļ","ĠCh ord","åħ¶ åİŁåĽł","æĽ´ éĢĤåIJĪ","Õ¸ÖĤ ÖĢ","áĥIJáĥ ķ","he ed","Ġm ites","ĠR end","èĩª æŁ¥","满 éĿ¢","æİĮ å¿ĥ","Ġassert ing","izz ato","æīĭæľ¯ æ²»çĸĹ","Ġcommission ers","ç¶² 絡","Ġcorrid ors","ĠC TS","ĠB EST","sp ots","Ġsol icit","Ġcyt oplasmic","áĢŃ áĢ","åĴĢ åļ¼","Î ¡","Ġr il","ĠO PER","亲 åıĭ","åī¯ æĢ»è£ģ","éħ¸ 奶","åĩĿ èĥ¶","rip ción","ான à¯į","ĠOun ces","ĠFerm i","Ġc rou","ç»ıæµİ åĴĮ","Ġdu plication","ĠSub mitted","èģļ åĬĽ","net e","Ġvolunte ered","B ranch","ig ail","æĶ¹ 建","ĠÑģа ми","èĥ¸ æĢĢ","([ [","Ġminist ries","S hip","Ġf p","uc aly","åıĺ èī²","è¯ģ çļĦ","åĽ½å®¶ 对","ç§» ä½į","Pl aying","Ġfem t","容æĺĵ 被","æĤ² è§Ĥ","Cong ress","N amed","о ÑģÑĤÑĮ","ĠJ ub","Ġrel ocation","æĹł ç¼Ŀ","èĦ± é¢ĸ","ÙħÙĪ ÙĤع","Ġnarrow er","çĻº çĶŁ","å¿ĹæĦ¿èĢħ 们","King dom","à¸ģำ หà¸Ļà¸Ķ","ĠL azar","Ġpl ight","Ġed n","åľĨ åij¨","æ°£ çļĦ","çŁĽçĽ¾ çļĦ","Ġexch anging","-develop ed","Ġc if","æıIJ åΰäºĨ","ç½® çĸij","ย à¸ģ","Ġ×IJ× ł×Ļ","ÅŁ t","ĠRecogn izing","Ġre w","est ine","Ġind eterm","çīĩ éĿ¢","\" ).ĊĊ","ce u","Ġ} :","æī¾ 人","ìĿ´ ê³ł","åĽłæŃ¤ èĢĮ","æŁĵ æĸĻ","éĺ´ æ²ī","gar an","CR C","Ġconstit uting","æĻ¶ èݹ","ĠBras ile","çĶµè§£ è´¨","f order","olog ii","ex isting","è§£ éĶģ","cul us","Ñī ении","ż yt","Ġber beda","çĮĽ çĥĪ","Ġmud dy","ĠÑĤемпеÑĢа ÑĤÑĥÑĢÑĭ","鸦 çīĩ","åıĺéĢŁ ç®±","ou re","ÙĪ Ø¦","ĠAr te","交 è´§","Ġlet zten","Ex ist",".c z","ĠAg u","èĪª 线","ĠGold smith","ĠпоÑģ оби","人群 ä¸Ń","ĠN er","åİ Ń","ass in","åĩº éģĵ","ä¹ĭ 士","å¾Ĺ åĥı","rad ed","। [","ĠMet abolism","Ass ignment","Ġadop ts","C her","Ġpre acher","Ġcor ona","é£İ åı£","ĠÑĤе оÑĢе","Ġrot ates","iÄĻ cy","ĠFront iers","ĠPil grim","çļĦ æĪĺ","oph ilia","Ġheart y","ме ÑģÑĤ","Ġ[] ;ĊĊ","Ġzitu zten","Ġengag ements","? s","b urs","çī¹ æĿĥ","ä¸ĩ å®¶","马 åĬĽ","ç¿ Ĭ","Ġmo ons","åľĭ çļĦ","L ex","Ġv amos","çĶŁäº§ èĥ½åĬĽ","ĠRem ark","Ġrout ers","ιο ÏĤ","åıĪ åı¯","Ġmanif ests","黼 çİī","Ġuncont rolled","ipel ago","Ġsp ong","åĨ· åĩĿ","ĠMon ter","Ġব লà§ĩ","ĠØ¢ ÛĮ","t oday","ĠC aps","ĠR IGHT","说 ç½¢","ath ione","fe el","het to","Ġઠµ","åĩº è´§","Ġra ids","اط ر","Christ mas","ä¼ļ åIJİ","ä½Ĩ è¿Ļç§į","ç͵ 容åύ","Ġer an","éĽħ çļĦ","Ġri ots","ĠIde ally","Ġej ec","Ġrever ence","ĠاÙĦبÙĬ اÙĨات","çĹĬ æĦĪ","\\ ----------------","Ġg our","ant as","æ°´ éĩĮ","Ġph ag","身 åľ¨","çľ¼ çľĭçĿĢ","ĠX ue","лÑı ÑĨии","ĠCor ona","ĠпÑĢед лага","Ġgrace ful","åįģåĽĽ æĿ¡","Ġdermat itis","E lev","è¿Ļ åıªæĺ¯","æ¬ Ħ",":ĊĊ ĊĊ","Ġset backs","éĢī éĽĨ","åĩī çļĦ","Ġconj unct","没 å¤ļä¹ħ","ĠFeb ru","åħĪ天 æĢ§","and ler","Ġا داÙħ","д з","=\" $","Ġgener ale","@ s","ä¼ļ å¾Ī","åıĮ åĩ»","à¸Ľ à¹īà¸Ńà¸ĩ","ุ à¹Į","ç¡®å®ļ æĢ§","ç§ĺ å¢ĥ","åIJĪåIJĮ æ³ķ","Ġmedi ating","деÑĢ Ð¶Ð¸","Exec utive","åįłåľ° éĿ¢ç§¯","Ġlivelihood s","Ġinfert ility","Ġ à¹Ģร","Ġyou tube","å± ¹","åı¯ä»¥ ä¸į","å±± æ´ŀ","Ġठ§","follow ing","R atio","} )(","åįĬ çIJĥ","Ġge hen","Ġvict orious","ém ie","åħĦ 妹","Reg arding","ĠFather s","å¤ĸåĽ½ è¯Ń","Ġsoll en","Ġnomin ations","Present ation","ter dam","åħ¥ é©»","ä¸įæĺ¯ ä»Ģä¹Ī","ĠSun rise","Calcul ator","çĸ«èĭĹ æİ¥ç§į","ç«ĸ 缴","ĠR asp","Ġclass ifying","ĠCon ver","åĮĨ å¿Ļ","ĠSt ro","主 å®°","med ical","宫 ä¸Ń","æĭ¼ æİ¥","×Ļ×IJ ×ķת","relations hip","pt une","-p rom","list ing","Ġsulf ide","Ġhes itant","@ implementation","çļĦ ç½ij绾","Ġch il","ÙĤ اس","çļĦæĹ¶åĢĻ å°±","ĠCor pus","å°¿ éħ¸","Ġhero in","Ġsang ue","åIJ¬åıĸ äºĨ","ĠP irates","å½ĵ 羣","ли ва","éģ¥ æİ§","Ġalve olar","Ġjó venes","ĠASE AN","_ ext","f ew","ag hetti","æľī éĤ£ä¹Ī","缸 åĬ©","è·¯ ä¸ĬçļĦ","æĬ¥ åºŁ","ÑĤи Ñİ","CI E","/L ICENSE","éĥĬ åĮº","Ġcontempor aries","ĠExped ition","d agger","ĠÑĥ ви","Ġcal am","Ġmill i","-h ost","uct ive","Ġpuzz led","Ġnort heastern","Ġবà§ĩশ ি","S ci","W u","çļĦ ç§įç±»","大 åIJį","çĤ¹ éĴŁ","ves ter","使ç͍ æĹ¶","Ġне пÑĢи","Ġcr ise","Ġsent imental","ĠÑĢаз лиÑĩи","ĠØ£ÙĨ ÙĪØ§Ø¹","ĠTur ks","ãĤ» ãĥ³","Ġs abe","Ġdis co","د اد","Ġobject ively","Ġconsum es","Ġmist ress","ĠJo ey","ĠSpace watch","æĦ£ äºĨä¸Ģä¸ĭ","V u","a ard","ĠB ef","æĭī çļĦ","ĠLa place","çī¹åĪ« 注æĦı","Ġdrop out","è§Ĵ度 çľĭ","ĠMono Behaviour","orget own","å®Ī ä½ı","åħ¼ å¹¶","Ġcycl o","æĺŁæľŁ ä¸Ģ","Ġwszyst kich","B its","è¿Ļ åı°","Ġres orts","ãĥ ¨","éĩij çīĽ","ÅĤ ów","æĴ «","çͲ éĨĩ","rig eration","Ben efits","ĠHir sch","åŃ¢ åŃIJ","Ġcaract é","Ġsynth ase","人 æĦı","Com mission","_l ast","ĠParliament ary","s pecific","次 äºİ","-f ilter","Ġبا ست",": flutter","Ġad anya","ØŃ ت","ä¼Ĺ 人çļĦ","Ġaccount ant","guna an","_ vec","Ġs eseorang","å¹¶ ç»ĵåIJĪ","å·¥ä½ľ ç»ıéªĮ","Ex am","\"> {","秦 å§ĭçļĩ","Ġmig ratory","Ġunter st","Ġvagu ely","+ âĢĿ","ĠF ail","Ġinter stitial","Ġsw amp","ĠGet ty","Ġpou co","Ġniv eles","B ST","T on","ĉ A","Ġh ikes","ĠF avorite","æĪij åıªèĥ½","æ´» åĮĸ","-s elf","Ġant iqu","ì§Ģ 를","认è¯Ĩ äºĨ","util isation","亨 åĪ©","å°±æĺ¯ æĪij们","ave z","ĠSp ani","ĠPar agu","ĠMass ive","หà¸Ļ ัà¸ģ","ĠMens ch","Ġt enses","ied e","æ·± åİļçļĦ","ĠاÙĦÙĨ جÙħ","Ġf osse","Ġdis belief","社 群","åķĨ è®®","ĠMe in","åħ³éĶ® æĹ¶åĪ»","çĶµè·¯ ä¸Ń","æ·® åįĹ","ĠEli as","ĠCitizens hip","- types","B at","P ear","æĺ¯ ç¾İåĽ½","ĠW WE","å¹¶ èĤ©","ä¸įèĥ½ å¤Ł","Ġcommunic ative","rav ings","ĠAB STRACT","ĠCM OS","éģ® æĮ¡","Ġembra ces","滤波 åύ","> ';Ċ","ĠOr ion","Ġcourse work","UM ENT","u encia","çļĦ æŃ»","åѦ åĪĨ","à¹Ģภľ","æ½ľ èīĩ","Ġe ins","Ġl ö","Ġk ort","éĩij éϵ","èģĶ éĢļ","ож еÑĤ","宾 客","Ġinvers ely","c ape","çħ ½","ç²¾ çĽĬ","ĠAnt io","Ġball ots","à¸Ńà¸ģ à¸Īาà¸ģ","æĶĢ åįĩ","Ġunres olved","w ant","å°ı æīĭ","Ġend block","çĭ¬ åħ·","讨 好","à«ĭ àª","Ġnomb res","Ġensl aved","ĠC ater","าภŀ","Ġë Ĵ","å̼ å®Ī","å¢ŀ 设","Ġhom ologous","sz taÅĤ","çĸ² å̦","ä½ıæĪ¿ åħ¬ç§¯éĩij","Ġrealiz ado","hte et","Ġam used","ĠSouth ampton","éĻĨ åľ°","è¯Ħ论 åĮº","press ure","สัà¸ĩ à¸Ħม","çļĦ éĹ®éģĵ","èĥ½ åģļåΰ","åĽĽ åįĥ","æĢ» æĺ¯åľ¨","ĠLe igh","à¸ķ ำ","ĠAct ivation","Ġsust ent","èµ¢ äºĨ","Ġ기 ìĪł","ĠEntrepreneurs hip","Ġunden iable","/ MS","ĠD up","梦 éĩĮ","ĠVer tex","èĻļ æŀĦ","æĮģç»Ń æĹ¶éĹ´","Ġgrass roots","Ġgru p","Ġintimid ating","on is","人 以","人 éĢī","cl oth","ĠHow e","æĢ» åħ¬åı¸","ĠGold berg","Ġни ж","ĠWOR LD","Ġconspic uous","ä¸Ģ æĥ³åΰ","ĠB ayer","ĠW ow","Ġver ifying","æĢ¥ ä¿ĥ","اس Ø®","Ġsynt actic","Ġpag ina","Ġshowc ased","o an","ol le","她 没æľī","两 å¼ł","ä¸ŃåĽ½ ç§ijåѦéĻ¢","çİĩ è¾¾","Ġà¦ķ à§ĩন","j uk","ĠS UM","ĠAm end","åįĥ ç§ĭ","Ġض ÙħÙĨ","ĠPra irie","Ġболез ни","Ġসà¦Ļà§įà¦Ĺ à§ĩ","ĠJ AMA","Ġun sc","Ġdet ain","Ġexper iential","ಠ¹","ĠEd monton","ĠInter ventions","LA ST","Ġru im",")/( (-","ar án","ĠR PM","ä¸Ĭ 空","åķĨ åŁİ","éļ¾ çľĭ","Ġbo is","Ġdiv ent","éĢĤ éħį","_d escription","×Ļפ ×ķ׾","Ġشخص ÙĬÙĩ","V IP","Ġc ords","Ġre vert","Ġcur t","mar ried","Ġма ÑĤÑĢи","Ġfirm ware","Set up","å¿§ 伤","对çħ§ ç»Ħ","? ?ĊĊ","Ġreg ión","ç»ĵ å®ŀ","oph armac","hab i","Ġë¶Ģ ë¶Ħ","S outhern","Ġ' [","-b rain","å®ĥ æīĢ","ĠBrand s","N el","Ġre juven","oll ah","Ġover expression","çĨŁ çļĦ","Ġvac ancy","Hel pers","Ġs akit","ist ische","åĮĸ åIJĪ","éĩij å¸ģ","ĠGu itar","ĠEqu ivalent","Ġfemin ism","åĦª ç§Ģ","Ġpharmac okin","ĠTunis ia","K ini","çļĦ åIJ«éĩı","ó ź","红 æŁ¿","åIJ¸ è¡Ģ","ĠG ABA","Ġch assis","urn ame","çĤ¹ å¿ĥ","æĺİ åªļ","Ch air","ä¼ļè®® çͱ","ĠEp hes","å±ł æĿĢ","rizz le","ãĢĭ ï¼ļ","åĵ¥ 伦","Ġrev olutions","å®ĩ æĸĩ","å¹³è¡Į åĽĽè¾¹å½¢","Ġà¸Ī ึà¸ĩ","Ġch iral","pl ots","ass uming","éģĵ åħī","ex ports","常 éĩı","Ġbu ena","åı¤ è¯Ĺ","Ġwel d","reci pe","è¨Ī çķ«","Ġacceler ator","å¿ĥçģµ çļĦ","å°± åıªæľī","ĠAf ro","ার à§įথ","ĠSign ature","ĠDick inson","à¸Ľà¸ı ิà¸ļัà¸ķิ","o pper","p olitical","ä¹ĭ åŁİ","åºĶ 纳ç¨İ","ops ida","Ġà° ¦","EX P","éĩĮéĿ¢ æľī","Ġchief s","ধ ান","кла дÑĭ","ĠINS ERT",". word","ĠS ánchez","Ġimport ing","fl ight","Ġsym phony","çļĦäºĭ 项","Red irect","åįģä¹Ŀ 竳","ä¸ĭ æłĩ","з он","co ord","æ´Ĺ 礼","Ġë§ ŀ","lock ed","Õ¸ÖĤÕ ½","Ġâĸ ¼","Ġthe o","åĪĨ æĭħ","Ġout ra","Ġinter és","åĬł åĵ¥","éĹ® ä½ł","ateg ori","å·¥ç¨ĭ æĬĢæľ¯","à¸Ĺาà¸ĩ à¸ģาร","Ġpilgrim age","Ġamel ior","ĠN olan","Ġha il","Ġا Ú©","æīĵ åĬ¨","åıijå±ķ ä¸Ń","ĠCol ony","ipp le","认å®ļ 为","he ra","Ġunder line","åij¨ äºĮ","åºĶå½ĵ æĮīçħ§","Ġquot ations","ä¸į è¯Ń","åľ¨ éĢīæĭ©","Ġsh rug","讲 åΰ","lick r","çļĦ ä»»ä½ķ","ä¸Ģ åį·","å¦Ĥ ä¸Ĭ","æĹł ç¼ĺ","éĢī ä¿®","çĨ µ","梯 å½¢","Ġ기 본","Ġsé curité","udd in","Ġh ides","ĠB RO","ĠL owe","Ġhe irs","Ġ\\( |","羣 åĪĩ","åıĸ äºĨ","åij¨ æľŁçļĦ","ered ith","è´Ł æľī","Ùİ ÙĤ","ĠOlive ira","ĠApp alach","é¾Ļ éŨ","Ġrev ived","ĠAltern atives","ĠConc ern","Ġlobby ing","il og","iz u","ĠCh loe","á» ī","ï½ŀ ĊĊ","OUR NAL","Ġrealt Ãł","p ng","åı¯ä»¥ çļĦ","ix es","ĠÑĢа ÑģÑĤв","Ġtre acher","è¸ IJ","åIJĮåѦ çļĦ","å¥Ķ èµ´","Ġverte bral","Ġп ÑĤи","产 çļĦ","åIJĥ 飯","æijĨ åĬ¨","ÑģÑĤвен наÑı","çļĦé«ĺ 级","å·¡ åĽŀ","ĠÑģеÑĢ ÑĮ","-ey e","-Un is","C ancer","Y E","ĠM ets","ore tic","å± ī","Ġpr ise","åİĨ æĿ¥","çĶµè·¯ çļĦ","=\"# \"","Ġpharmac ies","= M","没 éĴ±","æ°´ ä½ĵ","æĹł å¿ĥ","-f aced","ĠÙĬ ر","BO OL","િ àª","Ġprinci pe","æľī 声","å» Ł","-m enu","åIJĥ äºı","à¸ķ ล","建设 åįķä½į","éĢĢ åĽŀ","ĠRem ed","ĠSP SS","æĿŃ å·ŀå¸Ĥ","Ġadvers ary","â ł","çļĦ ä½ł","ig heid","-s elling","å¦Ĥæŀľ æĥ³","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","Ġrev ive","ĠAnn iversary","åĽºå®ļ åľ¨","Ġwear able","Ġtéc nica","æĺ¯ ä½łçļĦ","ĠD ix","Ġen n","çĶŁ åĬ¨çļĦ","æīĭ è¡ĵ","éĩij 丹",".âĢĿ [","ais on","æĦ¿ æĻ¯","ki ra","åĩ¡ 人","交æĺĵ æĹ¥","ip y","Ġem itter","æľĪ æľ«","æĢ» è¦ģ","Ġsl ap","çļĦæĺ¯ ä¸Ģ个","ĠDark ness","èªĵ è¨Ģ","ç쵿ķı 度",".EntityFramework Core","an ze","Ġr ites","天 ç¥ŀ","ĠÙĪ Ø¥ÙĨ","Ġnu isance","ר×IJ ×Ķ","å¿ ij","ĠJ F","Ġdes em","å½ĵ ä¸ŃçļĦ","iss es","ز ب","Ġje u","礼 æĭľ","æĢ»ç»ĵ äºĨ",". op","n ým","ç¬ ł","ob ot","ung tor","强 æĤį","Ġgro ÃŁ","æIJį 失","ĠHEL P","Ġp ää","ï¼Į âĢĿĊĊ","ĠI CD","åħ·æľī èī¯å¥½çļĦ",".T ime","å͝ çĭ¬","Coll abor","( View","d ong","å¹´ 为","rit a","Ġprop ulsion","åĿı çļĦ","ĠHor izontal","ĠHo over","Tra ditional","Ġsauce pan","Ġï¬ģ rst","former ly","Ġlangs ung","g uan","ĠG G","åįķ æį®","ĠÑĩ лен","åį¡ çļĦ","Ġqual idade","帮åĬ© ä»ĸ们","font s","Ġ ......","Ġg erman","ĠIn gen","Ġ ¯","ĠMar ines","éĢı éķľ","Ġassert ions","Ġми нÑĥ","ĠConc ert","ĠмаÑĤеÑĢи алов","- access","el ay","对 ä½łçļĦ","ĠSt ake","交 çķĮ","Ġsecond a","ĠاÙĦÙħ ÙĦÙĥ",".m atch","Ġod reÄij","Ġdos ing","ĠJo ão","Ġneuro science","Ġsh amp","ç¨ ·","pond er","绵 绵","éĽĩ åijĺ","Ġintrig ue","ĠGalile o","ä¸įåΰ ä½į","api ens","ĠLuc ia","Ġkar akter","Ġordin arily","над ле","Ġmendapat kan","ag greg","åѦ çłĶç©¶","-s u","-d em","Int o","ĠP ORT","åľ¨ æķĻåѦ","Ġà° ļ","Ġov at","üt zen","Ġapost les","| x","Ġh ym","ĠT act","ä¸Ģ åĽ½","Ġland fill","å¥ĩ çī¹","ĠMont essori","éĽĻ æĸ¹","at amente","Ġso aring","ĠCal der","ä¹ħ ä¹ĭ","ĠMon key","Ġtoug her","' art","Ġt ém","Ġh ottest","ا ÙĪÙĨ","éĢł æŀĹ","gl io","åħ¼ ä»»","Ġdefe ating","è¾ĸåĮº åĨħ","Ġbureauc ratic","ĠÙĨÙ쨳 Ùĩ","d ade","Out side","L l","ä»Ģä¹Ī 人","Con version","Ġসম য়","Ġincons ist","Altern ative","est hetics","Ġprogram mable","åı° è¯į","Ġallow able","Ġsin ful","ĠHy de","Ġsept embre","r insic","Ġg aug","åŃ ļ","ac ios","åħ¥ åľº","Ġdr unken","be havior","Ġda her","å°Ķ å¤ļ","Ġmot to","Ġdisapp earing","æĤł éĹ²","auss ian","ĠاداÙħ Ùĩ","P ixel","_ inter","ĠF reed","ĠL eng","çIJ ¶","è¿Ľ åŁİ","æĬĬ æĪij们","é£İ ä¸Ń","åıĤ éĺħ","ä¹Łä¸į åĨį","Ġclos ures","Ġscram bled","ĠHod g","_ ne","Ġo ps","ä¼ļ è°Ī","Th om","ÏĦ Ïīν","λ λο","ĠBel ieve","Ġbath s","éĪ ´","æĪij åΰ","æ°ij å¿ĥ","åħ» åĪĨ","计åĪĴ åĴĮ","Ġnar ration","ĉ let","ç͍ 好","åij¨ åĪĬ","éĢĢ ç¨İ","å°ļ æľī","çĸı 导","èĬĿ åĬłåĵ¥","ĠìĦł íĥĿ","她 äºĨ","ä¾Ľ è´§","å¯Į 豪","Ġhom age","Ġgrand eur","éĺ» æĸŃ","ĠÅ ŀ","æľ¬æĿ¥ å°±","âĢĶâĢĶâĢĶ .","\" So","_ const","i ada","ä¸Ģ çĶŁçļĦ","cc cc","èĢĮ å¼Ĥ","åı¯ä»¥ è¿Ľè¡Į","éħ ĭ","Ġpartner ed","个æľĪ åĨħ","è´¢åĬ¡ æĬ¥è¡¨","Ġà¦Ń াষ","ç¬ijçĿĢ è¯´éģĵ","æķıæĦŁ æĢ§","ĠпаÑĢа лле","ans ki","Ġacc ret","Ġо Ñĩи","èĤ² åĦ¿","à¸Ķ à¹Į","(f unc","Ġbright est","çĽĪ ä½Ļ","ĠHung er","ĠCategory TreeLabel","Ġl t","ĠS ECTION","ĠI ber","are nd","ä¹Ł åįģåĪĨ","åIJij æĪij们","å·² è¾¾","ĠÙħÙĨ ابع","Ġabs cess","æŁĶ æĢ§","éĤ® ç¼ĸ","éĢĻ ä»¶äºĭ","Ġtre asury","Ġза меÑĤ","Ġreason ed","Ġkel u","æķ· è¡į","Ñıв ление","% )Ċ",") --","ĠG om","èĥ½ ä¸İ","åĮĹ京 æĹ¶éĹ´","(\\ ,",") D","+ p","Ġ à¸Īาà¸ģ","ä¹ĭ ä¹ħ","Ġem inent","æĪĸ å°Ĩ","Ġк лÑİ","åıĭ 人","à¸Ĭ ัà¹īà¸Ļ","âĦ ĸ","ĠDEL ETE","Ġcondemn ation","Ġamplit udes","un iform","ore xia","å¿ IJ","åIJĮ æ¡Į","-pro ject","Ġfluctu ation","Ġuncon stitutional","Ġmathematic ian","Ġw ob","ide al","by t","Ġter ap","Ġpolit ik","Tr im","Ġоп ла","éĥij éĩį","Ġwet land","- web","re po","åı¯èĥ½ æľĥ","LE FT","ĠTechn ician","ĠÐĽ Ñĥ","Ġconserv atives","Ġاس اس","ĠP ec","ä¸Ĭ åĬł","IC Y","å°į æīĭ","çļĦé«ĺ ä½İ","强åζ æĢ§","Ġbenz ene","iv u","ĠC hern","act ed","ĠK afka","åIJİ åį«","Ġmat s","äºij çļĦ","imm el","大æ¦Ĥ çİĩ","åζ æľį","ĠÙĪ Ø§Ø³Øª","ĠAud ience","ĠðĿIJ µ","æĿı ä»ģ","çijķ çĸµ","ó ż","å¤Ħ 女","Ġম ার","çĽijçĿ£ 管çIJĨå±Ģ","F org","主è¦ģ 以","两个 人çļĦ","çŃĶæ¡Ī 为","åĽŀçŃĶ è¯´","æ¶īåıĬ çļĦ","æĭĸ çĿĢ","åĴ³ åĴ³","ä¹ĭéĸĵ çļĦ","à¹ģà¸ģ à¹ī","вле ка","O ri","ĉ count","an ey","Ġper ic","Ġdis respect","Ġsub space","-e v","æķij 人","Ġcas ually","Ġઠħ","Ġcowork ers","ĠM ug","ĠD ashboard","Ġhe ck","Ġr igu","åı¯ 羣","Ġreg ião","Ġпе Ñģ","ĠìĨ IJ","-round ed","ĠB ike","éĹ®é¢ĺ æĹ¶","é¢Ĩ çķ¥","çϾ æĹ¥","ĠEp stein",".github usercontent","Ġsurfact ant","' Brien","в ÑĪие","Ġrespons ibly","ä¿ĿæĬ¤ åĴĮ","Ġпов ÑĤоÑĢ","èī° å·¨","iop athic","Ġktóry m","R ATION","in x","çĶŁäº§ æĪIJæľ¬","è§ĦèĮĥ æĢ§","Ġpip ing","dig it","çĥĺ å¹²","å¿ĥ æĦ¿","arg as","à¸ķ à¸ģ","åĿĩ å̼","æĭį çļĦ","ĠSm oking","æ»´ å®ļ","é¾Ļ头 ä¼ģä¸ļ","нÑĨи клоп","( sp","G ab","ä¼ļ 说","å°ı èħ¿","çĸ Ļ","Ġsom me","max imum","寺 éĻ¢","Ġmour n","Ġawaken ing","are z","Ġfirst hand","çİ© äºĨ","ĠCard iol","缴æĴŃ éĹ´","гоÑĢ Ð¾Ð´","- fluid","Ġim position","Ġchild birth","Ġstruct urally","ĠAll ies","èĭ± å°º","-w rapper","éĸĭ æĶ¾","!! Ċ","第åįģ ä¹Ŀ","Ġcrypt ography","æĬijåζ åīĤ","ĠгÑĢа дÑĥ","ĠArgent ine","Ġrecess ive","Ġشر اÛĮØ·","Ġfibr illation","L ady","ĠF ever","ne hm","ä¿Ŀ æ´ģ","åıĹ éĻIJ","uf e","ä¸ĸçķĮ éĩĮ","åŃĻ æĤŁç©º","/ year","ok ka","Ġtemper atur","Äģ d","Ġimmun o","åįģä¹Ŀ å±Ĭ","- earth","ä¸į æĸĻ","Ġacc ión","èIJ½ åIJİçļĦ","rop ract","åį¡ æĭī","åģ¥åº· æĪIJéķ¿","æĭ¥ æľīä¸Ģ","ĠVo ices","ĠCele b","Ġsilic one","k atan","Ġe ut","å¤ĸ åħ¬","ĠAd option","éģİ çļĦ","ĠRiver a","ä¸Ĭä¸Ģ å±Ĥ","Ġcheap est","ç´«å¤ĸ 线","ĠÃīt ats","Ġläs st","! :","c pp","ĠE arnings","大 çϽ","ен наÑı","Ġend ings","Ġparas it","ĠPant hers","Ġbor on","> \\)","ar é","Ġtable au","ĠاÙĦÙĨ Ù쨳","ĠRef lect",". There","? >","ĠK ost","Ġlong o","éĨ ¬","人åijĺ åĴĮ","æ²ī çĿĢ","ï¼ģâĢĿ âĢľ","Ġاست اÙĨ","uy ên","èĿ Ļ","Ġà®ķ à¯Ĭ","ĠPend idikan","E ight","z uk","Ġgo alk","ä¸ī è½®","Ġserv ings","Ġر ÙĪØ§ÙĨ","Ġà¦ķ à§įর","ĠRec ruitment","ĠBr ush","Ġëĭ ´","çĵ¦ æĸ¯","ĠNE ED","æŀķ 头","Ġabb iamo","Ġh ukum","åľ¨ ä¸Ģ次","å¹³ æĪIJ","åĬ³ ç´¯","تر ÙĪÙĨ","ĠCard iff","- =","S afety","æīĵ åħ¥","Ġauthor ised","à¹ĩ à¸ĩ","Ġpu berty","d zi","ĠL un","Ġj aws","好 ç¬ij","èĥ ¥","Ġchar ger","åIJ¬ è§ī","Ġshort ening","Sh ader","æ²Ļ çī¹","æĨ ©","Ġenf ant","Ġconjug ation","ìķĺ ëĭ¤","Ġk ör","è¾¹ æ¡Ĩ","Ġг ÑĢе","Ġter race","IP P","ĠÙĤ Ø·","âĸ Ĵ","çĿ¡ ä¸įçĿĢ","ĠUnter nehmen","- fer","ĠR ental","ç¾İ éĩij","ĠSo vere","Ge ometry","ĠобÑīе ÑģÑĤва","ĠSina i","ĠM alt","åIJĪ æ³ķçļĦ","Ġdi jo","å¼ł å°ı","ç³»ç»Ł æĢ§","å¾Į ãģ®","ni ÄĻ","çĺ ©","à© Ī","Json Property","Af rica","ĠSad ly","Ġgior ni","ro ly","ĠA ED","ĠM X","åĴĮ è¡Į为","Ġtra inees","æĹł å¼Ĥ","èĤī ä½ĵ","ĠWal ton","Ġnaturale za","Ġlup us","= l","M ichel","ĠN es","og as","Ġch u","Ar k","åĮħæĭ¬ äºĨ","å¿ħçĦ¶ ä¼ļ","Ġunders c","िय ा","éĿŀçī©è´¨ æĸĩåĮĸéģĹ产","หà¸į ิà¸ĩ",": R","Ġpo pping","åıĭ åĸĦ","Ġgas ped","çķ¶ å¹´","ĠSun shine","wood s","arbon ate","ĠâĹ İ","ĠDead line","ol ism","qu ire","ile a","Ġform ação","IT DA","ικ Ïİν",".py plot","âĨĵ âĨĵ","çļĦ éĶĻ误","Ġh ardships","ĠG one","Ġsh oved","ä»ĸ åı¯ä»¥","åĪĨæŀIJ ä¸İ",")\\ ]ĊĊ","First ly","-com ponents","èĪªç©º åħ¬åı¸","- ru","- plan","ul ación","ĠF riendly","èĥ½ åĬ¨","Ñģк ог","çĶ· 士","ĠFl int","Ġship ments","V IR","ĠB raz","è¦ģ ç´§","åIJĪ ä¹İ","æĥħ è¶£","ä¼ĺ éĢī",".m ark","个人 æīĢå¾Ĺç¨İ","Ġautom obiles","æĮij çľī","çŁ¿ çī©è´¨","ativ i","Ġmic rons","Ġinters ections","轨éģĵ 交éĢļ","al ink","ä»ĸ ä¹Łæĺ¯","ire z","çݰ ä»Ĭ","ĠÑģ енÑĤ","è¿Ļä¹Ī ä¹ħ","Ġtransc ends",". );","d ater","get ting","Ġchild care","å¹² è´§","िठı","C Y","_ keys","ĠB aj","æľī æĹ¶éĹ´","th orne","oc ating","Ġpl oraly","å½ĵ å®¶","常 å·ŀ","Ġid é","èıľ åĵģ","Ġsort e","Ġcin ematic","Ġμε ÏĦα","大 éĺª","å®ī 康","åij¨ 身","ส à¸ļ","ç´¢ å°¼","ĠÑģво ими","éri ence","ĉ continue","ä¹ĭ åĬŁ","Ġmod elled","ĠWe bs","Ġза клÑİÑĩа","ç»Ī çĶŁ","Ġtrump et","Ġt ides","в ÑĪий","â̦ )","æĹ© å¹´","Ġge othermal","ĠNe cess","! âĢĻĊĊ","æ³ Ĺ","å·²ç»ı å¾Ī","ĠChar ity","Ġhat ten","Ġíķ ©ëĭĪëĭ¤","å¬ ´","ĠоÑĢгани зм","éĢĿ ä¸ĸ","Ġма ленÑĮ","éģ¸ æĬŀ","ï¼įï¼į ï¼įï¼į","A ust","Ġst itches","Ġon ge","em es","ĠÙĬ ÙĨا","ðĿij ĵ","ĠCast ell","Ġp iel","Ġz ost","æĪ¿ 举","де л","ĠÑħ и","ÑĤив но","{D oxy","ĠM ash","é¢ĺ åºĵ","Ġatt est","åħ± ç͍","Ġtemplate Url","Ġib id","Ġnue vos","Ġим мÑĥ","D V","ĠM imi","Ġ\" {","æĢ§ 質","Ġprov oked","Ġbu ku","æł¼ æł¼","红 éħĴ","ä½Ľ æ³ķ","ĠÏĥ ÏĦα","Ġpound ing","< meta","R oad","Ġp ágina","ans son","ä¸ī å®¶","Ġpo zw","æĮģ ãģ¡","Ġpass é","åį· ç§¯","ĠHot els","ĠÔ ³","Ġboss es","ĠY us","ER P","è¿Ľè¡Į å¤ĦçIJĨ","Pro ba","ĠاÙĦÙħ غ","积æŀģ æİ¢ç´¢","M olecular","_ handler","ĠM ice","ĠG osp","-b rown","ĠÑĢа Ñģк","ĠKey word","Ġboost ed","Ġbers ama","ĠL iga","åĪĨ 身","çĤ ķ","ع ÙĪÙĨ","Ġdi ameters","åIJij 她","æī¾ åĽŀ","ez ing","Ġส ำ","Ġprés ente","Ġunfold ed","Q T","Ġb ater","æ¯ı åΰ","---------------- --------","æĺŃ åĴĮ","Ùĩد اÙģ","-ne eded","+ v","ĠD abei","Ġcont emplate","Ġв ÑģÑı","åIJ¸ 纳","ĠEll iot","j alan","Ġp irates","Ġon Create","为 ç¡®ä¿Ŀ","Ġel oqu","红 线","åľŁ çļĦ","ç»ĵæŀĦ æĢ§","èµ¶ æĿ¥","ĠSyn opsis","ĠTransl ator","Ġredist ribution","æ±¹ æ¶Į","N ach","op olitan","ç¬ij äºĨèµ·æĿ¥","åįĬ å¤ı","fe it","neh mer","ĠдалÑĮ ней","as ers","ĠN Ps","åı¯ ä¸İ","强 å¼±","çľ¼ çķĮ","åįĹ éĿŀ","é¢Ħ 订","chan ics","ãģĭ ãĤĤãģĹãĤĮ","Ġimpos es","Ġfav ore","ĠBuy ing","Ġ à·ĥ","qu it","ĠW ent","ĠL ai","æĪij ä¹Łä¸įçŁ¥éģĵ","æ³ķ æľ¯","Ġes ophageal","ĠÐĶ Ð¾","ÐĿ о","éĩijèŀį å᱿ľº","Ġsubs ystem","åįģåħ« 大","worth iness","junct ive","st ay","Ġsol itude","ç²¾ æ¹Ľ","æŀĹ ä¸Ń","η μα","Pl ate","Ġsz ere","Ġpir ate","马æĭī æĿ¾","人 è¡Į","åķ °","Ġprec aution","å¢ŀéķ¿ çļĦ",".n c","Ġpoll uted","Ġshoot er","Ġward robe","Ġenlarg ement","Ġsh utter","女 æĸ¹","Re place","Ġка кие","Ġtax ed","ä»Ĭ天 å°±","Ġmagn itudes","Ġvoor al","Ġceremon ial","å®ŀå®ŀåľ¨ åľ¨","< E","Ġt d","Ġen listed","Ġsp ared","ç¡ ¼","Ġgr ating","Oh io","Eff ects","al ten","ud em","宣 åĤ³","ç²ī èī²","Ġca vern","ä½ĵç³» ä¸Ń","Ġcyst ic","ĠBrid ges","Ġdehydrogen ase","Ġskeptic ism","j r","Ġv illa","åij Ľ","Ġad joining","äºĭ åīį","ÑĪ ÑĤе","çİĩ é«ĺ",". En","_ account","ĠS utton","ç͍ åħ·","å¼Ģ éϤ","Ġδ ε","ĠЧ ÑĤобÑĭ","éĵŃ è®°","ден ÑĤи","Ġà¦ħব সà§įথ","ĠS IZE","Ġne oplas","ĠK emp","éĤ£ åIJį","Ġimpro vis","sh all","Ġsch wer","àµįà´ ª","Ġdens ely","v ariant","å·¥ä½ľ ç«Ļ","æĹ© é¥Ń","Ġestab ele","ĠGl oria","ðĿij §","åĸľæ¬¢ åIJĥ","缩 åĨĻ","ä¸īåįģ åħŃ","ĠпÑĢоек ÑĤа","F ox","Ġpre historic","ç½ ¡","Ġcar ving","Ġden ne","Ġenc ro","Ġcr ÃŃ","é¸ ¢","è°ģ éĥ½","åŁºéĩij çļĦ","Ġtransition ing","Ġsuc rose","Parent s","ĠPé rez","ĠBelf ast","ĠA X","åľ° è²Į","æİ¥ 管","åĩł å¹´æĿ¥","_t emp","Ġphot ographed","ĠجÙĩ ت","Ġsubscrib ed","ĠSvens ka","J oh","æµ· 绵","第ä¸Ģ éĥ¨","ìĽ Ģ","Ġни к","ĠAud itor","ĠÙĬÙĨا ÙĬر","Ġby ÅĤy","å°ı 车","Ġpar ch","Ġsol ace","表çݰ äºĨ","Ġpap ill","L N","m ock","Ġar cs","åĪ© 好","ç«ĭ äºİ","åĭ ¸","äºĶ èĦı","ाठ¶","ĠMat em","ĠÑģÑĤ оÑĢ","Ġdiagn osing","Ġreact ants","ĠChen nai","Ġt ama","ĠS art","æĿ¡ æĸĩ","ĠFr ant","Ġfort ified","A void","Ġв Ñģп","ĠSh am","å·²ç»ı æľīäºĨ","Ġje une","EF ORE","Ġли ÑĨа","èĢĢ çľ¼","Ġval e","è¿Ļ个 æĺ¯","ä¸ŃåĽ½ æĸĩåĮĸ","太 å¿«","ĠباÙĦ Ø£","èŀĥ èŁ¹","ä¸Ń å±Ĥ","Ġdis joint","èĢĮ è¿Ļç§į","Ġet ap","fl in","ย ี","Ġ×ŀ× ¢×","ĠIP L","Ġcontrast ed","Ġunf olds","Ġking doms","åķ¦ åķ¦","Ġtract s","Ġsuperv ise","yn aptic","Ġμ α","ĠOpen AI","躺 çĿĢ","ĠÐŁÑĢо Ñģе","- choice","Ġs ill","Ġk amp","äºĮ æĪĺ","ни кÑĥ","Ùĥ د","å¸Ń åį·","nex pected","ĠFly nn","å¹´ ä¼ļ","表 çļ®","Ġsc rolling","æµ· 滩","Ġpen is","缼 è¡Į","Ġsa i","Ġgl ove","æŀģ é«ĺ","ĠÙ¾ ÛĮد","ä½ĵç³» åĴĮ","eles a","Ġburst ing","æĪij åĨĽ","èµ Ĥ","ва ла","ä¸ĵ åĮº","cem bre","绿 çļĦ","ĠÑģо ÑģÑĤави","_S EC","ĠBerg en","Ġtril ogy","Õ »","ĉ throw","ç§ij 举","Com mercial","伤 åĬ¿","Ġdest abil","gl ob","ä½ľèĢħ çļĦ","ï¼Ĵ ï¼IJ","æ¸IJæ¸IJ åľ°","ĠFamil ie","ĠIndic ators","ĠNicarag ua","è¸Ĭ è·ĥ","ó i","ood s","ÃĹ (","åħ¨éĿ¢ æıIJåįĩ",".next Line","Ġly ric","æĻ´ 天","multi row","Ġrehab ilit","Ġconvolution al","ĠDivis ibility","O ften","ell ery","Ġund o","ng en","没 ç͍","ĠÙĪ Ø´","ÑĤе Ñħ","rag es","秦 åĽ½","K ab","å¾Ĺ è¿ĩ","æĸĩ è¨Ģ","own s","åı¯ä»¥ å®ŀçݰ","çħ ¦","ening en","Ġì ´Ŀ","äºĨä¸Ģ åıª","ĠPar al","ĠAg ar","èıľ èĤ´","Ġtong ues","åĭŀ åĭķ","Ġschn ell","R ab","å¸ ¯","per fect","两 æĸ¹éĿ¢","Ġrev olves","à¸Ĭ าย","è¡ĮæĶ¿ è¡Į为","Ġdial ects","ðĿIJ µ","à¹Ģà¸ģิà¸Ķ à¸Ĥึà¹īà¸Ļ","çļĦ çݰå®ŀ","è¦ģ æĪij","Ġqu oi","åı¸ 空","Prov ide","_fe atures","Ð Ĩ","Ġpres e","å¾Ģ å¹´","éĴ± è´¢","жа еÑĤÑģÑı","ĠмеждÑĥ наÑĢод","ĠG inger","éĩį æŀĦ","å¹³ ç±³","man a","AN E","ÑĨи ÑĺÑĥ","ÙĪØ± ÙĬØ©","Ġdire kt","Ġocean ic","ĠPut ting","Dist ribution","Ġas par","ĠD ruck","ill ust","Ġset Timeout","uc o","Im plementation","çŃ¾è®¢ äºĨ","oprote ins","Ġst anza","ĠF og","ĠG ER","Ġtra ctor","Ġо ÑĨени","-d ire","à¸ŀ ุ","ĠDid n","à¹ģà¸ķ à¸ģ","he k","Ġun idades","ue il","ä»ĸ ä¿©","æĸ° åĨľæĿij","ÙĪØ± Ùĩ","ocal ypse","èĭ±åĽ½ çļĦ","Ġa ired","ä¸į æľ½","åľ¨ åįĬ","ä¸Ĭ 岸","ä½ł æľīä»Ģä¹Ī","ĠCh ong","×ķ ×ķת","ull er","éĴ Ĭ","带 åĽŀ","âĹ ĩ","Ġadren aline","Ġpotrz eb","Ġonemoc nÄĽnÃŃ","z c","Ġ( /","ass ociated","Ġprot otypes","Ġsoc iais","Ġktó rzy","Ġvas cul","j aw","ro kee","Ùĥ رة","æĶ¾ ç¼ĵ","äºĨä¸Ģ 天","Ch i","Ġmatter ed","miss ive","Ġnicht s","Õ¸Õ ½","ĠHil bert","Ġâľ ĵ","å¿IJ å¿ij",". byte","; k","l iving","int i","Ġind el","Ġmon soon","Ġmus ÃŃ","Ġlow s","-st atic","åıĹåΰ çļĦ","æħĮ å¼ł","Ð Ī","Ġ| =","Ġdis infect","å°±æĺ¯ ä¸Ģ","ö h","Res p","-S mith","={ {Ċ","ç«ŀäºī ä¼ĺåĬ¿","Ġpione ers","Ġdischarg es","j te","缸åħ³ ä¿¡æģ¯","ĠBo ards","ĠPet ro","ä¸Ģä¸ĭåŃIJ å°±","Z K","Ġdis location","è¾Ľ è¾£","ãģ¡ãĤĥ ãĤĵ","нÑĨиклоп еди","K ids","Ġc ysts","çļĦ æĢ§èĥ½","ä¸Ń åįĹ",")ĊĊ ĊĊ","Ġexp r","å·¥ç¨ĭ è´¨éĩı","Ġsho ppers","Ġespa ço","èįĶ æŀĿ","кад еми","Go al","è¡ĮæĶ¿ å¤įè®®","Ġë³ ij","Ġpersu asion","æ·ĭå·´ ç»Ĩèĥŀ","Ľ ×ķ","Õ¡Õ ¢","ĠAssoci ations","Ġpun kt","ĠÄį esk","Ġанали з","ĠHond uras","T ak","ì £","ac ons","ĠÑĢа ÑģÑĤво","Ar row","ä¸ĸçķĮ åIJĦåĽ½","اÛĮ ج","ĠÕ Ń","æ²»çĸĹ æĸ¹æ³ķ","Ġmedi ators","ĠFamil ien","Ġdock ing","liwo ÅĽci","e val","ÙĦ اب","æĿ¥ åΤæĸŃ","ä¹Ł ä¸İ","form e","éĹ´ æŃĩ","没 åķ¥","Ġsal am","Ġber ada","Ġpow ders","ĠSam antha","Ġinsert s","ĠHindu ism","ä¹ŀ ä¸IJ","ĠSask atchewan","ut ti","Ġwor sen","è¿ĺæĺ¯ 没æľī","æĸĩåĮĸ æĹħ游","çİī çŁ³","ĠÑģÑĤа новника","éĽħ åħ¸","ĠØ· رÙģ","UM N","à¸ķร วà¸Ī","ä¸Ģè¡Į 人","ĠPent ecost","ĠKu bernetes","Ġpla ques","/ week","Ġnecess ities","ĠDr inking","å¹¼ èĻ«","éķľ åĥı","Ġκα ÏĦά","اÙĦÙħ Ùĩ","ĠзнаÑĩи ÑĤ","å°± ä¸Ģ缴","й диÑĤе","ĠAs ÃŃ","ว าà¸ĩ",".n ode","Ġtact ile","e ine","s leep","Ġr inse","Ġab stra","Ġtrou bling","Ġtheore ms","çĮĽ çļĦ","Ġëĵ ľ","c ault","c arb","åĴĮ æıIJé«ĺ","ĠK ras","ĠTh i","åħ¶ äºĭ","æĿİ å¤§","ાઠ¨","ĠEmb racing","ĠF TP","ĠHe arts","Ġco le","Ġsed an","H unter","Ġs zt","åIJİ åľ¨","è¿ij äºĨ","ae v","cz ÄĻ","积æŀģ 主åĬ¨","ä¸Ģ 楼","æĸ° åĵģ","项 éĵ¾","çݯ å½¢","æł¸ å®ļ","ाठĸ","形象 çļĦ","ãģªãģ© ãĤĴ","ĠCraft s","C nt","at ts","al as","Ġh ates","os z","ĠL unch","éķ ¯","Ġпо Ñį","éļıçĿĢ æĹ¶éĹ´çļĦ","Ġcin q","èį£èªī ç§°åı·","Ġmultid imensional",". He","Ġ ð","ĠP ipeline","ĠB K","ä½ł åij¢","å®ī å±ħ","ĠAl onso","åįķ 车","ĠEn ough","Ġune asy","Ġcraft smanship","ĠболÑĮÑĪ Ð¾Ð¹","Ġéconom ique","ĠM US","æ¶ İ","éĴ µ","å®ĥ 对","èĤ¡ çļĦ","å¿ĥä¸Ń æľī","ig its","å¸ ¼","ge ant","Ġcomplet a","æ¯ħ åĬĽ","ãĥIJ ãĤ¤","ãģŁ ãģı","ĠRel ax","Ġspray ing","Ġstamp ed","ĠClaud ia","ĠScandin avian","Ġf akt","ãĥ ´","Ġdist rust","éĩİ èĽ®","ĠاÙĦØ· بÙĬع","ĠDe e","ĠAb ram","ç¹ «","ĠAff ect","å°ıå§IJ å§IJ","ĠPul monary","ĠÐIJлекÑģ анд","Ġsh ading","Ġi Tunes","Ġtra pez","ĠSim one","ĠAnt ony","à¶ ¯","åľŁåľ° 使ç͍æĿĥ","Ġcontempl ation","H idden","n ian","а б","ĠB res","大ä¼ļ ä¸Ĭ","S olid","Ġh uh","Ġv iss","ĠH ilton","ik um","ph on","Ġer w","èµĦæºIJ éħįç½®","ĠAtt ributes","-an ak","fü hrung","f ine","ä½ł å¾Ĺ","Ġshe pherd","Ġsm elled","çϽ 马","åѤ åįķ","Ġdispon ible","ĠÙħار س","ãĥ ĺ","Ġag ora","æĹł æĦ§","à¸Ń ิà¸Ļ","Ġvol upt","åĽ´ çĿĢ","Ġang strom","ä¹Łä¸į ä¾ĭå¤ĸ","æĪIJ为 ä¸ĢåIJį","积æŀģ åıĤåĬł","Ġnan oc","åŁºå±Ĥ åħļç»Ħç»ĩ","-ex per","ĠRod rÃŃguez","å·´åŁº æĸ¯åĿ¦","[ name","åĩº 轨","èĢĮ å½ĵ","ä½ĵ èĥ½","Ġder iving","ä½İ çĿĢ头","rack er","Ðł Ðĺ",": \");Ċ","F IR","f ailed","æľī æĦıä¹ī","ĠN ath","yl abel","Ch urch","as en","st vo","ring er","èĩªå·± åģļ","æĿĢ ä¼¤","ĠWed ding","Ġwave guide","Ġresist ing","ä¸ĩ人 次","Ġunnot iced","O g","re dux","Ġl ama","Ġcon formation","Ġun st","管çIJĨ ä¸İ","çļĦ大 åѦçĶŁ","Ġled ger","Ġìķ ŀ","-en hanced","à¹ģà¸Ľà¸¥ à¸ĩ","re ason","Ġout flow","ÑĪ ÐµÐ½Ð¸ÐµÐ¼","让 åħ¶","é»ij äºĨ","æķ´çIJĨ çļĦ","贯彻 æī§è¡Į","ĠTol edo","ĠS icher","æľŁ 为","缴 è¾¾","èµ° åĩºäºĨ","ĠCor al","Ġcollabor ated","ĠઠĽ","Ġol factory","Ġjun ctions","ĠFA O","常åĬ¡ å§Ķåijĺä¼ļ","è¿ĺ å¿ħé¡»","éķ¿ åıij","eg g","©× ļ","å®īæİĴ çļĦ","gra du","ĠÙĦÙĦ Ø¥","è¿ŀæİ¥ åΰ","-co ated","R eb","ĠS UN","ĠR ings","åıį èĢĮæĺ¯","åĮĹ æĸĹ","ä¾Ľ å¥ī","ÑĪе л","å°¾ éĥ¨","ä¸Ģåıª æīĭ","ĠÑģем ей","- current","R ing","ĠC rane","ĠR amos","Ġsp el","çĤ¹ æķ°","äºĮ ä½į","Ġspec ie","å¦Ĥæŀľ æĬĬ","ĠнапÑĢÑı жениÑı","k az","н ениÑİ","ĠH emat","nt z","å¤ļ å°ıæĹ¶","åĬ¨ ç͍","ath olic","Ġà ´","ĠCrit ics","æŁľ åı°","Ġme ist","Ġob lic","èĢģ ä¸ī","ç§ģ ç«ĭ","å§ĭç»Ī åĿļæĮģ","Bas el","ĠSymbol s","าวิà¸Ĺย าลัย","Ġn t","Ġwh ispers","åĨį ä¹Łä¸į","Ġimp over","à¸Ĺ à¹Į","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ","çĻ» åľº","init is","> -","ĠR IS","主 è¯Ń","åĪ© å°¿","cient e","Ġhij os","ĠParticular ly",": ,","> [","Q i","ĠC BC","н оз","é«ĺ çĤ¹","转 è¿ĩ头","æ¯į æł¡","gin as","åΤæĸŃ é¢ĺ","ĠPlay Station","ĠRef lections","Ġhay op","k x","Ġb ucks","Ġbe ck","ä¸įåIJĮ ç¨ĭ度çļĦ","Count y","Ġвозмож ноÑģÑĤи","Ġpupp ies","c sv","l ut","Ġt ÅĤ","Ġp ami","Ġd rip","ر اÙĤ","Pro tein","af ar","Ġlog os","åıĮ èĩĤ","ĠÄij á»ĭnh","ÙĦا Ø©","ĠChemical s","Ġkur ang","L ate","ĠL ans","Ġme can","ĠY EAR","åĨħ éĺģ","Ġgood will","Ġconf ines","Ġdest ru","Ġfilm makers","Ġble ak","对å¤ĸ è´¸æĺĵ","_ API","wh ole","Ġма ÑģÑģа","Ġμ ια","ãģĬãĤĪ ãģ³","L uc","t ools","ĠS ofia","è¦ ĥ","ç©¿ æĪ´","å¼Ģå±ķ çļĦ","çĿ£ å¯Ł","ника ми","Ġshield s","ĠاÙĦدÙĪÙĦ Ø©","r outine","ĠT racing","ĠP unk","æŃ¦ éģĵ","ĠØ® ÙĪÙĨ","ï½ į","éī´ èµı","対 象","ĠЯ н","িষ à§įà¦Ł","im u","Ġend orse","}\\) \\(\\","åŃĶ éļĻ","ĠاÙĦÙĤ ÙĦب","بÙĦ غ","====== Ċ","ĠпÑĢиводи ÑĤ","d ain","m eters","åį« è§Ĩ","èĪį å¾Ĺ","ĠUnderg raduate","ĠاسÙĦاÙħ ÛĮ","W o","è¿Ļ è¾ĪåŃIJ","éĩĮ 头","æĹł æķħ","åħļ å·¥å§Ķ","ĠBl anc","ĠCar rie","Ġsie ve","ç¨į æľī","Ġbran ched","ëĿ ½","o itation","å¾Ĺ åħ¶","èµ° å¾Ĺ","æĢĿç»´ æĸ¹å¼ı","æĭĨ åį¸","èIJĮ èIJĮ","ĠS istema","ĠE ukary","Ġз Ñĥб","à§ĩঠª","ste ady","ĠEd ith","ĠMon ark","Ġtrou sers","ĠдÑĢÑĥ га","-review ed","n ienia","ĠB ret","ĠD FS","ĠRe gg","Ġallow ances","çĩķ åŃIJ","究竣 æĺ¯",". ly","表 çϽ","表 åĵ¥","п он","Ġinv ade","ÙĪÙĦ ÙĪ","-A ug","Ġgest ational","ãģĿãĤĮ ãģ¯","Ġতার া","ĠSurve illance","a eda","ĠC aleb","ع ادة","æķ´ æµģ","Ġins ulated","转 èĢĮ","ĠNe al","äºļ åİĨ","/f iles","ĠTR AN","ĠТак же","C ookie","k am","{ '","Ġ ï¼īĊ","çļĦ 缴æİ¥","Ġr anc","é»Ħ å±±","èĵ ¦","Col umb",".j upiter","ét ude","å¹ķ åIJİ","Ġਠ¨","ĠThank fully","ĠBag hdad","å°ı åĵ¥","uss es","AT US","à§ĩঠĹ","fect ure","Ġball oons","تر Ùĥ","Ġl ure","è¿ĺ ç»Ļ","æĽ´ éľĢè¦ģ","åı° è´¦","cz es","ĠSy racuse","Ġ×Ķ×ŀ× §","Ġps oriasis","S v","н ованиÑı","åĴĮ æľĭåıĭ","éĿ¢ æĹł","Ġinter v","æį »","Ġser ont","çľģ åĨħ","çζ çļĩ","Ġà° ¤","åķĨä¸ļ 模å¼ı","c ited","åıij èĩª","Ġprogram ma","åħļ ç»ĦæĪIJåijĺ","-e lement","Av g","çļĦ æīĵ","Ġв ÑĢед","ÑĪ ÐºÐ¾Ð¼","è¯Ĩ åŃĹ","Ġsens o","avor ites","= P","K in","éĩį ä»»","Ġbl an","ол ог","å¢ŀ åĩı","èī¯ ä¹ħ","æ¹ĸ æ°´","Ġord ained","àŃ ģ","มาภĪ","ãĥĸ ãĥŃ","Ġal iqu","ä¸Ĭ è°ĥ","æĹ¶ 髦","оÑĢ Ð¾ÑĤ","ĠSp rache","æŀģ æĺĵ","çľĭåΰ ä»ĸ","ĠInteg ral","ĠYah weh","Ġsquir rels","åıĺ å°ı","åIJĦ å®¶","èŀį æ´½","e ax","ang lement","Ġco vert","-g round","輸 åħ¥","èĿĻ èĿł","ä¹ĭ è¡Į","表 å¾ģ","ä¸ĩ 亿åħĥ","log ue","ĠاÙĦÙĨ ظاÙħ",".create Element","Ġv t","Ġhera us","Ġantico ag","F ri","ĠO man","天 é¹ħ","åĸ ³","å¸Ī éķ¿","åĸľ çαçļĦ","èµĦæľ¬ å®¶","à§ĩন à§įà¦Ł","æİ¥è§¦ åΰ","æ·»åĬł åΰ","Ġconfront ing","Ġdorm ant","Ġà¸Ķ ัà¸ĩ","ĠBever ly","èĮī èİī",") ãĢĭ","l ld","ĠS ib","ĠC ody","art ist","so f","身 çĿĢ","åģļ 为","å°ij åIJĥ","æłĩ è¯Ń","Re verse","So on","ĠDesign s","åĮĸåѦ åĵģ","çIJĨæīĢ å½ĵçĦ¶","ĠP ulse","æĺ¯ æĸ°","pl atin","ĠÙħ ض","åĵģ ä½į","ÑĤа й","宽 带","ë¶ Ī","Ġundert ook","ĠTon ight","å´Ń æĸ°çļĦ","éķ¿ å¤§çļĦ","sh ake","Ġvo ce","åIJĮæ¯Ķ ä¸ĭéĻį","f uel","çļĦ 缮çļĦæĺ¯","ĠG at","æľĢ åŁºæľ¬çļĦ","两 å§Ķ","æµ· äºĭ","ero on","åįļ ä¼ļ","ĠاÙĦØ£ خرÙī","PM ID","Ġdar ling","Ġgig antic","Ġtow ering","Ġauth ored","Ġunanim ously","ç´łè´¨ æķĻèĤ²","ĠпÑĥ ÑĤем","ĠBah rain","ç´§ç´§ åľ°","éĥ½ 对","Cont ains","ĠÑĢаз но","ระ ยะ","éĺ´ èĻļ","ĠExec ute","Ġì¶Ķ ê°Ģ","B ACK","ĠN ouns","ov iet","ks am","çħ§ æĸĻ","Ġcho is","ĠAugust a","Ġsin h","åĺī åħ´","æħĪ æĤ²","åĬĿ 说","ast on","æ¹ §","æĽ¾ åĽ½","Ġко Ñĺи","éĤ® ç͵","èIJ¨ æĸ¯","conf idence","Ġ문 ìŀIJ","ÙĨاÙħ ج","Ġодна ко","z és","大 ä¼Ļ","Ġen igmatic","åĽłä¸º è¿Ļ","éĶĻ è¿ĩäºĨ","Ġunf inished","ÑĽ и","Ġмноже ÑģÑĤво","ĠGENER AL","ĠMANAG EMENT","Ġrec ited","ä¹Łæĺ¯ éĿŀ常","æĮª å¨ģ","计æķ° åύ","ĠNavig ating","' ac","om ar","get ahui","åķ ¶","-f ire","ÑĪи ми","Ps alm","×ŀ ×Ļ","Ġsnipp et","n ict","} |\\","Ġd nia","æĿ¥ åİĨ","Ġpre z","ĠFl av","éĤĦ æľĥ","Ñģол ÑİÑĤ","J T","Q P","Ġd rowning","ĠRed is","Ġkn ights","Ġpra k","Ġmanual s","- unit","P ic","olog iques","ĠAb bas","Ġassess es","าม ารà¸ĸ","ãĤĪ ãģĦ","æĮº 好çļĦ","ĠImport antly","çļĦ å¢ŀåĬł","ĠO fic","Ġj ue","ç͍ å¤Ħ","Ġا ÛĮÙħ","åīį è¿°","Ġ` ĊĊ","ĠÐļ о","罪 è¡Į","Be i","ikh ail","ĠздоÑĢов ÑĮÑı","; **","Ġde ver","ĠL TD","让 èĩªå·±çļĦ","Ġlay outs","De leted","ĠGall on","Gre ater","Ġап паÑĢа","D ivid","äºī æī§","ç¯ ¡","åı³ éĶ®","ĠSim ult","çļ± çĿĢ","ØŃÙĬ ØŃ","Ġenfermed ades","åıĸå̼ èĮĥåĽ´","Ġestruct ura","N b","Ġa orta","ĠK yr","uc chini","ãģĤ ãģªãģŁ","åı¦ä¸Ģ è¾¹","é³ Ħ","ê· ł","ĠK Y","Ġsc ala","å¹¶ æıIJåĩº","ĠDe leg","ðĿij IJ","Ġкон ÑĨенÑĤÑĢа","éij ij","drop down","[ num","Ġcl asp","ä¹ĭ ä¹ī","ç¥ Ł","åıĺ æĢ§","ä½Ĩæĺ¯ æĪij们","UB LE","B ird","éĥ½ åºĶ","è¨ ³","ç»ĵæŀľ æĺ¾ç¤º","ä¸įæĸŃ å¢ŀ强","erd em","åĽ´ç»ķ çĿĢ","æ°¢ æ°§åĮĸ","สิ à¸ļ","Ġги д","Ġdread ful","Vert ical","è¯ ²","Ġen quiry","ä¹ĭ ç͍","ĠY ards","Ġco y","اÙħ ÙĬÙĨ","ç¨ĭåºı ä¸Ń","struct ural","年代 æľ«","éªij è¡Į","Oper ating","Ġinterven ing","IGH TS","L OR","Ġp inn","Ġп иÑģÑĮ","Ġacc eso","Ġpar ler","Ġpet its","Vis ibility","Ġkemb ali","v iii","ä¸į åħī","ä½ł 羣","af ia","夫 çļĦ","ĠOut er",".\\ ,","Ġнов ÑĭÑħ","ocent ric","qu a","ĠW rit","Ġind ig","æĶ¹ è£ħ","_t wo","(s rc","ĠØŃ ÙĪ","ç»ıè¿ĩ äºĨ","Ġsed ang","M ol","د ÙĪ","æĸĩ å¸Ŀ","Ġв Ñħод","äºĶ 人","ĠMe er","Ġر Ùħ","åįģäºĶ æĿ¡","ĠCiv ic","ĠSTUD Y","Ġanonym ity","Ġlượ ng","( position","= T","å°± åΰ","å°ı å®¶ä¼Ļ","ins chaft","ä½Ĩæĺ¯ çͱäºİ","æĹĭ åį³","è¿Ł æĹ©","×ķ×IJ ר","acqu a","乡 ä¸ĭ","ðĿij Ŀ","éĵģ çŁ¿","Ġpas ar","ĠQuest o","Ġot ten","Ġexceed ingly","ASC ADE","Ġprobl èmes","Vit amin","ĠÐł ÑĥÑģ","âĹĭ âĹĭ","ĠØŃاÙĦ Ø©","Ġc uff","Ġsl ash","çħ§ æł·","ĠCent uries","огÑĢа д","Ġagon ist","Ġitiner ary","ĠI EL","Ġat ual","é«ĺ é£İéĻ©","-l ocal","Ġabsol ut","اÙĤ ات",":** :","Ġbard ziej","or on","ĠB AL","rit te","Ġpe a","éĢļ åħ³","éĢ£ çºĮ","P OL","Ð ©","Ġsu k","ä¸ī äºļ","Ġsem in","Reg ardless","à¸Ľà¸£à¸° มาà¸ĵ","D IS","ent ie","co ins","åı¤ å¸ĮèħĬ","稻 èįī","ĠLev ine","ĠYugoslav ia","ĠR FC","for um","åºľ çļĦ","Ġemb ro","ĠJournal ism","à© Ĥ","ĠPRO DUCT","Ġparse Int","åĢŁæ¬¾ 人","! [](",". Format","ä¹ĭ ä¸į","-t w","ä½ı æĪ·","Ġlim a","ÄĽ ÅĻ","åĿı 人","ÑĢов ки","cr umb","Ġger ade","Ġstere otyp","Ġíķ´ ëĭ¹","Ġe gin","Ġst u","åħ¬ å¼Ģåıij","×Ļ× ©×","г он","æĶ¾ 宽","Ġav ian","举 åĿ¡","ás i","Ġpour quoi","ĠHS V","ĠnÄĽ kol","kc ji","Ġcraw ling","Ġ׼×IJ× ©×¨","et ten","æľº æ²¹","Ġب ÙIJ","书 ä¿¡","è¿Ļç§į 人","Ġparticip ates","Ġanimal es","conn ecting","æIJŀ ç¬ij","æģ¶æĢ§ èĤ¿çĺ¤","Ġverschied ene","ru ff","æĬĢ æ³ķ","ron omy","ÄĻ tr","ĠSc opus","-w heel","çļĩ 室","Gold en","S now","çµ ¶","Ġsem akin","_m ult","驾 车","çĭ¬ç«ĭ æĢ§","ä¸¥æł¼ éģµå®Ī","OH N","Ġting kat","Ġìĸ´ëĸ ¤","ĠÑģ окÑĢа","ãĢĤĊĊ Ċ","ت Ùĥ","åľº é¦Ĩ","ü cks","çļĦä¸Ģ æĸ¹","åºķ éĿ¢","è¿Ļæĺ¯ æĪij们","×¨× ij×¢",".st ore","ĠâĬ Ĩ","ĠW irk","ĠL OS","Ġint imately","æľĢ èĥ½","åĪĻ ä»¥","Ġmid fielder","Ġsel alu","ĠDeterm ining","charg ed","Ġp aving","太 ç¥ĸ","åIJĥ äºĨä¸Ģ","çŁ³ åύ","ĠNe on","Ġcontain ment","Ġfer mented","ĠEmp ower","моÑĤ ÑĢÑı",") t","Ġin und","Ġbe find","åĽ ¤","ĠG illes","ĠO nd","ä»ĸ 以","请 注æĦı","ĠMe hr","ãģĭ ãģ®","зи Ñı","ãĢĤ( ãĢĬ","ÙĪÛĮ ت","ajÄħ cych","Ġà´ ħ","Ġmild ly","ĠBeg inners","ĠSTAT ES","Ġus ando","Ġcomp añ","Ġত à§Ī","åį«çĶŁ åģ¥åº·","Loc ale","è°´ è´£","åıĸ èĥľ","Ġз ÑĢениÑı","ĠÑĢазлиÑĩ нÑĭе","ĠH ai","æĺ¥ å¤ı","ÏĨ α","sm art","Status Code","缸æ¯Ķ ä¹ĭä¸ĭ","YY YY","GRO UP","Ġalmond s","çļĦ çζæ¯į","è¿ĩ éķ¿","æĢ» åĨ³èµĽ","æľª 被","åı¦ä¸Ģ æĸ¹","缸å½ĵ çļĦ","Ġpartner ing","ĠTrib e","ĠET F","N ous","V AR","西 çº¢æŁ¿","Qu ad","IP E","éģį äºĨ","缼 å®´","Ġthread ed","Ġdetermin ado","-inter cept","ðŁĵ į","à§ĩà¦Ľà¦¿à¦² à§ĩন","Ġdestro ys","V F","_ active","w ash","ä¸Ģ æĪĺ","äºĶ åij³","åIJ« ç³Ĭ","æĨ IJ","ä¹Łä¼ļ æľī","Ġgran ules","P ray","Ġin iz","åĴĮ åľ¨","ä½ł éĤ£","èĢģ éĹĨ","κ Ïģα","Ġgly ph","arv ard","ÙĪÙģ ÙĬ","é pend","Ġres usc","æł· æĿ¿","æ¡ ¨","Ġsm ug","åıĸ æļĸ","èĬ± åĦ¿","Ġproject ile","Ġ×Ľ× Ł","Ġcow ard","ĠBAS IS","è¦ģ åΰ","(\" ../","Ġت ب","AP E","çĶ³è¯· 书","ĠTim ber","Ġprincip ale","airo bi","Ġun flagged","ĠSw im","Ġtransl ational","ä¹Į é²ģ","Ġcart e","详ç»Ĩ ä»ĭç»į","Ġsab wag","oped ic","C X","ä¸Ĭ æĸ°","æĮĩ äºĨæĮĩ","认 æ¸ħ","ä¸ĸ åŃIJ","Ġstabil izing","ĠоÑģоб ен","Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĊ","Ġw ards","Ġm uz","Ġl ids","ĠD K","ĠL och","Ġro v","à¹Ģ à¸Ńà¸ģ","æķ´ å¥Ĺ","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ","è°Ī æģĭçα","Ph ilosoph","Ùij Ùı","Ġdiscl osures","Ġparad igms","Ġb arr","ر اÙĨ","Ġsel v","_S ET","çİĦ æŃ¦","ĠERR OR","ra e","å±± å³°","Ġwater fall","ĠвÑĭ зÑĭва","沿 岸","mult iple","ĠìľĦ ì¹ĺ","------------ ---+","Ġkern els","C ool","Ġs inners","åĴĮ åѦçĶŁ","å°±æĺ¯ ä»ĸ","ke en","请 åģĩ","å¢ŀåĬł çļĦ","eks i","ĠCap ac","æ¯ķä¸ļ 论æĸĩ","_ exp","Ġ} .","ix e","Ġrese arches","积 èĵĦ","é¢Ħ åζ","-g rid","ML A","process ed","änd ern","ĠоÑĢга нов","ĠStra uss","ĠÑĤÑĢе ÑĤÑĮ","èIJ½åΰ å®ŀå¤Ħ","iss age","Ġس Ùģ","ĠTw ilight","åĽ°éļ¾ åĴĮ","主èIJ¥ ä¸ļåĬ¡","am pere","ç͍ çļĦæĺ¯","Ġп ен","ĠRes ident","ĠCommission ers","اج Ø©","tri angle","Nom bre","= N","P oll","åIJİ åįĬ","两 éģĵ","ush ima","âĿ Ĺ","h z","ro red","Ġch illing","Ġpe pp","å·² åĽŀçŃĶ","Ġserv ir","ä¸įæĺ¯ æĪij","ĠDes cartes","Ġapproxim ations","ĠSun ny","lem agne","Ġdoubt ed","if iz","Ġsuper intendent","ä¸įä¼ļ åĨį","绿 èĮ¶","ÉĻ ËĪ","èIJ½å®ŀ åΰ","Ġclock wise","Top ics","er land","çļĦ åĪ¶ä½ľ","ĠL LP","çIJĨ åıij","æľº 车","Ġtest o","ั à¹Īว","Ġworld view","Ġпо ÑĢаж","ĠGo ethe","ô i","orde aux","ĠS as","Ġbe gged","æľ¬ åįķä½į","æĸĩ éĿ©","س اÙĨÛĮ","Ġpo Äį","ĠSh all","èĸĦ èį·","åıĤæķ° çļĦ","Ġalc une","ĠFut ures","b ars","f ers","Ľ ×ķת","Ġ×Ķ× ¢×","³ à¯į","Ġا جر","Ġno ir","Ġins ider","ãģ² ãģ¨","ĠM IL","æľ¬ å®ŀç͍æĸ°åŀĭ","á lt","çĤ¹ æ»´","ĠÑģ еÑĤи","iew aż","Ġм еÑĢе","li w","ĠMar qu","éĢģ å¾Ģ","ĠSer ie","ä»·å̼ åĴĮ","æĪIJåijĺ çļĦ","éĢĻåĢĭ æĻĤåĢĻ","çŁ¥ ä¹İ","Ġconstit utive","ryst als","itos an","ĠSquad ron","it ars","Ġle th","ff iti","Ġdis min","ĠPh onics","åĽº æī§","å®ĥ们 æĺ¯","ĠAdv ocate","ä¸Ģå®ļè¦ģ 注æĦı","ĠEdu ardo","Ġd rowned","两 èĢħçļĦ","ض Ø©","رÙĪ Ø³","Ġkon stru","èĵĦ çĶµæ±ł","ĺ× Ł","åIJĪæ³ķ æĢ§","Ġintr ins","ue le","Ġint uit","ci an","å¹³ çļĦ","Ġins istence","æł¹ éĥ¨","An notations","Ġseason ing","Ġcred itor","IR ED","िठ¶","-sh ot","çµĦ åIJĪ","åįļ士 åѦä½į","Ġëł Ī","j är","Ġt innitus","vert ices","åģĩ åĨĴ","Ġrecomm ending","建çŃij çļĦ","ĠGreen wood","Ġvision ary","front al","нÑİ Ñİ","( en","Ġn ghi","çĶŁ ãģį","Ġinf init","è£ħ è½½","ذ ÛĮر","-pro duction","èģĮä¸ļ æĬĢèĥ½","ãģķãĤĮ ãģ¦","огÑĢаÑĦи и","ĠознаÑĩа еÑĤ","v ole","å¼Ģ åľº","rid ine","Ġза ко","ĠÑĤ ой","Ġstandard ization","ç§ģ ãģ¯","Ġnie ce","Ġrevolution ize","éģĭ ç͍","Ġобла ÑģÑĤÑĮ","Ġzpůso b","ĠH itch","æĮĩ æı®","ä¼ł åΰ","ä»ĸ们 认为","Ġdom ic","åĩĮ äºij","жд ение","ĠDew ey","Ġодина ков","Ġun att","Ġ{ -","Ġdo om","åħ¬ å·®","Ġreplace ments","æ´ĽæĿī 磶","çļĦ 女åŃIJ","Ġcl ing","å¼Ģ åĩº","Ġsub urb","çĭ¬ ä¸ĢæĹł","Ġaw al","Ġanaly zer","Ġpy game","ĠSep aration","æ¦ľ åįķ","Ġbias anya","ĠFern ández","æ·¹ 没","ak ume","Ġqu elli","æĢ§ 好","ĠÑĤÑĢÑĥ б","Play ers","/ config","ĠK eb","åľ° åĬ¿","θ ÎŃ","ান ি","è³ĩ çĶ¢","åľ°ä½į çļĦ","ĠSupport ed","om ie","ess ég","ĠN ass","对 æķ°","ink an","åıĪ åı¯ä»¥","çķĮ 线","Ġunf ores","conn ections","ĠاÙĨ ÙĪØ§Ø¹","èħ¹ èĥĢ","ÙĪÛĮ س","ethe us","ĠÙĩÛĮ ÚĨ","ĠоÑĩеÑĢед ÑĮ","çļĦ 第ä¸Ģ个","et ri","Ġj on","Ġcont rat","Ġdis may","çݰ 身","ä¿¡ å¾Ĵ","het amine","Ðķ ÐĿ","ĠHex apoda","ĠContract s","Ġelucid ate","Z o","Ġon a","Ġme isten","Ãł nh","uel lement","æ¤ħ ä¸Ĭ","ĠA ren","ot ers","ĠM MP","Ġac etic","ĠпÑĢи надле","Ġcut ter","伯 çε","å¼· èĢħ","Ġow es","Ġro k","ific ação","ĠØŃ تÛĮ","éĬ ³","ĠعÙĦ ÙĪÙħ","S av","[ tex","j l","å¿ ±","缸 èģļ","çIJĨ论 åŃ¦ä¹ł","Ġprop io","æ´Ľ ä¼Ĭ","d ou","éĥ½æĺ¯ 以","ET Y","è¿ĺæľī 人","æıĴ 座","Ġmurder er","Ġпоп а","ĠVers ailles","C ells","Ġth wart","Ġu Äį","æĿĤ çī©","ĠMit glied","ANG U","çļĦ çαæĥħ","ost ÄĻp","rit é","éĺ² åį«","éħį æĸĻ","-c ounter","ä»ħä¾Ľ åıĤèĢĥ","+ f","Ġst h","åľ¨ åĨľæĿij","ss ch","Ïģ ια","缴æİ¥ æĬĬ","伸 缩","-sc ore","âĢĿ( ãĢĬ","Ġlob es","াধ à§įযম","\" )ĊĊĊ",") p",". center","çļĦ åľ°åĮº","ĠRe leased","Ðļ он",". Con","G ray","m ens","Ġ{ $","éķ °","à¸ģ ิà¸Ļ","Ġ== >","Ġcontr ario","ĠìķĦëĭĪ ëĿ¼","аÑħаÑĢ Ñħой","ĠT oll","对 ä¸Ģ个",".S elect","ä½Ĩæĺ¯ 她",")= -","(b ool","Ġlandsc aping","ÑģÑĤви ма",".ex it","=[ ]Ċ",".Em pty","Ġживе ло","end orf","å¹´ 产","çļĦ人 æĺ¯","å½±åĵį çĿĢ","CO P","ĠSum mar","Co in","稿 ä»¶","z ug","() ),Ċ","æİ¥ ä¸ĭä¾Ĩ","\\) ;","oph il","认è¯Ĩ åĴĮ","丰å¯Į äºĨ","Ġinvent ive","åħļåĴĮ åĽ½å®¶","' ob","åľ¨ çĶŁæ´»ä¸Ń","Ġpre con","ific antly","In cludes","ature d","man ent","ĠCR ISPR","Ġkön nte","Ġuk ÅĤad","Ġinadvert ently","( ',","V in","Ä Ģ","Ġst essa","cre to","æĺ¾ èĢĮæĺĵ","Ġиз де","Ġorganiz z","aton in","ĠAdoles cent",". identifier","B ol","Ġsp acer","Ġbl ender","è£ħ åį¸","à¹ĩ ม","ĠاÙĦØ£ ساس","Ġjed not","使 èĩªå·±","ä¾Ľ çĥŃ","åįİ çļĦ","Ġrespond ers","ĠMil ky","Ġà¹Ģà¸Ķ à¹ĩà¸ģ","ç¾İ æĦŁ","æĮī è¦ģæ±Ĥ","Ġeffic ace","mm Hg",",' '","Ġار ائÙĩ","åĬłæ²¹ ç«Ļ","B RA","Ġp ave","Ġd izziness","ĠP ike","ien nes","EN A","é¸ ¥","}} -","Ġpend ulum","ĠPic asso","Ġangl ès","Ġcoag ulation","Ġartific ially","Ġgrocer ies","D Y","б ÑĢан","æķ°æį® ç»ĵæŀĦ","mm m","Rec ords","iesiÄħ t","> >ĊĊ","Ġs lick","ed iatric","æ² ½","çIJ µ","è¿ĩ ä¸Ģ个","éĩĮ åİ»","æ¯ı å°ıé¢ĺ","æĸŃ è·¯","æİĴ åľ¨","æĸ¹æ³ķ æĿ¥","åĬŁèĥ½ éļľç¢į","ĠMore no","ä¹Łæľī ä¸ĢäºĽ","躯 ä½ĵ","à¦ı à¦ĩ","Ġastronaut s","R ace","äºĨ çĦ¶","app iness",".C olor","Ġinvent ories","Ġét udes","ĠSeg mentation","ä¸įçͱ èĩªä¸»","ĠLED s","Ġreiter ated","Ġпедаг оги","ĠJ ules","éģĵ åıĭ","èį Ł","Ïħ νÏĦ","éħ Ŀ","绣 绣","Wh it","CH AR","Ġ׳ ת","Ġk V","Ġdist ancia","Ġgra bs","Ġdon né","Pro fit","Ġprim ero","sk á","æĶ¿åºľ 对","ĠÐĴ лади","å²ģ 以ä¸Ĭ","Ġadm irable","ÅĻÃŃ klad","tra ining","g te","r unning","ic om","ĠT RI","pl ine","Ġab re","Ġla x","好 ä¸ľè¥¿","ä¸īåįģ å¹´","çĵ· åύ","Ġì² ľ","åīį æ®µæĹ¶éĹ´","ss h","计 æıIJ","åºı å¹ķ","ĠઠĨ","ĠFem in","ĠArchae ological","Ġo min","Ġdr illed","ĠPol ski","æĶ¿æ²» å±Ģ","འĺ","Ġelabor ated","çī² çķľ","ĠÑģÑħ ем","Cho osing","Z m","ĠR PG","æİ¥ åĬĽ","éĺ² å¤ĩ","ส าย","ĠÚ© اÙħ",".T ab","Ġepigen etic","ĠÙħ ÙĦÙģ","å¾Ī æĺ¾çĦ¶","Ġб лок","Ġbook mark","羣çļĦ 好","رÙĬ Ùĩ","sl ides","åįģä¸ī äºĶ","åįłæį® äºĨ","å°ĭ æī¾","Ġre duct","ä¹ĭ æľ¬","Ġrest rained","Ġде ло","æįŁå¤± çļĦ","Ġশ à§įর","Ġadip is","Ġe ased","ĠB uzz","åħ¨ æĿij","æģ Ĩ","pro blems","æīĵ 交éģĵ","æ±Ł åĮĹ","iat i","ĠPower ed","ĠWil de","à¥ĭ à¤Ĺ","Ġди ÑĦ","bn b","ĠComb ination","er ase","ĠB é","pl acing","Ġher ds","Ġcomm ute","å¾Ī éĩįè¦ģçļĦ","×ķ× IJ×","æĬķ å°Ħ","èĴ ¿","ĠPa ÃŃs","Ġconstru cción","ĠÏĮ ÏĦι","S yntax","Ġh ype","ĠÑģ ейÑĩаÑģ","Ġam ely","ah uan","ãģĻ ãĤĮãģ°","çľģ çļĦ","è¿Ļé¦ĸ æŃĮ","\" },","ĠT ata","ĠF I","ĠW yd","ie ck","åĴĮ å¤ļ","Ġsh one","ç»Ļ åĪ«äºº","ris che","-c oll","ãĥ¼ ãĤº","è¬ ¹","åıĺå¾Ĺ è¶ĬæĿ¥è¶Ĭ","ĠHelp ing","ĠpolÃŃ tico","Ġelong ation","Ñ ķ","çļĦ éĿ¢åīį","Ġde an","Ġ ´","æĹł 为","æĶ¹ åĬ¨","Ġtem os","EF L","ĠNumer ade","Ġcran ial","M eg","Ġ ids","ä¸Ń èİ·å¾Ĺ","Ùħ بر","... )","af en","Ġ×ľ× ¤×Ļ","éĢĤåIJĪ èĩªå·±çļĦ","Ġsouth western","æī¿æĭħ 责任","Ġباز ÛĮ","Nut rition","ĠH ague","ok us","æ²ī åIJŁ","Ġing enu","Ġpromot ers","çªģçł´ äºĨ","n ich","Ġappro x","Ġcre cimiento","åħ± çĶŁ","Ġpost war","ĠÑĦ оÑĤо","æĮĤ äºĨ","< N","Ġb ons","èĢĥ 試","å½Ĵ æł¹","积æŀģ éħįåIJĪ","न à¥Ģ","oj as","ĠOrd inary","éĩijåŃĹ å¡Ķ","_ ip","ĠW iring","ç² ±","éļı å¤Ħ","æĮ½ æķij",": F","_ \"","åŃIJ ç³»ç»Ł","å·¥ åĨµ","ĠÙħ بت","å°Ĩ æľī","æ¯Ķ æĸ¹","ÑĪ Ð½Ð¾","ĠLex ington","ÅĤÄħ cz","Ġанали за","ĠÙĬÙĤ ÙĪÙĦ","ĠA ck","Ġal gu","åľ¨ åŁİå¸Ĥ","è¢ ±","Ġب ص","Ġcent ros","ê· ¹","è¿ĩ滤 åύ","ĠSach s","ĠBomb ay","Ġdeng ue","H ero","á ĭ","ĠA ure","ç» Ľ","ĠD ÃŃ","çŃī çī¹çĤ¹","Ñħ ом","Ġmod èle","被 åĽ°","ĠAl ps","ee ee","-m m","èĭ¦ èĭ¦","èĶ £","Ġpet itions","Ass istant","ĠSav age","Ġktóre j","- error","Ġe a","为 åĽ½","å¼Ģ åħ·","ÑĢе ÑĤ","åıĹ è´¿","è¯Ĺ ç»ı","Ġdas ar","part y","Ġliv res","ĠTry ing","er at","Ġm ala","ä¸į çķı","ab d","Ġhe par","åı° ä¸ĭ","Ġза пи","_c ounter","Ġextrem ity","åºŁ éϤ","åĪĨ享 äºĨ","ĠOcc asionally","K an","Ġt j","çļĦ å͝ä¸Ģ","ĠJ aw","çī¹ äº§","ĠZ usammen","å¹² ç»Ĩèĥŀ","Ġbreak er","ç»§æī¿ 人","çĸ¯çĭĤ çļĦ","' ils","U H","ap ur","åħ¬ çε","Ġfl ushed","éĹ® 她","িà¦ķ া","æĭľ çĻ»","-effect iveness","ç½ij æĺĵ","ĠÑĥ ÑģÑĤанавли","æĬķèµĦ èĢħçļĦ","ÑĤелÑĮ нÑĥÑİ","Ġlink ages","æĶ¯æĮģ ä¸ĭ","ĠLA W","å·¾ 帼","幫 å¿Ļ","Four th","exper ienced","ympt oms","w ent","Ġ ï","Ġim itate","ĠV end","ens ible","âĢĶ âĢĿĊ","Ġsol ución","WH AT","ĠFern andez","ãĢĤ ãĢĶ","人 å¿ĥçļĦ","æķĻ ä¹¦","èĬ± ç²ī","/s cience","éĢĤ ä¸Ń",".p one","Ġtown ship","aria h","ĠBay ern","ĠVisual ization","ĠFou cault","Ġelectroph oresis","Particip ants","zelf de","ĠS ust","id on","ĠU AV","Ġph ân","Ġmost rar","ĠLe uk","è¿Ļä¸Ģ éĹ®é¢ĺ","è² Ĥ","ĠAct iv","æĬĹ æ°§åĮĸ","dis k","çĽĸ ä¸Ĭ","宽 éĺĶ","Ġоп Ñĥ","uran ça","ĠHig gs","ĠDES IGN","æĹ¶ è£ħ","éĵ Ĥ","রà§įঠľ","ĠÑģво им","Ġtransf usion","ç«Ń åĬĽ","åħ¢ åħ¢","Ġa ange","Ġinter state","ĠCom pleted","Ïģ ή","亲 çİĭ","iol ary","Ġcut tings","ĠÑģÑĤа ÑĢиÑĺе","Atl antic","ĠU F","çĤ¹ 为",")) *","oph ytes","åı¤ æĸĩ","Ġ×ķ× ł×","æ´ŀ ç©´","ĠGer ard","ĠEnc ourag","Ġcoinc ides","ĠТа ким","Ġwz gl","- ly","âĢ Ł","ag lia","ĠW ander","åĽŀ æĩī","项 ç¾½","Ġhor rific","Ġли нии","èݱ åĿŀ","ĠÑĥп ÑĢаж","Ġlav ender","+ e",": -ĊĊ","çļĦ è§£","ĠS of","ä¸į æŃ£å½ĵ","Ġcont oh","-m eter","æ¿Ģ èµ·","à¸Ĭ าว","Ġtele com","ĠиÑģ Ñħод","Viet namese","am pl","æĺ¯ å±ŀäºİ","Ġdes con","书 é¦Ļ","éĿĴ å¹´çļĦ","èĤ¡ æĮĩ","æķ°æį® åĴĮ","æ¿Ģ åĬ¨çļĦ","Ġج غراÙģ",".P age","ä¸Ŀ ä¸Ŀ","鼶 鼶","è¿· 人","à±įà° ²","Ġrecon naissance","ĠMARK ET","L s","ĠR ox","ĠEn zym","æĻļ ä¸ĬçļĦ","ĠпÑĢо ÑıвлÑı","Ġfrag rant","ĠGrav ity","C W","get Name","ç¥ŀ çģµ","ç¬ij çľ¯çľ¯","åĬ³åĬ¨ åħ³ç³»","æ·±åħ¥ åŃ¦ä¹ł","èĻĶ è¯ļ","çļĦ æ°ĽåĽ´","ĠD over","åľ¨ å¿ĥ","der n","-b oard","Ġfull er","Ġìĺ ¨","Ġhabag atang","ĠÑģкоÑĢо ÑģÑĤи","æĪij æĽ¾","Ġem ulsion","æĹł èıĮ","äºĨä¸Ģ æŃ¥","-t ions","Ġlate x","Ġkle in","Ġchron ology","ĠEvel yn","- III","K s","Z a","éĢļ ç͵","ĠÙģ Ø±ÙĪ","çļĦ人 äºĨ","Ġsem ble","ä¸ĢåĪĩ éĥ½æĺ¯","ĠÅĽ wie","OUR CE","ĠO CLC","çľĭ æł·åŃIJ","ди м","çŁ¿ åĮº",".app ly","赫 çĦ¶","༠į","Ġ ........","ĠO ss","åı¯ å¼ķèµ·","ì§ ķ",".j ar","Ġкомп лек","Sn apshot","ĉ switch","ĠG omez","Ġun be","æĥ ¦","Ġsub class","两 æł¹","æĿij éķĩ","åŁºæľ¬ æĥħåĨµ","çͲ ä¹Ļ","Ġà¦Ĩম ি","Ġefect os","ĠпопÑĥ лÑıÑĢ","ĠEDUC ATION",", +",". bl","ps k","Ġsur rogate","ĠQ A","Ġactiv ités","Ġocc ident","Ġscre ams","èĤĿ çĻĮ","ĠShort ly","Ġestr ateg","ì ³","Ġm ij","id encia","se ver","Ġar isen","åģļ çĶŁæĦı","æ¥ Ĥ","æ²¹ èĢĹ","ĠÙĥ بÙĬرة","Ġ] ]","Ġtort ured","( values","ĠM embrane","æĺ¯ è¿Ļ个","åĴĮ æ³ķå¾ĭ","aut om","èᝠåºĹ","åĿļ 硬","arc ia","åijĨ åijĨ","Ġmilit ant","éĤ£ åı¯æĺ¯","Ġend owed","Ġна лиÑĩи","æĺ¯ä¸Ģ åıª",".f ield","Ġcourt room","ĠProm oting","Ġà¦¹à§Ł à§ĩà¦Ľà§ĩ","ĠVent ures","ĠPiet ro","ĠбезопаÑģ ноÑģÑĤи","¡ ת","ĠT id","ĠC ed","èµ° åIJİ","éĢģ æĿ¥","Ġcert o","æŃ¥éª¤ å¦Ĥä¸ĭ","D ATA","ĠB az","ä¸į æĢİä¹Ī","Ġsp arks","}^{ +","æ½ ¼","_CO DE","ĠHarb our","ĠSic ily","ĠR icht","act ivities","ä¸İ 被","æĩ ¶","asc us","æķĪæŀľ çļĦ","毫 çĦ¡","Ġble ed","Ġm ű","ĠS aaS","Ġris co","ĠпÑĢе з","Ġtan aman","Ġتع رÙĬÙģ","ĠBound ary","Ġবিà¦Ń িনà§įন","ert es","min utes","举 缣","为äºĨ éĺ²æŃ¢","hib it","æİĮ éŨ","cap acity","Ġà¦Ĩম রা","ĠMaced onia","ä¸Ģå¦Ĥ æĹ¢å¾Ģ",". zeros","se p","sc roll","æŁ¥ éªĮ","ĠSc hen","æĹħ éĢĶ","Ġded uce","Ġcollabor ators","èĩªåĬ¨ 驾驶","ĠMad onna","-k now","æ¶Ĥ å±Ĥ","ĠCart esian","Ġperc ussion","E uro","ill os","ap acity","ä¾Ŀ éĻĦ","contin ent","/ ',","ĉ vector","åĪĨ æł¡","æıIJ æĭĶ","é£İ ç͵","åıį æĦŁ","-n ational","([ \"","Att empt","Dep ending","ÙĪÙĤ ع","STR UCT","Ġpenc ils","Ġstew ardship","Alb um","ĠбоÑĢ ÑĮ","- if","è¿ĩ 硬","St one","Ġ/ .","ç»Ļ ä½łçļĦ","åħī 大","äºĨä¸Ģ å®¶",".get Id","åİļ éĩį","ç¯Ģ 缮","bes ondere","è¶ģ æľº","ĠÑįкÑģпе ÑĢи","S ad","w heel","¨ áĥĺ","et zen","ĠS inn","ä¸Ģ ç±³","ĠO CT","åĴĮ ä¹īåĬ¡","个 ä½ĵçļĦ","ĠK ut","åı¯ä»¥ ä½ľä¸º","ÑĪ ÑĥÑİ","åIJĦ æł·","ĠSe as","Ġsuper visory","åIJ« çĿĢ","åᰠ书é¦Ĩ","åĨł å¿ĥçĹħ","à¥įर à¥ĩ","Art ificial","ÑĦоÑĢ Ð¼","T al","im uth","大 éĽª","羣 æĮļ","çij Ľ","N m","ÑĢа м","Com panies","Ġ` Ċ","ĠпоÑģ вÑı","ĠPRO BLE","åĭ¾ åĭĴ","` :","çļ ĭ","ĠM ét","od il","ve g","Ġha iled","åħ³ åĪĩ","èĢģ çĪº","ER G","æīĵ åıij","Ġprá tica","çαå°Ķ åħ°","E conom","k p","m ere","ing ers","Ġb éné","Ġover arching","Ġfind et","ĠPh araoh","è°ģ çļĦ","vin yl","æ¯ħ çĦ¶","Ġnauc zy","( III","z yn","Å ļ","Ġoper and","ä¹Łæĺ¯ 个","çļĦå°ı åŃ©","人类 社ä¼ļ","Ġβ α","ä¸īåĪĨ ä¹ĭä¸Ģ","æ´½ è°Ī","ĠS MA","ĠT ickets","ĠH W","ĠSt ras","åĨħ 饰","ç± ĥ",".S chema","ling er","ges amt","ĠGra f","å¾Ħ 缴","ĠHy derabad","ால à¯į","d w","r ils","nd t","为 社ä¼ļ","åĴĮ ä½ľç͍","Ġac upuncture","ÖĢ Öĩ","å¿§ éĥģ","Ġrig idity","失败 çļĦ","ĠBern stein","Ġsalv age","çĸĻ çĺ©","å¹¶ åıĬæĹ¶","() ]Ċ","Ġatt rition","ä¼ij åģĩ","é̼ è¿«","Ġprofes or","- active","< \\/","_ entry","è¯ ¶","ä¸į åıij","vert ure","sk ý","ç¬Ķ 墨","Ġfeed er","Ġouts iders","åľ¨ é¦Ļ港","ĠF ate","åĩº å¢ĥ","åıij åĮħ","èĢĮ 产çĶŁ","Ġpol ishing","ì m","ĠоÑĤно ÑģÑı","Ġs ik","士 æ°Ķ","å·´ 士","å¼· 大","ĠBon n","Review s","gesch ichte","çł¥ çłº","Ġhither to","' att","ur un","ç® «","Ġup rising","Ġet ched","çŀ ¿","ĠNon linear","çĽĪ çĽĪ","çļĦ äºĮ","ä¸ī åIJį","è®ĵ ä»ĸåĢij","Ġchlor oplast","Ġth ru","ĠC ust","ĠP flanzen","ä¸į 满æĦı","ĠF ragment","为 ä¿Ŀè¯ģ","ä½ĵ è´´","Ġcompar ator","Art ist","ĠSyn chron","ĠMine craft","Ġог ÑĢом","it imate","te achers","ĠSt am","ê u","IQ UE","ĠÏĦÏį ÏĢοÏĤ","_ access","Ġb öj","ĠC ynthia","ĠV oting","ва ÑĤи","æĸŃ å±Ĥ","Ġdraw back","дÑĥ к","ç¨İåĬ¡ å±Ģ","Ġdoen ça","Ġunderest imated","æĺ¯ åı¯","Ġan gg","Ġk iedy","In nov","Ġac erca","社 åįĢ","èĭı ç»´åŁĥ","AB ASE","æĶ» åħ³","at hed","å¹¶ åŃĺ","æīį 对","-p iece",".A uto","å¿į ä½ı","ä½ľé£İ 建设","нал оги","Ġestrat ég",". ',","Ġt ujuan","ign ac","angu ard","åĩºçݰ è¿ĩ","éĥ¨éŨ åĴĮ","Class Name","ĠIns ider","åѦ åĴĮ","ĠPro ven","ä¿Ŀ èĤ²","ins ide","åĨĽ èIJ¥","åĪļ æĢ§","Ġelev ations","Ġsand stone","ĠмеÑĤ одÑĭ","Ġgri pped","Ġร วม","-equ iv","Ġv ient","ä¸į åħ¬å¹³","ä¿ ¨","å±ķ åİħ","çģŃ èıĮ","æIJ¬ åΰ","ĠConc erns","F er","ĠP OWER","ĠL od","so il","éĥ½ åı¯èĥ½","æģ ĥ","ĠEvery day","è¢ģ ä¸ĸ","好å¥ĩ å¿ĥ","ä¸Ń å¾Ĺåΰ","Ġle ases","ÙĨ ب","æĽ´ åĸľæ¬¢","æ¯ı ä¸Ģç§į",":: -","Ġeng l","æĶ¯æĮģ çļĦ","Ġesc rito","Ġmac rom","emat ics","Ġìĭ¤ íĸī","ĠëĦ ¤","åºĶæĶ¶ 账款","à¹Ģà¸ł à¸Ĺ","t emperature","ub script","Ġpl ata","ãģ¾ ãģł","Ġcas p","ĠRich ter","åī¥ ç¦»","Ġduct s","ä¸įçŁ¥æīĢ æİª","æĺ¯ ç͍æĿ¥","ãģ« ãģĭ","ย า","ĠAm ar","Ġexplos ions","ig li","ĠP ipe","ĠF ET","Ġim balances","mer k","ĠAl am","Ġport rays","ĠMicro sc","Ġroz wo","ĠмеÑģÑı ÑĨев","Ġex cludes","å¾ ĵ","yst one","--- |---|---","Ġstock holders","ĠEsc ape","c ja","ä¸Ģ æĻĤ","âĢľ âĢĺ","éĿ¢ æĸĻ","她 éĤ£","ج ÙĪÙħ","ĠAm mon","Ġл Ñĥ","ĠCliff ord","สุà¸Ĥ à¸łà¸²à¸ŀ","< \\)","Ġadd r","åľĭ æ°ij","Ġnumer ators","umin ous","èĦĬ æ¤İ","Ġjan vier","Ġá½ ģ","Ġmane ira","ĠC ory","Ñģ она","Ġcl as","çŃī éĩįçĤ¹","س ÙĪÙĨ","åı¯ä»¥ éĩĩç͍","С а","eli pe","èĦļæŃ¥ 声","ưá»Ŀ i","in ou","ĠF AM","ä½ł åΰåºķ","Ġparticular s","la ub","æĢ¨ æģ¨","Ġmorph ine","æ¸ħæĻ° åľ°","Ġp act","ĠE zek","In stitute","ä½ı æ°ij","wh ose","åŁ¹åħ» åŁº","é©» æĿij","νο ν","ĠпопÑĥла ÑĨиÑĺа","Ġt edy","ä½Ĩ è¿ĻäºĽ","å·¥ä½ľ æĹ¶éĹ´","ز ÙĬز","æĮī åİĭ","è¡¥ ç»Ļ","ı nda","UM E","ĠHard cover","-per iod","ĠпÑĥ ÑĤи","į ¨","Ġexp ires","é¦Ļ æĸĻ","å¨ģ 严","ä¸ĢåĪĩ çļĦ","羣å®ŀ æĢ§","system s","Ġpolymorph isms","iagn ostics","ĠMiche le","Ġre printed","ä½ľ æĪIJ","Ġsp illed",")) ?Ċ","åĨħéĥ¨ æİ§åζ","ĠSil ent","ĠREQU IRE","Ġt epat","al ias","ä¸į 讲","ĠR UN","ans ke","tr ust","EM P","ĠBoy le","Ġimperial ism","ĠRecycl ing","ĠT age","ure th","ä»İ ä¸Ģå¼Ģå§ĭ","ä inen","ĠSch warz","ĠØŃ ÙĬÙĨ","ĠWill ow","ÐĶ Ð¸","ĠGi ul","Ġenact ment","H ide","} \");Ċ","ol um","è¿Ļæł· ä¸Ģç§į","åħ» èĤ²","Ġس ÙĬاس","è¿Ļä¸Ģ æŃ¥","ĠFl ores","Ġdeg rade","èĮ¶ æ°´","æĹ¥æľ¬ èªŀ","ائ ÛĮ","ãĤ· ãĤ¹ãĥĨ","Ġflo ated","ĠÙħ ز","Ġconst ellations","Ġmil joen","à¦ķ à§ĩর","ĠÑģам ом","为主é¢ĺ çļĦ","ĠU PC","ip ated","å°± æĭ¿","Ġì ī","Ġgr ond","ä¿® 身","hold s","ек Ñģи","åħ§ éĥ¨","迪 士","æĹ¢çĦ¶ å¦ĤæŃ¤","Disc ount","ä¸Ģ åıĺ","Ñĥ Ñģа","ian os","æ·± æĦŁ","-the med","ĠCapt ure","Ġvag ina","Ġvolcano es","Ġméth ode","åħ¥ éĻ¢","-p rep","ĠStud ien","ogen etic","ĠТ и","æĬµ æ¶Ī","ĠAuthor ization","Fe el","Conn ected","Ġpromin ently","Ins ets","Ġov ary","Ġconten ido","íħ ľ","ĠزÙħÛĮÙĨ Ùĩ","Ġch ased","Ġus ize","In ner","ĠWe aver","urs os","Ġinf licted","Ġhabit ants","ĠSin clair","ĠMars hal","åı¯è§Ĩ åĮĸ","åIJij 西","Ġdist ressed","Ġacqu ires","Ġdra ining","ĠSmith sonian","Ġ×Ĵ ×ij","åĽ½å®¶åĴĮ åľ°åĮº","èĢĮ 产çĶŁçļĦ","ĠÙĦ غ","ats by","Ġа лÑĮ","ĠDel phi","ĠLook s","Ġ׳ ×Ļ×ª×Ł","å¿ħéľĢ çļĦ",". asp","ĠC ARE","人 ä¸Ń","Ġk and","Ġad verbs","aus ole","èĦ ¯","ĠFin als","ä¸Ģ天 çļĦ","Ġà´ ¨","Ġخاص Ø©","/ ap","g ames","Ġth ieves","Ġg emaakt","人 马","ĠU DP","æ³ķ åħ¸","μ εν","Ġbra ces","ESS AGE","親 èĩª","ĠHig gins","ĠCult ures","ĠاÙĨتخ اب","ĠF unc","ä¸ĭ 楼","Ġapp arel","ob ie","ĠRe placement","åĩĨ åħ¥","ðĿij ĥ","Ġcondition er","ç®Ĭ æĥħåĨµ","ĠRhe umat",". Configuration","/ it","y as","ä¹Ł ä¸įåIJĮ","ÑģÑĤ ок","Ġes ophagus","Ġgener ously","Ġà¦Ĩ à¦ĩ","æĤ² çĹĽ","Ġbath rooms","Ġhol iness","ĠUl tras","]== '","Ġbif ur","Ġd zi","Ġaff ront","ĠØ® بر","tu ple","à¸Ŀ ึà¸ģ","ĠHapp iness","æľĽè¿ľ éķľ",": |","ĠEd ison","ĠÙĪØ§ÙĦ Ùģ","net te","à¹Ģà¸Ĥà¹īา à¹ĥà¸Ī","å¾Ī éķ¿æĹ¶éĹ´","ax es","ĠCon structor","ิ à¹Īà¸Ļ","设计 äºĨ","μ ή","IO US","ĠSalmon ella","ĠB atch","å¹´ æĺ¯","ĠCon way","Ġmar in","Ġspecial ised","Ġcu anto","åͱ çīĩ","Ġmile age","Ġaccomp agn","Ġrever ed","ĠE H","ĠN CT","Ġgo ede","ä¸İ åºĶç͍","度 为","à¸Ĺีà¹Ī à¹Ģà¸Ľà¹ĩà¸Ļ","ãĥĸ ãĥ©","å°Ĭæķ¬ çļĦ","ad ir","ĠâĢ §","Ñģк Ñĸ","ä¸Ģ缴 éĥ½æĺ¯","ĠPsych o","ĠConf ederation","ÑģлÑĥ жи","ĠC oca","ĠE is","ĠY esterday","计 ä»·","è¡Ĺ åĮº","Ġ×Ļ ×¢","Ġru pt","áĥĶáĥ ł","-second ary","é¢ł åĢĴ","ĠSurv iv","Ġp iled","ĠB rem","н да","æľī 计åĪĴ","天 æķ°","iol i","éĢģ çļĦ","æĪĺäºī ä¸Ń","åݿ级 以ä¸Ĭ","Ġslipp ery","Ġreperc ussions","ĠL ydia","æĥ ±","ä¸ĭ æĸĩ","Ġprodu zione","车 éĺŁ","æĦ¿ ä½ł","Ġdark est","Ġpub li","Wal ay","Ġtrunc ated","' ){Ċ","/ icons","C el","le o","æ°´ æĸĩ","Ġ×IJ× ĵ×Ŀ","ä¹³ éħ¸","夺 åĨł","ĠEvent Args","Cle arly","ĠìĤ ¼","Ġпо ÑģÑĤав","-y our","ĠMac Donald","ĠPRO F","ÅĦst wo","å¤ļä½Ļ çļĦ","ólic a","Ġspole Äį","ĉ sum","Ġn en","Ġbrill iance","×ķ×Ĺ ×ĵ","ĠNich ols",") &","Z r","á Ĭ","Ġpe aked","第ä¸Ģ 款","áŀ Ķ","\\ hat","ĠV ille","å®ŀ åIJį","ĠÙħ Ú©","ĠØ£ ب","å¾® å¼±","æĹ¢çĦ¶ æĺ¯","ĠRefer ències","ĠÐłÐµ ÑģпÑĥбли","ãĥĻ ãĥ«","L an","ri ques","æĪij ä¹ĭåīį","ial ysis","åħħ è¡Ģ","à¹ĥ ส","Ùij Ùĩ","é¤IJ 廳","Ġcam ar","å¦Ĥæŀľä½ł æĥ³","Ġcolour ful","åįģåħŃ æĿ¡","s ym","an imate","im ed","Ġtrans cribed","ä¿¡ éģĵ","Ġза д","Ġprop ia","ÑģÑĤвен нÑĭм","à³įಠŁ","Ġcyt otoxic","psych ological","çĮ¶ 豫","Ġc rank","åľ¨ å®ŀè·µä¸Ń","ä½ł çľĭçľĭ","æĽ´ åĥı","да м","Ġital iano","à¸łà¸²à¸ §","Ġespañ ol","Ġélé ments","ac us","大 師","Ġpost modern","å¬ °","itsch rift",". he","h len","Ġc ations","åĴĮ çͰ","ä¸Ĭ è¡£","çĻ ĸ","èĢģ äºĮ","Ġmed ios","ä¾Ŀ åŃĺ","ç«Ļ éķ¿","ĠÐ´Ð¾Ð¼Ð°ÑĽÐ¸Ð½ ÑģÑĤвима","× §×","ä¸Ģ æĭį","ä¸Ģ åĪĨéĴŁ","Ġrep ell","anc ock","Ġcirc adian","éĢĤåIJĪ äºİ","ĠInvest or","ĠC annon","人 头","èĢĮ æĪij们","Ġо ÑĦи","Ġsc all","াঠī","-pro f","Ġdomain e","ĠDisc rimination","Ġrent ing","Ġhub s","ĠArg uments","ve el","ç»ı çͱ","Ġph i","Ġtrans duction","Ġcar ne","éĽĨ èµĦ","Ġhist one","Ġ% }","京 åī§","å®ĥ们 åľ¨","ĠاÙĦØ´ ب","çļĦåľ° çIJĨ","aaaa aaaa","( product","ĠM ILL","os aurus","ĠP ied","ç» ¢","te a","ö ffent","Ïĩ ο","}$ Ċ","ĠSylv ia","Ġt ipped","it he","em phasis","ĠÑģ на","Ġam ine","宣 æī¬",",\\ ]ĊĊ","Le af","à§ĭ ব","Ġbra very","رب Ø©","ĉ ĊĊ","Ġt ien","Ġa cept","ĠP ID","ver bial","Ġcl ut","æĪĸèĢħ åľ¨","δ ι","çļĨ æľī",".Log ger","ç¡®è¯Ĭ çĹħä¾ĭ","à¹Ģหล à¹Īาà¸Ļ","ra kt","ĠP GA","ĠV inci","交 èŀį","ĠØ£ Ø®","åĹ ĸ","All en","ĠStep hens","ĠÙħص Ø·ÙĦ","Ġath letics","å±ķçݰ åĩº","ĠдеÑı ÑĤелÑĮноÑģÑĤÑĮ","ĠL ern","åı¯ä»¥ æıIJé«ĺ","arn os","èĬ± 纹","Ġdu plicates","ĠCheck ing","Ġimmun otherapy","ĠUnter richt","ãģ§ãģĹãĤĩãģĨ ãģĭ","Ġ ÅĻe","Ġد ÛĮد","éķĩ éĿĻ","ĠSl ot","æĢĴ æ°Ķ","ĠOw ens","ĠÙĬØŃ ت","ĠØ´ÙĬ Ø¡","P arts","z iale","ĸ ׼","ar atus","ç»ĵæĿŁ æĹ¶","EO F","Ġinhal ation","ĠConstantin ople","ĠC et","ĠJ ets","Ġdes ider","åı¯ä»¥ ç͍æĿ¥","Ġت ÙĪØµ","Ġб из","å¸Ĥåľº ä¸Ń","è¡£ è£Ļ","çĨŁ äºº","Ġstd in","éĽħ æĢĿ","neg ot","Ġp lex","st wa","ä¸Ģ è·³","é«ĺ æĸ°","西 æĸ¹çļĦ","ั à¸IJ","-f ood","è tre","Ġsal a","ĠпÑĢи Ñģ","ĠÑĤÑĢан Ñģ","Ġenthal py","Ġf rench","æľī æ²Ĵæľī","ss l","Ùĥ ÙĬ","æĹ¶éĹ´ 段","ĠEn abled","à¸Ĺ ะ","à§ģ ধ","Ġmys qli","ÐIJ в","ĠIntrodu cing","ĠG ó","èį ¼","Ġsuccess ors","ç¦ı ç¥ī","Ġobjet os","æıŃ示 äºĨ","ĠP itch","в об","å½ĵ 她","ãģ® ãģ«","ĠØ£ Ùħا","è®ĵ æĪijåĢij","aks an","Ġê°Ļ ìĿ´","Ġокон Ñĩа","ĠThe mes","å°± åħĪ","è¿ĺ 被","å¾Ī è¿ľ","OC I","åĤ¨ çī©","åľ¨æĪij çľĭæĿ¥","Ġkul it","/ to","ĠD V","ain o","ĠCh andra","Ġret aliation","EC A","ĠPhys icians","ĠпÑĥ бли","ĠANAL YSIS","Ġcol itis","Ch ronic","ä½Ĩæĺ¯ 对äºİ","Ġje ÅĽli","è¿IJè¡Į æĹ¶","_b ound","Ġdesper ation","ĠZn O","Ġaddict ive","ĠOdys sey","è¯ħ åĴĴ","\" %","Ġ án","Ġcon gest","est ruct","ht on","AG A","ÑİÑīи ми","åĪĨæ³Į çī©","عر ÙIJÙijÙģ","çļĦ éĿĴå¹´","Ġtra umat","Ġins isting","pect ral","æľīä¸Ģ éĥ¨åĪĨ","T ile","åĴĮ åĽ½éĻħ","å°± 该","ĠاÙĦÙħ اء","åı² åѦ","åį± åıĬ","è¿İ éĿ¢","çŁ¿ äºķ","ÏĦο ν","Ġincident al","Ġcrypt ographic","Jac ob","åĵĨ åŦ","Ġدربار Ùĩ","ĠT LR","ĠI PS","Ġne k","èĢħ æĺ¯","è£ħ æľº","ĠAss im",".Cross Ref","åĮ ¾","und y","iss ing","ĠÙĪ Ø¸","ãģ¨ ãģĹãģŁ","Ġing les","æķħäºĭ çļĦ","= str","Ġw rought","iz end","Ġund ue","建 çļĦ","åĮº ä½į","Ġdifferent iating","è± Ī","è¾ĵ æ¶²","stit utes","èĦļ çļĦ","à§ĭ ষ","æĬ¢ éĻ©","æĤ² æĥ¨","ĠS ala","ä¸į éĢı","ä¼ģä¸ļ æīĢå¾Ĺç¨İ","èµĦ产 管çIJĨ","éĴ» åŃĶ","m j","ä½ł æĥ³è¦ģ","èĢħ 们","æ´» ç͍","ham mer","áŀ Ł","ä¼łè¯´ ä¸ŃçļĦ","er ick","ure ka","Ġass ur","åħĭ åĪ©","Ġfull ness","Ġge ographically","éĶĻ è§ī","æ²ī çĿ¡","Ġforward ed","ĠLand ing","Ġoste oarthritis","สั à¸ķวà¹Į","Ġenlight ened","^ x","en k","th ouse","ç§ij çļĦ","ends ection","об Ñīе","'. $","à¹Ģห มืà¸Ńà¸Ļ","Ġcrown ed","ĠMood y","ĠD ari","ä¸Ĭ ä»»","Ġj Äħ","æ´» ä¸ĭåİ»","åıĹ æīĺ","ÑĨи ма",".M od","Ġjo ys","Ġb ila","ĠC LA","Ġcl oned","Ġا ÙĩÙħ","Ġì ¸","ิ ศ","çĶŁæ´» ä¹łæĥ¯","Ñĩе ÑģÑĤво","æķĮ åĨĽ","ĠRav ens","P ant","ĠT ough","åľ¨ 设计","ĠJ asper","Ġcomp agn","æīĭ åĬ¿","ĠCol leg","åıĮ 缮","ত িহ","羣çļĦ æľī","ำ à¸Ļ","Ġtum ult","Ġдав лениÑı","ĠزÙĬ ادة","çļĦ 女æĢ§","æĺ¯ ä¸ĸçķĮ","è¿Ļ æŃ£æĺ¯","te chnology","ĠØ£ Ù쨶ÙĦ","Ġfile Name","_c ache","ĠWork book","Ġpou ze","Ġmountain ous","Ġb risk","åĬł åĪĨ","ex amples","Ġcor neal","ли во","Ġmaterial ly","ĠGu an","ĠоÑĤ ÑĢиÑĨа","éĵģ éģĵ","App lied","Ġapproxim ated","}}{ {","åĭ¢ åĬĽ","olk ata","åįĬ个 å°ıæĹ¶","ĠR acial","å·¥ ä¿¡","ĠEX ISTS","Ġhonor able","éĺIJè¿° äºĨ","M ET","R oles","ĠJ ord","Ġer red","Ġнеп оÑģÑĢед","Ġcaut iously","Fran cis","[ S","ach im","ual s","éĤ ģ","س اÙĦ","åĽ¢ åľĨ","Ġprim ates","स à¥ĩ","æĮ¤ åĩº","ĠT au","ĠĠĠĠĠĠĠĠ ĠĊ","Ġj ap","åİ» çļ®","ĠInteg rating","³³³³³³³³ ³³³","ĠPerm ission","\\ +\\","en oid","ik ki","åĽ½ å¤ĸçļĦ","è´Ł æķ°","[] )Ċ","Ġinstit utes","ĠSal ud","Ġcoun s","ĠLearn ed","à¸Ľà¸¥ ูà¸ģ","R i","åΰ æīĭ","æķĻ å¾Ĵ","den ed","éĺµ åĪĹ","Ġgarden ers","- CH","Ġo lymp","为 ä¾Ŀæį®","åĢ Ń","Ñĩ ением","AT G","壮 è§Ĥ","Ùĩر اÙĨ","ÐĹ Ð½Ð°","说åΰ è¿ĻéĩĮ","Ġkilob ytes","Ġparch ment","Ġb itch","ا ÙĬر","åĴĮ 缸åħ³","åĩº æµ·","ely n","ãģ¾ ãģ¨","é½IJ é½IJ","OM A","ĠÑģоб ÑĢа","ĠLeb anese","xxxx xxxx","Ġmalign ancy","- ROM","F unctions","çļĦ è®°å¿Ĩ","ul at","iz an","äºĭ åħ³","ĠÎ Ĩ","де к","æĭŁ å®ļ","ĠاÙĦÙĨ بÙĬ","åģľæŃ¢ äºĨ","ĺ× Ĺ","_ex ists","漸 漸","_ '","Ġch illed","Ġu sted","æŀľ åĽŃ","اÙĨ ÙĬا","ST A","Ġant id","è§£åĨ³ åĬŀæ³ķ","é¬ ±","ç¨Ģ åľŁ","ro de","ĠR ath","ĠL id","å¯ °","Get Mapping","çĸ¾ æĤ£","ë° Ľ","ĠDirector ate","Ġhug ged","Ġcompliment ary","Ġf ries","ur at","大 åį«","Ġj uta","io ids","Ġsub script","ĠAle j","Ġpier w","ĠнеÑģколÑĮ киÑħ","H W","\\ sum","ĠR OC","车 éĩĮ","Ġgl o","cos a","cycl ine","å²Ń åįĹ","ĠBuy er",". ?","C ultural","F el","ĠH oy","ен ное","com merce","Ġind ie","ven ge","çļĦ人 å·¥","ĠâĨ µ","ç»Ħç»ĩ ç»ĵæŀĦ",".M at","Ġanticip ating","ಾಠ¦","Ġaplic ación","æīİå®ŀ æİ¨è¿Ľ","Ġsurpass ed","is ia","çļĦ 女åŃ©","交 åĵį","è¿IJ è´¹","Ex pect","J i","n itt","Ġre ps","if ix","æĪij éľĢè¦ģ","æł¡ æľ¬","ĠÑĥ кÑĢа","sh ima","ма ÑĢ","ä»»ä½ķ çļĦ","Ġsubs istence","Ġvac ancies","å¿ł å®ŀ","Ġnob les","Òĵ Ñĭ","Ġantagon ists","Ġnewcom ers","y v","ĠB RE","Ġhel ix","ĠÙģ ÙĦس","å¾® å¦Ļ","Ġcó d","ĠHttp Client","Ġcré ation","ĠL J","Ġver der","æĺ¯ä»Ģä¹Ī åij¢","Ġhoof d","çªģå¦Ĥåħ¶ æĿ¥çļĦ","- output","< i","çļĦ éĹ®","纸 æĿ¡","Ġwy kon","-weight ed",", âĢĿĊĊ",". »","ĠA head","Ġal red","ãģ® ãģ¿","abs orption","ä¸Ģèĩ´ æĢ§","ಿಠĹ","ĠEisen hower","ok an","éĤ£ èĤ¡","-w in","ĠUN ION","Ġsedent ary","åħ¨ æĻ¯","æĸ° é£İ","und ra","表 éģĶ","æŃ£ æĺ¯åľ¨","Ġsupp ressing","å°±æĺ¯ 对","min imum","å¨ ĵ","ส ืà¸Ń","Ġtool kit","Ġinnov ate","×ľ× ŀ×Ļ×ĵ","Ġbrand ed","Ġrock ing","à¹Ģห à¸Ļ","Ġmacro economic","Ġvap our","ä¸Ģ æĮ¥","åºĶ æ¿Ģ","up art","Ġfr antsay","im n","ĠR ue","ors che","Ġcro cod","v ue","ĠP EG","Ġan imate","å¸ °","erm e","人åijĺ è¿Ľè¡Į","ĠMal ag","_l ocation","ÑĤив ной","ĠJac qu","Ġdiscretion ary","迪士 å°¼","Ġcon he","ĠD ani","ĠG ras","Ġout liers","æĢ§ æĦŁ","å½ĵ æľŁ","ÑĤа Ñı","åĨ³ æĪĺ","æ´Ĺ æ¼±","á¹ ĥ","Ġдан ного","ĠìĹĨ ëĭ¤","Ġperturb ations","Ġabsor bs","Ġt ari","Ġf ined","èĢĮ è¿Ļ个","è± Ĭ","ĠSh awn","Ġsw arm","Ġever green","ĠRob ust","Ġdess erts","Ġযদ ি","< ul","ĠT orn","Ġbl ir","Ġcol oc","æ´» åĬĽçļĦ","ĠÑģо ÑĨи","æĹģ è§Ĥ","ĠÐĵ оÑĢ","åĩºä¸Ģ æĿ¡","Ġয ায়","ожд ение","( pre","M otor","Ġk idding","ĠSt able","â tre","ENT ER","ĠEduc ators","anz as","> '","_ service","ĉ struct","ĠÑģ боÑĢ","åİŁ èijĹ","ת ×Ļ","å©ļ 纱","éĢŁåº¦ åĴĮ","{( }","à§Ĥর à§įব","ĠاÙĦبØŃ Ø«","( private","Ġne oliber","大 æĥĬ","ĠV AR","Ġinter esse","Ġco ales","Ġmed ically","Ġstr ives","åºķ æ°Ķ","çıŃ ä¼ļ","Ġfactor ing","ൠĢ","Ġweather ing","Ġ×§ ×ij","Ġrevers ing","n iz","ĠC lem","Ġpro let","ĠH IS","oc uments","Ġsa pp","Pro s","raft ed","ĠVer ification","Ġhyp not","å·¥ä¸ļ åĴĮ","æ¶Ī失 åľ¨","isl av","_ O","ĠL AS","Ġph il","åŁº çŁ³","Ġsm ashed","çłĶç©¶ 室","å¾· åĽ½çļĦ","åı³ ä¸ĭ","èĪª æµ·","Ġsand s","ì° °","wal ks","occup ied","Ġmik ro","ĠLä hteet","D iet","ul if","åĴĮ éĺ¿","èIJ ¦","-s al","éĽĨ å¸Ĥ","Ġopp ressive",".d is","ä¹Ŀ é¾Ļ","æ£ĢæŁ¥ åĴĮ","æĸ¹åIJij çĽĺ","ç¨Ģ çĸı","æIJľç´¢ å¼ķæĵİ","bold math","ĠLep id","æĺ¯ åĪ©ç͍","ĠD atuak","Ġì ±Ħ","éĩį è¿Ķ","Ġcar bs","Ġdistrib utor","æķ¬ æĦı","ç»Ŀ对 å̼","çĸı 忽","Ġroz d","çķħ éĶĢ","æĮ¡ ä½ı","-en abled","Ġattenu ated","ĠBacter ia","ĠJ T","å½ Į","ĠIn cent","ç³ ¾","æīį å¼Ģå§ĭ","è¿ĻäºĽ è¯Ŀ","è¿ŀæİ¥ çļĦ","Ġesp éc","Ġlact ose","Impro ved","B ool","Ġà °","éħ £","èĭ± ä¿Ĭ","Ġfull est","å¿ħè¦ģ æĢ§","ĠAlex a","Ġroz w","Ġud ziaÅĤ","Ġrif les","M aker","ad av","og li","åıĬ ãģ³","ÏĢ Î¬","ĠSO FT","Ġneces idad","mel on","缴åįĩ æľº","Ġsubl ime","f att","in om","Ġst aan","å·¥ä½ľ å²Ĺä½į","ogn o","åħ« 年级","æĮ¥ åıij","Ġmold ed","(` ${","le l","ra ke","è¿Ļ èĤ¡","ym a","çĥŃ æIJľ","ÙĴ تÙİ","éĤ» éĩĮ","ĠSom erset","ì ½","re comm","it zen","ä¸į éľĢ","Ġir resist","ĠMer lin","çļĦæĸ° åŀĭ","ähr ung","è°İ è¨Ģ","ĉ q","ab outs","Ġreg imens","ĠSch a","ĠEss entially","ÑĨиÑı Ñħ","ĠL java","åīį è¨Ģ","æĿ¡ 纹","论 çĤ¹","第äºĮ å¹´","ĠExpl or","失败 äºĨ","×ķצ ×Ķ","ĠпÑĢоÑĤив оп",". Order","; s","D ave","R x","end es","å¼Ĥ çī©","çļ® å¸¦","ĠBen z","ĠSuper man","UC K","èĬ¬ èĬ³","G ross","Ġt ending","Ġa uss","以 满足","对 åIJĦ","æĢ» 产å̼","éĿŀ常 大","Check ed","ĠASS ERT","g j","ren n","ж ем","èij ©","æĻĤ ãģ«","Ġded icate","áŀ ĺ","ĠìĿ´ 미","Ġdop ed","nas ium","æļ§ æĺ§","çIJ ¥","管 åĨħ","帮åĬ© åѦçĶŁ","éĢĴ 交","è¤ ¥","ĠÙħØ´ Ú©","P ATH","çļĦ æľ¨","Ġre create","äºĨ ä»ĸ们","æĦı æĥ³ä¸įåΰ","ĠAr lington","ä¿® éģĵ","Ġaud iting","èĤ¥ çļĤ","Ġθ ε","åķĨåĬ¡ åį°ä¹¦é¦Ĩ","hor se","Ġок ÑĤÑı","Kind ergarten","Servlet Request","\" ):Ċ","F ortunately","Ġr idd","ĠCh or","ung tod","ĠÐĵ Ðŀ","Ġburn er","Ġadj uvant","×Ļ×§ ר","Ġregener ative","ĠMär z","åĩº åĵģ","æĸ¹ åľĨ","å·² æĪIJ","åIJį èĥľ","RE AM","ãĥĥ ãĥī","Ġneuro pathy","ĠSerg io","\\ Omega","Ġا شار","åIJİ æĦŁ","éĥ½ 为","ä½į åĪĹ","å¼ł è´´","ĠÑĪ ÐºÐ¾Ð»Ðµ","Ġáĥ ł","ĠìĤ¬ ìĿ´","Ġdisproportion ately","åĩ¦ çIJĨ","ĠEmbed ded","G est","en ching","ĠB W","åħī 亮","åĪĻ éľĢè¦ģ","à¸Ħ à¹Ĥà¸Ļ","Ġر ئÙĬس","Ġq i","ĠBur ger","Ġcere als","ĠLuc a","æīĭç»Ń è´¹","-des cribed","ogra fic","Ġnanot ubes","- connected","É Ĵ","om bs","ĠR anger","ĠE Q","å°± åıªèĥ½","对 åı£","ah ami","Ġstr len","Ķ× Ĵ","å°½ èģĮ","åħ¨éĿ¢ èIJ½å®ŀ","ĠUnt ersuch","ĠNick el","ĠÑĢезÑĥлÑĮÑĤа ÑĤÑĭ","æĪĺåľº ä¸Ĭ","ĠÄijá»Ļ ng","B RE","Ġf url","ĠG us","çĶŁ æł¹","ä¸ĭ åľº","å¤ļ äºİ","åĮ» ç͍","oph ilus","æķ¬ èĢģ","æľīçĤ¹ åĦ¿","Ġtrad emarks","_ modules","ĠS cores","ĠC AGR","con i","åĪĨ äºĨ","好 èĩªå·±çļĦ","tr igger","asad pang","Ġcomp utes","åıĬ æĻĤ","éĶ Ħ","è·¯ çģ¯","ĠSp ir","Ġsuper im","ĠMa ÃŁ","Ġkab ungtor","Ġplag ued","ĠEVER Y","k owski","大 æĪIJ","ãĤĤ ãģĨ","ĠEst onia","Ġде ли","Altern atively","Ġappre hend","m ong","p ir","Ġon cology","-b i","æĿĥ è¡¡","Ġsucc umb","Ġunanim ous","Ġkabungtor an","ÃŃ k","缸 éĢ¢","æ´» å¾Ĺ","ĠHigh land","æ°ı æĹı","Ġfav oured","amil ton","æ¸Ĭ æºIJ","Ġredsh ifts","o pping","çļĦ æī§è¡Į","äºĭ åıĺ","igh bour","à¸Ń à¹Īาà¸Ļ","text tt","äºĶ 代","Ġиз ме","ä¸Ģä¸ĭ åIJ§","Ġdé b","OM O","k rieg","ĠB d","çĶŁäº§ èµĦæĸĻ","help ers","ĠFeature d","ill usion","æĻ ¤","-p y","Ġfilm maker","ä¼¼ä¹İ åľ¨","à· ļ","让æĪij们 ä¸Ģèµ·","ĠÔ ²","Ġconvey or","Ġغذ اÛĮÛĮ","ic eless","le ast","Ġen ch","å¾ģ åľ°","Ġlab yr","åŃĻ å¥³","Ġtherm odynamics","Ġmeng andung","ĠProv iders","ĠStaphyl ococcus","ĠIEL TS","Ġc atech","ä¸į èĢĥèĻij","ç»Ĩ èĩ´çļĦ","å·´ å°Ķ","Ġaud ible","пи ÑĤÑĮ","K enn","Ġrel ocated","两 åı£","ĠÑĥ ÑĢок","康 å¾·","çģµ çŁ³","Ġ» .ĊĊ","å±Ĭ ä¸ī","ä¸į对 ç§°","ĠRoss i","bere ich","ĠÑĢеали заÑĨии","Ġtect onic","pe ÅĤ","Ġsm oot","Ġé d","Ġé m","èĤī çļĦ","è·³ åĩº","ĠÙħج رÙĩ","Ø®ÙĦ اÙĤ","ĠBI OS","ĠMick ey","k id","ĠM arm","Ġpl unge","é¦ĸ æŃĮ","Ġpa ar","à¥įठŀ","Ġcut aneous","åĩĨå¤ĩ 好äºĨ","fe edback","ণ à§įড","åįļ士 çĶŁ","Ġgang s","Ġжеле з","ĠP SA","pl atz","ä¸Ĭ 个","ĠCh iang","Ġforward ing","ãĥ© ãĥ¼","-a uth","èħIJ çĥĤ","ĠExt raction","ĠConn ected","ĠFre i","Care er","Ġgad gets","çľ© æĻķ","¤ ×Ķ","ĠK ü","强 度çļĦ","åĿļ 强çļĦ","Ġà´ ®","/pro vider","ing les","è¦ģ ä¿ĿæĮģ","Ġprim ordial","äºĮåįģ ä¸ĥ","çģ¾ åĮº","Ġentitle ment","ĠL ens","Ġcharacter izing","缺 å¸Ń","ï½ ¥","ĠPet r","åĽŀå®¶ äºĨ","Ġprincip ais","-te am","ĠCommit ment",") }\\)","åĽ½ åºĵ","Ġet apa","izz ard","èªŀ æ°£","Ġescal ation","Ġplut ôt","Ġf ict","ĠIn gg","ĠMar se","atur ally","Ġmis information","ĠSal z","ERT Y","icol or","Ġfle eting","ια ÏĥÏĦ","ĠíĮ IJ",") ãĢĬ","ĠE bola","ĠF rid","ä½ł å¿«","ç´ IJ","æĦŁ äºº","åĬŁ åĬĽ","æĪ¿ è´·","Õ¡ ÖĦ","ìĸ µ","ĠÑģи лÑĥ","Ġnod ules","ç½¢ å·¥","Ġspo il","b ef","Ġb esser","ro ff","ast en","åĩº ä¸ĸ","form ations","ite ur","æĻĤ åĪ»","éĥ½æľī çĿĢ","ä¿ĿéĻ© è´¹","ĠMag dal","æĸĩæĺİ åŁİå¸Ĥ","æŀļ 举","R y","ĠB ars","çĽ ľ","ov ou","ick ého","Ġsent ir","Ġже л","çªģåĩº éĹ®é¢ĺ","ĠÑĤÑĢеб ованиÑı","ĠاÙĦÙĤر ÙĨ","q p","Ġg azed","Ġsub cutaneous","rid ged","äºĴ 为","Ġcomplet amente","ĠDE V","ĠVent ure","ĠPere ira","ÃŃp io","ĠS ü","ĠM ata","åĴĮ åIJĦ","ust ering","ç¤ ´","Ġra ining","ĠZ inc","çľ¼ è§ģ","list a","Ġκ ο","O I","ĠP CT","èĩª è¨Ģ","ç¥ŀ æĺİ","ON Y","ĠAng ola","ÐĴ о","(l st","èĪΠ奮","ĠHeide gger","Ġcirrh osis","Ġper nah","æł¼ 鼷","}} ,Ċ","IP C","身边 çļĦ人","ĠDoes n","бе лÑĮ","Ġblo ed","esters hire","}{* }{","Ġunavoid able","L etters","æł¼ åŃIJ","Or th","Cy cle","cro ft","ãĤ·ãĤ¹ãĥĨ ãĥł","ç͍ å®ĥ","AT EG","å°±ä¼ļ åĩºçݰ","严éĩį å½±åĵį","Ġanthrop ogenic","n odes","Ġdes erts","çī¹ å¤§","Ġes fuer","æĹ¶éĹ´ æĺ¯","离 éĢĢä¼ij","ĠSc her","Ġл оги","åį« åģ¥","鸡 汤","Ġmeg abits","åįģä¸ĥ æĿ¡","è´¬ å̼","Ġpalab ra","èħ İ","ä½İ è¿·","Ġtyp ename","ĠEm otion","èĮ¶ æĿ¯","ĠHil fe","çļĦ åIJĦ项","ä¹Ł å¾Ī好","管 åŃIJ","享 åıĹåΰ","ĠBal ancing","æŃ¦æ±ī å¸Ĥ","ĠÙĪØ¬ Ùĩ","ĠRN As","Ġstip ulated","+ A","_ head","ĠW ak","é«ĺ åĵģè´¨","éĥ¨ ä¸ĭ","Ġco ff","-T e","Sign al","ĠHom eland","/ https","ĠWh is",".n lm","éĻª æĪij","ĠPass ive","Ġdod at","Ġpanc akes","Ġvenge ance","Ġde formed","Ġas cent","ich ter","ç² ½åŃIJ","éĵ °","Ġcell es","åĿĩ ä»·","ĠMat te","Ġchromos omal","ĠEgg s","Ġunderest imate","Ġt ú","Ġfor age","ge ometry","éķ¿ åīij","åĮħ çļĦ","κ ά","icy cle","åı« ä½ł","åįĸ äºĨ","'' 'ĊĊ","ĠPen y","Ġgrasp ed","ãĤµãĤ¤ ãĥĪ","ĠB ett","æĹ¶ 许","Ġpart ed","ĠÙĪ ØºÙĬر","ij s","Ã¥ rd",".D isplay","社åĮº å±ħæ°ij","Ġми нима","opp ortun","Ġpear l","ĠPione er","辦åħ¬ 室","Ġmelanch oly","? **ĊĊ","ĉ arr","ĠD ess","ĠV and","è¿Ľ æĿ¥çļĦ","æĪ· åŀĭ","ĠAcc red","param etric","à¥Ģ à¤Ĥ","主é¢ĺ æ´»åĬ¨","泡 泡","åľ°çIJĨ ä½įç½®","ĠEu ph","Ġw es","Ġal at","ĠO c","éĥ½ ç»Ļ","åĿ į","æī¿ èªį","å°į 象","ðĿij ĩ","лен ноÑģÑĤи","Ġcolonial ism","æ©ĺ åŃIJ","ĠìłĢ ìŀ¥","ĠDivid ing","çļĦ ä¾Ŀæį®","ĠS per","ĠR SA","ĠH eld","ĠH UM","天 åij½","×ķ× ©×","å·¥ä½ľ éĩı","AN TS","AM D","-y il","Ġasym met","ol son","Ġg t","ä¸į åħ·æľī","Ġhe iÃŁ","ĠK ass","ĠK ats","cre ative","Ġmain tenant","ĠâĪ ¨","iy embre","( http","e ys","r än","ess ä","-f ed","Ġarm our","åħ® åħ®","NY SE","åijIJ åĸĬ","Ġmatern ity","ukun ft","L ik","n ite","çļĦ 被","æģ¯ æģ¯","Ġcustom izable","帮 她","è½´ ä¸Ĭ","èļ Į","ÃŃst ico","Ġarrog ant","Infl ater","Ġp éd","ig on","以 åĮĹ","Ġsa is","ĠHe ating","导 æķ°","za am",">< !","uh Ãł","DF S","ĠìĿ´ 룬íķľ","Ġà¦ħ স","ĠPf izer","o jo","ĠC alls","Ġch ina","ĠU A","ric ed","Ġco op","Ġest ilo","sw ith","isc ing","åįĥ 人","ĠGu in","OP EN","-he ld","rä ge","Capt ain","ĠBulgar ian","å¹³æĹ¥ éĩĮ","h ä","ent iful","ĠA CTION","ĠĠĠĠĠĠĠĠ ĠĠĊ","ĠD unk","ud uk","ä¼ļ èĩªåĬ¨","ä¿Ŀ é²ľ","ank ar","С Ñĥ","Ĺ× ł×ķ","- les","çļĦ æĶ»åĩ»",".. #","Ġна вÑĭ","ĠBl ocks","pre ting","èĭ¥ è¦ģ","Ind icator","à· IJ","ема Ñı","ĠJak ob","------------ -","Ġstyl ing","Ġail ments","qu iz","ĠCom ple","(g ame","Ġpou ch","Ġдов олÑĮно","Ġأث ÙĨاء","w et","å² ±","ÄĽ ji","Ġlo oming","Ġrefer encing","å±ĭ åĨħ","Ġtrack er","Ġnam un","Ġâĺ Ĩ","Ve hicle","Bibli ography","æĪIJæŃ£ æ¯Ķ","Ġin sofar","ĠS CR","ĠA UTHOR","ĠÙĪ ÙĦÛĮ","Ġsym posium","Ġsens ational","×ķ׾ ×ķת","ĠArchitect ural","ĠHart ford","Ġsacrific ing","ÑĦ еÑĢа","éĢīæĭ© ä¸Ģ个","Ġdistrib utors","ĠOl son","Ġdisrupt ing","æ¢Ĺ æŃ»","\" ...","ĠH utton","大 åΰ","Ġsub du","Ġgl aucoma","sk ill","Ġви дÑĭ","تÙħ اÙħ","汤 å§Ĩ","Ġtight en","å§Ķåĵ¡ æľĥ","ĠS ura","ra ient","ä¸į çľĭ","ĠG ael","ä¹Ł ä¸Ģæł·","å¤ļ å²ģ","ä¸ĵ èijĹ","Ġ· Ċ","Ġneg ro","ĠNe ighborhood","ĠRead ings","CI AL","Ġsuc ceeds","ä½İä¸ĭ 头","ort ical","Ġr c","ï¼ģ ï¼ģĊ","ĠPro xy","ŀ×ķ× ł×Ķ","Ġassemb ling","Ġì¶ ©","Ġcorrupt ed",". object","ä¸į è§ĦåĪĻ","Ġat a","ĠK rem","ä¸ĭ å®ļ","Ġmod em","é¦ ®","åı£ è¯Ģ","æķ°æį® å¤ĦçIJĨ","bo ost","éĻĪ ä»£è°¢","Ġsold er","çĩŁ é¤Ĭ","olu ção","è¤ĩ 鼾","Ġl eren","iv ore","aus ch","uk es","ger ufen","ĠBar rier","æľĢå°ı å̼",") ];Ċ","} $$","åΰ 缮åīį为æŃ¢","ast ra","éĩį åŀĭ","éĩij æ²Ļ","åĪ« åIJį","çķĻ ä½ı","ä¸ĥ 年级","Ġsy ringe","Ġfaith fully","ĠIP O","çļĦæīĭ æĮĩ","*** Ċ","åĸĺ æģ¯","ĠJP Y","ĠGink uhÃł","S ky","Ġw ah","ad m","ä½İ ä¿Ŀ","é¸ ½åŃIJ","è¿ĺæľī ä¸Ģç§į","ãģĹãģŁ ãĤĬ","اØŃ ظ","Valid ate","IND EX","- forward","is asi","le es","Ġn gan","Data Set","Ġell ipse","éĶħ éĩĮ","âĤ ģ","Ġmejor ar","os an","ä¸į æŃ£ç¡®","åľ¨ åĮĹ","д нев","äºĮ ä¸ī","aur ants","ĠObs erve","Ġγ εν","ĠMajor ity","æĺ¯ åħ³äºİ","ĠMed itation","_d iff","ĠíĮĮ ìĿ¼","ä¹Ł åı¯èĥ½æĺ¯",".m ove","Ġpain ters","see ing","æĹłå¥Ī çļĦ","åı¯æĥ³ èĢĮçŁ¥","/ issues",": p","Ġle aking","åħ¥ å°Ħ","ole cule","Ġка м","band s","Ġesc apes","ĠBas eline","Ġpel as","Ġprz eds","ĠпÑĢи ÑģÑĥÑĤ","ĠApplic ants","Ġeigen value","åıijçĶŁäºĨ ä»Ģä¹Ī","u atan","Ġw äre","Ġd ada","åı¯ åĨįçĶŁ","Ġcont iguous","ÛĮ ØŃ","ĠÙĪ ØµÙĦ","çĶŁäº§ åĬĽçļĦ","åıªæĺ¯ 为äºĨ","Ġappropri ation","ĠRad ial","Ġíijľ íĺĦ",". âĢĵ",". aw","ĠL ump","Ġprot rud","ä¹Ŀ 天","-h ole","Ġimmun ization","Ġrepro duc","Ġmamm al","æ·ĩ æ·ĭ","çļĦ 表达","äºĨ 两个","Ġj al","Ġam éric","Ġbu iten","çħ§ 缸","çŃĶ çĸij","Ïħ γ","ç²ī å°ĺ","Ġclean liness","å°Ī å®¶",".Ent ities","Ġà¦ķà§ĭন à§ĭ","_ current","he iro","åľ¨ éĤ£ä¸ª","ä½ł ä¸įçŁ¥éģĵ","arch itecture","Th u","Ġutil iza","Ġص د","_b uffer","Ġeste em","S EM","{ ~","Ġbe zeichnet","Ġsp es","ops ies","ĠTre as","Ġvolum etric","ce a","ĠH es","ri pe","__ ,","åħ¶ 对","ÑĢе за","ÙĪÙĨ ز","ĠаÑĢ Ð¼Ð¸","Ġlumin osity","å°Ĥ éĸĢ","D rag","I o","Ġs ied","Ġm ish","çļĦ åŃIJ","å· »","Ġmat hematically","Ġت ؤ","åĬŁ ç͍","çĥŃ è¡·","èᝠç͍","éĻį èIJ½","çŁ¥è¯Ĩ ä¸İ","Ġregular ity","ĠIns ulin","ĠNa omi","_M OD","Ġutter ance","ĠØ£Ùĥ بر","à¸Ħà¹Ĥà¸Ļ à¹Ĥล","Ġof ere","ä¸į å°į","Ġcoll oqu","ë¥ ł","лен нÑĭй","äºĮåįģ åħŃ","Ġcritic ize","çļĦåIJį ä¹ī","re lease","it ro","Ġn b","ĠR uf","ĠK ep","åıª åIJ¬","交 纳","AS I","è§Ĵ èĨľ","ĠMin erals","æĸĩåѦ å®¶","ìĤ ¼","Ġmaj ÃŃ","ä¼ĺåħĪ çº§","ç¡ķ士 åѦä½į","ap u","çľ¼ ç§ij","æĺ¾ éľ²","Ġprob ing","Ġvo z","-r anging","Ġ׾ ×Ļ","ĠNederland se","ĠÙĦد Ùī","ĠF owler","Ġch iff","Ġport ug","Ġvir ulence","Ñĩа н","ائ ÙħØ©","Ġপর িà¦ļ","Plan ning","t ow","iv ir","ck t","被 认为","ой ÑĤи","Ġsing iolary","ED I","çļ± äºĨ","Ġpian ist","Ġнеза виÑģи",") dx","- He","M ich","Ġ ï¼","çļĦ 绣ä¸Ģ","ĠT iny","ĠC ah","ĠK ov","å°± åΰäºĨ","ack et","Ġset Is","Ġrespons able","Ġleaf y","Ġö ss","ĠBlog ger","éĺIJ æĺİ","ĠDat aset","Ġanomal ous",".google apis","顽 åĽº","ĠAgen cies","çϽè¡Ģ çĹħ",". Delete","çļĦ ä¾ĭåŃIJ","çļĦ æĦŁè¦º","Ġsh a","åĩº ä»»","åıį åĵį","Ġplay list","Ġت Ùħر","Ġг ÑĢ","ç¨İ æ³ķ","ĠاÙĦØ£ صÙĦ","ÛĮد ÛĮ","Ġju icy","ĠPack aging","GP U","ãĤ¤ãĥ³ ãĥĪ","ä¸įæĦ§ æĺ¯","K ond","æīĢ éĢłæĪIJçļĦ","éϤ å°ĺ","Ġcharacter izes","Ġж ÑĥÑĢ","Ġdeal ings","SP EC","Ġflex ion","åħļçļĦ 建设","DE V","ëIJĺ ê³ł","ĠÐŃÑĤо ÑĤ","ìĺĢ ëĭ¤","-def ense","Ġt achy","Ġv ost","åĮħ 袱","Ġdr ifting","ĠÑį мо","ĠÐĴ еÑĢ","èĦļ åį°","Ġìłķ ìĿĺ","Ö´ Ö¼","为 代表","ah as","è·Ł ä»ĸ们","èIJĥ åıĸ","à¸Ķำ à¹Ģà¸Ļิà¸Ļ","+ l","e lements","Ġh ob","ĠL ena","Ġj adi","ä½Ĩ åĽł","åIJį åĪĹ","ç¦ į","ار ب","Ġcal ves","åı¯èĥ½ éľĢè¦ģ","æ²ī è¿·","ç»ıèIJ¥ æ´»åĬ¨","Ġà¦Ńার ত","o ÅĤ","ĠR AF","Ġpresent er","Ġmut ta","mo ire","าà¸ģ ร","ĠاÙĦج Ùĩ","ĠÕ° Õ¡Õ´","T ai","ad ar","×Ļ ×Ļת","Ġoff ending","Ġexam iner","æĴ ¬","ĠØ£ Ùģ","å°½ å¿ĥ","orph ism","Ġconson ants","à¹Ĥà¸Ħรà¸ĩ à¸ģาร","S us","Ġm v","åľ¨ å¾Īå¤ļ","大 åħ´","åīį çŀ»","ij× Ł","åı² è¯Ĺ","اÙĩ ر","è¯ij èĢħ","Ġسب ب",") `","p iel","çļ İ","çļĦ å¹³åı°","ĠP iece","pp m","æł¡ åĨħ","Ġorgan ismo","Õ¡Õ Ń","èĥľ è´Ł","ĠSupp l","Ġма Ñı","र à¥įत","ĠElis abeth","çļĦ 建çŃij","ĠS ys","ĠC oy","Ġper ubahan","åIJij åĮĹ","Ġinit With","è´µ 人","ĠFa ust","+ X","Ġe ens","ĠD aly","ĠR aja","ä½ł å°Ĩ","Ġcons équ","åıĤ å±ķ","Pe ace","çĤ® å¼¹","Ġboost s","Ġdict ates","ĠDest ination","I ran","Ġf ists","ĠK ron","ÙĤ ص","Ġне воз","ઠ¹","ĠFin ite","港 åħĥ","lab els","ic hes","Ġì £","èĽĭ é»Ħ","é¢ľ æĸĻ","Ġже лÑĥ","Inst ruction","ĠW AY","ä¸Ĭ 说","çĦ ±","éļı 访","å¾® å°ı","Ġа ÑĤом","ĠÙħع د","ĠSynt hetic","ĠíķĻ ìĬµ","æĪij å·²","ard in","è´ Ĭ","aj as","Ġdem i","Ġposs a","ĠAg nes","çݰå®ŀ çĶŁæ´»ä¸Ń","à¸ĵ ี","Dim ensions","ĠodreÄij enog","ĠT utor","é«ĺ éĽĦ","ä¸İ èĩªå·±","é¢Ĩ äºĭ","Ġ×ķ× Ľ×","ĠاÙĦØŃ ÙħÙĦ","ç°¡ åįĺ","ัà¸į à¸į","èįĨ å·ŀ","E gypt","ĠO CD","åĴĮ åIJĦç§į","ount able","Ġد ا","Ñī ений","Ġtop ographic","å¾Į éĿ¢","éĵ¾ è·¯","Ġsav age","ĠÙħس ئ","çļ Ļ","um é","ence g","ĠV ID","Ġbar ren","_m ask","ç§ĭ é£İ","Ġhero ine","Ġneck lace","ĠSir ius","ä¸Ĭ ä¸ĸ纪","èĥ½ 被","è¿ĩ æĹ©","inc er","è·Ł æĪij们","åıijå¸ĥ äºİ","ç²Ĺ æļ´","Ġnit ride","ĠDif ficult","ĠزÙħاÙĨ ÛĮ","refer ent","Ġplung ed","ĠT RE","ä¼ļ åıijçݰ","åħ¬ åħģ","马 ä¸ģ","çĬ¶ æ³ģ","ĠÐŁ и","ĠTer re","Ġઠ¨","Ġnu ova","ĠмеÑĤ ода","necess arily","ĠPharmaceutical s","Ġawa its","Ġp ense","éĤ£ æĪijå°±","ÏĢ Î¹","éģĹ æ¼ı","Ġshut ting","Ġexch anger","- arm","Ġas eg","ä»İ éĤ£","åıĺ é¢ij","-f acing","ĠGo es","ĠMe V","åıªæľī è¿Ļæł·","ĠAc res","ĠPost al","ĠArch iv","éĢĥ èµ°","_st ep","вид еÑĤелÑĮ","嬷 嬷","ĠIngg ris","åĩº çĤī","èĩª èĢĥ",".st d","Ġহ à¦ĵ","ĠRa iders","åı¸ä»¤ éĥ¨","×Ļ×ľ× ª","楽 ãģĹ",".pre vent","ĠO aks","æīĢ å¤ĦçļĦ","åħ¬åı¸ ä¸İ","Ġ×Ķ× ĺ×","èѦ åĬ¡","模å¼ı ä¸ĭ","ĠMal ik","åIJŀ åIJIJ","ĠнаÑĩа ле","Ġangi ogenesis","ĠÑĢам каÑħ","is ent","ĠIn fer","ä¹Ł å¸ĮæľĽ","Ġcomb inator","éħĴ çļĦ","_d etails","Ġ×ij× ¢×","çķ¶ åľ°","Ġvacc inations","à¤ĸ à¥įया","Ġinterrog ation","ä¿ĺ èĻı","[ self","it rile","çļĦ æ¡Īä»¶","æĺ¯ ä¸įåIJĮçļĦ","è¯ ¬","ĠH CF","æĪij æĽ¾ç»ı","lect ic","go al","Ġakt u","á»ģ n","ispr udence","is cono","Ġhand lers","Ñī емÑĥ","è que","Ġver a","讲 åłĤ","มาภ°","溢 ä»·","ĠвÑģ Ñij","Ġnarc iss","Ġceil ings","åĪĨ åıij","ологи ÑĩеÑģкиÑħ","ĠEN GL","Ġহিস à§ĩবà§ĩ","ĠÙħÛĮدÙĩ د","ĠC obb","ay ered","ĠJ ade","æĴ ¥","ç¦ģ ç͍","kom st","ĠMaur it","Ġmiracul ous",") -\\","Ġv m","ov ého","ob ility","æĸ° èĤ¡","Ġprov incia","uss y","Ġsk ating","ĠAP C","åŀĥåľ¾ æ¡¶","Ġonder wijs","ĠElig ibility","o ires","¦ ×¢","æĽ´ æĸ°çļĦ","æ°Ķ æĦ¤","uh l","H ung","h ope","ut ama","å¼ ¼","Ġcons ul","åı¯ä»¥ åģļ","arn i","è¿ľ åı¤","çļĩ çĶ«","积æŀģ ä½ľç͍","å®ŀéªĮ ä¸Ń","ãģł ãģĭãĤī","ĠEL ISA","Ġà¦ĩ à¦ī","Ġসà¦Ĥ à¦Ĺà§įরহ","Ġabbrevi ated","ĠT K","ä¸į çν","Ġcom rades","ä¸Ń ä¿¡","ÙĪ Ø¡","缸 è·Ŀ","åĩ» æĿĢ","ĠìĿ ½","Cons ult","< bits","S oph","h oles","u ces","z eg","Ġd art","ro let","st ats","ĠP ix","ĠP AL","çĤ¹ åľ¨","Ġг оÑĢи","ÅĽ ród","æĺ¥ æĻļ","Ġdire ito","augh lin","試 é©Ĺ","Ġutter ed","ĠEver ett","-supp orted","à¹Ģศ ร","Ġt aut","Ġl inger","Ġsal on","éĤ£ä¹Ī çļĦ","æ´Ĺ èĦ¸","cd ktf","ĠRom ney","ĠPROC ESS","Ġطر ØŃ","ĠÙĨØŃ ÙĪ","ĠHIST ORY","ĠF ahr","å°± æĹłæ³ķ","ç¥ IJ","Ġد Ùħ","ĠØ£ جÙĦ","ĠAb by","Ġtor rent","T YPE","æĪij 說","å®ī åįĵ","åįķ çīĩ","ĠZ ahlen","Ġ×ľ× ¢×","ĠØ¢ سÛĮ","Ġstri pe","Ġment orship","Ġrib u","Ġproc ure","ĠXX I","ĠÙħÙı عرÙIJÙijÙģ","ĠS ke","ä»ĸ 說","å¾Ī ä¸įéĶĻ","æĬĬ éĤ£","িঠ§","å¦Ĥä½ķ 使ç͍","åIJĪä½ľ åįıè®®","åĨ° å·Ŀ","/p df","à±įà° µ","ĠHead quarters","Ġpréc éd","åįļè§Ī ä¼ļ","Ġpiez oelectric","Ġc ÃŃ","Ġav aient","åĶ ¬","é¸ ³","-de ficient","ĠRot terdam","èĩªæĿ¥ æ°´","P ET","åı¯ è¦ĭ","说 她","ع اد","Ġterm ine","ãĥ¼ ãĥ³","å¥ĩ å¼Ĥ","Ġcommand ments","æľĢç»Ī çļĦ","注åĨĮ èµĦæľ¬","æľ¬æĿ¥ å°±æĺ¯","Ġperf ume","ou g","Ġqu as","éĢļè¿ĩ çļĦ","/d t","大å¤ļ æĺ¯","ྠ±","ĠDiff usion","Í ĺ","ĠRe yes","缴 ç«ĭ","à¸Ľ à¸Ķ","åħ¬åħ± åľºæīĢ","Ġtrat amento","Ġëĭ¤ìĸij íķľ","U d","on k","ĠEn um","æĪ¿ åľ°","ĠBe ef","ÅĽ lin","åĽ¢éĺŁ çļĦ","Ġhippoc ampal","æĺ¯ åĵª","Ġus uario","Ġpl upart","èĢĮ ä»Ĭ","ç¡ Ĵ","两 å±Ĥ","ull ed","________ ____","åĪļ 度","奥 æŀĹ","Õ¡ÖĢÕ ¤","ĠANSW ER","ãĢģ (","æĪij çĪ±ä½ł","Ġab a","ĠAn ch","æĪijçļĦ æīĭ","ĠBro oke","ĠÑģе веÑĢ","广大 群ä¼Ĺ","ä¼ĺè´¨ çļĦ","found land","ĠBren nan","ĠживоÑĤ нÑĭÑħ","Ġv ak","èµ· æºIJäºİ","Com ing","ĠSur rey","Z en","q n","am pling","å½ĵ éĿ¢","åħµ 马","âľ ¦","Ġbarb ec","Ġcód igo","N r","ile ver","Ġfe b","ĠÙģ ÙĨ","Ġminim ise","ĠСов еÑĤ","Ġnyel ven","é ¯","çļĦ åĩ½æķ°","å¹¶ 使","ĠâĨ Ķ","ĠSim ons","ï ve","éĢĽ è¡Ĺ","çļĦ èµĦæĸĻ","ĠH ut","Ġtra cer","å±ķ åĩº","ĠÙĪ Ø§Ø±Ø¯","à¸ģ à¸İ","æµģ æ´¾","åķĨ åѦéĻ¢","rac a","ĠPr att","Ġteam mate","Ġж а","æİī èIJ½","è¯ļ æģ³","atu ur","ĠBay es","ĠED IT","ĠÑĢоÑģ Ñģий","is ure","Ġin accessible","ĠP em","åĨ ¢","Ġdi odes","ĠPro gn","Ġdec oded","第ä¸ī æĿ¡","Ġmag ma","æģĴ 大","å®ĺæĸ¹ ç½ijç«Ļ","hig her","ੱ à¨","Ġì ·¨","åįģ åĩłä¸ª","uv ian","long rightarrow","G ly","Ġo trzym","Ġimport ância","åĪ©æ¶¦ çİĩ","é©ļ è¨Ŀ","วั à¸ķ","ĠD uc","к оп","å®¶ 主","ãĥ³ ãĤ¸","é¡¶ å±Ĥ","æijĦ åıĸ","/a uth","Mah on","acry late","S b","S kin","Ġh ymn","Ġout ing","ĠCh ak","ó rio","ä»İ è¿ĻéĩĮ","åĪ« å¿ĺäºĨ","Ġcare t","å¼Ģåıij åĴĮ","èĵĿ åĽ¾","Ġذ Ùĥر","-eff ects","ĠAn chor","Ġsl urry","ĠAtt achment","èĴ¸ æ°Ķ","Ġpedest rians","Ġb ony","Ġre play","lic a","éªij 马","Ġrz eczy","ĠUIT ableView","Ġì¡´ ìŀ¬","An onymous","ĠWar rior","å¡« åħ¥","Ġwet ensch","Ġburg ers","Ġaccru ed","/ no","ĠL ICENSE","æĥ ĭ","conf irm","æ²ī浸 åľ¨","V ision","Ġf ühren","Ġfor a","ĠH Q","Ġz war","æĹ¥ 讯","ĠAr d","Ġsom mes","æł¹ ç³»","åĽłä¸º 没æľī","ĠAm elia","Ġter abytes","Ġdim er","表çݰ å½¢å¼ı","Ġunivers it","Ġmá ximo","\\ int","un ku","ĠPh ilos","ĠSta ats",". ,Ċ","w olf","๠ĭ","Ġé tr","åĪĴ ç®Ĺ","IJ× ŀר","Ġcirc us","ç¹ģ çIJIJ","æĢĿç»´ çļĦ","াস à§įত","ĠMu eller","Ġling ua","= e","an imation","ĠT ric","äºĨ å¹¾","åħ¨ éĥ½æĺ¯","å¤į æł¸","Ġfashion ed","èĤ¡ä¸ľ 大ä¼ļ","ĠعÙĦ اج","ĠجÙħ ÙĦÙĩ","æ¶¡ è½®",". red","Ġle aked","Ġout c","çα æĪij","ÑĨи они","Ġfut ile","conf igure","ĠìĹĨ ëĬĶ","L AS","ĠF W","åĴĮ ç¾İåĽ½","åľ° 被","Ġcap ill","enn el","åĿĩ åľ¨","å°į äºİ","OP LE","B rief","us ätz","ĠB RI","åī ģ","éĢļ çķħ","OR A","è¿Ļç§į æĦŁè§ī","abel le","Sh adow",".e ach","âĢĻ ãĢĤ","æīĭ æĦŁ","èĢģ 夫人","ĠSe eking","ÏĦ ια","ats ch","Ġpress o","aff er","Ġhom osexuality","-n ormal","ĠLiter atur","ĠJahr hundert","наÑĩа ла","ä¸Ģ æī«","ĠF H","ä¹Ł ç½¢","ÑĢе би","ส าว","Ġconcent rates","à¹Ģà¸ģ ษ","-sp onsored","ĠÙħØ´ ار","Ġdess en","áģ Ĭ","-ne utral","à§ĩম ন","Ġpsy che","-determ ination","åľ¨ æīĢæľī","ä¸Ń ç«ĭ","å¼Ģå§ĭ æĹ¶","éĺ» æĭ¦","交éĢļ 大åѦ","è´´ å¿ĥ","riter ion","Ġbot an","éĥ¡ 主","Ġwit ches","b ranch","éķ¿ ä¸īè§Ĵ","Ġpod ium","æĺŁæľŁ æĹ¥","ĠÙħطاÙĦ عÙĩ","m illion","Ġa just","ĠJ unction","æĪ¿ ç§Ł","}} }}","èĩªçĦ¶ ä¼ļ","ĠÙĥ ÙĪÙĥ","})\\ ).","Ġunlock ed","Ġprovoc ative","j h","Ġo e","ĠG EN","ia v","ib its","Ùħ ØŃ","ose cond","Ġem iss","åħ¨ ç½ij","Ġsun flower","ĠاÙĦع ÙĤ","ĠMal awi","Ġме л","å°½éĩı éģ¿åħį","ä¹ĸ å·§","Ġcontempl ating","Recomm end","ent ional","Ġon ward","æĺ¯ åĽłçĤº","äºĨ åĹİ","Ġz av","ĠZ ent","-l argest","ä¸Ģèά éĥ½","ĠBlack s","འº","ãĤ° ãĥ«","Ġtren ches","è¿Ļ çŃī","г еÑĢ","çİĭ å®ī","èĶ ·","iert o","Ġmyth ical","ĠMAT ERIAL","Ġtecn ologia","/ types","Ġw ig","ä¸į æ³ķ","æĹ¶ èĩ³","éĢī åĿĢ","åħļ 竳","å°ı红 书","- ST","k ý","æ±Ĥ åĴĮ","æľ¨ é½IJ","-g rained","Ġrepe al","æļĸ å¿ĥ","ĠNor ris","Ġmol t","Ġexempt ions","b ran","å¦Ĥ çİī","ĠX iang","çļĦ人 åĬĽ","社ä¼ļ ä¸Ĭ","åı¯èĥ½ åĩºçݰ","ова н","çĪĨ æĸĻ","ìŰ 구","ĠInputStream Reader","ĠA cre","ĠP up","ĠN ets","ั à¸ķ","举 è¯ģ","æĬĵ èµ·","xim ation","ĠEp iscopal",". up","H ong","l us","å®Į ä¹ĭåIJİ","ä»ĭ äºİ","è¯Ĺ æĦı","×¨× ŀ×","ĠDec or","åĸĿ çļĦ","çĵ· çłĸ","-int ensity","å®Įæ¯ķ åIJİ","ĠRajas than","Ġre positories","ch ip","Ġqu als","Ġpres et","åĽĽ 项","å±± æµ·","Re q","iter ate","é¢Ħ çĥŃ","à§ĭ হ","-re ference","Ġди Ñģк","éĥ¨ä½į çļĦ","æħĮ ä¹±","ĠWait ing",". img","r outes","ĠâĢ ļ","Ġbet er","ĠFoot er","loss en","Ġperi ode","Ġnás led","ĉde fer","ł ×Ļ×Ķ","ä¸Ń æĺ¯","Ġconsider a","sh it","ãĢij (","çŀ °","ান া","ĠAccess es","äh len","Ġfibr in","gef ührt","ĠĠĠĠĠĠĠĠĠĠĠ Ċ","ä¸İ çݯå¢ĥ","ĠLa os","Ġlat ach","ĠDevelop ers","Ġcin emat","âĿ ¶","ãģ«ãģ¤ãģĦãģ¦ ãģ¯","Ġre organization","Ġres pe","缮 ãģ®","ä¸ŃåĽ½ ä¼łç»Ł","sk im","æľĿ 天","Ġmel odic","tan le","® ,","æ¤Ń åľĨå½¢",": ]Ċ","对 ä¸Ĭ","Ġsing led","äºĨè§£ æĽ´å¤ļ","亮 äºĨ","Ġ×ķ× Ĺ","éŃĶ æĹı","å¼ķåıij äºĨ","transfer ase","? ).","Ġb ouncing","ĠJ K","天 人","ä¿Ŀ å®ļ","åĿIJ åΰ","æŃ£åľ¨ è¿Ľè¡Į","ë§ ģ","Prov ided","ĠMP H","æ°ĶåĢĻ åıĺåĮĸ","K ay","L ake","ĠD ors","æĸ¹ ãģ¯","ph osphate","æ´» åľ¨","åIJį åѦçĶŁ","éĢĢ è¿ĺ","Ġma akt","-e fficacy","èᣠ幏","æĹłå½¢ èµĦ产","/ output","å¹´ ãģ®","åĽ½ åѦ","ook up","ĠUn icode","Ġins oluble","éĺ² ç©º","Ġsoft en","çļĦéĩįè¦ģ åĨħ容","æŀĦéĢł åĩ½æķ°","Ġin secure","çĿĢ æĥ³","å¸Ĥ éĿ¢ä¸Ĭ","ва Ñļе","é¡¹çĽ® ç»ıçIJĨ","æĪIJäºĨ ä¸Ģ个","Sl ow","æ½ľåľ¨ çļĦ","ĠبÛĮ اÙĨ","æ°ijæ³ķ åħ¸","ĠÑģÑĥб ÑĬек","Ġm ange","æľī è¿Ļæł·çļĦ","ä¸ĭ å±Ĥ","次 å¹´","raph ic","å¥ĭ åĭĩ","tex te","Ġaxi oms","Ġt ão","çļĦ èĦ¸èī²","ine craft","Ġ\" --","ata an","åºĶ ç«ĭåį³","奥 åľ°åĪ©","بÙĬ ب","ĠÑģоб лÑİ","ì¸ µ","b ilt","Ġw ohl","ä½ł æĶ¾å¿ĥ","ĠY ak","ä¸İ åĽ½éĻħ","å·¥ä½ľ æĥħåĨµ","æºIJ çłģ","Ġس رÙħ","ĠProgram a","ÐĽ Ь","ĠEle ven","ರ à³įà²","ĠRank ing","л ока","Ġìł Ī","Ġax le","ĠMes h","è¯ £","Ġun as","è¿ĺ ä¼ļæľī","çīĽ çļ®","~~ Ċ","تÛĮ جÙĩ","Ġwed dings","Õ¡Õµ Õ«","Ġ×ľ×¤× ł×Ļ","á Ħ","Ġget Name","Ġins ure","Ġve ut","è¶ħ é¢Ŀ","bl ast","ĠInter views","íĻ ķ","A uf","D ial","F ly","n og","ag ian","å¾Ĺ åĩºçļĦ","ĠSch m","ÃŁ erdem","ĠMet abolic","åIJĪåIJĮ ä¸Ń","ĠÑĥве ли","R IGHT","ĠD mit","Ġhome page","æĺŁ çº§","ĠØŃ Ú©","ĠSub scription","åħ§ å¿ĥ","Ġ}} >Ċ","ĠUND ER","èĦ±é¢ĸ èĢĮåĩº","R ather","} using","Ġcl ima","ĠV ue","Ġfun zione","Ġprot é","Ġiss uer","ĠRet rie","ĠMer chant","Ġfatal ities","Ġe ind","ä½ľ æ¡Ī","çĿ ij","èĢģ åħĪçĶŁ","åŁŁ ç½ij","è³ Ī","æĿ¾ åĬ¨","æĿIJæĸĻ åĴĮ","ĠÙĪØª س","Ġmun cul","- IV","c um","ī ´","we bs","п ÑĢави","éĶ µ","à¸Ń ายุ","åĨį è¿Ľè¡Į","è² ª","le hem","در س","bes ar","âħ ¢","Ġhing es","Ġappre hension","ob ook","ä¹Ŀ çϾ","æĭĽ æīĭ","dis abled","ati ivi","åĩºåİ» çļĦ","Ġbid ang","ĠиÑģполÑĮзÑĥ ÑİÑĤ","ĠÕ¢ Õ¡Õ¼","ĠTrou bles","çĭ© çĮİ","ĠB aden","Ġest udi","Ġcontent ious","åģľ äºĨä¸ĭæĿ¥","æĹģ 人","ä¸įåľ¨ æĦı","ĠCall ing","Ġmét odos",": t","Ġg els","ĠP au","ĠD iffer","ach o","In line","管çIJĨ ä½ĵç³»",".t ail","ç¶ĵ çIJĨ","æł¹æľ¬ 就没æľī","Ġoct obre","ĠUt ilities","ĠÑĨе ли","æĺ¥èĬĤ æľŁéĹ´","Ġquien es","Ġdispat ched",". result","b k","b ak","ch witz","æĪij们 ä¸į","() ),","æİ¨ ç®Ĺ","åįİ å±±",".t ar","èĹı 书","驱åĬ¨ åύ","ĠDeut schen","Pal indrome","ĠWhit man","çĥ ¬","Ġна зад","èŃ ½","èĭ¦ æ¶©","社交 åªĴä½ĵ","ĠWol fe","Ġdl ou","èĢĮ åħ¥","å·¥ä½ľ æĺ¯","Ġsl ang","Ġза кÑĢÑĭ","ĠRep ubl","Ġever lasting","ĠDi agonal","Ġjur id","å®ŀè´¨ æĢ§","æĬī æĭ©","ок ÑĢÑĥг","æķ£ çļĦ","ðĿij ħ","Ġ×Ļ ×ľ×ĵ","Ġà¹Ģภ¥","Ġ문 íĻĶ","Ġáĥĵ áĥIJ","v oy","ĠL itt","Ġна и","iter al","Ġang uish","ĠгÑĢÑĥп па","tim estamp",". Product","[ {","st mt","对 æłĩ","èĩª è´Ł","çͱ åĽ½å®¶","ĠZ ah","Ġcent red","×ķר ×IJ","Sk ills","åģļ é¢ĺ","åIJį èijĹ","ðŁ §","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","å¾ģ ä¿¡","ĠÑĢе да","ко ÑģÑĤи","à§Ģ ল","ĠText s","ĠAv iv","Ġgru ppo","ĠWy att","ĠÑĢай она","ĠR CT","ĠE SC","port e","åĽŀ éģĵ","emb les","Ġvar iances","ĠST E","-A f","Ġded uced","ĠÙħا ÙĬÙĪ","æĺŁæľŁ åĽĽ","æľŁéĻIJ åĨħ","æºľ æºľ","çŀŃ è§£","ĠAlm ighty","O il","é nd","ï¼ī #","Ġsub conscious","Ġest ers","Ġsim ulating","Ġfore arm","æİ¢ 寻","ĠBur d","tw enty","Ġn este","æĪĸ å¤ļ个","çī¹ å¾´","æľ¨ è̳","Ġorganiz ación","èĥ¸ èħĶ","ĠH AS","å®ŀ æµĭ","ä½Ĩ éĤ£","Ġest re","åĽĽ 次","Ġenc odes","æī¹ 示","è°Ī èµ·","饮 çĶ¨æ°´","主åĬ¨ æĢ§","æłı æĿĨ","é»ijæļĹ ä¸Ń","ĠU ri","ip ur","ici ó","Ġplas m","èŀº éĴī","Ġcircul atory","Ġcher ish","Ġdownt urn","uci ary","L j","} c","ĠS elling","se at","å¿ĥ å¢ĥ","æĸ¯ èĴĤ","Ø· ار","ĠÙĩ ÙĨÚ¯","én om","ĠOper ators","æIJ¬ å®¶","æ³Į å°¿","- ord","_ box","u els","ĠTh orn","Ġman ic","з оÑĢ","Ġfe udal","ĠSh ark","ĠEr fahr","Ġhunt ed","Ġpine apple","Ġinfest ation","ĠÑĦев ÑĢа","- era","l ude","æĺ µ","ĠG og","æĸ¹ æĸ¹éĿ¢","å¥Ĺ è£ħ","è´¦ éĿ¢","ĠTake aways","ĠKir by","ç£ħ 礴","Ġunbelie vable","à¸ĵà¸ij à¹Į","_ weight","r ž","åľ¨ æĹ¥å¸¸","对 ä¸į对","ä¹Ł 说","æĪij们 èĥ½","æĢ» æķ°çļĦ","Ġmult ilateral","ĠاÙĦÙħ ÙĨت","ĠÏħ ÏĢο","lene ck","ĠvÅ¡e chn","Ġrec ourse","é»ij æ´ŀ","Ñģи ÑĤÑĮ","Ġfire fighters","ĠÑħ о","æ¿Ģåıij äºĨ","Ġe osin","æľ¬ åĬŀæ³ķ","cre w","ï¼Ľ (","第ä¸Ģ å±Ĭ","ler i","è¡£ æŁľ","Ġsymbol ize","Ġpm id","åļ ĵ","vertis ements","æĺŁæľŁ ä¸ī","uwe ga","Ġthrott le","Ġا دار","åĪĨ éĺŁ","Ġtrans p","çłĶç©¶ æĬ¥åijĬ","æĿİ æĸĩ","Ġত থ","ĠPo ison","ĠCrit icism","iest a","Ġoxid ase","ĠHerm ione","éªĨ 驼","' una","Ġc apped","çļĦ æĪĺæĸĹ","ä¹Ł ç»Ļ","Ġbl at","ä chen","å½± éĻ¢","é»Ħ èĬ±","ç´¢ åıĸ","åį± æĢ¥","åħ¨éĿ¢ 建设","ĠFOR E","侦 æİ¢","éĵĿ åIJĪéĩij","he z","ou pling","ĠA ph","é ducation","ĠÙħ ات","Ġget Id","Ġimp ede","åĩı æ³ķ","ales e","Med ian","è¿ĶåĽŀ å̼","æĥ¬ æĦı","u omo","he p","çļĦ çĬ¶åĨµ","qu ee","æľ¬ 人çļĦ","èĭ± è¶ħ","å®ĺ åºľ","é»ŀ é»ŀéłŃ","Ġcin qu","ĠPRO JECT","å®ī 稳","åĨį ä¸į","Ġма ло","Ġpoly ester","Ġaden ocarcinoma","alcul ate","æĢ Ĥ","Ġ/ > ;ĊĊ","ä¸Ģ æĸ°","Ġha voc","建设 ä¸Ń","Ġexist en","å¼Ģå±ķ å·¥ä½ľ","ĠMor se","Ġহ à¦ļà§įà¦Ľ","би на","Hist or","Ġsé culo","Ġmant ra","ĠاÙĦعاÙħ Ø©","Ġböjnings form","Ġ à¸Ĺำà¹ĥหà¹ī","Ġthat s","ĠD ug","ĠR in","Ġ{ :","便 被","ãģ¨ èĢĥãģĪ","ÐŁ ÑĢед","ĠEst imated","Ġslow down","Ġ» Ċ","è¢ģä¸ĸ åĩ¯","/ product","Ġpro ponents","oc yst","Ġsp as","é«ĺ ä»·","pro tein","Jon athan","Ġneurodegener ative","ĉ long","Ġre int","ĠS op","ĠF t","app liquer","åĩł æĿ¡","ди ÑĤ","åĨ° æ·ĩæ·ĭ","] ).ĊĊ","åľ¨ 许å¤ļ","åĽ½ åħ¬","åºĶ çŃĶ","ĠThere after","æĮī åħ¶",".p rev","Ġkom puter","èĢķ èĢĺ",". Button","ar ck","Ġn ÃŃvel","ĠW ach","åĽ½ éļĽ","好 åĿı","å¤ĸ åĬĽ","(' -","éĢģ åħ¥","ĠAmb ient","Ġmart yr","à¸Ļัà¸ģ à¹Ģรียà¸Ļ","Ġre written","ik ko","ĠLe hrer","éĢĻ ä¸Ģ次","èĮĥ ä¾ĭ","Ġcomb ating","Ġáĥ ¬","}$ ĊĊ","intern ational","_ open","å½ĵ å½ĵ","管çIJĨ æ°´å¹³","inc orpor","à¸Ĺ ุà¸Ļ","ĠIm am","Ġprim eros","éļIJ å½¢","loc ations","åİĭ缩 æľº","ocon ut","âŀ ķ","à¸ŀัà¸Ļà¸ĺ ุà¹Į","è¿ Ĥ","ĠL az","ĠG aut","æĪĸ åħ¶","Ġë ĸ",".l en","é»ij å½±","oper ations","æĹ¢ ä¸į","Ġبر د","Ass ume","ç¡ķ士 çłĶç©¶çĶŁ","ĠÙħدÛĮر ÛĮت","en st","ve er","é ta","Ġп иÑģа","æĿİ æĺİ","è½» çĽĪ","åıijçĶŁ äºİ","à¸Ħร à¸Ńà¸ĩ","Ġباش ÛĮد","æļ« æĻĤ","Ġres ur","åıij èªĵ","Ġte asing","æķĻèĤ² äºĭä¸ļ","积æŀģ æİ¨è¿Ľ","Ġmetabol ite","Ġfebru ar","? ),","k ids","it ability","æĪIJ åįĥ","缺 æįŁ","çά å±±","( width","Ġm anga","Ø§Ø ¤","æķ´ 车","rig ation","ĠÄij á»ĵ","Ġkel ompok","-ac re","Ġlug ares","Ġelic ited","ĠAkt iv","ĠSOC IAL","ÙĪÙĦا ÙĬات","Ġhac en","Ġt á»ij","对 æĪijåĽ½","com pl","ä¸Ģ个 å°ıæĹ¶","ĠSh ak","ĠÄį as","设ç«ĭ äºĨ","Ġsek ali","诵 读","Ġà¦¬à¦Ľ র","ucaly ptus","- angle","ĠJ agu","åıį éĿ¢","ëĬ ĺ","оп ÑĢов","ç©¿ åŃĶ","Ġtough ness","å¤ĸ交 éĥ¨","hov ah","Ġs g","Ġs avor","iv ary","å°ı 鸣","å·²ç»ı åΰäºĨ","ĠMed ina","oss al","çĦ¡ æ¯Ķ","èĤ¤ èī²","Ġbloom ing","ĠÙĪØ§ØŃ دة","è¾Ĺ 转",") ।","è¿Ļ æĬĬ","Ġpl aza","Ġpl entiful","åľ° éĹ®éģĵ","æľ¬ å¹´","Ġz ig","æıIJ 纲","Ġhist ological","ĠNo el","ĠSome how",".R untime","Åij k","ĠSl ope","Ġstack ing","Ġком на","Ġpill ows","ĠдÑĢÑĥги ми","ä¹Ł åŃĺåľ¨","ĠCon verting","Ġsk ipping","æ¥ Ķ","Ġbre ve","à¥ģ द","\\ }\\)","ĠM AL","est imate","ek i","åĨį åģļ","çϽ èĻİ","åįĹ éĺ³","Ġmot ility","认为 èĩªå·±","à¸ŀ ูà¸Ķ","δ εÏĤ","ξ η","ĠBacter ial","F ol","Ġm ite","Ġk ong","ÑĢа б","åİŁ ä½ľèĢħ","便 å°Ĩ","ĠMan or","ĠÙĬ ÙĪÙĨ","IM AL","çѾ åıij","æĪIJæľ¬ åĴĮ","Ġorient al","Ġprecis o","Ġlibr arian","Ġдоба в","æĢĿæĥ³æĶ¿æ²» æķĻèĤ²","im in","ĠJ avier","we ets","ĠPro verbs","Ġpar all","áĥ ®","és i",".) .Ċ","ë² ł","ல à¯Ī","ĠE ighth","ç»ı èĦī","che nt","æĪ¿ ä¼ģ","ĠPol yn","Ġposit ivo","Ġbibli ographical","ĠAy urved","Ġspor adic",". rel","ab at","Ġspec jal","åįķ ä½ĵ","Ġcreat ively","Ġ×IJ× ¤×©×¨","çݰ代 åĨľä¸ļ","ìŀĪ ëĬĶ","ĠP ets","ĠL IVE","大 åIJĥ","ĠV alu","An cient","Ġvar ia","ĠEduc ator","part ition","ĠTim ing","employ ees","B V","æĶ Ķ","åħµ çļĦ","ãģķ ãģ¾","uther ford","Ġgloss ary","ãģ«å¯¾ ãģĻãĤĭ","Ġnouve aux","/ )ĊĊ","åĴĮ ä¸Ģ","rit os","æħĪ ç¦§","ĠÑįÑĤом Ñĥ","Ġköz ött",". How","C âu","Y LE","p redict","t ak","ĉ node","in ities","ĠY en","ert y","æį ¶","Ġover throw","Ġrel atable","ax el","Ġmen ace","Ġdu ra","åĸľ åºĨ","Ġ×ij× Ł","Ġpul sed","Ġa ula","æĺ¯ 社ä¼ļ","Ġpro actively","res olve","Ġad hered","Ġ×ŀ× ¡×¤×¨","åħ·æľī ä¸Ģå®ļ","ĠComp at","èѦ åį«","ĠRed irect","Ġlit re","Ġalg ún","rov iral","ĠMartÃŃ nez","ä¸į ä¹ı","Ġcur ios","ãĢĤâĢĿ *","z f","ch ronic","ĠR ang","使ç͍ 寿åij½","çķĻ åŃĺ","oms nitt","Ġpain fully","Ġpré cis","ĠÑĥÑģлови й","ĠHast ings","Ġcl ad","æĶ¹ åζ","空 èĻļ","è¯Ĭ æīĢ","æµħ æµħ","ìĻ ķ","ĠUNIVERS ITY","ĠCret aceous","B oy","ĠN ing","Ġse an","Ġu ur","ä¿Ŀ è´¹","ä»Ĭ çĶŁ","é¾Ļ çİĭ","ø m","Ġspo iled","Ġзаболе ваний","ĠExpect ations","漩 æ¶¡","' elle","- English","çļĦ æľªæĿ¥","ĠN EXT","ĠAd verse","å¸Ŀ åĽ½çļĦ","à§įয াস","ÐĽ и","< b","Ġt roph","çļĦ åķĬ","eb b","Ġconvers e","Ġod by","wa ÅĤ","é¹ Ń","ç£ģ æĢ§","æĻºæħ§ åĴĮ","ĠRam adan","Ġacknowled gment","ĠBuch anan","Ġад миниÑģÑĤÑĢа","ĠC PC","Ġtr imester","å¦Ĥ æ°´","ĠÎ ¥","åľŁ åĮª","arl os","ĠBo hem","åĪĨåĪ« åľ¨","rav ity","Ġencounter ing","ĠKön ig","Ġde pl","Ġcl ashes","ib ar","éĤ£ 裡","Ġrel inqu","å·¥ä½ľ éĺŁ","other wise","éļ¾ åħ³","Ġpa a",".d ao","æĶ¿åºľ éĩĩè´Ń","ত িà¦ķ","ı z","Ġvac u","Flor ida","ar coma","çŃī èijĹ","Ġhist oire","λ ίοÏħ","Member Signature","Rect angle","l ÉĻ","od in","äºĨ å¾Īä¹ħ","âĢĿ [","ell ä","Ġmas uk","çļĦ çĶ»éĿ¢","Ġh aci","åıĺ å¹»","ä»ĺ ãģij","When ever","追 åĩ»","gl ia","Ġcam ino","Ġalleg es","æłĦ é¤Ĭ","K er","å¤ļ å±Ĥ","éĥ½æĺ¯ ä¸Ģ个","zen iem","订 è´§","ĠHans on","Ġanth ology","ĠL oki","代 ä¹ĭ","Ġmeas les","Ġaut onomic","ĠUS P","Ġter us","Ġnorm ale","met rics","å°Ŀ å°Ŀ","Ġlip oprotein","碾 åİĭ",") }(","- ton","ro pe","om otor","ĠE SS","ĠK is","-m etal","è¶ħ åīį","éĢĢ åĮĸ","resh ape","æĮĩ导 æĦıè§ģ","Ġত াà¦ĩ","åĬłåħ¥ åΰ","ĠMuse ums","_D ATE","Ġশ à§ģরà§ģ","ç¶² åıĭ","課 é¡Į","ĠAlban ia","K u","i OS","ĠC edar","se hen","ert ian","è°ģ æĺ¯","Ġà° ¬","交æį¢ æľº","- ##",": @","{ J","Ġt orso","ä¹ĭ å¤ľ","æķĻ å£«","管çIJĨ æľºæŀĦ","Ġshort en","ú st","Ġconduct ance","Ġম াধà§įযম","ĠÑĨ аÑĢ","Ang el","ipe g","ðĿĽ ¼","Ġrel at","Ġо ÑĤе","ĠØ´ ÛĮر","è¿ĺæľī åħ¶ä»ĸ","Ġدر جة","åĬ³ å·¥","çͲ çĥ·","åºŁ å¢Ł","Ġ×ij×IJ ×ķפף","Ġlod ged","L ie","d uration","çļĦ 天空","Ġk ennis","åĩº æĸ°","Ġinter leukin","å°±æĺ¯ ä¸į","eb iz","ĠFig s","ĠMinor ity","ĠHamm ond","ĠhÃł m","Ġst alls","å¿ĥ æĻº","Ġins ufficiency","èĦij çŃĭ","Ġcapt ivate","_m ove","Ġвз Ñı","Mov ies","_ server","ä¸Ĭ ä½į","以 éĺ²æŃ¢","å°ı éĺŁ","ãģ® ãģĤãĤĭ","Ø® اÙĨÙĩ","æ¡Į ä¸ĬçļĦ","Ġ×Ķ×ŀ× ¢","Ġíķĺ ê³ł","à¹Ģà¸Ħ ล","ĠоÑģоб енноÑģÑĤи","Sud denly","Ġcomerc ial","Ġ' (","å¹³ æĻĤ","Ġд оби","enc oder","à´ ¨àµįà´¨","-w ell","Ġstd out","CM A","Ġש׾ ×ķ","ĠاØŃ تÙħ","ĠMunicip ality","n elles","Ġf ührt","Ġb ounced","le ist","Ġо де","æī¾ ä»ĸ","Ġaffirm ation","ĠACT IV","} ,ĊĊ","以 éģ¿åħį","cess ing","Ġvol gende","» .Ċ","ĠÑĤе пеÑĢÑĮ","åħ¨éĿ¢ ä»İ严治åħļ","Ġ×Ļ ×Ĺ","Ġáĥ ¨","ç¡« åĮĸ","Ġprv nÃŃ","\\---------------- --ĊĊ","Ġespec ies","Le v","ĠGerman ic","Ġnatur als","伺 æľį","D W","I ER","t os","z ny","ä¸į éĻIJ","åľ¨ 欧洲","Ġ- ,","èĩª ä¿¡å¿ĥ","åºĶ ç¨İ","äºĽ å¹´","交 çĤ¹","æµ· éĩĮ","Ġtable View","CH ANT","ĠÙĪØ§ÙĦ س","à¥ĩ त","é϶ éĨī","Ġrede em","he ure","太 å°ij","Ġap ical","ĠSte iner","ÙĬÙĨ ÙĬ","åĬŁèĥ½ åĴĮ","丰 满","ä¹Į é¾Ł","ĠоÑģнов нÑĭе","Ġges am","Ġprés ence","Ġf ünf","Ġget User","æĪĺ æĹ¶","Ġпо ÑĤе","ĠCont ra","Ġdownt ime","Ġinconven ience","ä¸Ń 线","亲 æľĭ","Ġе же","Ġdrop down","é©¿ ç«Ļ","ĠNewsp aper","Ġì¹ĺ ë£Į","转载请 注æĺİ","çݰ å̼","è®® éĻ¢","Ġide e","Ġthought fully","Ġtw elfth","×Ļר ×ķש","æ£Ĵ çļĦ","æŁ´ èĥ¡","ĠRow Box","ãģ¾ãģ§ ãģ®","çIJĨäºĭ éķ¿","Ġchim ney","Ġrehears al","- working","çļĦ æĦıè¯Ĩ","Ġsc and","åĨ° åĨ·çļĦ","ĠTw ins","ĠTit ans","ĠÑĢÑĥков оди","í ά","æĺ¯ 该","ĠL oyal","éĥ ¸","æĥ³ èijĹ","çķ¥ æĺ¾","Ġarch ival","çĥ¤ èĤī","è©ķ 価","Ġtant al","Ġdictators hip",") I","Ġre name","ĠS ark","ov nÃŃ","åĽ¢ æĶ¯éĥ¨","set up","Ã¥ nd","ĠA DA","ç͵ å·¥","èĦ¸ çļĦ","ðĿij Ĵ","à¸Ĥ à¹īาà¸ĩ","éĢĢ ç¼©","åĵĪ å¸Į","åģ· è¢Ń","说è¯Ŀ äºĨ","ĠTax onomic","Ġstem ming","amy cin","æĬĦ è¢Ń","Ġê´Ģ ê³Ħ","N OS","ĠF etch","éľ İ","éķĩ å®ļ","Ġব à§įর","ĠAnt oine","ĠHel ic","ĠCD s","è¿Ł åΰ","Ġtrav aux","èIJ¥ä¸ļ ç¨İ","Ġrubb ish","ĠEG FR","ĠFurn iture","ĠMozamb ique","çļĦ æĦŁåıĹ","çļĦ çĶŁéķ¿","iv et","ĠG eld","Ġpar able","ç»ĵ 对","Ġpres ión","åħī éĺ´","åĻ ¶","çα åIJĥ","ย ิà¹Īà¸ĩ","建设 çĶ¨åľ°","æ£ĭ åŃIJ","(null ptr","ĠRadi ology","Ġfierc ely","Ġh ortic","ag awa","æľī åħ³ç³»","åı¯ ä»İ","å°± å·²","ĠEx po","à¹ģ หล","é³ į","æŁIJç§į ç¨ĭ度ä¸Ĭ","?âĢĻ âĢĻĊĊ","ĠÑĥÑĩа ÑīиÑħÑģÑı","Ġprá ct","èĮĦ åŃIJ","Ġpont os","USS ION","in stead","ĠA FC","ĠD aughter","ĠN LP","æĿ¥ 计ç®Ĺ","æĥ³ åľ¨",".g it","ĠDel ay","¤× ł×Ļ","wer ken","H b","çļĦ åIJįç§°","st ellar","Ġte at","Ġpe ach","æİ¥ çıŃ","æ® ī","æĶ¶ åΰçļĦ","éģį åľ°","çļĦæĹ¶éĹ´ éĩĮ","Fin ish","] ),Ċ","z icht","ĺ ×Ļ×Ŀ","Ġm ash","Ġu ży","ä¿ IJ","åĽ½ éģĵ","pro jects","ä k","çĶ· æĸ¹","Ġе ÑīÑij","ç´¯ ç´¯","æĭĸ æ¬ł","Ġà¤ħ न","é¡ı èī²","= v","as ian","th on","æĸ° æĹ¶æľŁ","Ġpe int","AC Y","Ùİ Ø§ÙĦ","га ÑĤÑĮ","ĠST A","Ġhyp ogly","TR ACT","è§Ģ çľ¾","oplas ma","ĠOliv ier","_ encode","ØŃ اد","ĠFin ancing","èĻļ 伪","å®ľ å±ħ","ĠÙħر ض","Ġlaund ering","ĠÑģиÑĤÑĥа ÑĨии","& -","m bito","Ġar ches","æĹ¥ ç͵","Ġд во","ien iu","uk k","ĠاÙĦج ÙĨ","ĠKel ley","Ġал леÑĢ","μβ ÏģίοÏħ","( random","Ġr uch","rom atic","ä½Ĩ åĩ¡","è¡¥ èĤ¾","ĠÙĤ اÙħ","é¡¶ çĿĢ","ĠдÑĥ Ñħов","ä¸į å¤ł","Ġex on","Ġwork places","Ġpo ÅĤ","à¥įठĹ","京 å¸Ī","Ġиз бе","æķĻå¸Ī åľ¨","æĢª åħ½","ĠпеÑĢе ме","è¿ħéĢŁ åıijå±ķ","ει ÏĤ","bur ger","æĦĽ æĥħ","ĊĊĊĊĊĊĊĊ ĊĊĊĊĊĊĊĊ","nest js","and as","Ġpres enza","空 åīį","ä½İ æ²ī","æŀĹ å¤©","åºķ 座","æ¿Ģ åĭķ","Ġhom ology","è§£åĨ³ æĸ¹æ³ķ","å®īæİĴ äºĨ","ĠJo anna","eding ungen","Ġ×ŀ ×Ļ׾","æľ´ å®ŀ","_N OT","t ests","ĠK D","ym l","ĠAr gs","车 éĢŁ","à¸ķ à¸Ńà¸ļ","Ġза клÑİ","Ġstudent i","åį± æ©Ł","Ġনি রà§įà¦","Ġst ren","ant ro","ok ine","çħ§ èĢĢ","ĠMar ilyn","å±ħ å§Ķä¼ļ","ĠArt emis","ĠLaw son","ಿಠĤ","å¥ij ç´Ħ","å§Ķæīĺ 人","Ġcomun idades","Ġevangel ical","èĤĽ éŨ","Ġp f","ĠF iji","åħ¬ åĪĨ","å½Ĵ ç»ĵ","Ġalleg ing","ĠOs aka","âĸĪâĸĪ âĸĪâĸĪ","< float","Ġs ij","ĠG CC","åīį éĶĭ","eth anol","è·Ł 大家","Ġsimple x","Ġoblig ated","èµĦäº§è´ŁåĢº 表","ĠðŁĻĤ ĊĊ","r ü","Ġm ural","ä¸Ģ æĸ¤","Ġby ly","ç¥ŀ 殿","æĹ© å·²ç»ı","Ġmid field","åݻ年 åIJĮæľŁ","ĠобÑıза ÑĤелÑĮно","R oss","Ġin activation","est ablish","Ġ×Ķ×ŀ× ¦","Ġहà¥Ī à¤Ĥ","Ġdeut lich","-t emplate",".get String","Ber lin","ĠE MS","ĠL om","Ġz ullen","itt ings","åĪĻ åºĶ","åį· ç¬¬","åĪĽæĸ° åıijå±ķ","çķ¶ åīį","Ġà° ¯","åĽŀåİ» åIJ§","åŁĥ å°Ķ","ĠInnov ations","ac l","Ġk od","èĢĮ ä»İ","æīĢ å¤Ħ","æŀĹ åŃIJ","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","ucle otide","绿 æ°´","éĩįçĤ¹ é¡¹çĽ®","Ġren cont","N ie","Ġal ma","Ġpe anuts","å±ħ å¤ļ","ìŀIJ ëĬĶ","åįµ ç®¡","Ġdistint os","- around","Ġ ersch","Ġc ria","oc can","ĠâĢľ â̦","Ġ×ij× ĺ","åħ° èĬ±","ãģ¤ ãģ¾ãĤĬ","Ġdecor ating","om ini","em po","大 æĪ·","ç¼ ®","Ñģк оÑĢ","è§Ĩ åIJ¬","éĢīæĭ© åIJĪéĢĤçļĦ","éĻĦ 带","ĠDec ades","VID IA","çŀ¬ æĹ¶","claim s","ĠØ£ÙĬ ضا","Ġarous al","她 åį´","èĩªå·±çļĦ åŃ©åŃIJ","ä¾µ è¢Ń","Ġliber ated","Ġfre ak","dest roy","ĠChrom at","ĠSah ara","ĠëĴ ¤","f emale","ĠS add","а г","ĠB anner","ĠQ ian","اÙĦ ÙĨ","ĠAm os","é¬ ĵ","亦 åį³","Ġsoci etÃł","æľºåύ åŃ¦ä¹ł","ĠAccept ance","ro qu","ĠB rick","Ġch ast","çľĭ ä¸įèµ·","ä½Ĩ æĦ¿","为äºĨ æıIJé«ĺ","Ġqual che","ÐĴ и","Ġпод ÑĤвеÑĢ","以为 èĩªå·±","壮 çļĦ","ĠOw l","Ġà¦Ĩপন ার","Ġmillenn ia","M ic","z ers","çļĦ å¼Ģå§ĭ","om od","Ġk ro","ä¸Ń åĬłåħ¥","管çIJĨ ä¸Ńå¿ĥ","æ¦Ĥ è¦ģ","Ġmarvel ous","r ums","ĠB EN","Ġwith holding","all is","å±± å·Ŀ","æ¼ ī","Ġê ´ij","ada ÅĦ","×¨× §","Ġchem ist","θ εί","å®£ä¼ł å·¥ä½ľ","Sc ot","gun ta","Min ister","calcul ate","ĠTed dy","ĠRivers ide","éķ¶ åµĮ","' hui","/ question","G ro","b ir","l ungen","Ġt elle","Ġf ences","Ġl ore","Ġth umbs","Ġdev ient","åī¯ æĢ»","ĠSen ators","èĸª èµĦ","ĠT ess","op tic","è¿ĩ æľŁ","ĠAn ita","æ¥ļ 天","sm outh","ಿಠ°","ück e","ä¸ºåŁºç¡Ģ çļĦ","ãĤĪãĤĬ ãĤĤ","Ġintermedi ary","ĠSuff olk","Ġoverc row","ĠíĻĺ ê²½","ĠUnve iling","D ocs","Î Ħ","åľ¨ èĭ±åĽ½","Ġr ata","Ġ ¢","ĠGu il","缮æłĩ ä»»åĬ¡","Ġdeep ening","å¢ŀéķ¿ äºĨ","Te achers","ĠObst et","ĠNewsp apers","ä¸Ģ é½IJ","Ġad am","å°ı æĿİ","Ġag endas","sp iring","æĢ» éĿ¢ç§¯","岸 è¾¹","ĠInf inite","Ġphotos ynthetic","åĮĢ éĢŁ","Ġpap ier","Ġaccompan ies","ĠDow ntown","èĦĵ èĤ¿","/ value","ig ated","ä¸Ģ éļ»","èĢĮ 以","ĠY A","æ¯Ķ èĩªå·±","æ°Ķ ä½ĵçļĦ","åIJĦ å¤Ħ","ä¸ĵ å®¶çļĦ","æĺ¯ä¸Ģ èĩ´çļĦ","List ing","å¨ģ èĦħ","Ġjur ors","åıĽ éĢĨ","èĪī 辦","ĠназÑĭва ÑİÑĤ","= https","d j","ĉ ans","Ġy i","èĩª çIJĨ","Ġtr á»ĭ","åīį ä»»","ĠÙĦ ذÙĦÙĥ","Ġê´Ģ 리","åİ¿å§Ķ 书记","ĠCRE ATE","Ġchac un","Ġاشار Ùĩ","Z ip","Ġc read","ĠC ous","Ġad missible","好 åĩłä¸ª","yst ers","Ġcomm ons","Ġchar ismatic","æ¯ı ç§Ĵ","Ġbre v","æĹ© æľī","ĠDisc ourse","ĠEN T","ĠSent inel","ĠJenn ings","zeit ig","à°® à±ģ","Ġecu ación","\" '","K rist","Ġe Bay","ĠH ue","ä¸İ ä»ĸ人","б ÑĢи","Ġmat rim","Ġemb ell","å¹¶ä¸į å¤ļ","Bl ank","並 éĿŀ","×ŀ ×ķת","/ login","Ġp ien","ul ina","ĠF ailed","人 åΰ","Ġch ari","Re leased","Ġopt ing","éľĢè¦ģ æľī","ä¾µ åįł","Ġmac roscopic","]: =","جر اء","jug ation","Ġkilob its","D rawing","{ j","ĠA STM","ä¹Łæĺ¯ æľĢ","èŀ Ĥ","åΤ æĸ·","丽 æ±Ł","ÐĿ Ы","妻 åŃIJçļĦ","Ġimmun odeficiency","Ġflo ats","ãĢı ĊĊ","ÑĨион нÑĭй","æįį åį«","P ane","Ġbe ispiel","ĠH tml","ĠE AR","å¢ŀ èĩ³","Ġposition al","ĠØ® Ø´","åıĻ åĪ©äºļ","驳 åĽŀ","缤 纷","çļĦ çľĭæ³ķ","ĠB am","Ġdes erving","Ġpol a","ene ut","Ġsent ro","Ġин веÑģÑĤи","Ġhorse power","éĿĴæĺ¥ æľŁ","ĠPok ud","^ *","Ġg ib","é ch","åħ¨ åŁŁ","åģļ äºĨä¸Ģ个","æľįåĬ¡ å·¥ä½ľ","éĤ£ä¹Ī 大","à¸Ľ ริ","å¨ĺ å®¶","æ§ ĥ","á»ĩ m","ĠÕ© Õ¾","×Ļ×ŀ×ķ× ©","N ECT","ar ina","对 人ä½ĵ","ĠSt ocks","åĬĽ ãĤĴ","è¯Ŀ æĿ¥","åIJ« æ°´","Ġplant e","ĠSub stitute","å¹¼ åħĴ","çªĹ ä½ĵ","त à¥įव","Ġbio film","-or ang","esc ape","ĠTE ACH","Ġunfores een","Ġup bringing","ãģ® ãĤĪãģĨãģ«","Ġneg atives","åĸľæ¬¢ ä½ł","ä»į æľª","ĠCalcul ations","Ġtro v",", \\)","iss ances","âĪ §","æĸĩåĮĸ èīºæľ¯","Ġmi RNAs","ĠComput ation","è¾Ľ 亥","ĠÚĨ Ùĩار","بÙĦ د","Ġró wn","à¸Ľà¸±à¸Īà¸Ī ุà¸ļัà¸Ļ","J G","Ġc rane","ol ing","æľī è¿Ļä¹Ī","è¿ĩ åħ³","Ġsp anish","ми ÑĢ","Ph D","-n one","Ġpes ar","Ġà¦ĺ à¦Ł",". rest","ĠP ist","ĠF oo","ĠN arc","èĩ ¥","ç± Į","Ġser ÃŃa","课ç¨ĭ çļĦ","ĠURL s","at osis","ol at","ĠA insi","ĠThe rapeutics","Ġpres encia","羣 åģĩ","æ¶ī å¤ĸ","廣 åijĬ","棵 æłij","ĠSCI ENCE","è¡ ¢","Ġest u","uk t","-p ot","ĠTreat ise","å¯Ĩå¯Ĩ 麻麻","f ail","Ġd át","el an","Ġsp aring","ke its","Ġinf used","ĠMill imeter","ãĢĤãĢĤ ãĢĤĊĊ","ĠThor nton","à¹Ģรà¹ĩ ว","f h","Ġt ern","ĠC PP","ठł","æ°ij å±ħ","ĠAl umni","Ġid ade","ĠпÑĢе жде","Q g","s zer","Ġl int","Ġg at","太 大äºĨ","Ġbr ute","Ġsuff rage","Ġthread ing","ĠSem iconductor","мени ÑĤÑĮ","Ġcrian ça","ĠHepat itis","渤 æµ·","_ instance","v ich","åľ¨ 举","Ġnumber ing","enn ials","-P ierre","Ġಠĩ","æ°ij主 主ä¹ī","Comp ound","Ġsuit ably","Ġconstru ir","ÙİÙĬ ÙĴ","ĠArth ritis","_ SC","ä¸į æ±Ĥ","é¢ĺ çļĦ","åĪĻ åı¯ä»¥","ĠCarol yn","åľ¨æŃ¤ ä¹ĭåīį","Ġrub ric","( ref",") ^{\\","= R","n umeric","rom ise","ang l","ih ydro","æ¯Ľ åŃĶ","责任 å¿ĥ","ér êt","è´Ńä¹° çļĦ","ä¸įæķ¢ çĽ¸ä¿¡","å¸ħ æ°Ķ","Educ ational","T ogether","Ġn ylon","Ġl oot","ĠS ocket","Ġк ÑĬ","æĶ¾ çľ¼","åIJĥ ä¸ľè¥¿","à§ģ à¦Ł","æĿ¾ äºĨä¸Ģåı£æ°Ķ","Ġbra ille","Ġtak ie","éĤĢ è«ĭ","Ġneu rolog","ĠHarm ony","Ġencaps ulated","y embre","çļĦ æŃĮ","ĠD ive","ع ÙĨÛĮ","Ġimp urity","令 çīĮ","éĢı çĿĢ",".st at","å·¥ä½ľçļĦ éĢļçŁ¥","uzz les","éģ¸ æīĭ","Ġallerg ens","j ay","Ġt at","ĠB ind","Ġ= \"","ç»ı åĬŀ","Ġв кÑĥ","æŀģ åĮĸ","æīĢ以 å°±","он д","室 åıĭ","ĠInter ested","表çݰ åĩºæĿ¥","å°±ä¸į åĨį","Ġsed ikit","×ķ×ľ× ª","ä¾į éĥİ","Ġgr ud","è¾¹ çĸĨ","红 èĮ¶","à±įà° Ł","ÅĤo ÅĽÄĩ","Ñĩен Ñĭ","ĠваÑĢиан ÑĤ",". container","om ac","all ah","ÙĪ Ø¹Ø©","×Ļ× ŀ×ķ×ĵ","Ġза Ñħ","åįļ 主","Ġ×Ļ ×ª","èĻ« åŃIJ","ĠTH REE","% );","L isa","ä¸Ģ åħ«","-- ){Ċ","å®¶ é£İ","å¤į 审","å¼ł åѦ","ĠSu icide","çĽĺ åŃIJ","ding er","ĠNor uwega","å¹¿æ³Ľ åºĶç͍","Ġpac iente","è£ľ åħħ","vid ia","Ġproto že","< K","J oy","ot hed","ĠF rog","ip ak","ĠÑĥ ли","sh ield","ĠEl vis","åºĦ 稼","Ġbah kan",".contrib utor","à¸Ľà¹īà¸Ńà¸ĩ à¸ģัà¸Ļ","Ġfor ça","pp t","ĠU FO","Ġdi á","^{ -\\","\\) ),","Ġmen getahui","ìĿ´ ë©°","ĠÙĤ ص","ল à§ĩর","ä¸Ń央 éĵ¶è¡Į","ĠتØŃ ص","ĠBol ton","ĠFra ud","ĠEnviron ments","Ġw ow","Ġv ocation","ĠL ö","å°ı çĮ«","ĠRe bell","çłĶ 磨","ว ัย","Ùı ÙĦ","Ġpun ched","Ġverte brates","and r","å½ ¿","å¼ķ èĩª","åĽ´ å¢Ļ","cem os","-H T","Ġà¹Ģภ«","è£ħç½® çļĦ","Control s","Ġraz ón","Prob ably","ĠÙ쨱Ùĩ ÙĨÚ¯","O nt","_ mod","Ġc ors","๠į","æĥ º","è¾ĥ çŁŃ","ç»´ æĸ¯","ä¹ħ èĢĮ","Ạ«","è¿İ åIJĪ","Ġenviron nement","Ġoso by","us u","Ġall geme","æĹł çķı","Ġdel ir","Ġmem e","min os","å¼ł åĽ½","Ġrad ii","å½±åĵį åĽłç´ł",".t v","¤× ©","roph ot","åı¸æ³ķ æľºåħ³","ÐŀÑģ нов","Ġc unning","è¿Ļ æī¹","å¼ı åĴĮ","åħĥ å·¦åı³","æ¯ı éĢ¢","马 äºij","Ġmeng g","ರ à³ģ","rang ian","人 åİ»","æĬ ł","é«ĺ å¤Ħ","ey es","éĿĴ èī²","ĠاÙĦÙħ عÙĦÙĪÙħات","éķĩ çļĦ","ĠInstruction al","梧 æ¡IJ","ĠGyne col","- agent","Z u","åħ¥ åĽ´","ĠUn o","sh an","ĠAcc ident","à¥ģ ल","ĠBrown s","ĠBon nie","ujÄħ cych","âĤ ¹","æĤĦæĤĦ åľ°","/ [","V ED","大 象","èĥ½ å°Ĩ","å¹´ 以åIJİ","Ġн оÑģ","Re vision","-P r","ологи ÑĩеÑģкие","à¹ģà¸ķà¹Ī ละ","ê· ľ","Prot ected","Ġucz est","ä¸Ńåħ¨ä¼ļ ç²¾ç¥ŀ","Ġcommemor ate","\" H",". score","å¼Ģ åºŃ","ific ado","åĶ Ĩ","çα çļĦ人","Ġmic rometer","auf en","溺 æ°´","Ġarous ed","Ġgee ft","T c","åħ¬ è¯ī","ç¾İ éºĹ","æīĢ以 è¦ģ","è·Ł èĩªå·±","Ġze er","Ġlud zi","conc iliation","z ag","Ġo pl","对 çĹĩ","ov ies","è¿Ļ个 女人","rad iol","Ġfr ig","京 æ´¥","-M ay","ÅĦ ska","çļĦå¤ĸ éĥ¨","Ġdialog ues","< style","M OS","X A","è¿Ļ 使å¾Ĺ","åħ¬ åĬŀ","ع ÙĬÙĨ","ax e","go vernment","Ġarr ivals","Int el","Ġswe ets","主æĮģ ä¼ļè®®","�������� ��������","Ġobed ient","å¯Łè§ī åΰ","Ġgriev ances","G CD","ĉ Node","ĠL yme","ah rt","åĽ¾ 示","Ġref raction","è³ Ń","Ġpsych osis","ĠWil kinson","å¿ĥå¾Ĺ ä½ĵä¼ļ","Ġekonom i","= {Ċ","P enn","Ġc epat","id ung","ĠW ester","get ahuan","Ġpres ume","ĠAd ler","ä¸ŃåĽ½ 人çļĦ","иÑģ ок","ĠпÑĢед на","à¸Īะ มี","Ġrod ent","èĸª æ°´","H akut","ĉ max","on nen","å¥ Ħ","art ner","æĪIJ åĽł","Ġco ch","äºĨä¸Ģ å®ļ","ãĤĴ éĢļ","ĠAm id","ĠStud i","Ġcompl ied","ĠÐĵ а","_re place","uther land","à¦ķà§įত ি","Ġreluct antly","ar aj","ĠM our","ä¸Ģ å¼µ","est ä","åı¯ 使ç͍","åĩł åIJį","åĩı çģ¾","ĠPart ition","Ġপà§įর শ","ĠEll ie","ĠTele gram","è´© åįĸ","is ins","Ġd wa","ĠK ramer","ron i","è¾¾ 人","-f it","-l oss","æ²³ åı£","ĠIN FO","宫 女","触 çĤ¹","Ġpra irie","Ùħر ÛĮÚ©","ìĭľ ìĺ¤","èĩ³å°ij æľī","Ġ×ł× ¢","Ġpatron age","ĠDH CP","ĠEz ra","åį ½","ĠL inn","ĠJ ana","ç»ĩ çī©","à¸Ī ิà¸ķ","æĭ¿ çł´","订 è´Ń","çĭĤ é£İ","Comp iler","account s","ĠInvestig ations","doctor al","ĠÑıнва ÑĢÑı","ĠÑģеÑĢÑĮ ез","X M","ï¼Į âĢĺ","大 å°Ĩ","ï¼ģ âĢľ","asc ade","алÑĮ ного","åģļåΰ çļĦ","Sk ill","ĠI onic","Ùħ ÙĤاÙĦ","ĠâĢľ âĢĺ","ific ates","Pro per","}} \"","Ġmit osis",".re duce","ĠÑģодеÑĢ Ð¶Ð¸ÑĤ","Ġscreens hot","ĠSiber ia","Ġbisc uits","Hakut ulos","P UB","_ param","æľī å¦Ĥ","Ġres ol","ä¸ĭ 线","å¤ļ åģļ","å¤ļ è¾¾","еÑĢ Ñĭ",".s in","Ġart works","Ġap oyo","ĠInsp ired","ĠاÙĦعرب ÙĬ","plug ins","( check","A my","O ER","ĠF ULL","ast om","old t","Ġcell ul","ipp s","尽管 å¦ĤæŃ¤","Know ing","_ TR","åĽł æĸ¯åĿ¦","() {ĊĊ","Ġest ava","Ġed u","Ġden ken","ĠDo e","Ġка лÑĮ","çīĽ é̼","Ġcook s","β ά","TH IS","å§ĭç»Ī ä¿ĿæĮģ","éļ¨ åį³","sur face","Ġress ources","` .Ċ","h over","ä¸Ģ ç§Ĵ","ia ceae","Ġ[ ,","Ġcomm ens","Ùĥ اÙĦ","ret to","ĠString Tokenizer","My SQL","Ġze igen","辨 è¯Ĩ","Ġì¤ij êµŃ","Ġapost le","hyper link","_ answer","re on","ĠP ace","ell as","åΰ æĹ¶","循 åºı","}= -","Ġà´ ¸","lar ı","ĠNutrition al","าà¸Ī ะ","Ġparalle logram","éª·é« ħ","ak ai","Ġcl utter","é£ ¼","Ġм она","åį³ åĪ»","ĠCount ies","') ),Ċ","ĠÐĺ ÑĤа","Ġhi per","×Ļ×ĺ ×Ķ","Ġpatri otic","XXXXXXXX XXXXXXXX","ĠC CD","ç» ¾","ä¸Ĭ ä¾Ĩ","ĠV oid","Ġel bows","ĠDe vi","æīį ç®Ĺ","ĠFore ver","马ä¸Ĭ å°±","Ġt ann","Ġa ider","ou e","Ġhe g","Ġsa h","Ġdem ás","oph on","举 个","Ġesc uela","Ġremov able","Ġباز ار","â ¼","ê ¯","ic ent","ĠC app","Ġcan yon","Ġout burst","çͱ ä¸ŃåĽ½","åIJĦ å¼Ĥ","ĠпÑĢи Ñħоди","Res olution","临 æ²Ĥ","åŃ£ åIJİ","æ²Ļ é¾Ļ","Christ opher","ipot ent","ĠاÙĦخاص Ø©","Ġ ####","çļĦ éĿ©åij½","Ġl ernen","Ġfl ipping","ç¥ŀ å·ŀ","ĠShe ikh","åıĮ åıĮ","å½±åĵį åĬĽçļĦ","Ġnecess ario","ĠGen omics","ãģĹãģ¦ ãģĬ","اش ÛĮ","ĠDev ils","Ġδ εν","æģĭ 人","INT RODUCTION","Ġmood s","åįģåħ« æĿ¡","西å®ī å¸Ĥ","çĪº çĪº","Ġunderm ined","Lem ma","d ala","çļĦ éĵģ","好 è¿IJ","ä¸İ 她","ĠAl ma","Ġна ÑĪ","äºĨä¸Ģ åĪĩ","Ġste als","Ġvel vet","åij¨æľŁ æĢ§","ĠOwn ership","Ġgerm s","第äºĮ次 ä¸ĸçķĮ大æĪĺ","deg rees","itä ten","ĠPROC ED","ĠÑĤоÑĢ Ð³Ð¾Ð²","( address","ĠL us","Ġk ob","stand en","çİĩ è¾¾åΰ","Ġbenef iting","ç«ŀäºī åĬĽçļĦ","fa ith","Ġcart oons","ĠÅ¡ t","æĪĸå¤ļ æĪĸå°ij","D Q","ĠS igned","ĠT urtle","ä¸Ģ ãģ¤","Ġpot enti","amb ah","×Ļ׳ ×ķת","Ġبت ÙĨ","Ġw anneer","ĠY ok","éĩį åľ¨","æ£ £","ห ย","第äºĮ éĥ¨åĪĨ","cz ema","ÐĿ ÐIJ","gu ided","竹 æŀĹ","ä»° æľĽ","à§ĩত à§ĩ","Ġd iter","Ġl ider","Ġg ira","ĠC ecil","éĸ ²","èᝠå¸Ī","Ġdest ruct","æİ¢ 头","ä»ĭç»į çļĦ","èĵĿ çīĻ","Univers al","ĠLith ium","å°ı康 社ä¼ļ","l ux","ä¸į å¿«","ä¸ĭ åij¨","æĹ¥ å¼Ģå§ĭ","å®¶æĹı çļĦ","ĠSubst ances","ĠSurve ys","Ö ĥ","ä¸Ģ æĮĩ","ter es","ĠDe bb","ç½ij åIJ§","Ġterm inating","è¡Ģ èī²","ĠGu ards","Ùİ Ø³","à§ĭ à¦ľ","æķĻå¸Ī èµĦæł¼","æ³Ľ 滥","丼 æŀĹ","Assembly Version","Ġs ytu","åľ¨ åľºçļĦ","ian hi","Ġcor ri","ãģĦ ãģı","-l iving","è§Ĵ åĴĮ","Ġ×ŀ× Ł","ĠHigh lands","ĠRen al","Ġà¶ ļ","ĠLaure nce","Ġex iting","é is","Ġ< %=","à¹Ī ำ","ç¬Ķ è¶£","çĥ¤ ç®±","Ġgén érale","第ä¹Ŀ 竳","K elly","er us","ind ependent","ä¹ĭ äºī","å·¥ä½ľ äºĨ","æ® ¡","Ġcaus ality","amm en","ç½Ĺ æ±ī","è¨ĺ è¼ī","ì¹ Ļ","ä¼ļ åĴĮ","de x","Ġн еÑĢв","ract able","(\" ,","-f rame","åIJĥ èµ·æĿ¥","å±ħ æĺĵ","ĠÙĥ ار","éĺ´ æŀģ","Ġkom ple","ĠSil ence","Ġди нами","Ġseñ al",": M","z x","æİ ĸ","ru iting","ج اÙĦ","å»¶ æĹ¶","äºĮåįģ ä¸Ģ","à¥ĭ प","åĿ¡ 度","èĨĿ åħ³èĬĤ","Ġaument a","Feature d","Ġì¦ ī","t rade","ens ure","-b all","Ġد ÙĨÛĮ","ze a","ais es","Ġsurvey ing","æłª å¼ı","顯 å¾Ĺ","Ġub ic","Ġpharmaceutical s","Ġë³Ģ íĻĶ","y to","ĠT ube","res c","Ġun question","å®ī çļĦ","åĽŀ éłŃ","Ñİ ÑīÑĥÑİ","κ αν","第ä¸ī å±Ĭ","çŃĶæ¡Ī æĺ¯","zie hung","Ġда еÑĤ","åĽ¾åĥı çļĦ","Ġexhaust ing","Ġpalp able","! ##","Y R","r ink","st yled","og lu","ç»ı åıĹ","Ġع د","Ġmes ures","Ġоп ÑĭÑĤ","å°¤åħ¶ åľ¨","Ġvoc ab","ĠSN Ps","ĠобÑĢа ÑĤи","w rong","ĠR eds","æĿ¥ åIJ§","The ory","éĩij 屬","Ġbi asa","ĠDis crete","èĦī æIJı","æĮĩ导 åĴĮ","æľĢ好 ä¸įè¦ģ","èĤĨ æĦı","æªĶ æ¡Ī","\\ }","ĠM eredith","æĪ ¾","ä¸Ń æĸ°","з ме","ĠPr imer","è¡Įä¸ļ åıijå±ķ","ç³»åĪĹ çļĦ","ĠпÑĢед ÑĥÑģ","Ġimmun ohist","\\ l","Ġh uis","Ġdi pping","åĩ» ä¸Ń","Ġnature l","ı l","èģĮå·¥ çļĦ","Ġadequ acy","run ner","ĠAch illes","Ġde ities","ĠB V","Ġ\" ;","é«ĺ 大çļĦ","Ġsc out","raw l",".d es","èģļ ç±»","-n av","Ġlabor ers","ĠMater nal",". ph","W are","\\ ĊĊ","Ġs ni","Ġf oe","çĹ ŀ","chn ung","çϽ æľ¯","éħį èī²","课 ä¸Ĭ","IR T","Ġgene alogy","Ġye ux","Ġப à¯ĭ","ĠÙĨÙħ اÛĮ","Print able","æķĻçłĶ 室","Ġдав ление","ĠA FL","ĠM obil","urn a","arth a","æŃ¦ ä¾ł","Ġsimpl ification","Kl ase","Ġintrac ranial","Ġanh ianhi","' )ĊĊĊ","b earing","u Å¡","ch anged","åΰ åĮ»éĻ¢","çĿĢ æĪij们","让 åĪ«äºº","go vernmental","Ġunt ouched","å¾ħ åľ¨","ĠDis claimer","ĠVer ified","å·¨ åŀĭ","Û± Û±","ĠMalag asy","m ouse","ĠM ATH","est own","ĠK v","é«ĺ æĺİ","ex am","Ġsc int","Ġdown hill","ĠAdd iction","ä¹Łæľī å¾Īå¤ļ","arian ism","Su ite","Ġbe ide","Ġne ben","ĠV ý","Ġapp ara","ĠSe iten","åIJ¸ æ°Ķ","ëĵ Ŀ","sch ild","F o","ä¸Ģ 端","Ġturn out","ograph ique","读 èĢħçļĦ","åIJ¸ æ¯Ĵ","éĺ¿ å¯Įæ±Ĺ","åij¼ åķ¸","Ġин огда","Ġexacerb ate","D NS","W ave","åĬŁ åĪ©","åįĥ çϾ","æ¦Ĥ 论","ä¿ĥ æĪIJ","è¾Ľèĭ¦ äºĨ","Ġtis ÃŃc","/ new","Ġv oul","ĠH oughton","ĠV ort","çľĭ ç͵è§Ĩ","ä¸ī åħ«","ĠCl one","-------- -",".com pare","çĮ ¾","温 å·®","ousand s","å»¶ ç¼ĵ","ç»ĥä¹ł é¢ĺ","ĠÏĢληθ ÏħνÏĦ","{ class","ĠB are","ĠW ax","å£ ij","âĪ ª","ز ÛĮÙĨÙĩ","Ġocc ult","é½ ¢","cons istent","ìĬ¤ íħľ","å«© çļĦ","æłĵ å¡ŀ","H ook","Ġt ý","ĠSt uff","Ġadd icted","éĩij çŁ³","Ġgra cias","ĠRes idents","Ġcel u","ĠKan ada","×ķ×ŀ ×Ļ×Ŀ","B arn","s om","Ġ ï¼į","å°± éĢ£","éĿ¢ åĴĮ","Re uters","çīĩ åŃIJ","à¸Ľ รัà¸ļ","(t able","Ġthird s","Ġhydro ly","æĹłåı¯ å¥Īä½ķ","æĿ¥ åĨ³å®ļ","æĿ¥ 表达","æľĢ æ·±","но Ñģ","-l ab","åĶIJ å±±","Ġrespond er","æĻ¶ æĻ¶","Ġnan ocom","ä¸¥æł¼ è¦ģæ±Ĥ","éģł èĻķ","Ġtroubles ome","Ġfacult ies","[ â̦","Ġpar mi","Re peat","ãģ¨ ãģ¦ãĤĤ","Ø· Ùĩ","_d istance","ëł ¬","éĺ³åħī ä¸ĭ","éĨĭ éħ¸","Ġì§Ħ íĸī","Ġbored om","Ġlar val","B IT","让 æŃ¥","Com merce","-st ream","æķ£ å°Ħ","Ġম নà§ĩ","Ġод нов","াà¦ķ া","æ´Ĺè¡£ æľº","çĦ¶å¤§ æĤŁ","Ġbede utet","Y G","Ġf eline","ãģ® ãĤĤ","isc ount","Em ily","ĠAir ways","ĠLeg islation","å§Ķåijĺä¼ļ å§Ķåijĺ","/in ternal","Ġзада Ñĩа","Ġmacroph age","/ assets","j ohn","ÑĤ ка","Ġا Ø«","ĠÙħ ÙĥاÙĨ","Ġter abits",".get Value","ĠProf iles","áĥĺáĥ Ĺ","æĺŁæľŁ äºĮ","Ġrevel ations","\" ØĮ","' im","M etrics","Ġw sk","Ġr RNA","Ġapp ended","éģĵ å®¶","å¾Ī éĽ£","æĢĿ ãģĦ","arc ourt","Ab ility","лен ного","Ġdin ar","ĠPerson en","Web ster","ï¼ģï¼ģ ï¼ģĊ","ä»Ķç»Ĩ è§Ĥå¯Ł","çŀ§ çŀ§","ì° ½","è´® èĹı","线 ä¸İ","Ġserv icing","оп Ñĥ","ĠChe rokee","ĠباÙĦ ت","ĠCivil ization","Ġbak ery","- elle","< link","Ġt aj","Ġh ag","Ġl ends","ä¸Ģ çıŃ","åѦ äºĨ","羣 æ°Ķ","aw att","红 è±Ĩ","Ġsek ä","Doc uments","ĠÑĦÑĥн да","Ġs zyb","an cias","ä¸Ģ åĽŀäºĭ","ik as","Ġت اث","èª ĩ","ĠвÑĭ пла","ĠاÙĦÙģ Ø¶","ĠSP I","æįŀ åĩº","Ġanch ors","Ġpyram ids","Ġt api","为 æĸ°","ĠK ahn","éĢļ 红","女 主è§Ĵ","ĠAnd es","ä¸ĩ åİĨ","è¾ĥ éĩı","Ġcap az","ä¸Ķ æľī","åįģäºĮ å¹´","ಿಠ¤","çĭ° çĭŀ","th aca","æĻ® æĥł","Ġdé l","çļĦæīĭ æ³ķ","Ġkle inen","é£ŀè¡Į åijĺ","dz iesiÄħt","ĠEstablish ment",".pp tx","ித à¯įத","Ġd ÃŃtÄĽ","ĠH int","åľ¨ 产åĵģ","åı¦ä¸Ģ ä½į","ĠRoad s","ĠRod gers","Ġ ËĪ","æĢ ¼","Ġsh ack","cre ational","éĩį è¦ĸ","ĠÙħ Ùĩار","Ġvis as","Ġturn around","Ġspeed y","åĬªåĬĽ åŃ¦ä¹ł","ĠAssess ments","Ġжи во","-sp ect","á± ļ","typ ically","éĩij é¡į","Ġد ع","åĨĽ å·¥","ĠCar ry","ĠØ® Ùħس","Sub view","ä½³ èĬĤ","رب Ùĩ","sz ych","fu els","ĠнаÑģеÑĻ ÐµÐ½Ð¸","Ġzon as","Ġt aps","ĠG rain","éĤ£ éģĵ","è®® æ¡Ī","çĸ«æĥħ çļĦ","Ġuns aturated","ãģĹãģ¦ ãģıãģłãģķãģĦ","Class ification","é©» åľ°","த à¯į","ĠHas an",". pos","ä¸į çķħ","åĨħ åĬĽ","arch ical","ĠCon ventional","Al an","Ġdest a","ĠÑĦ ÑĢан","áŀ ı","uls a","Ġsab ot","rupted Exception","ĠDenomin ator","( the",". When","` );Ċ","主 人çļĦ","æĹł è¾¹","Ġeff luent","за Ñħ","æĭľ æīĺ","æŀľçĦ¶ æĺ¯","ĠакÑĤив ноÑģÑĤи","åıĹ害 人",". Query","æĺ Ģ","äºĭ åĭĻ","ç»Ļ å®ļçļĦ","sw ana","enc ji","ĠSm oke","é±¼ èĤī","å®ŀæĸ½ äºĨ","èĥ¡ éĢĤ","ebut uhan","Ġbrut ality","Ġεἠ°","' acc","S AR","b lick","Ġsu ites","ym p","Ġgl oom","Ġfam iglia","åºķ çĽĺ","è½» æŁĶ","å®ĺ åı¸","æĪĺçķ¥ æĢ§","éĿĴå¹´ æķĻå¸Ī","Part ial","çļĦ æĬĬ","ç« º","åĬ¨ åIJij","éĩĮ åħĭ","Ġart isans","Ġalloc ating","æİ¢æµĭ åύ","Ġinconsist encies","C old","ĠB atter","ĠN inth","ied enis","ĠCl ients","客 åľº","æ³¢ åıĬ","Ġmal adies","comp ile","åħħ满 çĿĢ","ðĿľ ij","ÑİÑīие ÑģÑı","ĠполÑĥÑĩ ениÑı","w ild","Ġint éress","ä¹Ł å¿ħé¡»","Ġsa ud","Ġam yg","åĨĻ åΰ","Ġcaus ation","ĠVer te","à¯ģà® °","ibilit Ãł","æŃIJ æ´²","Ġw k","ar ne","çļĦ èĢģå¸Ī","çļĦ è§Ĥ念","åľ¨ 两","ĠÙħ اÙĦ","ener ator","Ġav anz","ç²¾ é«ĵ","æĹı éķ¿","Ġbud ding","å·¥ä¸ļ 大åѦ","éģµ ä¹ī","Ġgrie ving","çļĦ åĩĨå¤ĩ","é ma","Ġп Ñı","-s ample","ES H","att empt","é¢Ĩ åĨĽ","Ġprevent ative","Ġdé cembre","CC I","è» ½","åĤ¨ éĩı","æĢİæł· æīįèĥ½","card i","à¹Ģล à¹Īà¸Ļ","ĠHE AD","பà¯įப à®Ł","im aging","è° ļ","åĨĻ äºĨä¸Ģ","ĠÑģо ÑĩеÑĤа","Ġspect rometer","Ġju illet","欣 çĦ¶","ëIJ ł","Ġcurs ive","Ġìĥģ íĥľ","Ġunlock ing","ĠÙ¾ÛĮد ا","ir at","ĠL EFT","æĬĵ çĿĢ","å¨ģ å°¼æĸ¯","riz ione","ĠSab ha","Ġlact ate","ĠSERV ICES","S igned","çļĦ ç§ĺå¯Ĩ","Ġch icks","ç² ij","è¿Ľè¡Į åħ¨éĿ¢","Ġbus c","à¸Ĺ à¹īà¸Ńà¸ĩ","-p ublic","ĠCal d","ken nt","зи ÑĤÑĮ","ĠÑĦи лоÑģо","åıĸæ¶Ī äºĨ","Ġenv oy","ĠاØŃ ساس","/ util","P ada","大 涨","æĪIJ 份","Ġam i","Re order","çĶŁæ´» è´¨éĩı","å®¶åºŃ æĪIJåijĺ","çĥ§ 伤","ìĬ¤ 를","Ġë³ ¼","è´¢æĶ¿ æĶ¶åħ¥","ĠTre asure","asis wa","> (Ċ","Ġd ne","Ġex erts","Ġk lin","ill ers","大 å°ĨåĨĽ","æīĢ åĪĹ","æĿ¡ å½¢","Ġcard io","çĸ¾ æİ§","Ġpropag ated","çļĦå¤ĸ è§Ĥ","ĠDrag ons","L W","çļĦ è¿Ļä¸Ģ","ĠC uc","ĠD ock","ä¸į 认è¯Ĩ","Ġout law","æľ¬ åħ¬åı¸","èµ· ä½ľç͍","缸 è²Į","åī§ ç»Ħ","ä¸įåı¯ 缺å°ij","mic ron","Ġsurf ing","-em itting","ĠFlu or","åľ¨ åĽ½å¤ĸ","åıij è´¢","æĭ ĭ","ex change","åĽŀ 转","éĿŀ å¾Ĺ","-d et","Ġperiod ontal","रà¥įठ¥","ĠSTAT US","stoff e","j id","st y","ÑĢ Ð¾Ð¹","ä¸Ĭ ä¸ĩ","æĿ¥ 个","åı¯ä»¥ èİ·å¾Ĺ","eng o","æ°ij åľĭ","az uje","irect ed","yl us","Ġarg ent","_c nt","Ġcoord en","çļĦç¡® æĺ¯","ä¸Ńåįİæ°ijæĹı ä¼Łå¤§å¤įåħ´","ĠÙĨسب Ø©","ĠHutch inson","Ġd na","çĶŁ å¾Ĵ","ĠاÙĦ ÙĪØ²","éĩij æĺŁ","Ġmet ode","Ġer hö","æŀģ çĤ¹","Ġне е","ĠÙħØŃ ÙĦ","রà§įঠ·","ĠبÙĪØ¯ ÙĨ","ç፠ç«ĭ","ĠвлиÑı ние","æĽ¾åĽ½ èĹ©","Ġw ield","ĠJ al","Ġj äl","ĠK ou","âĢĶ _","-s em","Ġreal iza","Ġvan ity","æĮ¥ èĪŀ","ĠRom ero","ĠCN Y","Ġenorm e","æµģè¡Į çĹħ","ĠN ucl","ĠV inc","á ci","åIJij 大家","Ġ% .ĊĊ","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ","çݯå¢ĥ å½±åĵį","Ñīе ÑģÑĤво","UL ATION","../../ ../",".prevent Default","- ring","Ġf ooth","Ġ- ------------------------------------------------","ä¿Ŀ å̼","åIJĦ ä¸į缸åIJĮ","è¿ľ è¿ij",".find ById","Ġacknowled gement","ì¡ Į","Ġappl ause","Ġhind ered","Ġlé ka","( form",") B",") \",Ċ","ĠB ov","ĠH icks","ĠE W","ä¹Ł æľª","åīį åı°","Ġke er","è¾ĵ è¡Ģ","works heets","ĠпÑĢе имÑĥ","×ŀ× ª","Ġবিশ à§ĩষ","Ġbound ing","rs chein","Ġפ ×Ļ","ĠاÙĦÙħؤ ÙĦÙģ","ä¸Ŀ绸 ä¹ĭè·¯","ĠRever end","ĠC anaan","å® ķ","Ġag reg","空 éĹ²","à°Ĥà° ¦","éĺ²çģ« å¢Ļ","Ġlumin ous","ç¿© ç¿©",".pe ek","+ F","in flammatory","Ġde x","Ġor naments","è¿ľ æľŁ","set ting","æĻļ å®ī","代表 ä½ľ","Ġseg uito","Ġprop hetic","기 ëıĦ","ér ations","ĠPand emic","Ġiç in","/ pre","ĠR aleigh","Ġcl ave","-b old","-p acked","Äį ky","ĠTrans parency","ç§Ģ æīį","Ġhot ter","ä¹Łæľī äºĨ","æī¬ èµ·","çѹ çłģ","äll en","ĠSlov akia","ĠHyg iene","iv re","ĠG w","ore ma","为 使","Ġacc ol","çĶ» åį·","opt ed","ĠGra ve","Ġinterview er","æĹ¶æľŁ åĨħ",". em","/ router","åĪĨ å½ķ","äºİ çĤ¹","ä½į æĸ¼","Ġза г","rog ens",", Z","- Ed","v ast","on ite","Ġs no","åħ³ ä¸Ĭ","å½ĵ ä»ĸ们","Ġequ il","ars ki","tr uth","éĺ³ æŀģ","Ġhor rors","иÑģ ан","Ġspr ang","Ġretard ation","\\ bar","Ġto fu","ut rients","ĠK ard","Ġ** [","ÏĢ Î¯","Ġnotebook s","Ġkaż de","缮çŀª åı£","V oice","ĠS üd","os um","ist ak","ĠN ied","Ġب اب","Ġgl aring","ane j","æIJ Ģ","è¿ŀ è´¯","éĿŀ常 好çļĦ","æĵį å¿ĥ","ĠâĦ ĥ","éĿĪ éŃĤ","æĦ£ ä½ıäºĨ","Mark eting","ÑĤен Ñģив","< typename","ĠK lasse","Ġgen om","纵 çĦ¶","æĮ£ èĦ±","in omial","ĠC p","âĢĿ ?ĊĊ","åĪĨ ç±³","ä¹ĭ éŨ","å¿ĥ æĢĢ","াঠ¯","带 åŃ©åŃIJ","读 å®Į","kt iv","}} $","Ľ× ł","Text View","ĠText ure","ÑĬ ем","風 æł¼","문 íĻĶ","spec ies","Ġv ox","ĠE fficacy","ä»Ģä¹Ī éĹ®é¢ĺ","åĩł å¹´çļĦ","ö z","~~ ~","á̬áĢ ¸áĢ","ĠεÏĢ Î¯","ĠBeng ali","VAL ID","M z","ĠP neum","åīį ãģ«","Ùĥ Ùĩ","åºķ æĿ¿","è½» è§Ĩ","ç¶ ¿","Ġব à§Ī","稱 çĤº","ĠCzech oslov","ĠBent ley","çļĦ åħĪ","ĠM ention","ĠN aw","å°ı è·¯","Ġgra zie","-b est","æĹ© å®ī","stit utions","Ġ×ľ× ¡","Ġarch iv","ĠSal ah","×ķ×ľ× ĵ","ĠPharmac ology","ĠBUS INESS","/ sc","W olf","d iss","s ound","çļĦ èIJ¥åħ»","她 è¿ĺæĺ¯","é¢Ħ åĶ®","éł ¸","sk irts","ĠST AR","ĠKey board","Ġઠ¹","otten ham","éĢĤç͍ èĮĥåĽ´","Ġdisag reements","Ġaquell os","h appy","} n","çļĦ è¿Ļç§į","æŀģ æĢ§","col onial","ett re","çͳ è´Ń","åĿ¦ çĦ¶","è«ĭ æ±Ĥ","Ġblind ed","Ġdile mmas","ĠAlexand re","éŀł 躬","Ġabnorm ality","- approved","F ilters","æĿ¥ è¿ĩ","å¹¶ ä»İ","Ġdro ite","স à§įà¦Ł","Me et","åįıè°ĥ åıijå±ķ","æī§æ³ķ 人åijĺ","Ġав ÑĤоÑĢ","ut é","Ġdet ergent","Ġë ľ","Ġка н","μο Ïį","Ġgebru iken","Ġkins hip","اÙĦÙħÙĩ ÙĨÙĩ","( op","Ġc ephal","çļĦ åĪ¶åº¦","us ión","ĠF av","ĠF OUR","Ġav ril","ÏĦ ικά","Or ange","è°ĵ è¯Ń","è¿Ľç¨ĭ ä¸Ń","éłĨ åĪ©","åĴĮ æķ°æį®","è¿Ļ åŃ©åŃIJ","æľ¬ çłĶç©¶","com o","(\" <","å¼ł åı£","ĠST AND","File Path","ç° ª","pot ential","Ġtelesc opes","ĠdÄĽ ti","ĠпÑĢеп ода","D ot","çļĦ åĪĨç±»","ĠJ as","eb en","Ïģ Ïİ","æŃ» ä¸į","æ±Ł åĮº","ĠAm ir","ĠSc enario","ç¨ĭåºı 设计","Ġcapital ization","æĬ½ å±ī","ĠAN C","ãĤı ãĤĭ","anal ytic","Ò¯ л","Ġsegreg ated","Ġqu eda","å¹¶ æĮī","plic a","Ġcarb oxyl","-le gged","Reg ional","éļ¶ å±ŀ","ĠCow boys","Ġleer lingen","ĠпÑĢовед ениÑı",".Ext ensions","ĠPhar ise","B less","èĩª è´¸","å¿ĥ çĹĽ","Ġwork piece","ĠCom ic","âĪĴ âĪĴ","çĺ Ļ","Ġcam ou","Ġзна ниÑı","Ġirre ducible","Ġì² «","Ġdol ore","Ġà¦Ĩম ার","ĠRah man","ĠSt ores","é«ĺ é¾Ħ","Ġend oscopic","æĻĤ ãģ®","第ä¸ī 天","δ ια","å¡Ķ å°Ķ","ä¸ĵå®¶ ç»Ħ","/n ull","Ġkem ampuan","ĠlÃŃ qu","âĢĵâĢĵ âĢĵâĢĵ","ç»ħ 士","ĠÑĨелÑĮ Ñİ","( Console","ĠG iles","ier archy","-t raining","оÑģ п","à¸Ĥ ัà¸Ļ","ìķ ¡","ĠEff orts","ĠÄį ty","Õ¥Õ ¦","Ïģι ÏĥÏĦ","Ġméd ic","ĠÑģооÑĤ но","w ik","ä¸į 建议","ĠW EB","åĩº åIJį","Ġatt ic","Ġlater ally","Ġdemand a","اÙĩ رة","A o","{ eq","re land","il ig","åѦ éĥ¨","Ġar senal","表 象","æ¼ ģ","Ġimp regn","Ġgre ener","RO LL",".A cc","Ġrot ten","ãĤ¯ ãĥŃ","ĠAlex is","æ¼Ķåͱ ä¼ļ","<> ();ĊĊ","rom o","ä¸Ĭ åľº","ik ation","æľįåĬ¡ æľºæŀĦ","Ġexc ursion","ĠAst on","Ġcort e","ĠOm aha","UIC olor","ĠSovi ets","Ġ ãĢĪ","ĠP ÅĻ","ri ot","Ġk ennen","å®ħ åŁºåľ°","ffff ff","ĠплоÑīа ди","Ġanál ise","on ych","ult on","åĵ ½","éĤ£ ä¸įæĺ¯","ax ios","cept ual","Ġpost o","ĠØ£ ساس","Ġvan ish","éĩįçĤ¹ æĺ¯","ĠUl tr","Ġspray ed","M ess","N obody","Ġ icing","Ġwe eping","表 éĿ¢ç§¯","-s upport","Ġprof il","éϤ å¤ķ","èϽ æĺ¯","ips ych","cal a","ĠDomin ic","? _ĊĊ","he se","oc ious","è¿Ļ åħ¶ä¸Ń","æ³ķ 西æĸ¯","Ø· ÙĤØ©","ðŁ ĮĢ","ĠÕ İ","å¥ĩ èij©","çľĭçĿĢ æĪij","è¡ĮæĶ¿ è¯ī讼","Ġmig rating","à¥įर à¥Ģ","Ġï¼ī ãĢĤĊĊ","Ч ÑĤобÑĭ",". line","an am","ĠN é","Ġform ule","rupt cy","how ever","Ġpel let","ĠSel ain","Non Null","H N","un ächst","ĠP ATH","Ġcon clusive","eb el","æŀģ é«ĺçļĦ","ĠEconom ist","ĠPet itioner","ĠPR INT","ëħ Ģ","ìĪĺ 를","èģ¯ çĽŁ","Ĺ×§ ר","à¦ŀ à§įà¦ľ","+ '","ä½ĵ é¨ĵ","ç§į 群","Ġform es","Ġо во","Ġcur ly","ĠDo ch","Ðļ ом","رÙģ Ùĩ","Ġни ми","çļĦæĦıæĢĿ æĺ¯","ĠпÑĢимеÑĢ Ð½Ð¾","ĠDepart ments","B right","Ġc ached","ĠS onic","Ġsh immer","iss ima","â̦â̦ â̦","Ġseed ed","Ġmerg ers","ab ra","æľ¬ èī²","æ¯ı ä¸ĢæŃ¥","-------- --","ส ืà¹Īà¸Ń","class ified","å¸ĮæľĽ ä½ł","Ġsin ister","汽车 çļĦ","ĠPot ato","ĠSem antic","åħįè´¹ çļĦ","STR ING","Ġp ula","å¿ĥ æĤ¸","å®ŀ åĪĻ","yl ated","åĿĩ æĺ¯","åŁİå¸Ĥ è§ĦåĪĴ","触 åĬ¨","Ġcad mium","ãģĹãģ¾ ãģĹãĤĩãģĨ","Ġinfrast ructures","ĠD odge","æľº 身","aw ks","Ġver st","aim an","Ar gent","ĠÙħÛĮ ÙĦÛĮ","-ch ip","Ġتر ÛĮÙĨ","ä¸İæĹ¶ 俱","H em","g ments","ĠC ors","ĠB rat","ĠN EC","pl anned","以 ä¸Ģç§į","å°± æĺ¾å¾Ĺ","Ġall ot","Ġall erdings","Ġar rib","const ants","Ġharm ed","ĠاÙĦØŃ ÙĤ","ĠEmploy ers","Ġredist ribute","ĠпÑĢодол жи","Ġg ithub","ĠK ult","é«ĺ èĪĪ","Re cipe","æķ£ å¸ĥ","è½® çļĦ","åħ³éĶ® çļĦ","åħ½ åĮ»","龸 æ°Ķ","Ġzm ÄĽ","á»ĥ u","Ġпоб е","n ienie","ĠP ued","ĠD z","å·¥ä½ľ æĸ¹æ¡Ī","-s ector","ĠÙĦ ازÙħ","éĻIJ éĩı","Ġste alth","ä¸įæĸŃ æī©å¤§","CC A","Ġpow od","å¿ĥçIJĨ åѦ家","ç¥ĸ å®Ĺ","ĠSimpl ify","st ige","Ġ( ),","æĪij éĿŀ常","Ġ[ []","èµ· å±ħ","æľĢ 常è§ģçļĦ","ä¿¡ ä»¶","头 é¢ħ","ĠSp ray","â̦â̦ ãĢį","IC C","æ±Ł åİ¿","ĠGe orgetown","ç£ IJ","æĮij åīĶ","yn ie","åĩł æĹ¥","ä¹Ŀ 年级","å®Ī æģĴ","Ġà¦ī দ","å¹²åĩĢ çļĦ","Creat or","ĠOP EN","-read able","Ġford ÃŃt","pps ala","ĠاÙĦØ´ÙĬ Ø®","w aves","Ġp ry","çļĦ 顺åºı","èĥ «","社 åijĺ","æ±Ĥ è¯ģ","aw ia","大åѦ æ¯ķä¸ļ","IL abel","ĠاÙĦØ¢ خر","C ele","ĠC indy","iz m","д Ñĸ","ä¸İ å®ŀè·µ","Ġac oust","Ġac adém","ĠPh o","Ġbus car","èµĽ åľº","িà¦ķ ার","Si O","Ġidi om","ĠÑĤÑĢебÑĥ еÑĤÑģÑı","E h","T ensor","j ahr","v oke","Ġd item","Ġg m","åģļ å·¥","ĠMin imal","Ġmor se","ר×ij ×ķת","/ ep","ate a","yn ch","hor izontal","ĠThe men","ort ing","ĠG J","ub o","éļı åľ°","主è¦ģ è´Łè´£äºº","Ùİ Ùĩ","éĢı æŀIJ","é¼ĵ æİĮ","Ïģα γ","ĠLab els","ĠCONCLUS ION","ĠM ST","ĠO ce","è¿Ľè¡Į ä¸Ģ次","è¿Ļ个 æł·åŃIJ","å¼ķ çͳ","èĩªå·±çļĦ æĥ³æ³ķ","Ad just","ĠTw ain","Ġdelay ing","ĠClub s","ĠRam sey","è£Ŀ ç½®","ÙĨس ا","Ġнов Ñĭе","ĠReserv oir","н оги","åĴĮ 第","ç͵ ç«ŀ","Ġhard ening","ĠBal let","ĠRen é","ĠMP I","è¿Ļé¦ĸ è¯Ĺ","åĽłæŀľ åħ³ç³»","M ás","ul ly","Ġess es","AM B","ൠ»","ä¸ĺ éϵ","ĠÑĥÑĩеÑĤ ом","p Ã¥","Ġad ore","å°± 对","ile en","ĠÑģ ви","Ġdisc ol","Ïģ εÏĤ","Ġsem antically","ĠEr r","Ph ill","ĠForm ats","æĹĹ ä¸ĭçļĦ","èĥĥ èĤłéģĵ","Ġdri pping","\" How","< $","r zym","y te","Å ©","ĠAr che","åĽĽ çĤ¹","Ġcond ol","åħ¬åħ± åħ³ç³»","ĠGall agher","Ġgig g","å®ŀç͍ æĢ§","ĠKat rina","prov ider","Ġcuid ado","Ġa ções","er ian","ĠH orses","ĠF AA","oc re","ĠAr b","åĩł å®¶","Ġsleep y","á»ij n","ãĤĴè¡Į ãģĨ","Ġellipt ical","Ġejerc icio","å°± è·ij","ĠV ega",".p redict","é¸ ł","ĠItal ians","çļĦé«ĺ æīĭ","ĠFlex ibility","æģį çĦ¶å¤§æĤŁ","ĠButter fly","çļĦ åľºæĻ¯","ag ric","ĠD arm","ĠW IN","Ġor acle","man age","ĠÑĤе кÑĥ","åѸ æľĥ","æĿ¨ æŁ³","æ·±åĮĸ æĶ¹éĿ©","ĠCycl ing","ACI ÃĵN","( The","/ inter","= ï¼Ī","C ir","Ġre but","ä¸Ģ æľ¬ä¹¦","人 æĢ§çļĦ","oc io","大 åŃĹ","æĿ¥ æĿ¥","ib re","aw asan","Ġmus ik","主ä¹ī åĴĮ","ذ ب","æĭĽ æĥ¹","ĠPat rol","éĢı æ°Ķ","ç®Ĭ æĢ§","ĠCross word","ycz nych","Ġstere otype","Ġencuent ran","Ġhod not","H oly","j obs","Ġm ã","ĠB ray","ä¸Ń 度","all owed","Ġem pez","Ġes os","éĸ ij","Ġut ile","ઠ£","ros a","Ġbed side","ĠJew el","Ġnan ometers","éĢĨ åIJij","ĠVen et","åIJķ å¸ĥ","Ġ( âĢĵ","Ġal as","ĠK and","ä¿Ŀ ä½ı","èĬĤ 缮çļĦ","åį° è¯ģ","ĠпÑĢо ÑĤи","ĠT out","Ġv ener","æľī äºĮ","åĴĮ éĻĪ","Ġв ÑĬ","rit ure","ĠZ ukunft","åħĪ åľ¨","ĠØ¥ ÙĦÙĬÙĩ","ä¸Ģ è§Ĵ","ĠP DP","Ġsub sp","常 è¦ĭ","éĻ¢ éĩĮ","é»ij 客","ç§ĺ è¯Ģ","ĠìĿ´ íĽĦ","Ġtail le","åĬ¨çī© åĽŃ","-vol tage","Ġc io","ok ines","æĺİ æľĹ","æĹł åĬ©","St ra","Ġmon oton","ĠEx ist","åIJĥ æİī","è¿ĺæľī å°±æĺ¯","Ġprop elled","ĠSk inner","ëŀ µ","Ġал гоÑĢиÑĤ","Ġparab ola","ĠS print","ĠS IP","ĠT os","ä¸Ģ æŀ¶","em aker","å¿ĥ æĪ¿","ĠY ORK","Ġbott led","综åIJĪ èĢĥèĻij","ᣠĭ","ĠPoly techn","ÐŁÐ¾Ñģ ле","ä¹Ł æĪIJ为","å¤ļ çĤ¹","Ġcre eping",".âĢĿ #","Ġleg umes","EC E","Ġmar rying","ĠNot able",".get Instance","缸åħ³ éĹ®é¢ĺ",".T ag","ekt or","ár nÃŃ","K r","S j","e on","Ġacc using","æŃ¤ 举","Ġد Ùģ","Ġpath ophysiology","æŃ¦ æĺĮ","cz ny","Ġmoy enne","Ġì¤ Ģ","ä½ł 为ä»Ģä¹Ī","ä½ł ä¹Łåı¯ä»¥","天 ä¸ĭçļĦ","ãĢĤâĢĿ (ãĢĬ","åħ¶ä»ĸ åĽ½å®¶","ê ts","Åį ng","_se q","-product s","å¾®éĩı åħĥç´ł","Ġinverte brates","ic ule","Ġal am","ov asc","Ġmod ulating","Ġhere after","æ»ij åĿ¡","ĠDer ived","çά ä¸Ĭ","ä¼łéĢĴ ç»Ļ",".class List","orsch ung","Ġskew ed","Ġdemol ition","äºĨ ä¸ĭä¾Ĩ","ä¸ĭ è¿°","åı¯ä»¥ è¾¾åΰ","ĠAl ien","èĩªæĪij ä»ĭç»į","Ġresist ivity","ĠÙħر تب","ĠApost le","æ« »","ĠP AGE","ĠF ighter","ĠاÙĨت شار","#if def","F ord","z ett","Ë ļ","Ġy rs","ĠB lick","æľī çļĦæĹ¶åĢĻ","ĠQ String","é© ¹","讲 æķħäºĭ","ĠLi pp","èĺ ĭ","×ķ×¡× £","Ġv äl","д ка","å°ı åĿĹ","Ġback lash","æĶ¾ çĸĹ","ç´§ è¦ģ","é£ŀ èĪŀ","Ġten or","ĠRed dy","驾驶 è¯ģ","Ġflour ished","á»ĩ t","ĠاÙĨÚ¯ ÙĦÛĮ","ĠM ikhail","Ġsk im","лÑĮ нÑĭми","Ch ars","(d ir","åĭ¾ èµ·","ĠIh nen","àªĤ àª","oprop yl","ä¸į æŃ£å¸¸","å¿ĥ æĥĬ","Ġro bbed","äºĮ 审","г Ñĭ","Ġ×Ķ× ¤×¨×","éĴĪ åĪº","ĠSw imming","çĽ¸å¯¹ çļĦ","éģ¥ æĦŁ","ustain ability","æĺĤ è´µ","prot ocol","çĪª åŃIJ","ĠÑĥÑĢов не","ĠLond res","åľ¨ ä»Ĭ","Ġdel ights","ĠMin h","cz nej","è§Ĵ度 æĿ¥çľĭ","çαæĥħ çļĦ","Ġaccent u",": c","F ounded","S AT","ĠS ous","ä¸Ģ æĭĽ","è¦ģ 约","æŃ£ åĩĨå¤ĩ","oth or","åIJĦ åĽ½çļĦ","ç¥ŀ çµĮ","Ġaud itors","IM ENT","ĠNor se","çĵ¦ è§£","Fin ance","Ġtah u","Ġmascul inity","éĤ£ 头","ä½Ĩ ä¸įæĺ¯","Ġfe cal","ĠPh yll","Ġر ب","×Ļפ ×Ķ","Sing apore","GRAP H","人 åı¯ä»¥","ĠN airobi","ä¸Ń 以","-st udy","èħ¾ èħ¾","çļĦä¸Ń å¹´","Ġlors qu","æ½į åĿĬ","( async","L aura","Ġs ushi","Ġw akes","Ġm Ãł","ç¬ º","Ġdel a","æĮĩ æĺİ","requ ests","Ġinflu encers","第ä¸ī æŃ¥","åºĬ 头","Ġtim elines","Ġà¦ħন à§įয","it im","-s afe","He ading","Ġ×©× ª","اش ر","ĠShow ing","ç´§å¼ł çļĦ","Ö· ×Ļ","ĠиÑģÑĤо Ñĩ","ĠHarm on","Ġellipt ic","us ual","ĠM VC","Ġro ÅĽlin","èģĶ å¸Ń","çIJĥ å½¢","Ġsat ire","ĠAst hma","Ðķ ÐĿÐĺ","ĠLat via","ĠEq s","üt zt","Ġalred edor","M etric","Ġc ông","ol ang","ĠD IG","Ġque ues","Ġmicro tub","CH O","Ġutil iser","å°ĩ æľĥ","äºĨåĩł åĪĨ","ابر اÛĮÙĨ","Ġoutf its","_ gen","Ġch illy","é ria","ä¹ĭ é¦ĸ","ä½ł ç»ĻæĪij","å®ŀ å½ķ","åħ¶ 人","æŃ£ è§Ĩ","work ed","å¸Ĥåľº åĴĮ","Ġlo osen","åħį è´£","å̾ éĶĢ","æĺ¨å¤© æĻļä¸Ĭ","Ġpon ad","Ġproyect os","Ġun ification","Ġgl ued","èģĶ åĨĽ","Ġdu plicated","Ġí ά","æķ£ åİ»","秦 çİĭ","æĿĥåĪ© çļĦ","Ġcompound ing","ĠLy ons","Ġauc un","Ġadipis icing","Ġle asing","Ġঠł","åĨħ åIJij","ĠÙģ Ø¹","äºĨä¸Ģ åĿĹ","æ¹ Ĭ","ãģ¾ ãģ¾","æĭ¿ èijĹ","Ġste j","éĢı å½»","讨论 äºĨ","èĥĥ çĻĮ","Ïģα ÏĤ","Ġanton yms",". ]Ċ","Ġb x","Ġв д","ĠSe ems","ĠZ ucker","ĠಠĨ","æµĵ 度çļĦ","Ġrect al","ĠAL S","à¸ķร ี","Ġlu ogo","Ġté to","Ġwszyst kim","ĠWrest ling","Ġj ot","ik os","hen ol","Ġ_ **","缸 éļĶ","æµģ éľ²","Ġstyle Urls","çļĦæīĭ èĩĤ","ĠFlash cards","Ġhast ily","\\ langle","æĺ¯ åĴĮ","为 å·±","ĠInter faces","åĹ Ķ","åľĪ çļĦ","åĩºçīĪ çī©","Ġrend re","æĭĵ æīij","æľīæľº çī©","ĠAuto CAD","æµĩ çŃij","åŁİ乡 å±ħæ°ij","Arg uments","Ġмилли онов","Ġn ú","Ġne hmen","éķ¿ åŃIJ","ле г","à¹Ģà¸Ľà¹ĩà¸Ļ à¸ģาร","æĦŁæĥħ çļĦ","Ġwet en","MR C","à¥Ĥ ल","ç´§å¯Ĩ ç»ĵåIJĪ","å¦Ħ æĥ³","G rowth","Ġle quel","éĩį éĺ³","Ġrec ap","æĶ¾ ä¸ĭäºĨ","ĠÙģ Ø±ÙĬÙĤ","è´¹ åĬĽ","OR IES","æ¯į 女","éĿŀ常 éĢĤåIJĪ","é¦Ļ çĶľ","çıł æ±Ł","Supp orted","Ġenerg i","K ir","ĠI GF","eth oxy","æĬĬ å°ı","Ġsol a","So il","ائ Ùī","ĠпÑĢе дела","æ´¥ æ´¥","çı¾åľ¨ çļĦ","åıijè¨Ģ 人","å¼§ 度","Develop ing","Ġendeav our","å¤ĸåķĨ æĬķèµĦ","C arm","çļĦ çľĭèijĹ","ĠB ordeaux","Ġset back","ç² ķ","Ġ... âĢĿ","æĿ¾ äºĨåı£æ°Ķ","é¤IJ åħ·","ĠGi ac","åľ¨è¿Ļ个 æĹ¶åĢĻ","ĠKit ty","à¸łà¸²à¸§ ะ","A ctor","ar ist","Ġm én","åĬł åİĭ","vers ed","-m oving","ĠMan ag","ĠAnt ioxid","าà¸ģ าศ","éĶĢåĶ® çļĦ","Ġposit if","ĠHon ors","ASC AR",": id","Ġb iking","æĪ į","æµ ļ","çŃī é«ĺ","Ġlook out","å±± æ¥Ĥ","åĵª ä¸Ģç§į","Par allel","ĠExp and","åľŁåľ° ä¸Ĭ","Ge o","Ġnh ư","Ġwit ty","\\ Entity","_ manager","cl imate","Ġgl u","æļ Ī","bre cht","ë r","ĠConst antine","ĠÙħج ÙĦس","_ iterator","Ê ²","ro zen","are an","ен нÑĭм","åŃ¦ä¹ł èĢħ","ĠDar cy","Ġrever ber","V ia","ĠD AN","çŃī æİªæĸ½","Ġlocal Storage","åĽº æĢģ","æ´Ľ 夫","ĠDif ficulty","ĠDur ante","Ġpyl ori","ĠSanct uary","ay ashi","res olution","ç¾ ¯","Ġfl ute","Ġquest ão","Ġcond ições","Ġnecess ário","å®¶éķ¿ çļĦ","жд Ñĭ","áĥĿáĥ ij","ĠNam ibia","Ġmeteor ological","L H","âĢ ¼","Ġpol o","åĪĻ è¯¥","Ġlist ens","ón ico","麻 è¾£","éĥ½æľī äºĨ","ĠباÙĦ س","ĠPet ra","åŁĭ 头","åŁĭ ä¼ı","Ġexplor ers","Ġscrat ched","% .Ċ",": _ĊĊ","] \",","Ġl angu","ĠT omb","Ġen pres","ÃŃ ng","ĠAl one","ç§ģ èĩª","ãĤ¢ ãĤ¤","ré al",") the","M ental","Y B","ĠT av","ĠM im","ĠоÑĤ меÑĩа","æłij å¹²","-F ran","å½ĵåīį 离线","ĠIll ness","èĤĸ åĥı","ĠTro jan","ĠÑĪе ÑĢан","teil ung","v ac","ĠC CC","Ġhe ats","Ġj argon","Ġob y","rap ist","éĥ½æĺ¯ ä»İ","æ½ľ æĦıè¯Ĩ","اÙĪ ÙĬØ©",".res et","Ġiv ory","Ġfen omen","Ġco ffin",".S upp","åħ« è·¯åĨĽ","æıIJä¾Ľ æľįåĬ¡","çł´åĿı äºĨ","ĠWild erness","ĠØŃد ÙĬØ«","太æŀģ æĭ³","çľĭ ä¸Ģçľ¼","urs ing","å®ĺ çļĦ","Ġmi RNA","θ ν","Ġpoly gons","åħ© åĢĭ人","åĮħåIJ« çĿĢ","лож ение","ĠEle phant","M is","h onderd","æİ Ļ","á ž","æĪij们 åĨį","Ġprim i","鸣 ç±»","f v","m ongoose","Ġref le","ĠCont rovers","ĠBer ks","comp ress",".H ome","èªį çŁ¥","- el","} p","ÙĬ ØŃ","Ġdis content","ident ique","è¿Ļæł· åŃIJ","ä¼ĺ å¼ĤçļĦ","Ïĥ κ","åĪ· çīĻ","Ġauthent icate","é¡ŀ ä¼¼","Ġto án","æł¡ åĩĨ","Ïģ ÎŃ","Ġpolit ica","ĠMc Int","çĺ ĺ","ãĤĤãģ® ãĤĴ","\" My","( with","[ current","_ INT","ĉ str","le it","ĠE FFECT","ĠIn hal","à§ ·","ient i","ÑĪ Ñĭ","å°±æĺ¯ æĮĩ","ĠDev Ops","ajÄħ cy","Ġrecall ing","P ho","Ġ Ì","ĠM AD","ars ely","Ġpor ter","iten ess","nost ÃŃ","M ort","çļĦ åİŁçIJĨ","ĠR BC","å°± é¤IJ","æĸ° ãģĹãģĦ","èĢģ çİĭ","ĠMat emat","ê°Ģ ì§Ģ","ĠAM L","çµĦ æĪIJ","Ġfest ivities","Ġbot anical","ĠPythag orean","Ġb oven","对 éĺµ","Ġper usahaan","ĠSe ine","Ġlocal ities","æ²³ 举","Int ers","äºĨè§£ åĴĮ","æģIJ é¾Ļ","æĩĤ äºĭ","+\\ ,\\","Ġsed e","ĠCatholic ism","ĠTu ber","Ġl ire","åIJĮ æĹ¥","åģļ çĿĢ","Ġcare less","Ch ap","ones e","ĠÑĩа Ñģ","Õ¸Õ ¿","Ġchamp agne","Ġতাà¦ģ র","æīĢ ä½¿ç͍çļĦ","åħ¬ æĬ¥","åıĬ æĹ©","Ġpass a","åĥı ä¸Ģ个","ĠвÑĭ вод","\") )ĊĊ","åIJĪçIJĨ å®īæİĴ","Ġfost ered","Ġзакон ода","å¦Ĥä¸ĭåĽ¾ æīĢ示","Ġc rib","ar om","ĠR oe","ĠO verse","èĢģ æĹ§","åıĪ éģĵ","Ġ×Ķ× Ĺ×","-c ells","λ Ïİ","ç»Ŀ ä¸įæĺ¯","ĠвÑĭ биÑĢа",".M AX","å¥Ĺ æĪ¿","Ġphilosoph ies","Ġreg ained","åıĹ ç²¾","åĺ ħ","class ification","ĠFr antsay","æī« é»ij","ç´ħ èī²","Report ing","ĠاÙĦØ¢ ÙĨ","( content","u uid","ĠW alls","天 å¹³","æľº ä¸Ĭ","社ä¼ļ åIJĦçķĮ","Pro blems","éļı åı£","ãģ¾ ãĤĭ","ä¿¡æģ¯ åħ¬å¼Ģ","æ¦Ĥ念 çļĦ","çĽĪ äºı","á»į n","ä¸ĥåħ« ç³Ł","Ġиг ÑĢÑĭ","ĠLars en","Ġب اÙĨ","éĢī æ¡Ĩ","Ġ; ;","ব িদ","Ġত à¦ĸন","绣ä¸Ģ æĪĺ线","s imp","Ġt av","ĠS app","ĠT uring","ĠÑĥ бе","è§Ĵ éĢIJ",".w orld","Ġng On","ĠÃľ bers","Ġisot ropic","ĠT ent","ени н","Ġprof iss","ĠÑĤ ÑĥÑĢ","Act iv","Ïģα κ","à¹Ģà¸ŀ ราะ","Ġa ñ","Ġb Ã¥","ĠT err","ĠB ubble","ĠU ll","é«ĺ 举","Ġع ضÙĪ","ENT IAL","èĦ± åı£","ĠEst ud","Null able","Ġraz or","Ġdilig ently","Ġcreep y","Ġp auses","两 åĿĹ","é¦ ĭ","æļ Ħ","ÙĪØ± ÙĪØ¨","çŁŃ æĸĩ","è¡£ é£Ł","åħ³äºİ åĬłå¼º","Ġsurv ives","ĠÑħ Ñĥ","à¹īว ย","Ġescal ating","EMA IL","ĠRobb ins","人 æµģ","ä¸İ æĸ¹æ³ķ","çŃī æĸ¹æ³ķ","ع ض","建 åĨĽ","Ġs red","ç»ı å¼Ģ","Ġsub lim","æłĩ æĺİ","çĹħ æĤ£èĢħ","ĠQu iet","æ¼Ķ 说","sk in","ĠConn ecting","Ġconjug ated","åĨ¤ æŀī","Ġdiz zy","c q","or ra","Ùĥ ب","ĠNew foundland","åĨ³ ç®Ĺ","ĠÙĨ ÙĤÙĦ","ĠOl sen","ĠStart up","Ġstick ers","S oci","m ény","um ably","è¿Ľ åİ»äºĨ","å·¥ä½ľ å¼Ģå±ķ","Ġfoot wear","Ġпод Ñħод","-Americ ans","/ The","I OS","ing t","ÑĢи ÑĤе","acc a","éĿ¢æĹł 表æĥħ","ĠO j","ren al","åıĸ åIJį","ĠSu omen","\": [","жи ÑĤе","社ä¼ļ主ä¹ī æł¸å¿ĥä»·å̼è§Ĥ","aws ze","خر ج","ĠíĮ ¨","Ġназ вание","æ¯Ľç»Ĩ è¡Ģ管","ĠD AM","Ġz ahl","éĤ£ åĩłä¸ª","Ġind is","Ġsub mar","ç«ĭ å¾·","èij Ĩ","注 è§£","å§ĭ èĩ³","ãģ¨ ãģĨ","ĠEl aine","éĺ» å°¼","æĬµ 触","æ°¸è¿ľ æĺ¯","çĦĬ ç¼Ŀ","ĠÑĢÑĥков од","G rowing","R on","u ais","人 åIJį","æĪij åĪļ","ä¸ŃåĽ½ 梦","游 è¡Į","ú c","å§IJ 夫","ĠиÑģп ÑĭÑĤа","çīµ æĮĤ","T ek","um ina","Ġch oke","æĿ¥ ä¹ĭ","常 åľ¨","éĢł åĮĸ","ла ÑĤ","ç»Ń 表","Ġstruct uring","vol ved","g w","{ matrix","Ġde ceptive","äºĨ 大éĩı","are kin","建 åĬŁ","Ġcol t","ĠÙģ ØµÙĦ","åĽ¢ åĽ¢","Ġey ed","ĠоÑĤ пÑĥ","ete en","çļĦåıijå±ķ åĴĮ","æµģç¨ĭ åĽ¾","微信 群","è¡Į éķ¿","Ġte ó","èµ° åĬ¨","Ġfam ed","åĮ» æ²»","Ġassoci ative","åĬŁèĥ½ æĢ§","ãĥ¼ãĥ Ħ","ĠGent iles","ĠоÑĨен ки","Ġent anto","Ġм одÑĥ","áĥ łáĥ","Ġvis ite","åĩĢ èµĦ产","Ġbank er","Ġপà§įর ব","åįģä¹Ŀ æĿ¡","Catal Ãł","ç¬Ķè®°æľ¬ ç͵èĦij","ĠбÑİ Ð´Ð¶","y am","Ġf y","ĠWe apons","Ġdire t","OT HER","ĠØ¢ ثار","ĠHel per","èĢIJ ç͍","èİī èİī","/sh are","= j","ĠSt rain","ä¸ī æĸ¹","èĢģå¸Ī åľ¨","æĢª çļĦ","Ġrob es","aud i","åĮĪ çīĻ","è¡Ļ éŨ","ĠAUT O",". With","H art","in atal","çļĦ åĵģçīĮ","ay ama","ÙĨ ادÙī","æģ Ļ","Ġrem nant","ç»Ļ ä»ĸçļĦ","åĨį è¿ĩ","ж еÑĤÑģÑı","aut y","à¹ģ à¸ļ","ĠCreat ure","åij¼åIJ¸ åĽ°éļ¾","Ġdescript ors","A sp","æĪij åºĶ该","Ġmodel os","Ġпо ÑģÑĤе","æ¿ ®","da q","åIJ¯ 迪","ĠRoll s","ĠÐŀÑģ об","วิà¸ĺี à¸ģาร","é· ¹","çļĦ èµĦæºIJ","Ġun balanced","éĩij åįİ","-b eta","åį´ æľī","æī¿ åħij","ĠOff ers","-pro p","Ġplug s","ĠмаÑĢ ÑĤа",". th","Ġcon ical","ust ion","Ġdes ember","æľįåĬ¡ ä½ĵç³»","á» «","Ġter l","len a","Ġpilgrim s","åħļé£İå»īæĶ¿ 建设","C od","н наÑı","und ing","Ġmes enchymal","Del ay","çĽ¼ æľĽ","gre SQL","ĠInfect ions","ĠSold ier","ĠT ears","ath lon","ŀ× ¢","åĺ Ī","æĬķåħ¥ 使ç͍",".se lected","-mod al","ì²ĺ ëŁ¼","ãĤ¨ãĥį ãĥ«ãĤ®ãĥ¼","缮çŀªåı£ åijĨ","Ä Ħ","at ars","ĠV oll","ä¹Ł éĢIJæ¸IJ","æ°ij æŃĮ","太 æ¹ĸ","éľĢè¦ģ ä¸Ģ个","Ġrece ivable","ĠSc orp","Ġampl ifiers","Ġhal ogen","Ġdrum mer","-te chn","Ġexpans ions","à¦Ĺà§ģল à§ĭ","ĠComplement ary","' o","ĠH ari","Ġres ize","å¿ĥ å®ī","çľ¼ äºĨ","ĠTr is","çĶŁäº§ åŁºåľ°","Qu arter","èĩªçĦ¶ åľ°","æĺ¯åIJ¦ ä¼ļ","roy o","ĠStat ist","-L ab","ĠÙħد ÙĬÙĨØ©","èĦĸ åŃIJä¸Ĭ","Ġaument ar","سر عة","Ġgloss y","Ġtyr anny","I AS","ĠS EN","个 æ¡Ī","ç͵ æĦŁ","Ġα ι","åģ¶ æķ°","( ?","L IST","ĠW ohn","Ġk é","è¿Ļ åı¯èĥ½","ach able","å®ļ è¯Ń","ek om","è¿Ļ个 ä¸ľè¥¿","éľĢæ±Ĥ åĴĮ","Ġalleg ation","鸣 åĦ¿","ĠPrior it","åįĶ åĬ©","æĻ¶ä½ĵ 管","ing ale","ĠT ad","Ġk ru","åĨĽ å§Ķ","NA P","--- |---","Ġét abl","ĠBow en","çŃīåIJĮ äºİ","Ġv ents","ĠB Ãłi","ag ged","å¿« æĿ¥","ä¸ĵ æĶ¿","Ġsw ine","à¸Ħ à¹Īาà¸","åħŃ ä¸Ģ","ĠEl m","à§ģ ণ","æ¯Ķè¾ĥ 容æĺĵ","ĠEr g",".L ocal","ĠAP R","æIJľ æŁ¥","Ġ à¸ģร","Ġst umble","ist os","ç͍ æĹ¶","å¹´ 头","Ġت ÙĩراÙĨ","Ġris chio","ĠFarm ing","ın ın","ĠÑĨенÑĤ ÑĢ","Comm unic","éĩĮç¨ĭ ç¢ij","¤ ׾","äºĨ çĤ¹","çĻ £","çĥŃ æ°Ķ","Ġtreat ise","Ġdoll s","ç©· 人","Ġlob ster","äºĮæīĭ æĪ¿","ĠRepro ductive","è¦ģ 被","Ġad herent","++ ;ĊĊ","Ġ! [","Ġ×IJ ׾×Ķ","è´´ çݰ","Ġphen olic","å̼å¾Ĺ æĪij们","Ġdisag reed","äd agog","ĠFell ows","Ġnatu url","ä¸Ģ åīij","æľ¬ 以为","çͱ åĽ¾","But tons","Get ty","ĠDepart amento","ĠTox icol","å¯ĦçĶŁ èĻ«","Ġê·¸ëŁ¬ ëĤĺ","ĠH ubb","æĺİ æĻ°","Ġrem embrance","èĥ¶ åİŁ","ĠÎł ο","ÈĽ Äĥ",";\\ ;\\","Ġst ellt","ab u","æľī æŃ¤","int ed","åħ¨ 线","ĠAl ph","è¯Ŀ åī§","积 éĽª","oph ore","çł´ å£ŀ","çͱäºİ åħ¶","(b uf","Ġпод ÑĢоб","并没æľī ä»Ģä¹Ī","天èĬ± æĿ¿","_ board","Ġst res","ĠH id","ĠE i","Ġr uss","çĶŁ çģµ","åıij åĩºä¸Ģ","ens ky","oph ile","äºī åģļ","åºĬ è¾¹","اج ع","ĠCr ash","-rec ord","Ġglycer ol","Ġp ics","Ġg out","ĠL aut","è¦ģ åѦä¼ļ","åı¯ è¨Ģ","é¢ĺ å¹²","管çIJĨ 模å¼ı","Ġart igo","ux e","åıĮ è¾¹","Ġpor celain","Ġhom olog","Ġutil isation","帮åĬ© ä½ł","åζéĢł çļĦ","ä¹Į äºij","ĠCam eroon","ĠاÙĦÙħر Ùĥز","æĿİä¸ĸ æ°ij","Ġ ÑĪÑĤ","æ¯Ķ å°Ķ","ä¼ģä¸ļ åĨħéĥ¨","é»Ħ è±Ĩ","ĠCor b","ĠÙĪØ§ÙĦ Ø´","Ġsun screen","/d ownload","ĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĊ","Ġhur ricanes","Ġalloc ations","çĸ¯ åŃIJ","Ġresidual s","Ġdich o","Ġharness ing","Ġhin aus","Ġgoalk eeper",", @","F ra","ĠS PR","pe a","ĠV amp","Ġdi x","Ġ) .Ċ","Ġwater ways","Ġع ÙĪØ§ÙħÙĦ","éϤ æģ¶","ml ung","ĠMon arch","Ġধ র","Ġdeduct ible","' ad","Ġn ess","ä¸į é«ĺåħ´","Ġcl enched","Ġت ÙĤد","æ´¾ 对","ĠMor an","åŁ¹è®Ń æľºæŀĦ","//////////////////////////////// ////////////////////////////////","p one","he ids","Ġe w","ĠG ull","Ġsh alt","ĠTh romb","è¿ĩ èĬĤ","ON DS","Ġname eee","Ġmus i","æķħ ä½ľ","ä¸įæĸŃ å¢ŀåĬł","é²ľ æ´»","ò ria","çļĦ éŃħåĬĽ","ĠC yl","ĠW yn","ĠO E","为 åħ¬åı¸","çĶŁ åŃ©åŃIJ","åħ³ ç¾½","æł¡ ä¼ģ","Ġম ন","çĸ«æĥħ å½±åĵį","-sh irts","Del ivery","Ġtecn ologÃŃa","ÅĦsk iego","an en","Ġto ch","åĪĨ éĶĢ","å¿ĥ çĶµåĽ¾","Ġت عد","ç½ij çIJĥ","ĠBe et","Ad dition","åĢŁ è®°","бо ÑĤан","âł Ģ","^ k","e al","Ġb im","ä¸Ń æĮĩåĩº","Ġtr atta","ä¸İ åIJĪä½ľ","ä»İ 天","åħ³ç³» ä¸Ń","ĠGu err","oly b","Ġод нÑĥ","Ġà¹ĢภĦ","arroll o","Ġdistint as","æĪij çªģçĦ¶","èĩª æĪIJ","æĪij们 èĩªå·±","äºĮ çͲ","å°ij æŀĹ","è¿Ļ个 æ¶Īæģ¯","ĠNe ptune","乡 åľŁ","ĠпÑĢе вÑĢа","åĺī éĿĸ","ART MENT","Ġë¶ ģ","(id x","à§ĩম à§įবর","éĦĻ è§Ĩ","typ ical","çļĦ æ°ijæĹı","ĠW olver","pe er","è¦ģ èĢĥèĻij","int osh","æĹł æŀģ","ĠÚ© اÙħÙĦ","ĠÙĨ تÛĮجÙĩ","æŃ¢ æįŁ","ĠTra che","-w rap","ĠاÙĦÙĨ بات","ãĥĥ ãĥģ","Ġminim ization","Ġப à¯Ĩ","âĻ ª","Ġpob re","Disc uss","Ġef ek","สู à¸ķร","Ġaccus ation","Ġery the","ĠIncorpor ated","inguish able","F ix","S Q","çļĦ ç»ĵåIJĪ","ä¼ļ è§īå¾Ĺ","Ġper g","é«ĺ è¶ħ","æ¯ı ç»Ħ","Ġgu itars","éĢł ç¦ı","令 ä»ĸ","ha al","Ġsyn ergy","ä¹Ļ éħ¸","འ£","å¼· 大çļĦ","æĬ¬ é«ĺ","æŀĿ æĿ¡","Ġspr outs","设ç«ĭ çļĦ","åĿļå®ŀ çļĦåŁºç¡Ģ","Ġk all","ä¼ļ è¯Ŀ","å·² äºİ","Ġmon omers","éĢł 纸","ä¸ĵ åįĸ","æĹı 群","Ġfa uc","Ġgrass lands","ĠÙħØ« اÙĦ","ĠNucle ic","Ġb ok","èĩª åį«","使ç͍ æĸ¹æ³ķ","ص ØŃ","åĽ½å®¶ éĺŁ","è¶ħ æłĩ","Ġcivil ized","×ķ׳ ×Ļת","Ġcré er","ĠPAP ERS","Ġcoerc ion","åŃ£åIJİ èµĽ","Ġc Åĵur","è¾ĵ åįµç®¡","ĠRep roduction","Ġmi ÄĻ","Ġstru ktur","ĠJean ne","Ġprod utos","Ġtus en","= E","D alam","Ġst ag","ĠJ ol","ä¿ ij","æ°´ ä¸ĭ","å¾· æĸ¯","ĠDes criptive","Ġgeneral ize","åĵ¥ 们","Ġkom m","×ķ×ĵ ×ķת","ĠPARTIC ULAR","Ġth o","and re","Ġmet ac","ä¼ł æĿ¥çļĦ","å®īåħ¨ å·¥ä½ľ","表示 æĦŁè°¢","Ġ×¢ ×ķ×ĵ","æIJŃ æ¡£","Ġesp resso","Ġinterf acial","Ġসমà§įপ রà§įà¦ķ","Ġativid ade","_ SE","m ight","Ġv ows","æĪij éĤĦ","åĪĩ å¿Į","åĨĻ è¿ĩ","EST AMP","漫 天","æIJħæĭĮ åĿĩåĮĢ","-sur face","Initial ize","æ¯ĶåĪ© æĹ¶","ĠGreen wich","Ġì§Ģ ìĽIJ","åĮ® ä¹ı","F ried","çļĦ éĢ»è¾ij","ĠM OR","te ÅĻÃŃ","å¹´ èĢģ","çłĶç©¶ æĸ¹æ³ķ","Ġdol phins","Ġíĸ ¥","Ġs alsa","Ġin ductor","çİ ®","ins ured","åİŁ åIJį","ĠÑĥ де",".m p","Ġоб Ñīего",".D is","ª× Ŀ","ĠÏĢ ÏĮ","Ġgi ác","প à§ģর","ĠPartnership s","Anth ony","- ep","Ġdi abet","åį³ å°ĩ","äºij 端","é¸ ¯","èģŀ è¨Ģ","æ³ķå®ļ 代表人","Ġto ma","äºĨ ä¸ĬæĿ¥","Ġk ron","ub untu","å°ı çĶ·åŃ©","iss ements","п еÑĢе","sc ar","æ¸ħ åĩĢ","à¸Ľ à¹Īวย","ĠDisc ord","ĠÑĤи п","ĠRap hael","ãĥĥãĤ¯ ãĤ¹","ĠÑĨвеÑĤ а","P as","at im","Ġp ony","st ance","æĺ¯ 两个","ĠR ach","ty w","ĠZ ones","ç¥ŀ 社","ĠPh arma","ĠEr asmus","ĠStat utes","Trans late","ĠOcc ur","ĠÑģооÑĤвеÑĤ ÑģÑĤвенно","Ġdru h","Ġech ocard","ĠíĤ ¤","' esp","b inding","ĠD und","ĠD SL","Ġpre clude","çľĭ åģļ","æīĵ æĪIJ","空 空","ÑĤи ÑĢÑĥ","åįĥ èIJ¬","ips es","ĠØ® اÙĨÙĪ",")ãĢģ (","áĥĿáĥ Ľ","verb ose","ĠSlow ly","ĠÐŁÐµÑĢе вод","E z","Ġd usk","se us","Ġne bul","è¿Ļ åĽĽä¸ª","对 æīĢæľī","åħ± åŃĺ","-f ront","è´¨éĩı æİ§åζ","å¢ŀåĬł å̼","Ġê° Ŀ","ĠاÙĦØ· اÙĤØ©","溶液 çļĦ","åįĹ京 å¸Ĥ","ĠIncorpor ating","ĠR ally","æľĪ åľ¨","Ñħоди ÑĤÑĮ","Ġexport ing","Else vier","c row","q x","åΰ 她","Ġdis grace","å¤ĩ æĪĺ","(c ode","Ġpolic eman","gest ellt","ĠPur due","à®ķ à®°","ĠFer ry","Ġdz iew","ĠS uf","ĠE rie","æīĢ éľĢè¦ģ","ç¬ij èµ·æĿ¥","çϾ 计","åī§ çĥĪçļĦ","à¹Ģà¸ģ ิà¸Ļ","åĤ² æħ¢","åīµ ä½ľ","Ġtrab ajar","над ÑĨа","%%%%%%%% %%%%%%%%","ĠìĥĪ ë¡ľìļ´","Ġhou den","en zen","Ġp ère","ĠH ancock","çŃī 她","-d uty","ĠÙĥ ÙĪÙħ","Ġbin ocular","Ev olution","Ġobs essive","- /","h ara","y per","ĠN IC","ä¸Ĭ è¯ģ","ze um","à¸Ĺ à¹Īาà¸Ļ","ĠBe coming","éĥ½æĺ¯ 为äºĨ","Ġtool bar","dis able","IST ER","ĠLem mon","ĠÑģоÑģÑĤоÑı нии","ĠUtil ize","Z I","ĠR abb","以 示","ä¸İ 该","åĬł èµ·æĿ¥",".\" ;Ċ","åĪĺ æŁIJ","Ġêµ °","Ùĩر س","Ġê± ¸","C ursor","说 ä¸Ģ说","Ġsp ines","空 èħ¹","amb re","ĠاÙĦÙħ Ùĩ","éĢĢ ä¼į","Ġthin ning","åĭĺ æŁ¥","Ġprat iques","å°ıä¼Ļä¼´ 们","éĩįè¦ģ讲è¯Ŀ ç²¾ç¥ŀ","em ps","çłĶç©¶ 对象","çĶ» ç¬Ķ","ĠÙĬ Ø£","ĠIS IS","ĠÑĺе д","Ġ×Ķ×Ĵ ×ĵ","Ġpupp et","ĠTOD AY","s ig","çļĦ çģµéŃĤ","ĠK g","å·¥ åķĨä¸ļ","fl uss","-m ost","ĠÑĩ ÑĤ","ди ка","Ġstar ving","Ġkl ub","z ij","× ŀ×ķ×","in ÄĽ","Ġw issen","çļĦ è¯ģæį®","æīĢ åħ·æľī","Ġent angled","èģĮä¸ļ åѦéĻ¢","ÑĨа ми","Ġpals y","' Connor","c ancel","Ġsp ills","åĽĽ åĢĭ","Ġappro ving","èĤ² ç§į","æĸŃ å®ļ","ita ção","çĪĨ çϼ","Ġפ ר","å®ıè§Ĥ è°ĥæİ§","éĽį æŃ£","Ġn emen","ä¸į éĹ®","Ġqu is","æĸ° 模å¼ı","æīĭ èīº","Ġaffili ations","à¸ŀื à¸Ĭ","ug o","ST E","ĠGe ographical","ĠMor ales","è¿· 人çļĦ","åªĴä½ĵ æĬ¥éģĵ","ç©Ĩ æĸ¯","Program ming","-ad justed","ĠÑĢаÑģÑĤ ений","æľ¬ æĥ³","ä¸ī åĨľ","æĽ´ ä¸įèĥ½","å¼¹ èį¯","Ġκ Ïħ","ĠLow ell","Ġmedi ates","ĠAstroph ysics","Ġfron te","f am","Ġd uke","âĢĻ -","åΰ 头","ä»İ åĵªéĩĮ","å®ī 妮","Ġconst rain","让 åŃ©åŃIJ们","离 åĪ«","ĠÙħÙĨ ظ","ĠMon aco","æĽ¸ ãģį","Ġban ning","ós ito","Ġdisproportion ate",": m","M ut","ĠT ory","åľ¨ çİ°åľº","th orn","ak ses","éĤ£ 群","西 å±±","æĮģ ä»ĵ","Ġhand held","åıĸ ä¸ĭ","é¢Ĩ åľ°","å®īåħ¨ æķĻèĤ²","ĠEm issions","ä¸įè¿ĩ ä»ĸ","γ ά","Ġdim ost","лен ноÑģÑĤÑĮ","OW S","ãĤīãĤĮ ãģ¦ãģĦãĤĭ","óln ie","_ \\+","ĠB uh","å¤ļ 头","æķĻ ä¸»","å¹¶ è¦ģæ±Ĥ","Ġmaterial es","Ġmind ed","ĠOff ering","Ġà¹ĢภĹ","ؤ اÙĦ","Ġawa ited","= `","ĠS erm","ĠW ad","ĠU PS","æĪij们 åıijçݰ","ä g","ĠZ H","åıĪ ä½ķ","女 æİĴ","夫 åŃIJ","计ç®Ĺ ç»ĵæŀľ","æ¶² æĢģ","åĪĺ æµ·","Ġber dasarkan","èĥŀ èĥİ","ô ne","map sto","ãģ¨ãģĦ ãģ£ãģŁ","R t","Ġc ia","Ġn Ãły","ut ility","Ġun recogn","èĥ½ åĴĮ","Ġso ort","ci ation","åħ¬ éģĵ","æĸ° æĹ§","æĪij们 ç͍","اÙĦ Ø´","Ġoriginal ity","ĠاÙĦس ÙĬ","æ·±åħ¥ åŃ¦ä¹łè´¯å½»","èĦı èħij","Ġdepartment al","çªĹåı£ ä¸Ń","ĠBull s","Ġinterfer on","ĠÑĤеоÑĢи и","Ġsplic ing","G ib","ar ı","ĠS my","Ġla z","ĠAl varez","IC ATIONS","ĠпÑĢи гоÑĤов","Ġce ases","æķĻåѦ è¿ĩç¨ĭ","宽 广","æĭĴ ä¸į","ç»ĻäºĨ ä»ĸ","Ġvra ag","; import","b ang","v ette","Ġt els","Ġpro kary","é« Ļ","ĠÙħ ÙĦÛĮ","Ġд оказа","ĠAl pine","æīĵ 车","è£ħ ä½ľ","à¸Ħ à¹Īา","ĠÙĤ اعدة","CP A","Ġbatter ed","î Ģ","ith a","ber a","cc io","æĭ ļ","Ġob at","Ġна ÑĤÑĥÑĢа","Ġsl ugg","ĠSp ine","åĩºçīĪ åķĨ","æĸĩ竳 æĿ¥æºIJ","sl ice","èĭį èĿĩ","ĠPMC ID","ĠÏĩ α","ĠWel ch","Ġincarcer ation","èłķ åĬ¨","Invent ors","ĠFIT NESS","ĠTuc son","B n","_ Q","x in","Ġp aj","åĴĮ ä¿¡æģ¯","主 讲","è¿ĺ èĥ½å¤Ł","æ¶Ī çĤİ","æĬķèµĦ åŁºéĩij","ĠLog ical","Ġreact ant","Cong ratulations","çļĦ å®¶ä¼Ļ","æ¯ ¡","è¿Ľ è´§","Ġд ек","åıijå±ķ æĸ¹åIJij","ว รร","Ġза веÑĢ","Ġsequ enced",".in cludes","Ġovers hadow","çĺĭ çĭĤ","_ space","æĢ§ æĺ¯","Ġrec or","-c ert","主è¦ģ 表çİ°åľ¨","ĠÐŁ ÑĢов","ĠÐĶ Ð°Ð½","би ли","è¿ĻæĿ¡ è·¯","大 大å°ı","åIJİ æİĴ","ç»ı åķĨ","Ġover power","交 éĽĨ","çŁ¥éģĵ ä»ĸ","ä¸ŃçļĦ åľ°ä½į","ĠØ¢ ÙħرÛĮÚ©","æĶ¹åıĺ çļĦ","sch l","å°¿ æ¶²","Ġretrie ving","ĠاÙĨد ازÙĩ","oplast y","Ġsynerg istic","I oT","ĠB ok","ä¸į ä¸Ģä¼ļåĦ¿","ier ungs",".com mand","管çIJĨ æĿ¡ä¾ĭ","Ġline ages","лÑı ÑİÑĤÑģÑı","æľīä¸Ģ èĤ¡","èľ Ĵ","ä¹Łæ²¡æľī ä»Ģä¹Ī","Load ed","Port al","ĠдÑĥ ма","Ġlod ging","åīĶ éϤ","ĠENG INE",". mean","åľ¨ 马","ठ«","ah at","ĠZ an","åĵį 声","ä¿® çĤº","Ġtyp u","AG C","è½° è½°","Ġá¼IJ ν","Ġkond ado","ĠBax ter",", name","ĠS istem","使 å®ĥ","å¹³ æģ¯","å¹¶ 被","ร à¸ĸ","æµ· çĽĹ","èᝠä¸ļ","Ġann uity","ĠÏĦ ῶν","_l ines","vd ots","Ġn är","ĠT ie","ĠB ones","æĺİ äº®çļĦ","åIJĦ åĮº","åIJĽ çİĭ","ç§ģ ãģŁãģ¡","å¥ĭæĸŠ缮æłĩ","Ġhover ing","B attle","j ou","Ġm ÅĤod","ĠM uj","ĠW atching","form al","ä¹ĭ 举","à¸Ĺ à¸Ķ","ĠMod ules","or z","éĢī æ°ij","ĠìĿ¸ ê°Ħ","Ġmarch é","ĠBh ag","Ġbeispiel sweise",". Common","æĹ¥ åĩĮæĻ¨","ĠAll ocation","建çŃij å¸Ī","رÙĪ Ø¬","è¿ŀç»Ń çļĦ","ĠاÙĦر س","_re port","ĠCro hn","ĠÑģозда ниÑı","æłĸ æģ¯","le ine","ĠA UC","âĢĿ ).ĊĊ","ÃŃ amos","ع Ùģ","æĽ´ ä½İ","éĵ İ","ж нÑĭе","à¹ģ สà¸Ļ","Ġcr éd","ĠCarl son","èŀįåIJĪ åıijå±ķ","Ġer otic","æĢ» 管","AM ENT","ĠÑĢе Ñĩи","è°ģ æĿ¥","èĥ¡ 说","éĵº åŀ«","Ġpued es","Ġfed eration","ãģªãĤī ãģªãģĦ","} P","çļĦ æĬĹ","le itung","æ±Ĥ ãĤģ","éĻ¢ åĨħ","rand s","Ùİ Øª","å»ī ä»·","éĹº 女","Ġforesee able",".nc bi","Ġn ám","åı¯ ä½ľä¸º","ge om","ĠCh ir","å®Į ç»ĵ","书 åIJį","ĠGe ophys","ç§» éϤ","Ġten ÃŃa","ĠMcC ain","æ³ķ åĬĽ","æ¸ İ","rit ann","åıĹ çģ¾","oot s","ait o","à¸ļ วà¸Ļ","温 æĥħ","åŃ¦æł¡ åĴĮ","ĠTrans plant","ĠMet z","ĠPal ae","×ķ×ĵ ×Ļ","ĠKir che","ãĥĢ ãĤ¤","M ix","} $$Ċ","çļĦ èĢģ人","ol ari","大 好","Ùħ Ùĩ","å°ı 妹","å¦Ĥ æĦ¿","Ġi P","ÃŃ sk","éĢł èι","ĠRes ume","af s","é»Ħ æ²¹","èŀº æ¯į","akh ir","Ġingenu ity","D ad","ç±» æ¯Ķ","Ġmus ique","ubl ique","çĶŁäº§ è¦ģç´ł","ĠJan uar","Ġbio active","رار Ø©","= g","çļĦ é¢Ĩ导","äºĨ ä»Ģä¹Ī","ĠH ib","Ġ\" ^","ere quisites","产 ç§ij","ä½Ĩ ä¹Łæľī","Ġpost graduate","comm ons","Ġemb ar","-se arch","wa arden","æĪ´ åı£ç½©","tag Helper","ĠAber deen","Ġmagist rate","Ġdistort ions",": String","ĠC rack","æµ Ĵ","av ad","è° §",".D omain",".T itle","Ġintegr ative","ĠCy bersecurity","æĶĢ çĻ»","F ew","Ġpol i","to le","ĠHar ley","åIJĮæĦı äºĨ","йÑĤе ÑģÑĮ","Z L","Ġs acks","ÑĢ ÐµÐ½Ð¸Ðµ","ĠG V","ç© Ģ","å¾Ī å·®","rem os","çĭ¬ æľīçļĦ","èι éķ¿","ĠSal is","ĠWater loo","åįıè®® çļĦ","Ġstrat ég","ĠSter e","Ġkelu arga","ĠH AR","ĠSte ele","åģľ äº§","oe lect","çĸı éĢļ","æıŃ å¼Ģ","_w rite","âĢļ ¬","ĠÏĥÏĦο ν","èĩĢ éĥ¨","ä¸įèĩª ç¦ģ","B ang","D ry","第ä¸ī ç§į","ĠHor ror","ĠRh ine","åį°è±¡ æ·±åĪ»","èħ³ æŃ¥","Univers it","F ür","Ġt ud","an nten","åĬł æĪIJ","Ġterm es","Ġda o","Ġmax ima","Ġinform azioni","Ġentre prises","Ass uming","اط ع","æĴĴ å¨ĩ","Ġbroadcast s","ĠC ure","od oxy","ĠH anna","éĥ½ çͱ","åİ» åĵª","西 欧","Ġد ÙĤÛĮ","ĠØ® دا","Ġvalid ator","Ġfif teenth","ĠPlant ae","Ġби ло","ĠBrief ly","Ġদà§ĩà¦ĸ া","M otion","æķ ķ","Ġcomp iling","ĠAb igail","-se q","ĠìŀĪ ìĿĦ","æĢ»ä½ĵ è§ĦåĪĴ","ĠDam ascus","prof its","วั à¸Ĵà¸Ļ","Ġbast ard","ĠHistor ically","个 åĽ½å®¶","Ġdep ressing","管çIJĨ åѦéĻ¢","Ġpaper back","çIJĨ论 çŁ¥è¯Ĩ","Ġsn ail","Ġspect roscopic","ä¿ĿæĮģ äºĨ","æĮ¯ å¹ħ","ĠговоÑĢи ÑĤ","ĠAj ax","_ print","Ġâ Į","ÑĤÑĮ Ñı","ова Ñļа","bit os","å¯Ĵ åĨ¬","kl är","Ġwa ived","Ġú t","æĴ¤ åĽŀ","Ġcompanions hip","-set ting","Ġw iping","åı ¢","ÙĬ ص","Ġind isc","ом ен","ли ÑĤе","ä»ħ åľ¨","æ¡ĥ åŃIJ","à¹Ģห มาะ","Op ening","ĠдокÑĥ менÑĤа","çĬ¹å¤ª 人","å½· 彿","st he","th i","iz al","åĩº ãģĻ","æ³ķ è¯Ń","Ġneed y","æ¯Ķ æŃ¦","åĻ ĵ","Ġland slide","è¤ ²","ĠAbs olutely","è¾¼ ãģ¿","ĠÙħÙĦÙĬ ÙĪÙĨ","າ àº","( words","F REE","he ws","an um","ä¸Ĭ 交","æľĢ 容æĺĵ","ĠAl uminum","æį¢ çĥŃ","Sub scription","ç¿» æ»ļ","ĠMor i","ĠNO AA","ĠRandom ized","ĠBorrow er","Rear range","B OSS","H ill","Ġa lem","Ġd addy","个 好","Ġsa ils","æĪij们 没æľī","ä¼ĺ åĬ£","æ²Ļ åŃIJ","æ²Ļ æĭī","Ġshaft s","Ġexpres ión","? !ĊĊ","Ġto ho","ĠH J","è¿Ľ é©»","Ġв оде","Ġco lect","çݯ æ°§","Ġbi etet","ห าย","è´´ åľ¨","èĭĹ æľ¨","ĠPo et","Ġrail ways","ĠFar ms","ĠíĻľ ìļ©","- \"","] ++;Ċ","ĠI J","Ġat ra","以 å®ŀéĻħè¡ĮåĬ¨","åľ° æľĽçĿĢ","ĠSt ations","Ġpar ach","åĨ· éħ·","Ġзна ÑĤÑĮ","Inst ructions","ാഠ¯","ĠдÑĢ Ð¶Ð°","ĠCann abis","åĴ¬çīĻ åĪĩ","Ġì» ´",". rs","/ dev","O w","åΰ åľº","Ġpoint less","Ġisol ating","алÑĮ ной","Ġker atin","_by tes","as zt","Ġsh unt","Ġatt aining","åħļ 群","ç¼ĸ èĢħ","pre pare","ĠGl ossary","Ġcritic ised","Ġassemb l","Ġresemb led",", in","- onset","ĠK urs","å°ı å·§","ä»ĸ们 说","Ġlik eness","æĿĢ çļĦ","amin ated","ĠÐIJ меÑĢи","å¨ĺ çļĦ","ĠÅĽ wiad","ĠкÑĢÑĥ г","ĠUtil izing","ĠDres den","ug no","åĨį çͱ","è®° è¿°","Ġcy tochrome","æĶ» åħĭ",".P ath","path ic","ĠدÛĮ گرÛĮ",", null","S ets","re ja","ĠT rap","Ġv ase","ĠE I","å±± ç¾Ĭ","Qu esta","Ġ×ľ× Ľ×ľ","Ġкомпа нии","N X","Ġform azione","ĠQ UE","ĠMar vin","Ġر ÙĨÚ¯","ĠDis p","æ¯Ľ è¡£","pa ñ","ä¹Į 鸦","Ġeste emed","abb ing","ĠCub s","ĠSepar ate","ĠPeb rero",". click","w arts","ut ting","ĠE MB","op i","è¿ŀ å¤ľ","åįĩ å̼","ĠاÙĦÙħ ÙħÙĦÙĥ","à¸Ī ุà¸Ķ","ĠпеÑĢе Ñħод","Ġroof ing","Ġinfant il","วั à¸ķิ","à¸łà¸²à¸¢ à¹ĥà¸Ļ","ä¸į å°ıäºİ","Ġeng ulf","over rightarrow","çͲ åħ¬åı¸","ĠSw ap","Ġcool ant","Ġsac r","Õ¸ÖĤÕ µ","ĠбеÑĢемен ноÑģÑĤи","ĠK orn","Ñħ ождениÑı","Ġac um","Ġwa iter","Ġwidth s","འł","Ùĩد Ùģ","Ġle ased","Ġwe e","夺 å¾Ĺ","æĸ¹ç¨ĭ ç»Ħ","Ġ'../../ ../","% ï¼ī","+ T","âĢ ī","ĠB ord","è¿Ļ ä¸įä»ħ","å°± æĪIJ为","大å¤ļ æķ°çļĦ",".Cont ent","Multi plication","ĠJohannes burg","c odes","ĠB ACK","ik oa","ateg orie","æŃĮ åī§","à¸Ĺีà¹Ī à¸Ķี","'] ))","ĠBet rieb","-al one","à§§à§ ®","(: ,","Ġimproper ly","'aut re","Ġ×IJ×ľ× £","- To","in at","ut down","åIJ ¡","ĠP ERSON","qu iet","ĠK G","éĽĨ 约","å¸Ĥåľº ä¸ĬçļĦ","Ġmag giore","Ġing ested","ìĸ´ ì§Ħ","åĩŃ çĿĢ","-act ing","ĠQuad ratic","ĠÑĢеак ÑĨии","มาà¸Ī าà¸ģ","Ġm ister","ĠB ism","Ġse xt","èĥ½ ä»İ","Ad j","éļĶ ç»Ŀ","áŀ ·","äºĮåįģ ä¹Ŀ","ĠExp enses","Ġstar red","Ġét ude","ÙĪØ¬ ÙĪØ¯","ĠÑĢабоÑĤа ÑĤÑĮ","ĠColomb ian","Ġfals ely","Ġtranqu ility","Ġsung lasses","Ġk teÅĻÃŃ","以 åĨħçļĦ","æĭ ´","æĮģ å¹³","è¿Ļ个 æķħäºĭ","æķĪçİĩ åĴĮ","ĠMel anie","Õ¥Õ ¯","i ators","ĠN amen","大 æ±Ĺ","ĠIn jection","ï¼Ī ï¼īãĢĤĊĊ","emb ros","åĨľä¸ļ 大åѦ","ĠÚ©ÙĨ ÙĨدÙĩ","西æĸ¹ åĽ½å®¶","Ġdzie cka","ĠBos ch","ÑĦика ÑĨии","ë¸ Ķ","Ġst ÅĻed","Ġk osten","Ġad quir","å¦ Ŀ","ठĻ","Ġz g","ó rd","Ġcap itals","æ¶Ī éĢĢ","Ġelect orate","Pre pare","Account s","Ġlin ux","Ġperk embangan","ĠMongo DB","brevi ations","R ome","ow aniu","ver g","Ġfl ax","被 æįķ","åį³ ä½į","æĶ¯ æķĻ","çİ°åľ¨ å°±","åį´ è¯´","ÑĤелÑĮ ном","ĠNue va","ĠпÑĢоÑĦи лакÑĤи","\" When","T ro","Ġf ray","Ġb ola","ä¸į ä¸İ","ĠR ear","éģŃ åΰäºĨ","Ñļ ено","ĠLess er","Ġ(... )ĊĊ","Hig hest",") âĨĴ","H OME","ĠM olecules","ast re","æľ¬ æºIJ","éĩį å¡ij","å½ĵ 好","å°Ĩ æĮģç»Ń","çϽ çļĻ","ĠWor cester","è¿ĺæĺ¯ æĮº","åºĹ éĿ¢","-P er","æııè¿° äºĨ","Ġgrass land","Ġscra ps","Ġহà¦ļà§įà¦Ľ à§ĩ","+ P","ĠS AC","ĠS itting","åĮĸ çŰ","ĠPro jek","身 亡","æ® ĩ","åŃĺ åıĸ","象 éĻIJ","Ġtot ality","éķĩ éķ¿","éĺ´ æļĹ","تر ÙĦ","Ġsimpl istic","-r unning","Just ice","使åij½ æĦŁ","Ġphosphat ase","' all","çļĦ æ¯Ķè¾ĥ","ĠG OV","天 å±±","åİŁ åıijæĢ§","çı ŀ","za Äĩ","é»Ħ è¿ŀ","æıIJä¾Ľ ç»Ļ","è¡£ è¡«","享 ç͍",")\\ ).ĊĊ","Ġ×©× ł","CA ST","ಿಠ¨","ĠSE Q","ĠÑĨе лом","ĠÑĥÑģÑĤÑĢой ÑģÑĤва","- engine","/ components","F U","un er","åŁº è°ĥ","Ġx n","AL A","ift ed","å®Ŀ åīij","åŁ¹ è¨ĵ","ä¸ĥ æĺŁ","Ġci erto","ĠJackson ville","ãĤ¦ ãĤ§","Ġté mo","ĠL ef","Ġ{ },","对 å°ı","çα åĽłæĸ¯åĿ¦","ĠØŃ Ù쨏","鼨 天","çļĦçĶŁæ´» æĸ¹å¼ı","ĠAppro val","-dis covery","ĠавÑĤом аÑĤи","èµİ åĽŀ","ĠQUEST IONS","A a","ä½ł è¿Ļä¹Ī","åħ¬ å°º","åİ» åIJij","æĶ¾ ä»»","Ġactiv ator","Ġline back","ĠQu el","读 è¿ĩ","Ġsitu ational","/d etails","ĠDon ovan","æijĩ æijĨ","rij ke","ãĤīãĤĮ ãģ¾ãģĻ","íĿ ¬","Ġc est","Ġh l","Ġst ale","ĠD zie","Ġpre face","头 çĽĶ","Con verting","ç®Ģ æĺİ","Ġpolit ely","ĠGe V","äºİæĺ¯ ä»ĸ","PL AY","Supp l","æĴĩ åĺ´","ड ़","ĠHind us","ÙĪÙĬÙĥ بات","_ helper","Ġв ода","ĠØ£ ÙĩÙĦ","Ġfac ade","ĠاÙĦت Ø£","çļĦéĩįè¦ģ åĽłç´ł","éĤ® å¯Ħ","ạ ng","باش د","R n","x on","åħ¨ åĨĽ","Ġsecond ly","Ġfond o","两大 ç±»","à¸Ħà¹Ī ะ","} C","çļĦ è®Ńç»ĥ","æĶ ¤","к ÑĬ","æīĢ åģļ","Ġpo chod","åıĹ è®¿","ÏĦ ικÏĮ","да Ñĩи","å¸Ĥåľº 主ä½ĵ","èĥĮ å¾Į","ĠWil kins","æijĦåĥı æľº","ĠизмеÑĢ ÐµÐ½Ð¸Ñı","id us","è¿ĩ ä½İ","æĪij们 çľĭåΰ","ä»ĸ们 è¿ĺ","Ġcre pt","Ġد ÛĴ","åĽ´ æĶ»","åºŁ æ°Ķ","åħļå§Ķ å§Ķåijĺ","ĠLect ures",", !","u itive","ĠP NG","å®¶ éķ·","ite kt","ĠRe cht","ä½Ĩ éļıçĿĢ","åħĥ 代","ä¼ł è®°","Ġج دا","楼 æĪ¿","éĸĭ åķŁ","/d l","ãĤĪ ãģŃ","ÃŃn as","ĠDou glass","cut ta","াষ à§įà¦Łà§įর","referent ziak","H J","O racle","id ious","ä¸Ģ æ´¾","Ġout skirts","ç»ĵ è¯Ĩ","ym b","ĠâĢĺ âĢĻ","ãģĽ ãĤĭ","Requ irements","ĠBeth lehem","/ ~","_ TH","Ġf printf","çļĦ å¿«","ĠP ocket","ĠR MS","Ġform ato","led ged","è¿° èģĮ","ĠÙĬ ÙĪ","ç¹ ³","Ġwel ke","ĠCamp o","ãĥ³ãĥ Ģ","åŀĤ缴 äºİ","ĠмÑĥ зе","åįĶ æľĥ","ĠDent istry","éĹŃä¸Ĭ çľ¼çĿĽ","ĠÙ¾ÚĺÙĪÙĩ Ø´","g li","en ko","Ġs ifat","ou w","Ġwith held","èİ ĺ","ĠÑģи ла","åĪĨéĴŁ å·¦åı³","Gen esis","ánd ose","æ±ķ 头","Ġdazz ling","Ġc iento","ig ual","æĿ¥ 形容","Ġsp azio","åıΠ以","æĸĻ åΰ","Ġsubject ivity","AP PL","ĠÑģо Ñħ","ĠLu igi","æĢĿç»´ èĥ½åĬĽ","Ġodd ly","ï¼ģï¼ģ ï¼ģĊĊ","Ġà¸Ħ ุà¸ĵ","Ġsucc inct","Ġramp ant","ĠEstablish ing","çķĻå®Ī åĦ¿ç«¥","Ġzomb ie","çļĦ åĩłä¸ª","ĠT anner","ع Ùī","Ġpos ición","红 çģ¯","Ġvo it","OT T","empl os","å̾ åŁİ","_R ES","ĠIceland ic","ĠLaur ie","å¿ĥå¾ĭ 失常","çĺĻ çĹĴ","ĠP fe","åľ¨ å¼¹åĩºçļĦ","ĠAr ter","ç½Ĺ 伯çī¹","Ġnight mares","Ðł аÑģ","漫 漫","ĠAuthor ities","è´¢æĶ¿ å±Ģ","سÙħ بر","éļĬ ä¼į","lat est","ĠHB V","Ġhepar in","Ġth al","Ġj ohn","Ġme adow","ĠRe ception","ef eller","Ġche ering","sh own","Ġap an","å´ĩ é«ĺçļĦ","Ġল à§ĩà¦ĸ","Ġdivert ed","Ġetx ek","V ous","r ů","ĠM MA","ĠL akers","Ġret reated","-s an","Ú© ÛĮÙĦ","è¨Ģ æĥħ","èĩ´ æŃ»","èİ« è¿ĩäºİ","Ġ×Ļ ×©×","æĬ± èijĹ","Ġ[' ./","å¤ļ项 å¼ı","- users","ol one","ä¸į å̼å¾Ĺ","iz adas","ĠPro portion","常 人","ĠSe asons","Un s","draw al","Ġfut ur","ĠUncertain ty","P ont","Ġb ib","Ġand ra","Ġmay ores","è¿ĺæľī 许å¤ļ","çĶļèĩ³ åı¯ä»¥","软 çļĦ","ĠPres idents","å¹´è½» 人çļĦ","Ġjun io","C f","èĢĮ ç«ĭ","æ¸ħ çļĦ","å¾Īå¤ļ äºĭæĥħ","é¡¿ äºĨ","Ġré ponse","ç¼ĸè¾ij åύ","æīĢå¾Ĺ çļĦ","âľ ĵ","ĠConsult ation","ĠTransl ated","ĠRosen berg","ä¸įèĢIJ çĥ¦","u racies","ä»ĸ çªģçĦ¶","-n ode","Ġwave let","ĠPRO P","ÃŃs ica","Ч ÑĤо","è¨Ĭ æģ¯","èī°èĭ¦ å¥ĭæĸĹ","Ġh aya","qu ina","ä»ĸ åıª","æ¸ħ å»ī","\\) ).","ĠPl uto","ĠEl on","å¸Į çī¹åĭĴ","ĠNow adays","çģ¯ åħ·","ç° ¸","à¸ł ั","Ġretic ulum","( #","V iol","st ral","ĠR NS","ä½ı å¤Ħ","ç¢ ©","Ġvo i","ĠÑĦ оÑĤ","Ġalien ation","ĠAdvoc acy","Ġintrins ically",". Not","ĠJ h","åİ» åĵªéĩĮ","Ġserv icio","à¸Ĭ ุม","-C D","ĠAD P","ÑĢова но","ấ y","ĠÑĤеÑĢ Ð¼Ð¸","ĠLif etime","C ases","Ġre ak","ig te","Ġdel ving","Ġexec utor","лÑĥ а","MS O","ĠAnaly se","ĠповÑĭ ÑĪен","Liter al","Ġsanction ed","S om","S usan","Ġg uts","Ġis to","å¾Ĺ å¾Ī好","æľ¬ èĬĤ课","Ġoff sets","åĽĽ åĪĨ","è¿ĺæľī 个","æĬĹ è¡¡","Ġcomputer ized","Ġcast ell","ĠSche matic","ä½£ éĩij","çĹħèĻ« 害","b elt","Ġl uce","è¦ģ åĪĩå®ŀ","hat ikan","åĮħ åĮħ","è¾ĥ å¼±","å¤į åİŁ","Ġدر اسة","Ġpurpose ful","' or","C ass","T icket","Ġd inners","ra ga","Ġbefore Each","è§Ħ模 åĮĸ","çŁĽçĽ¾ çºłçº·","çĽ£ çĿ£","Ġmaior ia","- jud","p ont","Ġn omenclature","ĠF DI","ĠHe ck","Ġsim ul","Ġdoes nt","æĶ¹ ç͍","да в","Ġdou te","å·¦ ä¸Ĭ","ئ ÛĮ","ìĦ± ìĿ´","ĠCS I","/D ay","Ġscrap ing","碳水 åĮĸåIJĪçī©","ĠW AR","æľĢ 主è¦ģçļĦ","ع ÙĨ","ĠØŃ سب","key words","iy ah","Ġshore line","Saved Point","D ATE","il h","ĠF uzzy","Ġhum ane","Ġtransform ers","Ġcomprehens ively","tre cht","ल ा","Ġdeleg ated","çħİ çĨ¬","ĠCho ices","Ġsincer ity","ĠheiÃŁ t","# line","_ FL","Ġf ps","ĠL ets","åĴ Ħ","å·¥ä½ľ è¦ģæ±Ĥ","çļĦ人 éĻħ","Ġplace ments","é¢Ħ å¤ĦçIJĨ","Ġproblem i","ĠпÑĢо ÑĤÑı","æĺ¯åIJ¦ æĺ¯","缼 å¼Ģ","orb idity","жа ÑĤ","áv ÄĽ","åįĶ èѰ","Ġtremend ously","ĠÑģ видеÑĤелÑĮ","åģľ ç͵","Ġlat itudes","кÑĥ лÑı","Ġtit ration","sex ual","ç»Ļ人 以","ĠGrad ient","W EB","] he","Ġm arty","Ġfl amm","éľ ı","社 éķ¿","åıĪ éĹ®","Ġз ол","ãĤĴ 使ç͍","μ ι","ĠWar wick","Set SavedPoint","à¤ķ ार","Ġcart a","Ġзада ниÑı","Ġdéc ada","Ġeben falls","ä¸į 妥","act ually","Ġme glio","åĵ §","ĠEn rique","Ġне Ñĥ","æ¼Ķ ä¹ł","âĢĶâĢĶâĢĶâĢĶ ĊĊ","Ġশ র",".* ĊĊ","Ġincons istency","ç¡®ç«ĭ äºĨ","Ġunrest ricted","Ġbloss om","å§Ĭ 妹","- Christian","ĠS IL","设 å®ļçļĦ","åħī åIJĪ","об е","æĭī åΰ","æĻ¯ æ°Ķ","Ġho op","顺åĪ© å®ĮæĪIJ","f us","ĠN ec","Ġad el","éĢļ åIJij","ε λ","ĠChrist i","Ġpas a","CE P","æľīæīĢ æĢĿ","ä¸įç͍ 说","Ġpu issance","ĠWat kins","ĠMand ela","ĠMand arin","à¹Ģà¸Ħ ราะ","Ġescal a","Invest ig","Ġextraordin arily","ĠC one","ĠM á","ĠF as","åĴĮ çݯå¢ĥ","ĠU W","ä¸İ 大","ä»» æķĻ","æ¡Ī æĥħ","apt op","Ġdise ño","æĺ¥ 鼨","oud re","اÙģ ÙĬ","å¹» è§ī","é¸Ń åŃIJ","çĿĢçľ¼ äºİ","Ġблаг одаÑĢÑı","Î Ĵ","ä¸ĭ é¢Į","好 èݱåĿŀ","表 çİĩ","Ġ×IJ× Ĺת","æijĦ æ°ı","Ent ries","ĠPs alms","ĠDest iny","ĠPam ela","ãĢĤ ï¼īĊ","åIJİ å°Ĩ","èĩ³ é«ĺ","Ch allenge","çİ°åľ¨ æĪij","æ±Ł æ³½","Qu ando","ĠSuper vision","Ġ×ŀ×IJ ×ķ×ĵ","Ġdecid uous","il ver","Ġv ite","çĶŁ å¹³","ĠTh é","åIJĮ ä½į","×ķ ×Ļ×Ļ×Ŀ","Ġaut ores","Ġpast ors","ios ync","ĠاÙĦÙĤ در","Off er","ĠPas o","Ġfot ograf","Ġuninter rupted","Virgin ia","n age","Ġm ailed","ĠR het","éĤ£ 两个","å¼ł ä¸ī","med io","Ġune quiv","软 åĮĸ","Ġзна ком","Ġbloss oms","or ov","ur ricular","ĠU TF","Ġdata frame","Re illy","éĿŀ常 é«ĺ","Ġdire cción","Ġrefer encia","ষ à§įà¦Ł","à§ĥত ি","ĠÐľÐ¸ Ñħа","СÑĤа новниÑĪÑĤво","Ġprue ba","z we","Ġd ude","ĠR ican","æ°´ æ·±","æĬĬ ä¸Ģ个","ĠEqu ilibrium","丹 çͰ","åij½ä»¤ è¡Į","ÃŃm bol","ĠпÑĢÑıмо ÑĥголÑĮ","à¹ģà¸ľ à¸Ļ","Ļ àµįà´","ion y","ä¸į 顺","ĠW inners","ge v","å¾Ĺ å½ĵ","Ġза ме","Ġprec arious","Ġন à¦¿à§Łà§ĩ","è±Ĩ æµĨ","Ġtut ta","Ġcycl ists","æµģåĬ¨ èµĦéĩij","Ġ'@ /","Ġoc as","ĠHig hest","Ġevacu ated","ĠÙħÙĤد ار","æĺ¯ å¦ĤæŃ¤","å§ĭ çµĤ","à§Ģ দà§ĩর","tz mann","Ġembark ing","ä¸į åĴĮ","å·¥ä½ľ æľºåζ","Ġpat hetic","ĠLe aving","ĠPh antom","æ¥ļ åĽ½","æĥĬ éĨĴ","Ġamb iance","缼 çļĦ","交æµģ ä¼ļ","Ġwood y","ĠEU RO","è¿Ī è¿Ľ","æľĢæĸ° 竳èĬĤ","Ġzir con","v án","ĠL arger","Ġ\" \")Ċ","ĠK up","å¸Ĥ 人æ°ijæĶ¿åºľ","ey a","è§ģ æķĪ","ä¼Ĭ å§ĭ","ãĥ© ãĥ³","ĠExt ensive","ĠExpress ible","Ġcom um","-b usiness","AN O","æī¾ å·¥ä½ľ","ਠ®","ĠMat hemat","Ġjack ets","Ġempt iness","Ġdemean or","c ash","Ġr ant","ĠAl tra","åıĪ æ²¡æľī","Ġav ersion","åĪĿ 审","Ġsw ore","ĠDis yembre","å®ģ åİ¿","Ġপà§įর য়","Ġpool ing","ĠPlatform s","è©¢ åķı","ĠÑģамоÑģÑĤоÑı ÑĤелÑĮно","m q","ol ome","ä»Ĭ å¤ľ","ĠDep os","_f older","è¿Ķ æł¡","Ġinject ing","ovan é","Ġprophyl axis","B ow","åħ¨ åħļ","Ġfe ces","åįģ åĩłå¹´","Ġref urb","Ex pr",".P ost","éĹ» åΰ","Ðļ ÐIJ","Def initions","çļĦæĸ¹å¼ı æĿ¥",".sh ort","{ sub","çݰ å¦Ĥä»Ĭ","Ġproject or","Ġsaf est","Ġá¼ Ħ","Ġbatt alion","Ġsesu atu","Ġvæ re","S ed","çļĦ èģĮä¸ļ","ĠE tymology","Ġha wk","éħį æľī","èĩªå·±çļĦ 身ä½ĵ","Ġplant es","åĨ² 天","-e volving","误 导","å³° ä¼ļ","र ण","ÙIJ ÙĬÙĨ","Ġsto ichi","Ġperman ente","Ġnod ding","ĠP ASS","ĠH ors","åľ¨ å½ĵåľ°","çŁ¥ åIJįçļĦ","æį¢ è¨Ģä¹ĭ","ĠØ´ Ùħار","åĪ¶åº¦ åĮĸ","lim p","Ġà¦Ĩ দ","Ġসর à¦ķার","Ġprojekt u","\" ][\"","S ender","ic ar","åIJį å½ķ","Ġbu en","é£İ å¯Ĵ","æ½ º","ĠÏĦ ὴν","ä¿ĿæĬ¤ 好","çļĦæĹ¶éĹ´ åĴĮ","èħ° éĹ´","Ġalcohol s","Ġgé nero","ĠÑģимпÑĤом Ñĭ","ĠBeit rag","ropl asty","Ġy acht","Ġk up","çĶŁ çĶŁçļĦ","é conom","ле в","ব à§įয","æļ´ èºģ","Ġdefe ats","-fe ira","çľĭä½ľ æĺ¯","t id","Ġun i","éĢł è¡Ģ","è·Ł éŀĭ","ato on","伤 çĹķ","åįģäºĮ æĮĩ","çĮİ äºº","Ġконе Ñĩно","Ġtama ño","F riend","t ol","Ġt roll","Ġs ú","Ġst umbling","ĠG ud","Ġinv ading","ä¸įèĥ½ 让","ä»·æł¼ ä¸Ĭ涨","åijĪ çı¾","IO Exception","滿 æĦı","ĠRo oms","ĠKon stant","v ara","ĠHe ads","pro ble","Ġت بد","ŀ× Ł","å¼ł æĸĩ","ç»Ħç»ĩ äºĨ","æ²³ çļĦ","è¡¥ æķij","Ġhom estead","Ġcert ify","åĶĩ è§Ĵ","åľ°çIJĥ ä¸Ĭ","Ġreflex ive","Ġconte ú","T K","Ġm appings","ĠT ack","æľī æĪIJ","ĠIn hibition","æĮĩ åĩºäºĨ","yt est","产ä¸ļ éĽĨ群","Ġcm p","æĬĺ ä¸į","Ġoptim ally","åı¦ä¸Ģ åįĬ","itz ació","æģ° åΰ","ĠÑģлÑĥÑĩа ев","ĠCroat ian","as io","ĠC ups","ĠD SP","and emic","åħ¥ åĬĽ","Ġsystem at","ane a","ĠOr ch","Ġter reno","Ġоб Ñģ","çĽij åIJ¬","Ġâĸ ½","Ġ×ĸ ׼","Ġê°ľ ëħIJ","nd en","ĠT rit","åľ¨ åīįéĿ¢","Ġinv ocation","ĠLe ase","rm ann","åħį è²»","Ġod k","çĴ ŀ","à¥Ģ न","èħ¿ ä¸Ĭ","æĿľ é¹ĥ","ç»ŀ çĹĽ","ĠSold iers","Ġse ep","åİ» å¹´çļĦ","ع ÙħÙĦ","Th irty","ä¸ĩ 象","Ø´ رة","رÙģ Øª","æī£ æĬ¼","ĠProm ote","ĠMcG ill","ropract ic","- icons","çĤ ľ","uc os","oh m","Ú¯ ÙĪ","ĠRel ay","Ġبر ابر","åľ¨è¿Ļ åľº","ĠÙħر Ø©","ĠBol she","æĥĭ æĥľ","G K","Ġl apse","ĠC CS","ĠPl ays","æľª å®Į","pon en","ĠPar an","Ġasp ire",": d","Ġc actus","çļĦ æĪ¿åŃIJ","op era","à® ĩ","\", ĊĊ","ç§ij æ¯Ķ","Õ¶ Õ¥ÖĢÕ¨","onom ia","ĠMcC orm","Ġperpet rators","Ġtö bb","ĠAccom mod","Ġmisunderstand ings","Ġj at","è¾ į","å°Ĩ ä»ĸ们","Ġdem ikian","à¸ļ ู","ett lement","å¹¼ èĭĹ","ä¿© 人","Ġepid emi","ĠContrib utor","ĠDiss ertation","Ġem pre","app ers","еÑĢ Ð¾Ð²","ä½Ľ éĻĢ","丽 ä¸Ŀ","бли ÑĨа","ĠSelect ing","develop er","ĠChile an","ĠIllust ration","Ñĭ дÑĥ","ĠSt ur","Ġdu ż","ä¸ĵä¸ļ 人士","Object ives","àµįà´ ļ","स म","Char Array","åŁºåĽł ç»Ħ","æ²§ æµ·","ĠMack enzie","Ġwp ÅĤyw","ç¼ħ æĢĢ","为é¦ĸ çļĦ","B ull","K ate","Ġd rown","æľī åĢĭ","å¿ ¡","cl o","èĩª ä¹ł","Ġev oc","çϽ å±ħæĺĵ","Ġke adaan","åħ´ 建","æĩĤ çļĦ","çĤ¼ åζ","åħĦå¼Ł å§IJ妹","Ġlymph atic","( height","d ling","al ignment","Ġd ni","Ġk val","ower ed","ä¸ĩ èĤ¡","Ġimpro v","à¥įठ¡","Ġod m","Ġentre v","Pre ferences","Ġê´Ģ íķľ","λε Ïħ","ĠGlac ier","Ġaccret ion","Ġth orn","åľ¨ æ¯ı个","Ġk odea","åĴĮ æľī","act in","æĦı 念","æ°Ķ 缸","ĠAb normal","å¸Ĥåľº è§Ħ模","ih ak","vis er","å»¶ 误","Ġ×ķ× ©","ĠBel ize","Ġgro ep","Ġliberal ism","ĠÑĦÑĥнк ÑĨий","REF IX","ικο ί","c w","| ^{","or in","Ġr in","å®ļ åŀĭ","erv ative","ä¸Ģ个 åŃĹ","eng agement","ла ва","CO OH","Ġà¦ı à¦ĸন","ĠVir al","èµı æŀIJ","åĪĽå»º çļĦ","Ġপà§įরà¦ķ াশ","Ġpertain s","ÏĮÏĦη ÏĦα","Ġt l","ä»ĸ ä¹Łä¸į","çĻ «","Ġfl ere","Ġfl ung","Ġpartic ulièrement","åŁİ åįĹ","çĭ¬ åѤ","ĠاÙĦت س","åįĸ ç»Ļ","ĠTables poon","Ġczas u","Ġjel as","ĠСе веÑĢ","ĠRut gers","id io","ĠM ord","è¿ĺ 对","äºĮ åı·","éĵ ¿","çİĭ 大","Ġgovern s","æłij ç§į","æĺ¯åIJ¦ åı¯ä»¥","à¹Ģà¸Ķ ืà¸Ńà¸Ļ","Ġfrecu encia","Ġruth less","Ġre open","Ġal te","æľº æŀª","éļı å¿ĥ","表示 çļĦ","éĻIJåζ äºĨ","以æŃ¤ æĿ¥","æıī äºĨ","ĠBron x","Ġmyel oid","ĠEins atz","ĠA ten","ĠW age","è¦ģ 大","ï¼ļ âĢĺ","á ss","å¹¶ å°±","ĠData Frame","實 è¸IJ","Ġhypot en","Ġmoist ur","ĠÂłĠÂł ĠÂł","ĠF elipe","ition ers","缴 çļĦ","女 åŃIJçļĦ","太 éļ¾","æĺ¥ è¿IJ","æ²Ĵ äºĭ","âĨ µ","ĠÏĢ Î±Ïģ","è®¤çľŁ èIJ½å®ŀ","ĠRod ney","éħ¿ éħĴ","ĠDemon str","-Col a","ĠS lavery","èĢĮ åIJĮ","æķ° 次","Ġcar ers","ÅĽ ni","ĠÕ ¹","ĠAnn ounce","ĠPra xis","æĴ° 稿","-gener al","Mag ic","ĠженÑīи н","ĠMisc ellaneous","åĻ© 梦","S IM","re kt","Ġtr atar","å¦Ĥ åīį","é«ĺ 楼","åIJĦ çıŃ","çļĦä¸Ģ å®ļ","ä¸Ģ缴 éĥ½","åĵ² çIJĨ","Ġdeux ième","ĠIter ator","( view","Ġreg rets","eng ed","up mu","ĠTr igger","åĨľ æŀĹ","è¯Ĺ éĽĨ","éĸĵ çļĦ","Count ing","Reg istered","Ġital iani",".res olve","T am","h are","é«ĺ æĸ¯","âĢĶ âĢĿ","ĠZ ust","', $","Ġav alan","ä¸įä¼ļ æĺ¯","Ġstress ing","ãģı ãĤīãģĦ","ĠSupp lier","ĠLear ner","Ġcorpor al","è¿« 害","ì¹ ¨","Sty led","ĠÙħØ´ خص","ĠTrain er","ĠTud or","Ġremun eration","/ <","E ither","b idden","m ur","è· ·","课 åīį",".f ont","æİ¢ æŁ¥","اض ر","Ġels Åij","ĠиÑģполÑĮзÑĥ ÑİÑĤÑģÑı","åħĪéĶĭ 模èĮĥ","Ġund ist","ĠÙĦ ÙĤ","åį¡ éĢļ","åĢĴ éĹŃ","Ġbrilliant ly","aille urs","Ġj ub","åIJĦ éĥ¨","ε Ïħ","Event ually","ĠK K","èĢĮ 她","ys ÅĤ","åĬł åĢį","ĠDe le","Ġins ensitive","æĪĺ ä¸Ń","Ġб еÑĢ","ĠÙĥ تب","çIJĨè§£ äºĨ","Ġcov ari","æ¼Ĥ æµģ","Ġà¶ ´","ĠFat igue","ä¸Ŀ毫 没æľī","Ġinfl ow","Ġج ÙĨÚ¯","æĺ¨ å¤ľ","ç¨İåĬ¡ æĢ»å±Ģ","dep artment","Vari ables","Ġex termin","èĢħ åı¯","Ġprov a","Ġhel fen","åıĺ çݰ","ĠPl atinum","Ġpop ulate","Ġsum mons","iet a","åıijçĶŁ çļĦäºĭæĥħ","Ġব à§ĥ","æľ± çĨ¹","تÙħ د","Ġkit chens","ãĥģ ãĤ§","ĠBurn ing","ongs To","ĠзнаÑĩи ÑĤелÑĮно","奥æŀĹ åĮ¹","çļĦ æıIJé«ĺ","ĠL OW","ĠO lig","). #","èĢĮ åħ¶","ä½į ä¸Ĭ","-s i","new command","è³ ľ","Ġconfig uring","Ġhall mark","çĽĨ èħĶ","ĠкÑĢа ÑĤ","Ġmotiv ates","Ġsquee zing","ĠResp ir","J our","r ification","} ')Ċ","ĠW oo","èĩ §","Ġac claim","Ġ# ĊĊ","èģĶ æĥ³åΰ","Äħ Äĩ","ĠMed ication","à´ ³","Ġdise ased","Ġbar ang","ĠÛĮ عÙĨÛĮ","ĠRef lex","áĥĶáĥ ¡","Ġsubstit utions","çĶŁæĹ¥ å¿«ä¹IJ","æµĵæµĵ çļĦ","Ġpro gres","ĠN omin","没æľī éĤ£ä¹Ī","让 ä½łçļĦ","Ġmult it","Ġcalcul ators","Ġmicro environment","æįĨ ç»ij","Ġkidn apped",". +","D omin","_ true","Ġl ø","ess ere","ر ت","cl s","é«ĺ åĪĨåŃIJ","èĩªå·± è¦ģ","è£ħ åľ¨","Ġtim etable","ĠاÙħ رÙĪ","Ġtres pass","Interest ingly","ĠAdvance ment","F V","L am","ĠM k","ĠH inter","az an","Ġchang er","-st ud","æĦıè§ģ åĴĮ建议","å¼· åĮĸ","Ġneuro s","Gener ate","ĠFac ilit","ĠGru ppe","Ġbez pie","Ġdern ière","ĠMeet ings","ĠDIST RICT","- road","ä¹ĭ 交","ä¹ĭ æģ©","ĠCom es","两 ä¸ī","à¹Ħ à¸ĭ","Ġconvert ible","ĠDevelop ed","Ġtang led","çļĦ å½¢çĬ¶","ĠW rap","åĴĮ å®ŀè·µ","å¦Ĥ èĭ¥","Ġ×Ķ× §×","æĿİ åŃIJ","åįĩ èĩ³","éĻĪ çļ®","ç©¿ è¡£","è¬ Ļ","æľīä»Ģä¹Ī åħ³ç³»","éĴ» äºķ","ĠAus chwitz","ĠRout ing","pay load","ç¬ĶèĢħ 认为",". active","ar oo","Ġا صÙĦ","ĠRe inh","åıĬ çŃĶæ¡Ī","Ġac ab","æµ· å°Ķ","áĥ Ĵ","Key board","ende z","à¸Ľà¸£à¸° à¸Īำ","éļ¾ä»¥ 置信","ĠOs borne","Ãī tat","superscript subscript","ĠNathan iel","( options","al era","Ġre used","ä¸į 详","se v","说 ä¸Ģä¸ĭ","Ġfe ud","çŁ³ åŃIJ","ĠAb del","col s","la id","Ġrh ymes","ĠPH YS","çĿģå¼Ģ çľ¼çĿĽ","çIJĨ èµĶ","ree ze","de ath","ÏĦ Ïİν","Ġgl ances","าร à¸ĵ","ĠArch itects","rend e","æĸľ çİĩ","åķĨåĬ¡ éĥ¨","ĠدÙĩ ÙĨد","Ġvertebra e","( iv","Ġc é","好 æ¯Ķ","ĠÙĨ د","æĭ¿ åİ»","ä¸ĩåħĥ 以ä¸Ĭ","ĠÙħÙģ Ùĩ",", Q","ong ru","д ÓĻ","éĤ£ ä¸Ģ天","æīĢ以 她","Ġthin ly","Ġfon te","Ġ구 ì¡°","J n","_ ms","åľ¨ å¸Ĥ","Ġra ging","ãģ® åł´åIJĪ","Ġrequ er","Ġter rest","ëĬ IJ","å¯Ĵ é£İ","×ľ× Ĵ","åħ³éĶ® åľ¨äºİ","Par agraph","æĬµ æī£","çĶľ åĵģ","ĠCatal unya","äch lich","à¸Ľà¸ģ à¸ķิ","à¹Ģà¸ģษ à¸ķร","& =","ĠF N","è¿Ļ个 çĶ·äºº","èĬ± æľŁ",".S printf","Ġmother hood","ÐĿ и","ĠOrth op","Ġszko ÅĤy","à Ķ","id ou","äºİ 人","çĿĢ å¥¹çļĦ","çŃī éĥ½","Ġph antom","çĹħ æ°Ĺ","eter ia","ĠSc and","ĠPaul ine","Ġá¼ ¡","×ķ×ij ×ķת","ĠTai pei","衬 æīĺ","ĠHold en","Ġouts ider","çķľçī§ ä¸ļ","Ġapprentices hip","ĠDebb ie","ic ating","Ġl izards","Ġv yp","ay at","æĭ ®","ä¸ĩ è¾¾","è¿ĻäºĽ äºĭæĥħ","åĽ¾çīĩ æĿ¥æºIJ","ĠNi agara","è¾ĥä½İ çļĦ","- price","} b","å¹ ¡","ia x","å±ķ ä¼ļ","åŀĭ ä¼ģä¸ļ","AT IC","-t ri",".t oken","åī¯ åİ¿éķ¿","Ġbuff et","çļĩå¸Ŀ çļĦ","Ġmism os","ĠÑĢаÑģÑģ ÑĩиÑĤÑĭ","Ġecclesi astical",") y","he er","Ġn imi","以 å®ŀçݰ","Ġdi j","æŃ¥ æŀª","åī¯ äº§åĵģ","-st at",".M in","æ³ķå¾ĭ åĪ¶åº¦","åĽłç´ł çļĦå½±åĵį","æĽ¿ ä»ĸ","éĩįè¦ģçļĦ æĦıä¹ī","Ġtac it",".Hash Map","Ġsufic iente","Ġsu elo","åĩº å¾ģ","å͝ å¿ĥ","Path Variable","æ¡ĥ æºIJ","æ¯ģ äºĨ","Ġepid ermal","ĠAx el","( client","_ mean","ess ler","ç͍ å°ı","Ġem per","cy d","çŁ¥ éĿĴ","ä¸ĩ èĥ½","åĬŁ èĢĹ","éļ¾ å¾ĹçļĦ","{{ {","Ent ities","æĻºèĥ½ åζéĢł","ĠìĪĺ íĸī","Ġperm is","Ġrent als","ĉt mp","ĠвелиÑĩи нÑĭ","à¹ģà¸Ĺ à¸Ļ",", ooo","_ prefix","以 æľŁ","Ġem its","å½ĵ ä¸ĭçļĦ","æľº ç¼ĺ","çĸ Ł","å¾ħ 人","æĿ± æĸ¹","è·¨ 度","ĠNan op","ðŁĴ °","Ġdiscre et","à¸ŀัà¸Ļà¸ĺ à¹Į","ĠQUEST ION","Ġc iencia","ĠL TE","æĪij åIJ¬","æĪij æĺ¯ä¸Ģ个","å°± 以","Ġwill en","ĠSt abil","åĮĸ éªĮ","éĩį ç͍","æĹł æĿĥ","ç¾İ å¦Ļ","ç§ij åįı","Ġdon na","Ġpot rebbe","第ä¸Ģ éĥ¨åĪĨ","ä¸įèĥ½ 满足","èĤ¿ åĿĹ","Ġses ame","noÅĽci Äħ","éĴ¢çŃĭ æ··åĩĿåľŁ","ĠHolid ays","Ġre think","ĠS erving","ld on","ĠDep osit","产çĶŁ å½±åĵį","ĠÑĢаз ÑĢÑĥ","æľĢç»Ī è¿ĺæĺ¯","Ġital iana","åħ¸åŀĭ æ¡Īä¾ĭ","Ġcra bs","å¸ĪèĮĥ åѦéĻ¢","ĠlÃŃ der","éĽĮ æ¿Ģç´ł","ĠPeg gy","/ )Ċ","| }","ter al","ĠJ em","Ġsub contract","اÙĦ س",".S pring","éĿĴ èıľ","Ø· ÙĬع","_c ard","roid ery","æ·¡ åĮĸ","Ġthr ives","éĶ» éĢł","Ġpúblic as","è¶ħ声 æ³¢","æĻ®æ´± èĮ¶","éĤ¯ éĥ¸","ber ta","Ġab iotic","Ġtra iled","ä½ľç͍ æĺ¯","å®ŀæĸ½ ç»ĨåĪĻ","å·¥ä¸ļ åĩºçīĪ社","çī¹çĤ¹ åĴĮ","Ġjej ÃŃ","+-+- +-+-","Ġoud ers","obac illus","ĠMemor andum","ĠDEVELOP MENT","( child","n iki","ä¸Ģ个 æĸ°","Ġbet re","èĢģ çι","Ġer as","Ġhum iliation","irc ular","åΤ åĪ«","çĮ® ç»Ļ","Ġsz á","ĠUN C","av l","ĠX Y","ĠX ing","å¾Ģ æĹ¥","ĠAb ril","ाठ§","ĠÑĢе ÑĪи","ĠÑģÑĤа нов","ä»İèĢĮ 导èĩ´","ĠEX T","æĺĤ æī¬","Ġnh ất","ãģ» ãģ¨","Ġги пеÑĢ","ĠпоÑĩ емÑĥ","à¹Ģà¸Ħราะ หà¹Į","N GC","Ù «","ä½ł è¿Ļæĺ¯","åīį åįģ","ов е","失 äºĨ","ĠBl ogs","ä½Ĩæĺ¯ ä»ĸ们","Ġant igu","ĠÙĥ ÙĪØ±Ø©","以ä¸ĭ åĩł","िठª","ìĭľ íĤ¤","Ġcomplain ant","ĠзаÑīи ÑĤÑĭ","Ġgénéral ement","Ġì¸ ¡","Ġc ac","çļĦ 巨大","Ġto l","åѦ è¯Ĩ","Ġhelp ers","æİĴ 便","................................ .","Rel igion","æĪĺæĸĹ æľº","æ¡Ĥ æŀĿ","à§Ĥ ম","ĠìķĦ ëĭ","Ó© ÑĢ","à¸ŀุ à¸Ĺà¸ĺ","at m","Ġb art","et code","ĠCh olesterol","Ġsur ged","osp atial","ä¸ĸçķĮ ç»ıæµİ","UR Y","èĤī è´¨","æķ´ä¸ª è¿ĩç¨ĭ","ĠEss entials","Ġb é","çļĦ åΰæĿ¥","ct ype","æİ¥ éĢģ","ĠPr zy","åĽ¢ èģļ","Ø· ÙĨÙĬ","ç©¿ èijĹ","ĠØ¢ ز",".out put","ĠSal vation","忽 æĤł","Ġpun itive","ç¬¬åĽĽ 次","æĸ¹ç¨ĭ 为","ãĤª ãĥ³","ĠاÙĦÙĪØ·ÙĨ ÙĬØ©","Ġ ĉĉ","up aten","æij Ĵ","è¿ij çϾ","æĪ¿ åŃIJçļĦ","ÑĤÑĭ м","åĿļæĮģ ä¸įæĩĪ","å¿į èĢħ","è°ĭ æ±Ĥ","ĠMir iam","Ġlam inate","F IN","T reat","ar ach","iz ando","Ġso i","еÑĤ еÑĢ","èĩ´ çĻĮ","Al bert","è³ ¬","å¦Ĥä½ķ çľĭå¾ħ","é¤ ĵ","ĠMo ist","ĠпÑĢодÑĥк ÑĤов","ĠHait ian","ĠRasp berry","w asser","åľ¨ æĸ°çļĦ","Ġun idad","Ġapp art","ä¿Ŀ 驾","» ØĮ","ĠEd mond","Ġbul ly","ĠStre ets","PP PP","èĤ¾ çĤİ","ĠHal ifax","ĠFriends hip","compet itive","ĠAdjust ed","ĠاÙĦدر اسة","ĠZusamm enh","W is","e ating","Ġs uture","ĠR X","好 书","Ġtrans missions","Ġcar ic","ç³»ç»Ł åľ°","à¸Ī ีà¸Ļ","缮åīį åľ¨","ĠÙĪØ§ÙĦ ÙĤ","æľīä¸Ģ 段",".re verse","æĢ»ä½ĵ ä¸Ĭ","ugin osa","Ġprefix es","ĠмаÑģÑģ Ñĭ","( email","ĠI MD","ĠH ogan","Ġint oler","Ġz acz","éĢļ ãĤĬ","西 è·¯",".m ock","Ġж ена","ĠKe pler","Ġshelter ed","ä½łçŁ¥éģĵ åIJĹ","ÅĽcie j","Ġglyc ogen","b v","Ġdis ple","Ġknow ingly","éĹ®é¢ĺ äºĨ","ìĹ ĩ","Ġinit iates","å®Įåħ¨ ä¸įåIJĮ","è¾ĵåħ¥ çļĦ","ĠAR C","Ġindel ible","m oment","Ġ วัà¸Ļ","es imal","å·¥ä½ľ è¿Ľè¡Į","è¾¹ å½¢çļĦ","}\\) \\(","æĺ¯ä¸Ģ éŨ","åIJĮæĹ¶ 对","ĠMod er","Ġsurn ames","ĠWARRANT Y","æ·Ħ åįļ","H arm","g els","Ġp ep","Ġyear ning","æĪij们 å°±åı¯ä»¥","ä rm","ems et",".add ress","cor por","Ġtransplant ed","Ġtys iÄĻcy","Ġëģ Ŀ","Ġinteroper ability","ĠC en","Ġv ene","л Ñijн","è¦ģ åħħåĪĨ","å¤ļ å±Ĥ次","Ġ' ,'","天 ä¹ĭ","Ġtra ys","åĪĩ 身","çªģ èµ·","EM PL","æ»ij 稽","渡 è¿ĩ","Red is","loc ale","Ġutiliz ando","ĠíĻľ ëıĻ","ĠSiem ens","Ġf ret","ĠF K","åIJİ ä¼ļ","éĤ£ å°ı","ĠCon cerning","é¦ĸ éķ¿","æĶ¿æ²» å®¶","Ġfresh ness","|| ||","Has Column","ç¥Ī æ±Ĥ","Ġa and","Ġk itt","ug as","æŃ¤ æ³ķ","æĬĢ å¸Ī","-d oped","åŃ¦ä¹ł æĪIJ绩","ç͍æĪ· åIJį","ĠUN IT","éŁ³ä¹IJ ä¼ļ","çļĦæ°Ķ è´¨","ĠÑĢо ÑģÑĤа","- client","ĠR ÃŃo","ak ak","ä¸Ń åı¯ä»¥","å°± ç»Ļ","Ġall otted","é¾ Ī","请 åľ¨","}\\) /","avig ate","å¿ĺ è¨ĺ","ĠAN N","Rem ark","财产 å®īåħ¨","ĠAltern ate","ĠÑģÑĤÑĢа не","Ġgem acht","Ġtoss ing","žit ÃŃ","» ê²Į","ed ited","ĠB ihar","è¿Ļ 表æĺİ","å¤ļ åľ°","ĠRe pt","å¹³ 庸","ç¡® ä¿¡","ج اÙĨ","æ´Ĺ å¹²åĩĢ","اÙĩ ÙĬÙħ","Ġkn ob","Cor porate","ĠLE VEL","è©ķ åĥ¹","ãĥ¯ ãĥ¼ãĤ¯","Ġnewborn s","ุษ ยà¹Į","§ ש","-b el","é£Ł è°±","æĭī å¼ĢäºĨ","è¿Ļæĺ¯ ä»ĸ","Ðľ Ñĭ","Char acters","Ġprzy czyn","Access ed","\" S","L ot","¦ ×Ļ","ic u","ĠH ahn","çī¹ åĬ¡","ĠSe ñ","æīį æľīåı¯èĥ½","ç´§ æī£","ĠLa ud","ãģĭ ãģij","à¸Ĭ à¸Ńà¸ļ","Ġhub ungan","Ġcock tails","Ġb ounty","çļĦ é£İæł¼","ä¸į åŃķ","ä¹Ł åĪ«","ç³ ł","ä¿Ŀ è´¨","Ġgu er","Ø´ اء","èĩªçͱ è´¸æĺĵ","Ġgro aned","åı¹ äºĨä¸Ģåı£æ°Ķ","寥 寥","Ġbuz zing","Ġt ë","为 客æĪ·","åĴĮ æĶ¹è¿Ľ","Ġbi oc","ĠDis patch","幸 åŃĺ","Ġà¦Ĩ à¦ľ","å¾IJ å¾IJ","æĢĴ äºĨ","Ġfont Weight","è§£æĶ¾ æĢĿæĥ³","ĠЦ енÑĤ","ĠGastroenter ol","Ġlabyr inth","D OC","or h","Ġc ÃŃm","Ġin ÃŃcio","ĠS b","ĠS GD","ĠT ung","ans ky","çIJ °","cre ases","Ġsub ter","ĠAn o","ãģ® ãĤĪãģĨãģª","ç±» åĴĮ","æ¸ħ æľ«","èµ° åħ¥","åı² å¯Ĩ","Me eting","å¹½ çģµ","éĨī äºĨ","ÐĽ Ðĺ","Ġerm ög","l án","ĠM AS","Ġu uid","ĠK T","åĬĽ éģĵ","åĮº åĮº","è´¢ ç¨İ","帮åĬ© æĪij们","Ġwrong ly","ê² ¨","ĠBud dy","×ķ×ĵ ×Ļ×Ŀ","åı¹ æ°Ķ","ĠBuck ingham","ĠParad ox","Ġf film","éĤ£ æĹ¶çļĦ","ĠZ r","å·® é»ŀ","çģŃ ç»Ŀ","主é¢ĺ åħļ","ĠOffic ials","Ġdwell ings","N os","ĠL ESS","æīĢ åŃ¦æł¡","å¼Ģ 端","éĤ£ åĿĹ","ä¹IJ åĽ¢","ä¸ĵåĪ© çĶ³è¯·","Ġante ced","åĺĹ è©¦","ĠàªĽ à«ĩ","çļĦ æ¯ĶèµĽ","Ġcomm as","åıĹ éĺ»","æľį å½¹","Ġmen cap","Ġconcept o","CT S","Ġrend ah","OV ER","éŁ¿ èµ·","ĠSubs idi","ĠاÙĦا Ùĥت","H erm","e ck","ĠC PA","ঠĿ","åıijå±ķ 为","ઠ²","log s","ä¸ĵä¸ļ 课","_T EST","å®ŀè´¨ ä¸Ĭ","Ġgeomet ries","observ ed","H AM","ri ko","Ġhe ure","Ġsom a","-S axon","Ġfast ened","cher y",".pro ject","Ġcs ak",". with","F ax","_ ]","Ġ ial","ĠT alm","Ġdis ordered","ert ools","ĠSp ending","å¾® é£İ","ĠÙĬ Ùĥ","light ly","sub stant","ç¿° æŀĹ","Ġprejud ices","Copy With",". «","in crease","ĠC arly","大 头","ĠEn rollment","çį ħ","æľ¬èº« å°±","Ġheter osexual","ĠJon ah","ಾಠ¨","飵 åij³","quer que","amps ia","opath ological",") ·","çļĦ ç»ĦåIJĪ","ĠP Q","Ġpro jets","ĠV ALUE","åĪĨ éĥ¨","Ġup he","Ġsc rit","Ġpower less","Ġsing ly","Ġsam men","ĠÐŁ ÑĢави","è°Ī ä¸įä¸Ĭ","ãĤ¹ ãĥĿ","zo a","Ġemphas ised","Ġextrem ities","Ġdeter rent","Ġvern acular","U g","c annot","Ġh izo","Ġj eg","lic zba","åIJĥ èį¯","ç»ĵæŀľ 为","Ġcoord in","Ġram ifications","ãĤ« ãĥ«","ĠMind fulness","ĠаÑĢÑħи ÑĤек","ĠOun ce","CHANT ABILITY","L X","ot emporal","å¹´ å¹³åĿĩ","åľ° éĿ¢ä¸Ĭ","ઠª","icht et","Ġsac ra","Ġtub ig","éļ¨ æĻĤ","Ġдан ном","å¼ĵ ç®Ń","Lab our","Ġexplos ives","ĠS EE","arn ish","ĠVis ible","å±ħæ°ij çļĦ","Ġpossess ive","åĪijäºĭ æ¡Īä»¶","à§ĩল à§ĩ","Ġmö g","ĠÑĢоди ÑĤелей","Dam age","Axis Alignment","ĠS crib","ĠT ons","åΰ æĻĤåĢĻ","çģ« èħ¿","èijĹ æľī","án ica","Em ma","ĠOR GAN","ĠÑĤи ÑģÑı","尤为 éĩįè¦ģ","Ġaneur ysm","ĠSain te","ch arts","ع ÙĦÙħ","Ġsl apped","éĢĻ éĩĮ","æŃ£å¸¸ 人","ĠPhil ips","ĠFred die","ĠPros per","ul ing","ĠIn clusive","éĽ ij","ла йн","ĠÙĦ ÙĩÙħ","Se ed","ĠString s","éĥij å·ŀå¸Ĥ","æĺ¯éĿŀ常 éĩįè¦ģçļĦ","Ġgehö rt","ar od","Ġk ota","ĠSt off","ç¶ Ļ","fin ancial","} d","Ġd uc","ig rants","ĠK ins","æīĢ ç§°","æ¯Ķ åħ¶ä»ĸ","Ġdef lect","лÑĮ Ñı","ãĤĴ ãģĬ","ĠBo is","ائ ج","è¶³å¤Ł äºĨ",". header","O u","t ur","Ġ ÉĻ","Ġs ón","ĠE SR","åĴĮ åIJİ","ä½ľ å¼Ĭ","èĩª åªĴä½ĵ","å¿ĥ åŃĺ","reg istered","log os","ÐŁ ол","à¶ §","jet o","Ġcro pping","Ġmol te","ĠÑĢ Ð¾Ð´Ð°","ؤ ÙĦ","Ġsummar izing","ĠвозÑĢа ÑģÑĤе","Ġlum ière","Ġa leg","Ġin cess","ĠA ES","ĠC AB","Ġha ze","๠ķ","åĩº çϼ","ä¹ĭ èĻķ","çĿĢ åij¢","æĥħ åķĨ","ä»ĸ们 å°Ĩ","åĽ´ æ£ĭ","é¢ij è°±","åĢŁ éĴ±","Ġutil ised","ìĭĿ ìĿĦ","à¤ľ ़","é«ĺå°Ķ 夫",". tt","A ld","C ouncil","Ġ_ {\\","In sp","-m en","Ex erc","Le od","Ġcounter act","Ġ§ §","Ġburg l","Ġwrink les","ĠآزÙħ اÛĮØ´","æĺ¯ å®ŀçݰ","Ġun popular","ä¸ĭ å²Ĺ","ÃŃ me","áĢ ľ","åįĥ æĸ¹","_f ull","С е","ĠProt ective","Gener ation","ĠTan aka","Ġdemol ished","Ġanisot ropy","< any","W el","Ġem ulate","æĬ¥ 社","çļĦ人 çļĦ","ĠEm ission","åĩı æĮģ","éĺ¿ æĸ¯","Ġskin ny","è·¨ çķĮ","ĠRun ner","Ġzak res","Ġeru ptions","ĠпÑĢÑıм ой","ĠpÅĻÃŃpad ÄĽ","å¼Ģ éĶĢ","她 èĥ½","æİ¥ ä¸ĭ","aj ÃŃcÃŃ","æĸĩåĮĸ 大éĿ©åij½","è¡£ çĿĢ","Ġdw ar","ĠÄį asto","ãĥĹ ãĥ¬","åΰå¤Ħ éĥ½æĺ¯","Ġsuck ing","out side","被 ä½ł","sh oe","ĠEm m","å¸ĮæľĽ åľ¨","åİħ éķ¿","ĠLab rador",".r b","there fore","isse z","à¸łà¸²à¸© า",". action","S ensor","x k","or os","Ġw rench","æĪij ç»Īäºİ","Ġinter cepted","è´¨ æĢ§","Ġbest owed","Ġpay off","第äºĮ éĺ¶æ®µ","ÙĪÙĦ د","ðĿij Ĩ","ÙĬر اÙĨ","ming ton","æ·±åħ¥ çłĶç©¶","ĠRequ irement","Ġê²½ ìłľ","ĠнеболÑĮ ÑĪ","L ost","¡ ×ĵ","´ Ħ","ill é","Ġjust e","åı¤ 迹","èĭı éĨĴ","ذ ÙĬØ©","CO D","è°· çī©","Ġnuest ras","R W","n ake","ĠL ONG","Ġr ych","ÃŃ ce","空 éļĻ","ih in","Ġkil ka","æĻ®éģį çļĦ","ĠSoc io","ĠмÑĥ ниÑĨипа","-Rel ated","d ock","ch ial","ĠM itte","ind ÉĻ","æīĭ æŁĦ","fl ags","触 碰","æ§ ¿","fil ms","ê¸ ¸","initial ize","ythm ias","çIJ¥ çıĢ","æĭ¿çł´ ä»ij","< th","f ö","Ġt rom","Ġw ikipedia","ĠH ui","ব াদ","é«Ķ åħ§","ĠRout ine","à®ķà¯įà®ķ à®®à¯į","Ġห าà¸ģ","ÏħÏĥ ιαÏĥÏĦ","m ile","ed ip","ub i","class Name","ás ok","Ġ×IJ× ĵ","å̾ è¯ī","ĠConst raints","-ex istent","ế p","ĠMarx ism","Ġtravers al","Ġtweet ed","ĠSOFT WARE","Ġt ep","qu ite","ä¸į éĤ£ä¹Ī","ä½ł å¾Ī","ĠÑģ ÑģÑĭ","ä¸ī ä¸ĥ","Ġsens ibil","ĠÏĥ Ïį","ĠSH O","ĠобÑĢа ÑīениÑı","глÑı д","ĠPseud omonas","ĠLIM ITED","Ġfórm ula","ĠF iled","Ġform ulating","Ġaut ant","ç»Ħç»ĩ ä¸Ń",".d emo","æĺ¯åIJ¦ 为","리 ìķĦ","Ġaqu aculture","گاÙĩ ÛĮ","Ġash ore","ĠDemon strate","ç¥ŀå¥ĩ çļĦ","ac ija","åľ¨ åĮ»éĻ¢","èĭ Ĵ","éĺ² æ´ª","Ġза пол","-M arie","Ġпов ÑĢежд","ĠÙĨÙħ ÙĪØ¯","ðŁĺ Ĥ","ĠCly de","B uf","x m","çļĦ æĿ¡ä»¶ä¸ĭ","ĠL ois","缸 éĢĤåºĶ","æ¶ ĵ","æĹł ç͍","æĪij们 åıĪ","ĠWe iter","å°±æĺ¯ è¿Ļä¹Ī","å¥ĩ æķ°","cred ited","ĠTeas poon","ĠT unnel","iv ative","äºĨ çīĩåĪ»","åĩº åħµ","天 æĢ§",")) ]Ċ","Ðļ о","nik ov","Ġaugust i","Ġbic arbonate","c ells","| _{","æĪij éĥ½ä¼ļ","Ġer ased","Ġspe ck","人æ°ij çĶŁæ´»","ĠCons ensus","ĠSam ar","Ġcycl ical","åħ·å¤ĩ äºĨ","Ġobs ah","-fl ight","Ġinert ial","é¢ħ åĨħ","¿ IJ","Ì £","ic ola","ĠB IT","Ġj wt","ĠÙĪ Ø§ÙĨ","æĬĬ ä½łçļĦ","äºĨä¸Ģ å¹´","ĠShe ep","Ġbr ine","Ġ×IJ ×Ļש","мÑĥ ли","ç¨Ģ å°ij","áv ánÃŃ","ĠFu ÃŁ","Ġglut athione","ĠLot us","ĠLaf ayette","ĠP oh","åŃŠ第","ĠâĪ Ļ","Ġair ways","åIJĪä½ľ åħ³ç³»","Ġstru kt","åĨĴ åĩº","Ġsan itary","Ġvra gen","_ offset","il ess","ĠP asc","æŀģ åĵģ","èĭ± ç¾İ","ĠÙĬ ا","At Index","æĸĩåѦ çļĦ","Ver b","Ġacceler ates","< P","为 æķ°","ĠRe chts","åºĶ éĩĩç͍","æĹł æĺİæĺ¾","Ġmon os","åij¨ å¯Ĩ",".p oll","å³° çļĦ","ĠLand roid","ĠIr vine","Ġmir rored","ĠвÑĭпол нениÑı","( argv","_ bar","Ġs vol","ä¸Ģ å°ıæĹ¶","ĠW arb","th or","æĢ¥ åĪĩ","èľ ¿","Äģ l","ĠJul io","è´¢åĬ¡ ä¼ļ计","Ġvy Å¡","Ġcasp ase","Ġglo omy","\\ M","Ĥ à¸Ńà¸ĩ","Ġb ern","åĽł 人","ä¼ģä¸ļ ä¸İ","(b uffer","çľ¼çĿĽ éĩĮ","Ġphen ol","ĠBre ndan","ĠAff iliation","å¦ĸ æĹı","tem ps","ĠÅŁ i","G ui","w ashing","im id","ĠB IM","管çIJĨ çŃī","rid ium","ä¸įèĥ½ åľ¨","}} _{\\","ĠìŀIJ ë£Į","çķľ ç¦½","æĥŁ ä¸Ģ",") ï¼Ľ","ä¸Ĭ ãģĮ","ĠSch le","å¤ĦçIJĨ æĸ¹æ³ķ","ĠÑĦ едеÑĢа","ç§ijæĬĢ çļĦ","_pro ject","æįĤ çĿĢ","Ġbicy cles","Integ ration",". trim","_ ass","is odes","ĠS ÃŃ","ite iten","天 åºľ","áĥ ij","äºĶ åĪĨ","ઠ¸","ĠоÑĤ ве","ç§» éĢģ","æİĮ 管","ĠÑģи л","- IR","ĠP ey","ir ut","ĠPro spective","ä»» ä¸Ģ","ĠBl vd","-S aint","å·¥ä¸ļ åĽŃåĮº","ĠPot assium","Ġrég ime","ĠSatisf action","B ridge","on in","åIJ Ĵ","Ġ} ;","å°Ĩ æīĢæľī","ĠGold stein","Min or","ynt hes","abb at","/ CD","P ri","Ġw ych","Ġg c","Ġg azing","æľī è¿Ļæł·","and ar","ĠK ota","Ùģ Ø§Ø¸","ü ng","Ġvan ishing","æĸ¹å¼ı è¿Ľè¡Į","AP T","æĶ¿åºľ åľ¨","غ ÙĬرة","åıijæĮ¥ çĿĢ","æ·±åĪ» åľ°","Ġlibr arians","ĠконÑĤÑĢ Ð¾Ð»ÑĮ","( last","Ġam az","å¹³ æĹ¶çļĦ","åħĥ éĴ±","sc apes","Ġgl ancing","Ø® Ø©","Ġbi ore","çİ© å®¶çļĦ","ä»İå°ı åΰ大","ĠRank ings","ĠÑģÑĩиÑĤа еÑĤÑģÑı","h ank","ĠM org","身 æīĭ","Ġvis ibly","Ġdom u","_T IM","/n ode","Ke eping","Ġpopul ace","ĠSOL UTION","H yd","Ġd agger","åı¯ æģ¶","åıij åijĨ","Ġب س","par ad","æĿij éĩĮçļĦ","æĢª å¼Ĥ","ĠPal grave","Arch ive","ĠWol ff","ĠÑģÑĥм мÑĥ","Ġrelacion ados","Ġshamp oo","r ion","Ġl actic","è¡Į è¿Ľ","第ä¸Ģ éĺ¶æ®µ","ëĭ¤ ëĬĶ","ãģĭ ãĤĮ","ç»ĵæŀĦ ä¸Ń","à¸ŀ ลัà¸ĩ","çªģçĦ¶ éĹ´","Man age","Ġtrend ing","ĠاÙĦب شر","à¹Ģà¸Ĺ à¸Ħà¹Ĥà¸Ļà¹Ĥลยี","Ġtack led","ä»İæĿ¥ 没","ĠпеÑĢв ÑĥÑİ","- thing","F inder","[ ]ĊĊ","ĠS MP","pro z","马 è¾¾","ĠEr de","ĠDen ise","ĠSeason al","ĠnÄĽ kter","+ V","j pg","ri i","ĠSt ages","ember g","ìĹ Ĩ","端 çĿĢ","uj an","æľ« æĹ¥","лов ой","ĠRoman o","czy k","- rad","[ cur","Ġl izard","æĪij åĢĴ","ure tic","å°± åĪ«","åħ¨ èĥ½",".p ow","Ġsubs urface","è¿ŀç»Ń æĢ§","åľ¨è¿Ļ个 è¿ĩç¨ĭä¸Ń","Ġল à¦ķà§įষ","åѦåīį æķĻèĤ²","P el","Å ı","Ġre ed","缸 ä¹ĺ","æ¸ħ æŁ¥","æ¤ ¿","æ¯Ķè¾ĥ 好çļĦ","æľīæķĪ æľŁ","鬼 ç¥ŀ","æľºåζ çļĦ","Ġnan ow","ãĥ¼ãĥ ĸ","Ġdissem inated","ĠÑģÑĤаÑĤÑĮ и","p owers","Ġm apper","é nek","Ġev okes","交 æ±ĩ","Ġread ability","ä¾Ľ æĩī","Ġpolit ic","åİĨåı² ä¸ĬçļĦ","积æŀģ åĪĨåŃIJ","å´ Ĺ","åģı é«ĺ","å¢ĵ åľ°","Ġvy u","ĠKend all","ĠMöglich keit","Ġfantas ies","åIJİ åı¯","iment ation","-p arent","Ad s","const rained","çļĦå°ı æľĭåıĭ","æĭ¥ åłµ","çĽ¸å¯¹ åºĶçļĦ","èIJ¥åħ» ä¸įèī¯","Ġobsc ured","\" C","D avis","§ ×Ļ×Ŀ","Ġd ziaÅĤa","Ġpre cio","Ġhel ical","éĽĨ ä¼ļ","ç»ıæµİ æ´»åĬ¨","ä¸Ģå®ļ ç¨ĭ度","éĵ¶ æĿı","çĹħæ¯Ĵ çļĦ","ĠдеÑģÑı ÑĤи","Ġhypoten use","D ied","Ġp óÅĤ","Ġv ars","åħ³ ä¹İ","Ġâ ĭ","}\\) (",".l ib","Ñģе да","课ç¨ĭ æłĩåĩĨ","Ġdisp ensing","éļ¨ ä¾¿","Ġgrate fully","çķ« éĿ¢","ológ icos","Ġmening itis","( order","ra du","OR TS","Ġ×ľ× ł×","è¯Ĺ æĸĩ","ãĥķ ãĤ§","ä¿ĿçķĻ äºĨ","Ġconstru ção","åĸ§ åļ£","ĠÙĨدار د","n ard","ĠS ow","ĠK ohl","Ġا تÙģ","ز ÙĪ","Ġlo fty","Ġر ÙĪØ³","ĠText Style","éĺģ ä¸ĭ","Ġcomplement ed","-pl atform","Ġoppos ites","Jenn ifer","Ġst il","In strument","ï¼ī âĢľ","Ġmod ifies","å¨ Ľ","å¹³åĿĩ æķ°","------------ ---","ĠÕ©Õ¾ Õ¡Õ¯Õ¡Õ¶","S ECTION","T amb","çļĦ éĥ¨ä½į","ĠC BT","est imated","æŃ Ĩ","ard y","ä¹Ł 称为","ep artment","è¿Ľ æ°´","æľº å¯Ĩ","Ġwar p","å®Įåħ¨ çļĦ","ĠJe hovah","Ġод ном","Ġho pping","ĠEX PER","λι ÏĦικÏĮÏĤ","ĠÑģодеÑĢжа ние","\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\","ĠпÑĢедна зна",", ]","_ flag","c ida","| <","est ration","ĠH BO","ĠBl a","循 çĴ°","Ġdispos itions","ĠÙħست ÙĪÙī","ĠHoff mann","ç¬Ķè¶£ éĺģ","- ending","Ġall ora","主 è½´","åĽ¾ ä¸ŃçļĦ","Ġbest im","ί ζ","ìĤ¬ íļĮ","Ġconsegu ir","Ġbrom ide","' ar","ĠT ide","her st","ä¸į è´¥","ab leness","iku uta","ellation Token","direct ory","á rs","åŁº æºĸ","Ġident ificar","åľŁåľ° åĪ©ç͍","æIJŀ æ¸ħæ¥ļ","Dec oder","ç´łè´¨ åĴĮ","Num mer","æĪ¿åľ°äº§ å¸Ĥåľº","rez ent","Ġbamb ino","Ġstal ks","poss ibly","Ġb aw","åºĶ è¯ķ","à® ¨","æĬĬ æīĢæľī","ä¸įåIJĮ ç¨ĭ度","å³ ¦","Ġmut able","ÙĦÙħ ات","ĠBh utan","Ġeer st","ĠForsch ung","> ()Ċ","M BA","ĉ ĠĠĠĠĠ","Ġc aching","ig lio","Ġqu attro","å¤ļ åľ¨","Ġnum a","ãģª ãģľ","Ġgen omics","Ġ×ij× ¤",".A pi","ĠLaw yers","স à¦Ĥ","Ġtrig onometry","Ðľ и","lu or","Ġê·¸ ê²ĥ","åĽ½åľŁ èµĦæºIJ","Ġаб ÑģолÑİÑĤ","c ée","ä¸Ń è·¯","为 åħĪ","Ġme u","ile vel","å¹¶ 讲è¯Ŀ","æĬĢæľ¯ çļĦåıijå±ķ","æ´»åĬ¨ çİ°åľº","bol t","Ġcapac itÃł","çļĦæĹ¥åŃIJ éĩĮ","ĠÑģлов о","Ġenpres ak","\" ])","ot roph","ĠD iverse","ĠH ao","ĠThe ological","大 人çļĦ","Ġpol yn","å°ij åĦ¿","è¯Ń å½ķ","è¿ľ å¤ĦçļĦ","夫 人çļĦ","Ġbehav ed","Ġà¦ķ ব","Ġnorth western","Ġdesc endant","ĠDar ren","å¸ħ åĵ¥","æĦŁåħ´è¶£ çļĦ","Ġcompost ing","Ġtatto os","ĠwÅĤa ÅĽci","ĠRebell ion",") ',","F arm","ĠS ik","id t","ĠN abi","ç͍ è¿ĩ","èĢĮ ä¾Ĩ","å¾Ī çŁŃ","ĠС ÑĢед","æĪIJåĬŁ çİĩ","ör ter","è¡ĮæĶ¿ 许åı¯",".B uff","åĵŃ çĿĢ","ĠCast illo","éĦ °","Ġayud ar","F light","p ies","al ers","ĠC yrus","æľī ä¸īç§į","éĥ½ 好","åĪ© çī©","éĢģ äºĨ","ĠIS P","Ġbes økt","Ġpok ud","ëł¥ ìĿĦ","à¸ķัว à¸Ńยà¹Īาà¸ĩ","-trans form","à¸Ķู à¹ģล","Ġoutrage ous","ANGU AGE","& C","f ran","{ l","Ġd á»ĭ","ä¼ļ å¢ŀåĬł","Ġ[ {","ĠRe active","å¹³ éĿľ","ĠÙĪ Ø²Ø§Ø±","ĠAnd hra","Ġver ific","ĠMc Gu","ĠPower ful","abs ent","Ġuno fficial","ĠоÑĤно ÑĪение","Ġo cz","Ġm io","ro bi","Ġl ÃŃm","ul ner","ĠL orem","ree ks","åıį èħIJ","ä¸ĩ åIJį","comm ands","ाठ·","Ġrev olving","Ġpret ended","æ¶Īè´¹ ç¨İ","ç»Ĩèĥŀ åĨħ","اØŃ د","ÖĢÕ ¯","Av atar","ĠUtt ar","@ media","P GC","åľ¨ å¾Ī大ç¨ĭ度ä¸Ĭ","èĥ½ è¾¾åΰ","ĠاÙĦ اخ","__ Ċ","Ġpre frontal","åIJĮ é¾Ħ","她 å¾Ī","æĬĬ ä»ĸçļĦ","é£İ æ³¢","arm ee","ï¼ļâĢľ â̦â̦âĢĿĊĊ","è¯ķ ä¸Ģè¯ķ","çĶ· 篮","kt ions","设计 æĸ¹æ¡Ī","-g rowth","ba o","ĠÚ¯ ÙĪØ´","Ġplug ged","Ġhij o","Ġë² Ķ","Ġfisher y","every thing","ĠDod gers",", input","ĠK ne","ÃŃ cula","ĠTra ff","Ġfoot note","ĠÑĩа Ñģа","åľĸ çīĩ","æĸij çĤ¹","è©ķ è«ĸ","á»ĥ n","Ġfacil itation","} );","Ġl Ã¥","ĠG av","Ġп алÑĮ","×ķ× Ĺ×","Ġes cl","æīĵ åŃĹ","æīį æľĥ","Ġsk incare","ä¸įåIJĮ ç±»åŀĭçļĦ","åıĮ ä¾§","伤 æ®ĭ","mm ol","ĠMor occan","Ġtend ons","Ðļ Ðŀ","start ing","Ðķ Т","Ġpued a","ĠCore y","ĠмаÑĤеÑĢи алÑĭ","ĠfÃŃs ico","LOB AL","Ġm nie","Ġ( £","ch ol","åIJĦ 乡éķĩ","ĠGl u","åģı åĥ»","Ġauthors hip","Ġpel ig","ls x","à¥ģ न","à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ à¸Īาà¸ģ","Ġbroch ure","< %@","t reatment","Ġ urs","ĠL iet","ä»ĸ ä¸Ģçľ¼","Ġz na","å¹¶ åı¯","è§ī å¯Ł","åŃ¦ä¹ł ä¸Ń","Ġ×ŀ× ł×","åĶIJ ä¸ī","اÙĩ ÛĮ","åħļçļĦ é¢Ĩ导","说è¯Ŀ çļĦ","ĠMic rowave","ĠÔ ¿","( queue","ra ven","ä¹Ł è¡Į","çªģ åħĢ","ĠDes ire","æĿ¥è¯´ æĺİ","åīª çº¸","辦 çIJĨ","Ġ×ij×©× ł×ª","R p","c oding","m ese","s ales","ĠI CP","æĺ¯ æĮī","ï¼Ł ï¼ģâĢĿ","åıª åı¯æĥľ","Ġdon ating","ĠDeut er","P ero","Ġc ách","个 头","ĠOr te","play ing","alf a","å·´ åħĭ","ÑĢÑĭ й","table View","浩 çĢļ","ĠWal ay","Ġjou les","ĠAlban ian","æĺ¯ æĪij们çļĦ","Ġal ap","ä¹Ł åĴĮ","éĶ Ń","Ġback track","ĠFr ans","çĭĤ 欢","ĠHor ace","Ġscar let","Ġróż nych","ĠÑģли ÑĪком","е б","ĠJ óz","éĢĤ äºİ","Data Type","Ġmut ated","was her","Ġgig abits","Ġsubtract ed","Ġpriest hood","fas st","Ġmathematic ians","ĠH anoi","ä½ł æľī没æľī","ĠCh annels","ah oo","æıIJ æĹ©","Ġes pect","åħī å½±","çľ¼ èĬ±","Ġopt s","å¼ķ æĿ¥","ĠÐļ ол","ĠDec imals","æļ´ 涨","æĤł çĦ¶","Õ¸Õ ¬","del imited","Kond ado","\\ varphi","ant or","éģ ´","-p olitical","Ġر سÙħ","ĠPres idency","ola ire","èĢIJ åıĹ","æĺ¯ä»Ģä¹Ī æł·çļĦ","Ġneces idades","we k","да ÑĤ","Cl osing","èϽçĦ¶ 说","Ġsn ails","aks a","inst ruction","ণ à§ĩর","義 åĭĻ","Ġdag li","V on","и в","Ġan them","åľ¨ åIJĦç§į","itt ance","éĢī èĩª","åıijå±ķ éĺ¶æ®µ","à¸Ħ ะ","ä¸Ģèά èĢĮè¨Ģ","ä»·æł¼ 为","Ġuns uitable","ĠAster oid","ĠWinn ipeg",", len","P RE","ĠT iffany","ĠL ester","ãģ§ ãģĤãģ£ãģŁ","Ġbo oming","红 å°ĺ","äºij éĽ¾","Ġsam o","And y","ĠÙ¾ ار","ĠText book","ĠVis iting","ĠпеÑĢе Ñģе","æĿ° åĩºçļĦ","Ġà¹Ģภ§","à¯ĩ à®°","Ġtrim ming","Ġarqu itect","ĠBulld ogs","ĠÙħØ´Ú© ÙĦات","Ġsubdu ed","Ġتاث ÛĮر","al ien","ä¸Ģ æİĴ","Ġag ama","ĠÑģ мож","æľº æĻº","Ġset e","书 çĶŁ","åĦ¿ ç§ij","Ġday a","Ġleg ge","ç¾İåĽ½ æĢ»ç»Ł","ĠPet it","Ġintellect ually","ĠSens ory","dec ision","ĠÑĪкол оваÑļа","_CO MP","ĠMerc er","Ġanecd otes","któ ber","an at","ĠP oc","Ġwas her","èĢĮ ä½ł","åıĬ çļĦ","åıĪ ç§°ä¸º","ĠInd irect","ĠList e","struct ures","æĮº 好","详ç»Ĩ äºĨè§£","Ġcust od","Ġdere g","ĠHeaven ly","าà¸Ĥ à¸Ńà¸ĩ","Ġpatriot ism","E K","X u","er ad","il aren","ang kat","é£İ åIJij","è¶³ çļĦ","ĠAng elo","åĢĴ ä¸ĭ","ĠIsrael is","ðĿIJ ¶","Co ordinate","-ex ec","à¹Ģส à¹īà¸Ļ",".assert True","Ġconcert ed","ç¶ł èī²","Ġevapor ated","Ġch rome","ĠK olkata","ĠDe af","èĢĥ å®ĺ","åıªæľī å½ĵ","åľŁåľ° çļĦ","Ġpand emia","ĠHu bert","éģ® æİ©","Ġmencap ai","G ran","it ely","ĠL OT","ä¹ĭ æĢ¥","oth ic","äºĨä¸Ģ çīĩ","âĪĴ (","å°Ħ é¢ij","ĠÙ¾ ÙĨج","è̳ 缮","æıĴ æīĭ","ĠPo em","Ġà´ ¤","Ġf od","çĶŁ æ°£","ä½ľ æ³ķ","ll en","çĦ ¯","ุ à¹Īà¸ĩ","éŁ³ä¹IJ çļĦ","æĮĩæłĩ ä½ĵç³»","Ġrepro ducing","_L IST","ãĤ¦ ãĥł","Ġngh ìn","è¡Ģ红 èĽĭçϽ",". pr","he on","äºĨ åĩłä¸ª","èĩ ¼","ĠU IImage","ä¸ĩ å²ģ","ç¬ij è¯Ń","ĠSch ultz","ок и","Ġmiddle ware","ä¸ī个 代表","ä¸¥æł¼ æİ§åζ","åĪºæ¿Ģ æĢ§","contin ence","çļĦ è¾ĵåĩº","om ét","ch uk","Ġ\\ ;","Ġठĺ","ÅĤ am","div isions","Ġlog e","Ġdu oden","Ùĩا ÙĬØ©","adem ia","Ġpen icillin","Ġprop el","á½ º","Ġtur meric","Ġcyt otoxicity","Ġpon ieważ","ĠCondition al","Ġmell om","READ ME","âĢį âĢį","Ġ ãĢķ","Ġm arm","ĠM ULT","ĠF ACT","Ġk idd","ä»ĸ å¿ĥéĩĮ","Ġtr unks","å°ı å®Ŀ","In stitut","ten ir","Ġresult a","Ġess as","it ets","Ġe ject","im plement","ĠL ama","åĴĮ ç»´æĬ¤","æĮ Ļ","æīĢ éĢī","ĠRe is","管 è·¯","ä¸įä»ħ åľ¨","奶 éħª","chen ko","Ġcatch ment","ĠFre und","çłĤ ç³ĸ","Ġìłķ ëıĦ","ĠVi rol","å¹³åĩ¡ çļĦ","è¿Ħä»Ĭ 为æŃ¢","K arl","è¦ģ 强åĮĸ","å¦ ©","Ġper ovsk","éĥ¨ éļĬ","be am","Ġbreak out","æĭį æĶĿ","ĠSol ved","æ»´ æ°´","Ġru pee","ĠVan essa","çī§ å¸Ī","ãĤ³ ãĥŁ","Ġcontra ception","ĠRub ber","Ġ문 ìĦľ","iw ers","ãĥķãĤ¡ ãĤ¤ãĥ«","( username","G n","æĪij æĦŁåΰ","в Ñĭй","åĽ½ 度","Ġdet alles","(\" @","è¿Ľè¡Į åĪĨç±»","æŃ» è§Ĵ","ĠFl ags","Ġsem iconductors","Ġли ÑĩноÑģÑĤи","ĠMem ories","onna ise","ĠبÙĪ Ø§Ø¨Ø©","ĠP rak","âĢĿ âĨĴ","ist ro","Ġcur ls","ç»Ħç»ĩ é¢Ĩ导","Ġt onic","ĠP AS","Ġle ans","An imals","na eus","ส ีà¹Ī","ĠÙĨ ب","ä½ľç͍ åĴĮ","ÖĢ Õ¡Õ¶","ĠSupp lies","ĠAtt end","Ġpeu ple","å¸Ĥå§Ķ 常å§Ķ","ĠFem inist","åĹ¡ åĹ¡","åķ§ åķ§","< sp","Ġt rol","ra ils","ec ks","åıĭ 们","ä»Ļ å¢ĥ","ĠاÙĦØ· ب","ĠDom ingo","ĠInequ alities","çļĦ éħĴ","Ġle icht","Ġ{ ĊĊĊ","éĥ¨ 主任","ĠBe aver","Ġaud ition","æĵįä½ľ æĢ§","ĠPort smouth","Ġ×Ľ× ĵ","ĠKath ryn","ĠVID EO","A th","M ission","R MS","¦ ׾","Ġs izing","ä¸į åıª","ĠE rit","ĠF are","大 å¼Ģ","Ġsh aky","Ġdes ks","çĤ¹ ä½į","wo hl","å·² ä¹ħçļĦ","åIJį åīį","åıĹ åİĭ","-p olar","Ġgard ener","Null Or","Ġadvers arial","Ġaproxim adamente",".Fore ign","ĠOsw ald","Ġin order","Ġv all","ark ing","åħī åŃIJ","Ïģ ία","ĠMan aged","bug s","D ies","Ġha pl","Ġpl ating","åѦ çķĮ","éĤ£ ä»ĸ","ĠAl ém","åĮ» åĺ±","Ġmus cul","/m odels","set minus","ĠStud ying","View Holder",".get Item","book ing","uh nya","Ġ×Ļ ×","ìĥģ ìĿĦ","äºĨä»ĸ ä¸Ģçľ¼","Ġfon ctions","ĠEver est","åīįæıIJ ä¸ĭ","Ġatroc ities","R oy","Ġar ra","Ġam ber","Ġent anglement","AT C","Ġpur port","çľī çľ¼","èĪį å¼ĥ","Sum mar","Ġশ ত","åŃĿ 顺","isex ual","c ivil","ï¼ °","Ġcon ect","ĠR uns","æĪij æĦ¿æĦı","ĠاÙĦ اÙĪÙĦ","äºĭ éłħ","表 å±Ĥ","ä¼ł è¨Ģ","太 éĥİ","ãģ« ãģĻãĤĭ","åį· ä¸Ģ","è¹ Ĭ","ĠÚ¯ زار","ç»ĵå©ļ äºĨ","ĠWin ning","Ġktóre go","ĠTak ah","Ġexcerpt s","- **","- change","J ordan","P aint","Ġs ane","ĠW orship","åĬĽ åĽ¾","ÃŃ culas","ĠMark us","Ġsil encing","apan ese","Ġstri pping","ĠBack up","Ġestud os","ĠNap oli","ãĥĿ ãĤ¤ãĥ³ãĥĪ","åŃIJ宫 åĨħèĨľ","P ap","P redict","re ment","Ñģ он","åľ¨ 以","Ġpre zent","uk h","ĠSim s","Ġep hemer","æ·· æĿĤ","ĠBar at","اÙģ ÙĤ","Ste ven","Ġcele bra","تÙħ بر","- url","Y U","_ player","Ġth yme","am end","æľĢ åĸľæ¬¢çļĦ","Ġins ulator","Ġstand ings","Ġaqu ifer","æįĤ ä½ı","ĠWare house","ünst ler","Ġwan ita","e asy","Ġn ore","è¦ģ ç»§ç»Ń","Ġ[ ?","one g","å°ı èĤł","Ġsl am","åı° ä¸Ń","åŃ© ç«¥","requ ires","åŁºæľ¬ éĿ¢","åĬłå¼º ä¸İ","ÐŁÑĢи меÑĢ","Tim estamp","身å¿ĥ åģ¥åº·","Ġintest ines","à¹ģà¸Ĥ à¹Īà¸ĩ","L CD","Ġde em","ĠF ries","çľ ŀ","天 é¾Ļ","Ġappro vals","Ġthink er","اÙħ ÙĦØ©","sk ap","综åIJĪ åĪ©ç͍","ãģł ãĤįãģĨ","ĠÚĨ ÙĪÙĨ","zek o","Ġutilis é","ĠCann ot","ĠCoul omb","B W","C alled","r ünd","çļĦ å¼ł","æĪij æĥ³è¦ģ","Ġab ound","St ress","èµ° ä¸Ĭåīį","Re ce","ç¼ĸ çºĤ","ãĤī ãģļ","Ñĸ ÑĢ","å·¨ çŁ³","ĠInf inity","rogen ic","ĠEns emble","ĠподдеÑĢ Ð¶Ð¸","! )Ċ","ut um","ĠR x","å°Ĩ è¿Ľä¸ĢæŃ¥","缴æİ¥ å°±","ĠÐĴ ол","ÑĤелÑĮ нÑĭми","ç©¿ æıĴ","Ġsn ug","ĠLog arithms","Ġhex agonal","Sw ift","Ġhydroph ilic","ĠTempor al","Ġtekn ologi","( Map","D ROP","_ rel","Ġy er","ĠU ps","å¤ĸ æĺŁ","å°±æĺ¯ æĥ³","æĸĩåĮĸ çĶŁæ´»","çļ® éĿ©","Ġಠ°","Ġnan ometer","ĠTri um","çļĦæ°´ æŀľ","åºĩ æĬ¤","Ens ure","B ruce","ost as","Ġint ending","ĠSt anton","Ġbl aming","ense ignement","æ·± åĪĩ","ä¸ĸçķĮ åIJĦåľ°","ĠStud ie","EL A","è» Ģ","/b uild","Att r","é¼» èħĶ","Õ½ Õ¿","ä¸ļåĨħ 人士","ĠH au","åľ¨ çϽ","ä»ĸ 没","æīĢ åIJ«","fl are","Ġparticip ación","Ġmember i","-D r","عÙĦ ÙĤ","Ġ×Ĺ× ĸ","æ²¼ æ³½","ĠChev rolet","æĬĽçī© çº¿","Ġrése au","æıīäºĨ æıī","D ates","ĉ y","Ġe cht","ip ers","天 éŨ","åķ Ħ","æĹł éļľç¢į","ç»Ļ å®Ŀå®Ŀ","Ġred ness","Ġvalid ating","欧 åĨł","âĦ İ","ç²ĺ 度","è¼ķ é¬Ĩ","ĠHard ing","ĠاÙĦØ« اÙĦØ«","< S","Y OU","ĠS ao","est het","ac etyl","Ġag itated","åįİ å°Ķ","ĠBy rne","顾 å¿Į","- ste","ï¼Į âĪ´","ĠM LS","ĠB ets","åĴĮ ä¿ĿæĬ¤","å°ı é¢Ŀ","è¿ĺæľī çĤ¹","Ġmoment arily","Ġinterpret ive","è¡ĮåĬ¨ 计åĪĴ","ãģ© ãĤĵãģª","ĠVis itor","èµ°äºĨ è¿ĩæĿ¥","ÛĮت ÛĮ","Est im","sted t","Ġspraw ling","B ah","çļĦ èį¯","常 说","Ġent orno","Ġsl ated","ĠSte phan","åį° ç¬¬","ĠBar cl","çĽ¸å¯¹ æĿ¥è¯´","å¼Ħ æ¸ħæ¥ļ","Ġר×ij ×Ļ×Ŀ","Ġstagger ed","( answer","} s","Ġa cesso","åĴĮ æĶ¿çŃĸ","Ġoff season","ç»Ħç»ĩ æľºæŀĦ","ĠCO UN","Ġground work","ĠÑģÑĤеп енÑĮ","al p","大 ç±»","大 红","ik uti","ied o","Ġet iquette","å°ij éĩıçļĦ","车 è½½","áĢ ĻáĢ","çļĦä¸Ģ 款","Ø® Ùģ","çͰ åľ°","Ġtransl ucent","Ġর à§ĭà¦Ĺ","èĪĮ å°ĸ","ĠConscious ness","t asks","Ġt erg","Ġf ian","ge e","å¿ĥ èĤº","ä»İ çİ°åľ¨","Ġeff ets","ç¾İ è¡ĵ","ĠSch war","ä¹Ļ èĤĿ","ĠاÙĦÙĥ ÙĪÙĨ","Ġpanor amic","å¤ Ļ","ĠD uties","cl one","åŃIJ äºĨ","ä½ł èĭ¥","åħ¶ ä¸Ĭ","éĩı åĪij","åıΠ好","让 ä½łä»¬","Ġоб ÑĥÑĩа","Ġbul ky","ÑĤив нÑĭй","æĮĩæłĩ çļĦ","ĠпÑĢоиз вед","pher ds","Ġsuiv ant","ĠTas mania","B uzz","å°± 容æĺĵ","缸 ä¾Ŀ","led on","çľ¼ å¸ĺ","害 èĻ«","ĠØŃ جÙħ","åŁºç¡Ģ æķĻèĤ²","Ġclos eness","亿 åIJ¨","æĪij们çļĦ çĶŁæ´»","Ġsau ces","Hot el","è´¿ èµĤ","Claim s","Ġirresist ible","_ process","ĠI da","ä¸Ģ åĢį","ord nung","Pro cedure","-c ancer","ĠEl iza","ĠPass over","ĠعÙĦÙħ ÛĮ","Ġserm ons","crum bs","çļĦ 說","ĠC andy","à§ ±","б л","Ġles qu","æĿİ åĺī","AC HE","åıĮ åŃIJ","æīĺ è¿IJ","Ġpra ises","ĠLE AVE","ĠCarn ival","Ġkontrol a","Ġ ïº","åĬł æĭī","ĠSh ay","Ø® ÙĪØ§ÙĨ","Ġrad iative","æĿ¡ä»¶ æĺ¯","稳 妥","å« Ķ","trans port","è¬Ľ 座","ĠImmun ology","Ġresur rect","ro gram","è¿ ¥","am ina","Ġv ex","Ġا ÙĤ","ax ia","示 å¨ģ","åĩł ä¸ĭ","Ġimp ulsive","大家 对","æĶ¿æ²» ç«Ļä½į","Ġly rical","ãĥŀ ãĤ¤","èıł èıľ","Ġhabil idades","F ee","Ġm uted","ou is","çļĦ çĹħ","ĠT AB","æİ¨ 论","ä¿® ç½Ĺ","å·® åĪĨ","ĠTra ditionally","éľ² èIJ¥","Ġhost age","imm ung","Ïģο Ïį","ç½IJ 头","à¹Ģà¸Ħ ย","ĠCi udad","ĠÑĦак ÑĤоÑĢов","Arab ian","Phil adelphia",".doc x","Q D","r type","Ġc á","ĠC ary","ĠB odies","ult ad","å¤ļ åĬł","æ³ķ åĽ½çļĦ","èĬ ®","old o","èĩªå·± åĸľæ¬¢","ç«ĭ éĿ¢","Ġstr ife","åĨį éĢł","éĢļè¿ĩ åIJĦç§į","å±ŀ ä¸ĭ","ÏĢ Î·","Ġje Å¡tÄĽ","(f ilter","å¥ĸ æĥ©","åħ¨åĽ½ çļĦ","Ġta vern","è®°èĢħ ä»İ","Ut ility","Ġcurs os","é£İæĻ¯ åĮº","C ro","Ġele venth","Ġع ض","Ġmar bles",".M an","arrow s","æĽ¸ 館","Ġdow ng","Ġling ui","ĠпÑĢави ла","supp orted","V OL","g all","j az","åľ¨ 身ä¸Ĭ","Ġen ero","åѦ é£İ","old ed","Ġfl air","äºĨä¸Ģ æł·","éĩĩ çŁ¿","èĮĥ å¼ı","Ġpay out","Ġexc ision","éĸĭ åĤ¬","å¦ĩ å¹¼","ĠProt ected","ĠìŀĪ ìľ¼ë©°","Ġwy ra","çªģçł´ åı£","梨 èĬ±","ót ár","端åįĪ èĬĤ",". ',Ċ","R iver","q d","id ores","Ġg ond","æľ¬ æł¡","åı¯ä»¥ çͱ","å¦Ĥä½ķ å¤ĦçIJĨ","기 ê°Ģ","Ġantic ancer","æĻºæħ§ çļĦ","转åĮĸ æĪIJ","oderm a","éĩį éĩijå±ŀ","Ġext rud","åĪĩ åĭ¿","') [",".to LowerCase","ún cia","æīĢä½ľ çļĦ","çļĦ 說éģĵ","ĠL ara","Ġen kele","ç͍ æĸĻ","ç͍ ä¾ĭ","... [","ç»ıæµİ æĬĢæľ¯","Ġsw irling","Ġdire tt","å¤ı 令","ĠNet z","åĮĹ京 çļĦ","ĠDirect ed","Ed iting","Inst ant","Ġpear ls","ãģ«ãģ¨ ãģ£ãģ¦","ĠFORE IGN",") c","C OR","m eyer","es or","åij ±","erv en","ä¸İ ä¼łç»Ł","é»ij æĿ¿","çīĽ çļĦ","åį± æľºçļĦ","Ġreck on","_ over","g ut","å¤ļ åıĺ","б ов","Ġtom bs","ä¸ĭéĻį äºĨ","m ins","æĸ¹ æŃ£","ä¸ī éĩį","Ġco isas","Ġmem ang","Ġprob iotics","â s","AG T","ĠAtt itudes","à¸ĸ าม","è¨Ń æĸ½","åħĪè¿Ľ æĢ§","âģ ¿","æ¶Į åħ¥","Ġprohib iting","éį Ľ","ĠContin ued","Abs olute","ĉ test","ä¸Ģ åĪĻ","ä¸Ģ è·ĥ","ir led","ĠR ider","被 æµĭ","å®ĥ åħ·æľī","åĮħ æīİ","ĠÙģ ØªØ±Ø©","ĠDel iver","plan et","ĠкоÑĢ Ð¾ÑĤ","< x","\\ qquad","_ users","Ġs apiens","st ress","ĠH s","ap are","ä¹Ł å¾Īéļ¾","Ġд ÑĭÑħа","æīĵ çIJĨ","ات ب","ä¹³ åĮĸ","éĶħ åĨħ","ĠRequ ires","æıIJèµ· è¯ī讼","Ġadolescent es","Upper Case",".ext end","E Z","E astern","çļĦ ç¼ĺæķħ","ĠS os","Ñı еÑĤÑģÑı","Ġser geant","How ard","åºĹ åijĺ","èįī 丼","Ġrat ification","ĠاÙĦØ£ Ùģ","Ġdiagn óstico","Red uce","è®°å¿Ĩ åĬĽ","à¹Ģส à¸Ļ","G all","} v","ol m","ĠC un","Ġcl inging","м м","åįķ éĢī","Ġsl ain","ÑĤо ÑĢиÑı","åIJ« èĵĦ","ÏĮ Ïģ","ĠWork force","ô mes","Ġsou ps","åħĦå¼Ł 们","ĠMit ochond","-trans fer","à¸Īำ à¹Ģà¸Ľà¹ĩà¸Ļ","ç¾½æ¯Ľ çIJĥ","-circ uit","åIJĦè¡Į åIJĦ","Ġd up","ت ÙĤد","Ġ_ $","éķ¿ çº¦","ä¸ī æŃ¥","ä»· 为","Ïĥ Ïĥα","ĠYork er","åĨ² åĨ²","ĠSer ious","Ġker es","å·¥åķĨ èģĶ","Ġreass urance","Typ ically","ὸ ÏĤ","à¹ģà¸ŀ à¸Ĺยà¹Į","J l","_ module","çļĦ æīĭæľº","åľ° çĽ¯çĿĢ","Ġtrans itive","ÑĢÑĥ п","æ´¾ çĶŁ","ç²ī çµ²","Ġber p","Ġsang re","鲤 é±¼","çĭĻ åĩ»","B eg","D iss","p v","ĠD l","ĠCh arm","ĠPro g","-c ross","ä¸ĵä¸ļ 人åijĺ","rec ision","Ġìłģ ìļ©","Ġculmin ating","n ata","Ġre organ","Ġro asting","Ġop ioids","Ġcome ç","ç®Ģ çŁŃ","çķĻ ä¸ĭä¸Ģ","åħ© ä½į","Ġdoctor ate","è¿ħéĢŁ çļĦ","Ġtér minos",") ^{-","Ġpl ume","Ġme iosis","æĤ ´","æīĵ ä¸ĭäºĨ","ม à¸Ńà¸ĩ","Ġtext os","Ġfree ing","éħ¸ éĴł","-S i","met al","ĠVol t","æĮģç»Ń æĢ§","ĠUN ICEF","Ġਠľ","Ġবিঠľ","ĠShel f","Ġneglig ent","Ġpsy ched","P seud","_ bl","ĠD ID","æľī 幸","Ġk sztaÅĤ","ry ch","æĪĺ åĬĽ","ä½İ 级","Ġgen ere","临åºĬ è¯ķéªĮ","è¢Ń æĿ¥","_count s","ĠиÑģполÑĮзова ние","Ġки ÑĪе","ic lop","çļĦ ä½ľèĢħ","ä¸Ń å¼ı","â̦ #","ĠÑĥ Ñĩе","ок о","ĠComp end","麻 çĸ¹","çĥŁ æ°Ķ","ĠPath ways","ä¸įä¸ĭ åİ»äºĨ","ார à¯įà®ķ","ĠÏĢÏģ Ïİ","ĠðŁij į",". instance","; ++","ï¼ ¢","ĠG um","Ġman power","çĹ £","éĩij åŃIJ","ĠFor get","ðĿ ļ","å·´ èIJ¨","åı¯æĺ¯ æĪij","Ġgeb ied","å®īéĿĻ çļĦ","ĠÑģооÑĤвеÑĤ ÑģÑĤвÑĥÑİ","ĠJour nals","Ġréal ité","Buk id","d ependent","d ashboard","Ġd uality","ĠF ool","ĠL oh","ĠN og","èĥ½ æīĭ","ĠAl ta","Ġ-- Ċ","رد Ùĩ","Ġве дÑĮ","Ġjed na","Ġquot as","ä¸Ģ群 人","ĠHumph rey","ĠD AC","Ġch oking","大 å¦Ī","ç®Ĺ ä»Ģä¹Ī","ĠBe handlung","æĺŁ æ²³",".D iagnostics","Ġзна ний","Action Result","Ġphon ological","Ġcalend ars",". white","X B","åĴĮ æ²»çĸĹ","以 使","ip so","å°ı åĵģ","ä½Ĩ åĽłä¸º","é» Ĵ","éĢł åıį","æ·± æµ·","ÑĤи ви","Ġdu plex","å¿Ĺ åIJij","ĠоÑĤ веÑĢ","ĠInter ests","åĬ³åĬ¨ äºīè®®","Ġliber als","ĠDra co","ĠOrt iz","Ġcyn ical","h ousing","pl and","ĠاÙĦ ÙĪØ³","æĬĢæľ¯ æĶ¯æĮģ","PR ESS","zn Äħ","ĠBR CA","ibilit é","Ġrebell ious","Ġkasar igan","/ list","> >Ċ","ì ¿","ä¸į æĶ¯æĮģ","eb ra","of en","اÙģ ØªÙĩ","(y ear","ÈĻ ti","Ġnost ri","Ġwil ayah","Ġoss erv","ent os","ãĢĤ ###","ĠF ilters","大 è·Į","çľĭ ä¸įæĩĤ","ĠPro spects","åĽŀ æĹı","ä»ĸ们 å·²ç»ı","ö ff","äºĨä¸Ģ èά","ze ichen","éŁ³ èĬĤ","ĠChrist ensen","_p op","Ġsto let","ìĿ¸ ìĿĺ","èĺ ¿","Ġsan ity","Ġko ji","Ġpemer intah","+ S","ar rays","Ġg enders","Ġv ara","à¸ģ ว","ç²¾ æ²¹","å᳠以","çĮ ¥","è¿ĺæĺ¯ ä¸Ģ个","稳 æĢģ","åıªèĥ½ ç͍","त à¤ĥ","Ġét at","è¾£ çļĦ","ç§ijçłĶ æĪIJæŀľ","ள à¯įள","Graph ics","西éĥ¨ åľ°åĮº","Ġrooft op","åĮĪçīĻ åĪ©","N ich","p oor","Ġc x","ĠV ERY","ä¿Ŀ ç¨İ","ç¡® æľī","第ä¸Ģ åĢĭ","ĠCal c","èĦij ä¸Ń","(( -","Ġ__ ('","ĠEnd angered","é³ Į","辨 æŀIJ","×ijר ×Ķ","-bl ood","ĠWi ener","Ġanisot ropic","\" ));ĊĊ","Ġm ong","Ġex cret","ph ilis","æľ¬ éĥ¨","å¼ı 计ç®Ĺ","åĥı æĺ¯åľ¨","åĮ» æľ¯","AN I","ĠPr és","ĠMon aster","ĠвÑĭ ÑģÑĤÑĥпа","تر اض","Us uario","trans ition",".ed it","v ana","st icks","ol and","ĠD ish","ä¼ļ计 æł¸ç®Ĺ","Ġrust ic","ĠпоÑģÑĤоÑıн но","Ġáĥ¡ áĥIJáĥ","åłķ èIJ½","- ho","q z","on ent","Ġd ziÄĻki","ou lli","ä»» æľŁ","Ġser es","ï¿ ¥","Ġimpro b","åī¯ è¯į","è¯Ńè¨Ģ æĸĩåŃĹ","Ġ×Ķ Ö·","ä¹Łåı¯ä»¥ ç͍","ĠLic ensing","æ£Ģå¯Ł éķ¿","ĠTher mod","Implement ed","' Or","et ako","ĠS ST","æĪ Ľ","ç» «","Ġapp ellants","åºĶ ä»İ","ex c","å¹³ åľ°","Ġche que","åĢĴ å¡Į","èϽçĦ¶ æľī","Ġecho ing","踪 迹","Function al","ĠدÙĩ ÛĮد","بØŃ Ø«","Ġprzeci w","åĽŀè¿ĩ ç¥ŀæĿ¥","ĠطبÛĮ عÛĮ","ª ר","ĠS od","um na","åĩº 游","cc an","Ġtr ast","æīĢ å¼ķèµ·çļĦ","ott es","Ġlo ft","Ġemp ires","¡× Ĵ","Ġkin ases","Ġdanger ously","Ġadult os","Ġham pered","ÑĤеÑĤ а","èĭ¥å¹² 个","indust rial","Ġepoch s","# ,","c ional","Ġb inge","çļĦ åħ±åIJĮ","ent ar","åľ¨ åĴĮ","ew el","Ġco ž","顾 åıĬ","pa Repository","ĠNor wich","éģµ çħ§","isi ème","åĮª æµħ","æĸĩçī© ä¿ĿæĬ¤","ĠWebs ites","Ġt ij","Ġa ang","Ġf encing","ĠB oul","ĠW olves","çŃī åħ¶ä»ĸ","ark ed","éļ¾ ä¸įæĪIJ","Ġorig ine","Ġimm oral","(- \\","ĠGr ill","Ġни Ñĩего","èĦļè¸ı å®ŀåľ°","鸳 鸯","Ġfor sk","ĠO ll","æĿ¥ ç͵","Ġpres que","åĪļ èIJ½","ĠMarket place","According ly","Ġmoon light","Ġ×¨× Ĵ","åĨ¶ çĤ¼","Ġdilig ent","ĠAppropri ate","' er","Ġd red","è¿Ļ é¢Ĺ","举 å±±","çͲ 骨","Ġcycl one","讲解 äºĨ","Ġneutroph ils","ĠArbit ration","- occur","_ device","ro chemical","Ġn äch","åŃIJ æĽ°","åħ» çļĦ","Ġshort cuts","纤 ç»Ĩ","éͦ 绣","ÄŁ i","å¤įåIJĪ æĿIJæĸĻ","ĠVik tor","à¹ģà¸Ĥ à¹ĩà¸ĩ",". ro","_ err","ĠD ane","ĠK J","à¸ļ าล","Ġر Ø£","å¯Ĩ éĹŃ","ĠMc Mahon","èĶ ¼","æŃ£å¸¸ å·¥ä½ľ","èĢĥè¯ķ çļĦ","ĠPract itioner","ç½² åIJį","arsh all","Ġban quet","ä¸ŃéĹ´ çļĦ","_B U","Öī Ċ","ĠDermat ol","Islam ic","ĠоÑģи гÑĥÑĢа","Ġpige on","æľī åĬŁ","Ġpat ented","Ġtechn ologie","æ¶Ī çĺ¦","äºī 端","Ġnorm a","Un ity","è¿Ľä¸ĢæŃ¥ åıijå±ķ","ĠSi oux","Ġadj our","Ġмо Ñĩе","ycz Äħ","Ġassault s","/ default","[ B","if ol","ĠH G","ĠH ert","åľ¨ 主","ĠG ron","å°± æĦıåij³çĿĢ","è¿Ľè¡Į çłĶç©¶","IN TS","text it","á» Ĺ","Ġair planes","åĿļ æŀľ","æ½ °","Ġà¦ķর à§įম","ç¢İ çŁ³","Ġthank fully","ĠCross ref","íı¬ íĬ¸","MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM","सà¤Ĥ à¤ĸà¥įया","Ġte legraph","æĥ³ åģļ","ॠī","åŁº è´¨","ris y","Ġfaith fulness","ĠHol z","Ġtit ik","Ben ch","éŁĵ åľĭ","ĠLore ntz","ĠtÅĻ eba","ĠÙħرب ÙĪØ·","C li","H OW","Ġf ide","ĠA lem","书 å±ĭ","str ut","åı¤ å·´","Ġey ew","Ġrich ly","-ex amination","Ġcosm ological","Ġwe gen","ors ke","pro xy","ĠIs les","Ġpract icable","饮 åĵģ","System s","Ġjur ÃŃd","-per forming","Ġdias pora","ĠInequ ality","éĤ£ çīĩ","In crease","Ġent renched","åķĨ ç͍","æµĭ 温","å¾· åľĭ","Ġstory t","èĥĮ 书","æĽ¾ æĺ¯","åĵĪ èIJ¨åħĭ","å°¿ 管","Ġä n","éĹ² æļĩ","å¼Ģåı£ 说éģĵ","Abs olutely",". Input","K g","Z ur","f äh","Õ °","re ve","un ner","åľ° ä¸Ńæµ·","æ°´ çĵ¶","æķĻåѦ å·¥ä½ľ","ãģ£ ãģĭãĤĬ","ï½ ħ","åıĬæĹ¶ åıijçݰ","è¾ŀ åħ¸","ிர à¯ģà®","ä¸Ńä¹ĭ éĩį","F avorite","f ills","ĠO y","ost ÃŃ","å¦Ĥ ä¸Ģ","ä¸įæĺ¯ ä½ł","Ġbook store","ãĤĤ ãģĤãĤĬãģ¾ãģĻ","éļIJ æĢ§","Ġmedic ina","à§ĩল া","ĠCompl aint","ĠÑįлем енÑĤÑĭ","Ġhelic opters","ä¸Ń级 人æ°ijæ³ķéĻ¢","Cred entials","æĭĸæĭī æľº","Ġc ape","ĠH ector","以 åĩıå°ij","æ¯Ķ ä¸įä¸Ĭ","yl ie","çIJĨ论 çłĶç©¶","lam ide","Ġaura it","æĬ¤èĤ¤ åĵģ","Ġprak ty","c j","r Äħ","} [/","ad ditional","便 åı¯ä»¥","_d ataset","èĩªçĦ¶ 人","à¹Ĥ à¸Ń","ĠÙħع ÙĨ","ĠиÑģп ол","ĠKurd ish","Ġaspar agus","ĠUltras ound","ĠÙħصطÙĦ ØŃات","( action","p unk","ĠC ognition","ĠThe resa","Ġqu ark","Ġpat ter","Ġoper e","Ġ% Ċ",".p k","åĪĨ享 ä¸Ģä¸ĭ","ĠتØŃ د","ç´§ç´§ çļĦ","Õ¡Õ£ ÖĢ","- admin","m ort","p ile","Ġd agen","te v","de z","ĠÑģ ва","che cks","åħ¥ ãĤĮ","æīĵ åİĭ","åĽłä¸º è¿ĻäºĽ","Ġbehav ing","Se pt","奥 ç§ĺ","豪 éŨ","Ġlu as","Ġunm istak","ĠGRE AT","ĠзнаÑĩ ений","J y","M CA","Ġn uit","Ġcom plicate","Ġun named","åĬ¨ éĩı","-s olid","-c ritical","ส ูà¹Ī","Ġbenef icios","ाठĥ","Ġrout ed","Ġdil ation","çļĦ主 è§Ĥ","乡æĿij æĹħ游","Ġleis urely","ĠÙħÙģ Øµ","æ¶ħ æ§ĥ","A sc","F ab","Ġd va","ĠT itus","ad is","缸 声","æĺĵ çĩĥ","Ġbest selling","hr te","é½IJ åĽ½","åĪ¥ çļĦ","ĠInsp ire","æijĶ åĢĴ","Ess ential","ĠاÙĦاØŃ Ùħر","aksan akan","Ġprotrud ing","ä¹ħèĢĮ ä¹ħä¹ĭ","ign eur","çľĭ æ¸ħæ¥ļ","æľĢ 常è§ģ","å°±æĺ¯ ä¸Ģç§į","িঠ¯","Ġold s","ൠĬ","Ġmis interpret","ç¾İåħĥ çļĦ","ç²ĺ è¿ŀ","ĠTele communications","Ġslic ing","Ġd ul","ï¼ ³","ĠF is","mer zen","ĠSh iva","åįĹ è·¯","è¯Ń è°ĥ","åĬ¿ åĬĽçļĦ","æ¼Ķ æĪı","æĢĿæĥ³ å®¶","Ġhistor i","Ġconsult ancy","Û± Û´","à¸ĭ ี","Unt uk","Ġà¦Ľà¦¿à¦² à§ĩন","b ron","çļĦ æĶ¹åıĺ","ab ets","ust ed","ä¹IJ çļĦ","Ġge vo","ç´§ éĹŃ","åĺ´ è¾¹","illa ume","ĠIV F","åĤ¬ çľł","Ġquad ru","ĠForest s","Ùħس ار","×Ļפ ×ķר","å̤ ãĤĴ","çµĮ é¨ĵ","ac ie","ĠF IELD","æĪij åı¯æĺ¯","ome gal","éŁ³ éĩı","æ²¹ çĤ¸","åĽº æľīçļĦ","uel as","éļĨ èµ·","ĠÐ¥ а","apit re","B aker","H Cl","{ item","ir u","ĠE ternal","ub y","å°± è§ģ","xt ies","Ġpass o","hem atica","Ad ult","严 éĺ²","hel ps","çīĽ æ´¥","ĠVI EW","ĠкаÑĢ ÑĤи","Ġnag y","éģĹæĨ¾ çļĦæĺ¯","ĠJur assic","æ¤ľ æŁ»","Ġense ign","R achel","q e","ĠS ø","Ġg f","Ġв окÑĢÑĥг","æĪĸ 以","ox ys","èģĶ è°Ĭ","ula ção","è¿Ļå°± è¦ģæ±Ĥ","ĠÙħد ت","à¸ķà¹Īà¸Ń à¹Ħà¸Ľ","Ľ ×Ļ×Ŀ","om ach","ĠG K","大 æ´ĭ","ä¸Ĭ 身","In creasing","ĠÙĨ Ú©","ili han","лен но","-re al","Ġroll ers","ĠTim ur","ĠCardi ovasc","id elijk","ä¹Ł åıª","å°ı äºĮ","çŃī 离åŃIJ","è°ĥ åΰ","ва нии","gen stein","Ġbo a","读 åΰ","Ġill usions","â ge","èĻļ å®ŀ","Ġveget ative","çݰå®ŀ 主ä¹ī","Ġrail roads","Ġsig ue","èĤļ åŃIJéĩĮ","Ġë¹Ħ êµIJ","ĠGem ini","ĠDipl om","Ġtung sten","ا ÙĪÛĮ","è¦ģ é¢Ĩ","ĠY or","ĠAl ain","ĠWh ites","Ġhard y","ãģ£ãģŁ ãĤĬ","ᣠĨ","ĠÙħست ÙĤ","Ġbew ild","Ġank les","缸è¾ĥ äºİ","M ID","on im","çļĦ 妻åŃIJ","st up","ĠT odos","ill ar","éĥ½ ä¸įè¦ģ","éĤ£ è¾¹çļĦ","à§įঠ£","åĮĹ çº¦","gu est","çİĦ å¹»",".sub str","éŀŃ çĤ®",". na","E ye","_ shape","x F","äºĨ 她çļĦ","the st","ار ÙĬØ©","gin x","ĠPay Pal","{d ocument","Ġannoy ance","çļĦ ç»ıèIJ¥","Ġch oked","为 ç͍æĪ·","Ġus ur","ĠAr ten","Ñİ Ð·","ĠIs ra","িঠı","ĠPr ussia","ĠAm es","çĨĦ çģŃ","ä¿Ŀ驾 æĬ¤èĪª","p oke","Ġb ian","Ġfor ging","æľĢ éļ¾","und ai","æľª å°½","åĽ½éĻħ ä¸Ĭ","Ġqual itatively","é¢ĨåŁŁ ä¸Ń","ĠPresent ed","/ âĪĴ","M b","l ach","à ķ","æĪij åľĭ","ÑĤи ÑĩеÑģкие","Ġgroup es","纳 å¾·","à¥ĩ ष","Ġö ffent","নà§įঠ¡","Ġuph ill","Ġuczni ów","z nej","he esta","un u","ĠP p","ä¸Ĭ æĸĩ","æīį ä¼ļæľī","ä¾Ľ æļĸ","åŃ¦æł¡ æķĻèĤ²","ĠJe ÅĽli","ா஠®","æ´ª èįĴ","ĉĉĊ ĉĉĊ","×ķצ ר","Ġirregular ities","åįģäºĮæĮĩ èĤł","are k","ĠV OC","ÑĢи ди","Ġdown wards","åķĨ äºĭ","象 æ£ĭ","Ïĥ κε","åħ³ç³» åĴĮ","åı¥ ä¸Ń","wh atever","CT C","çļĦåİŁåĽł æĺ¯","Ġfruct ose",", but","al ais","Ġn ude","ä¸į æĢĿ","ĠU h","ç»ı 纬","è®°å½ķ çļĦ","çĶĺ å¿ĥ","Ġgew orden","Ġار زش","udd led","stoff en",".Foreign Key","w ania","è¿ĺ ç͍","åĽŀ 车","声 åĬ¿","ĠÙĦ ب","лÑı ем","arc ane","ĠFran ça","ä»Ĭ天 æĪij们","å½Ĵ æ¡£","ĠÑģа мÑĭм","water ing","Ġbek end","k B","çļĦ çĹħ人","ĠC arth","ĠP ty","éĹ´ è°į","ä½Ĩ ä»ĸçļĦ","Ġel ke","åij¨ è¾¹çļĦ","(t op","æİ¢ åºĹ","åºŃ 审","ĠFe ather","æķ¬ 请","ĠоÑģнов ном","Follow ers","Ġë§İ ìĿ´","ĠAthen a","Ġì¦Ŀ ê°Ģ",") R","= lambda","f ighter","çļĦ çŃĸçķ¥","Ġen im","ulf ide","è¦ĸ éł»","Ġabstract s","Ö¸ ×IJ","ĠTele com","çĨı é϶","bou w","ĠMesopot amia","V N","çļĦ å®ŀçݰ","el age","ĠL ips","大 åºĨ","è¦ģ 害","åIJĮ å±ħ","æĢ§ çĶŁæ´»","าภĵ","åħ» çĮª","æŃ» ç¥ŀ","æĬķèµĦ é¡¹çĽ®","å¤ı 侯","Ġdro g","Ġmé can","忽è§Ĩ äºĨ","Ġmultim odal","ĠTrou ble","ĠRegist rar","ĠاÙĦÙ쨱 ÙĨس","à°¿à°Ĥà° ļ","ĠGosp els","Ġs enc","un ay","åħ¨ éĻ¢","æĸ° å¥ĩ","è´£ å¤ĩ","ðĿ ĺ",".c ategory","é£ŀ æĿ¥","åģı å¿ĥ","Ġdepend able","¤× Ĵ","CM C","ĠTransl ations","A WS","ĠB aba","ä¸İ åѦçĶŁ","æĹł èĢ»","容 é¢ľ","åĩł åı¥è¯Ŀ","Ġrest ricts","Ġobject ivity","Ġза ÑĢабоÑĤ","Ġج ÙĪØ§ÙĨ","çĨŁ çŁ¥","ĠProcess or","Ġverb ally","Ġaer uginosa","ĠÑģоз на","Ġdevast ation","åį°ç¬¬ å®ī","d ire","ĠN okia","ĠK arena","Ġì º","ä¸ī çľģ","eng k","两个 å°ıæĹ¶","èħ¿ éĥ¨","çħ¤ å±Ĥ","Ġcod on","Ġ×Ķ×IJ× Ĺר","Ġeu ph","ĠRick y","ogg les","ãģ®ä¸Ń ãģ§","Ġfid uciary","ì µľ","our ism","é¡ Ķ","举 è·¯","ĠCl an","羣çļĦ 太","ĠSer ra","ĠAnt wort","马ä¸Ĭ å°±è¦ģ","Ġpubl ique","å©· å©·","辩è¯ģ æ³ķ","Dif ficulty","z ust","ĠF ruits","å¹´ éī´","她 åİ»","马 éĩĮ","Ġaff luent","orth and",".f etch","×ķ×ĵ ת","-J ones","Ġaffection ate","Ġdoub ly",") âĢĻ","ĠS IN","ĠK opf","åζ æ³ķ","æ¯Ķ 为","æĽ´ è¿Ľä¸ĢæŃ¥","该 åĽ½","áĢ ħ","åij³ åĦ¿","Ġcook er","Ġtall est","ĠобнаÑĢÑĥ жи","Ġo któber","ĠM SC","Ġhe res","ht ar","为 群ä¼Ĺ","åıĹ åĤ·","ĠEm my","å¯Į äºİ","åıªè¦ģ èĥ½","ãģ§ãģĻ ãģĭ","Ġhepat ocellular","Organ ic","åѦ çĿĢ","å°ı å±±","Ġbl i","Ġresp osta","å®ĥ å°±","IN ING","ĠSch re","ĠCont rary","çĬ¯ 人","IR D","ĠÑĢезÑĥлÑĮÑĤа ÑĤов","ĠØ£ØŃ Ùħد",".comp any","Ġconsom mation","ĠÑĤеÑĢа пи","Ġhuv ud","< Product","} T","Ġa ção","Ġk ub","ï¼Ł âĢľ","п оÑĩ","交 æīĭ","Ch anging","ĠApp aratus","Ġsem en","æľĢåIJİ è¿ĺæĺ¯","æ±ī åł¡","го е","为ä»Ģä¹Ī åij¢","ĠCath y","æ»ĭ çĶŁ","ĠPra ise","ĠMother s","···· ····","ĠÑħÑĥд ож","H ou","\\ epsilon","on uclear","ĠK iller","Ġme i","ä½ĵ ä¸Ń","æĺİ æĸĩ","Ġque ens","Ġes cre","ز ÙĨ","Ġbre ached","èijĹ ç§°","ä¿¡æģ¯ æĬ«éľ²","åħ·æľī éĩįè¦ģæĦıä¹ī","ille urs","Ġchem o","计åĪĴ ç»ıæµİ","ĠFe ast","åĥħ åĥħ","ĠHem ing","Ġdeput ies","Ġchor oby",". ok","z ac","ì ·¨","ä¸į æ¯Ķ","åľ¨ ä¸Ĭè¿°","cl a","é£ Ĵ","ä ge","Ġext ingu","å°±æĺ¯ ç͍","ä¸ĸ äºĭ","åIJĥ åĬĽ","Ġbroad ening","çŃĭ 骨","çĢ Ľ","Ġpharmac ist","ĠTot ient","ĠHels ing","Ġeyel ids","Ġ Õ¡Õ¼","he i","ĠT idak","Ġst int","ag ini","-c ause","ka ÅĦ","çľĭåΰ 她","æĺ¯åIJ¦ æŃ£ç¡®","ä¼Ļ 计","éĥ½æľī åı¯èĥ½","ĠFound ed","Ġtang an","/ as","it helial","Ġw retched","as one","Ġal ia","ĠR amb","Ġcent imet","æĿ¡ çļĦ","Ġprot i","è£ħ æī®","ĠIP C","ÙĨا Ùĥ","bu querque","å¼¥ éĻĢ","积累 äºĨ","ĠBeat rice","Ġunb ear","èıł èIJĿ","Ġмл н","Ġdici embre","< void","it ats","Ġin bound","am am","Ġv ign","Ġhome owner","ãģ¯ ãģĦ","Ġben öt","Ġdat as","oz illa","Ġ(! $","C rypt","iv itis","æĸ¹ ä½įçļĦ","è¾ĥ æĹ©","å¼ł åĺ´","Ġnight time","ве ли","äºķ ä¸ĭ","Ġà¸Ļ ัà¸ģ","åİī害 çļĦ","Y i","iz y","ĠV IS","ens ibly","Ġdis qual","Ġdis burs","åİ» å¤Ħ","inal e","西 åĮ»","è± Į","ĠIs mail","æĻ® 京","æ¥ļ æ¥ļ","宽 çļĦ","m essages","Ġy u","ä¸į æıIJ","ĠJ G","éľ ģ","ict ures","æ±Ĥ çŁ¥","Ġhand made","ĠCl arence","ĠТ огда","大å¤ļ æķ¸","Ġpró ximo","æĶ¾ç½® åľ¨","Ġb ạn","ol ed","æĹ¶ 为","-f ired","积 èģļ","Ġalt itudes","Ġweak est","-a verage","ĠBa ud","å¼Ħ æ¸ħ","ĠMillennium s","ĠS EP","ak ang","ä¹ĭ æºIJ","Ġп ÑıÑĤ","Ġب Ú¯ÛĮر","约 çijŁ","çĹĽ åĵŃ","ικ α","ĠExpert ise","ĠDiagram s","তিহ াস","ĠA the","Ġex asper","ä¸ĭ æĸ¹çļĦ","èĤ ±","Ġ$ Ċ","iss n","Ġer oded","Al right","ä»ĺ 诸","å°¾ 声","ĠÚ¯ رÙħ","ĠName eee","纵 éĺŁ","Conf igure","Ġinse gn","techn ical","w itch","he arted","le žit","ĠA PS","Ġal ue","æŀ ³","Ġher nia","olog ico","Ġdi ï¬Ģ","ĠÐŀ д","ãĤĦ ãĤĬ","çĶŁæ°Ķ äºĨ","Ġrede emed","ĠмоÑī ноÑģÑĤи","åı¯åĨįçĶŁ èĥ½æºIJ","R oz","Ġf évrier","ĠS ass","Ġbe ak","è¿ĩ çĥŃ","天 äºĨ","Ġi edere","ä¸ī éĥ¨","缴 线çļĦ","ว ล","Ġmus culoskeletal","对äºİ éĤ£äºĽ","Ġ-- ĊĊ","åĪº çĹĽ","ĠPlan etary","ÑĥÑĪ ÐºÐ¸","ĠGru ppen","å®ł çα","Ġpartition ed","Ġwr ists","Ġware houses","ĠëĨ į","yste ine","ĠÅĽrod ow","x C","al ic","ĠP rab","ä¸ĭ åįķ","头 åĥı","åĽ¾ åĴĮ","Ġsch le","ĠGovern ors","Ġpi ety","ç¢İ çļĦ","äºĭå®ŀ è¯ģæĺİ","ĠëĦ Ī","Ġimperfect ions","f ork","ĠS LE","ĠP ern","ĠP IC","ĠSt aat","ä½ł è·Ł","å®ļ çĦ¶","èĢĮ æľª","被 åıijçݰ","çͱ è¡·","å±± éĩĮ","ĠBl u","Ø® ÙĪØ±","(a q","æIJŃ çIJĨ","ĠVir gil","çıį èĹı","ĠBig Decimal","×ij× ¢","ĠпÑĢимен ениÑı","Ġh ive","åĩº å®¶","Ġ[ ĊĊ","ä ck","代 人","æĬĬ åħ³","ior i","Ġaff ine","æĤ¨ çİ°åľ¨","Ġpen a","ĠSl ack","è°ĥæŁ¥ æĬ¥åijĬ","ĠGar land","ĠGram my","åħļ建 å¼ķé¢Ĩ","Ġutiliz ado","ола га","à¸ļริ หาร","ĠÅĵ uvre","Ġpanor ama","D isk","Ġan ew","ä¸į åĿĩåĮĢ","æĪij ä¹Łä¼ļ","çIJĨ åĮĸ","红 èĤ¿","ho res","Fl at","ĠMerr ill","ä¸į èĩ³","ä¼ļ æĪIJ为","ç® Ķ","ex ist","代 è¯į","Ġrest itution","è¿Ļä¸Ģ 段","ÅĽ nia","Ġmanufact ures","éĶĻ误 çļĦæĺ¯","Ġdesarroll ar","Charl ie","Ġcredential ed","j unction","re ma","ĠS lam","大 æ¸ħ","ng el","Ġsc rape","两 æĹģ","èµ° ä¸ĬäºĨ","Ġaut ore","ி஠ªà¯įப",".st op","ĠMer ry","ĠCON S","é»ijè¡£ 人","- ten","ĠP onte","ers h","qu am","ĠH umb","ĠF unk","Ġj im","Ġun structured","å°ı 鬼","æĸ° çŁ¥","ç¾İ èĤ¡","ç«Ļ åĩºæĿ¥","çªģ åİ¥","Ġadminist rations","-re dux","æ»ĭ æ»ĭ","Ġpatriarch al","> T","we ge","ident ally","Ġcell ar","-c ircle","çĥĪ çģ«","ĠCou rage","rah ydro","Ġbipart isan","p rav","ar be","ĠN ug","ä¸ĩ ç§ij","ĠÙģ Ø´Ø§Ø±","æĿ¾ æĩĪ","pert ensive","èĪĴ ç¼ĵ","åģ· ç¬ij","Ñīа ÑĤÑĮ","ä»İå°ı å°±",") \")Ċ","M organ","g ere","çļĦ åĨħéĥ¨","Ġto h","Ġj apon","ठ³","de cimal","Ġpo ids","æ¡ ģ","Ġeconom ÃŃa","端 åŃIJ","ä¹ĭåIJİ å°±","宫 éĩĮ","é«Ķ ç³»","Ġneut rino","Ġбе л","ãģĿãĤĮ ãĤĴ","åı¯æĢľ çļĦ","- De","- images","= input","çī© åĬĽ","ms on","ave c","æĶ¯ æī¿","_d etail","ĠØŃ ÛĮ","hund erts","ĠCoe fficient","' Ar","ĠC oke","ر ÚĺÛĮ","ÙĪ ØªØ±","ark a","Ġer am","ĠPl atz","æ£Ģ åĩº","读 åĩº","ĠMed ications","å¥Ĺ 管","RA INT","Ġত à§ģল","åIJĪçIJĨ åľ°","ĠMagn esium","à®± à¯įà®ķ","åĪ©çī© æµ¦",": \")Ċ","Ġy ummy","ä»ĸ æĢİä¹Ī","天 å°Ĭ","åĨį 说äºĨ","ĠEn emy","Ġdig ested","Ñģки ми","èıľ çļĦ","æķ´çIJĨ äºĨ","Ġprod otti","adapt ive","ĠЯн декÑģ","Ġসà¦Ĥà¦Ĺà§įরহ à§ĩর","ĠSalis bury","çĦ ĸ","è· º","æĸĩ åĪĽ","äºĮ ä¸ĸ","ec imal","æĪĺ 绩","ĠاÙĦÙħ رض","where in","æĢª æĪij","ãģł ãģ¨","First Name","èĤ¡ä»½ åζ","بÙĬ ÙĤ","åĦĴ åѦ","Ġerad icate","S erve","Ġt ipping","Ġm ère","ĠW ür","æľī ä½į","Ġam igos","ier es","ม à¸ŀ","亲 çĶŁ","Ø« ÙħاÙĨ","æ²Ĵ æľī人","éĻĦ åĴĮ","Ġheat ers","åīij æ°Ķ","Ġà° °","ĠMad ras","ĠCic ero","à¹Ģศร ษà¸IJ","Ġ Åij","or relation","åľ° éĿ¢çļĦ","社ä¼ļ æ²»çIJĨ","ص ائ","ĠпÑĢи вÑĭ","Ùħا ÙĬØ©","оÑĤоÑĢ Ñĭе","Ġà¦ħ à¦Ĥশ","TR AN","è»į äºĭ","ĠNig el","à®® ிழ","ĠCOMP ANY","ĠاÙĦÙĩ ÙĨد","à²ķ à³įà²","P W","ou b","Ġk reat","ĠK not","Ùħ ÙĬ","Ġdec ad","ĠSh iv","åij¨ åΰ","éĩĩç͍ çļĦæĺ¯","aur i","èι åijĺ","ĠAnn u","-R ay","ĠLiber t","Ġglad ly","Ġcoex istence","Measure ment","Ġa λ","ĠA eron","æľī ä»·å̼","ident ification","è¿ĻäºĽ éĥ½","ĠEm otions","(b ody","Ġnone x",")$ .","ĠValid ate","p il","u ire","åĴĮ åĪĽæĸ°","æľĢ ãĤĤ","该 书","å¼ķ èĦļ","ĠComm ittees","add ers","ê tes","èľ ·","æľ« æľŁ","sz ág","Ġconce ivable","ktion en","Ġorche str","; <","v endor","ed uct","çļĦ èĩªå·±","ĠW arn","Ġorgan ising","ĠØ£ ج","Cl ark","éķĩ çĹĽ","Ø« ÙĤ","Ġmer ry","模åŀĭ ä¸Ń","ĠÚĨ ÙĨÛĮÙĨ","Ġprecip itated","-pl ugin","ëłĪ ìĿ´","Ġawaken ed","Ġdisgu ised","çĥŁèĬ± çĪĨ竹",". They","\\ Controller","Ĩ ãĤ£","Ġt p","Ġw x","ĠS SS","Ġme in","ced ing","chn en","Ġdecided ly","纵 æ·±","é³ ĸ","丢 å¼ĥ","Georg ia","èĭ± éĩĮ","çķĻ æľī",".t ask","irm ing","ogen icity","åıij表 æĹ¥æľŁ","-v iol","ä¸Ģéģĵ éģĵ","Ġnanop article","ãģ¨ãģª ãĤĬãģ¾ãģĻ","ĠXCT Assert",".aw t","Ġo ily","ĠW ochen","ĠK iel","port al","we isen","åģ¥åº· çĬ¶åĨµ","æĽ¸ è¨ĺ","ĠUl rich","Ġнал огов","èľ¿ èľĴ","ĠоÑģигÑĥÑĢа Ñļе","ĠA ks","åĴĮ åİĨåı²","Ġsh outs","Ġun identified","éĻ¢ èIJ½","æĶ¯ åĩºçļĦ","Al le","å®ĭ æ±Ł","Ġli ar","Ġscript ing","ĠÙģÙĩ ÙĪ","æºĿ éĢļ","ĠVaugh an","w ali","ĠS loan","ĠM endoza","ve au","ĠK ran","æĹ¥ ãģ®","è°ĥ çļ®","Å¡ ka","åı« ä½ľ","-st ates","ĠAcc um","Ġμ L","大è§Ħ模 çļĦ","N d","Ġin clusions","çļĦ çĶ·åŃIJ","ut m","Ġg ör","Ġde ceive","ĠL ies","Ġk ultur","大 å·´","Ġet en","çŁ³ æĿIJ","ĠÙĨ شر","-g rown","Ġvar ie","b z","f older","{ split","Ġfor za","ih ilation","給 äºĪ","çĤ¼ 丹","ĠCD T","pons es",".de code","Ġpant ry","Ġdoen ças","Ġimpover ished","ab al","ĠR K","åı¯ æĪij","ens ively","社 ç§ij","Ġfil aza","Ġconvert ers","çĹĽ çĤ¹","ĠDep ot","ilit é","è¶ĬæĿ¥è¶Ĭ é«ĺ","Ġsie bie","åĪĨå¸ĥ çļĦ","Ġclar ifying","æħĮ å¿Ļ","ĠÛģ ÛĴ","Ġrelic s",". il","Ġa is","æĸ ĵ","ĠD ijk","æıIJ æĮ¯","Ġdire itos","ãĤĤ ãģĤãĤĭ","çĿ¡ å¾Ĺ","ç«¥ åŃIJ","çĵľ åŃIJ","WE EN","Ġнем ного","åĴ§ åĺ´","ĠнепоÑĤпÑĥ ним","% s","< title","ir ubin","Ġse hat","Ġle hen","çļĦ人 åĴĮ","oph an","Ġneg ation","å®Ī çĿĢ","ĠExp anded","å¼Ĺ 鼷","Ġcomport amento","ĠвелиÑĩи на","Kas ipak","ĠBlo oms","R at","f rey","æĺ¯ é«ĺ","Ġch ocol","å®¶ æľī","å®ī é̏","iel lement","ĠAr range","ĠÑĤ ен","Ġimm utable","Ġod ious","ĠSk ate","UL A","çħ® 沸","ĠìķĦ ëŀĺ","Ġdot yczÄħ","Ġtorn o","Ben jamin","Ġolig onucle","Ġinduct ance","æ·ĭæ¼ĵ å°½","/ dis","åĴĮ å¸Ĥåľº","åħ¶ å®ĥçļĦ","å°Ĩ æŃ¤","ĠBe acon","çļĩ 马",".T able","ãĥª ãĤ¹ãĥĪ","_C ONT","Ġrou ge","åĶ®åIJİ æľįåĬ¡","Ġo yster","is ar","èµ ¡","å¿ĥ çĥ¦","æĦı åľ¨","è¶ħ åĩ¡","ĠAt kinson","ĠDes de","è¿ģ å¾Ļ","ĠOpt ics","Fin ished","Õ¥ÖĢ Õ¥Õ¶","cephal us","Ġarrog ance","Ġunexpl ained","h oun","ãĢĤ ãĢĤĊĊ","ĠK lu","ä¹ĭ 乡","ç¼ ¨","è¾ĥ è½»","便 åľ¨","ê r","æ¸ħæ¥ļ æ¥ļ","owa ÅĤa","Ġprincip ios","Ġgam ble","çĿ¾ 丸","% C","ĠR utherford","ac ro","å¼ ģ","åĬł åĪ©","cy an","没æľī ä¸Ŀ毫","åıĸ è¯ģ","æĮī æľŁ","çŁ³ æĿ¿","aring an","×¨× ¢","å½Ĵ ç±»","à§įয à§ĩ","ç¶ĵ åħ¸","нÑı еÑĤ","né v","å°Ī éĸĢ","Ġbold ly","Fund ing","f k","æľĢ 强çļĦ","Ġinf irm","Ġrest rain","ering en","Rec all","Ġdram as","è¿ħéĢŁ åľ°","ĠTown s","åºĶæĢ¥ å¤Ħç½®","Ġt ÃŃtulo","Ġest ados","åıĪ ä»İ","Ġmed yo","Ġaff licted","Ġsch w","×ķר ×Ĵ","äºĮåįģ ä¸ī","åħĪè¿Ľ 个人","à¥Ĥ प","< class","ĠP erg","ĠP vt","åľ¨ æī§è¡Į","è¦ģ 为","å®¶ 大","æĬĢæľ¯ æ°´å¹³","ĠاÙĦÙħ شار","å¤ľ å¸Ĥ","ãģĵãģ¨ ãģ«","ĠÐĵ лав","ĠNS Log","ãģ¨ãģª ãĤĬ","ĠNeder lands","Navig ate","ĠExcess ive",") âĢĶâĢĶ","m achine","om ina","Ġse ism","æĪij å±Ģ","ia al","åѦ åΰäºĨ","é«ĺ 声","und ant","åĨį çĶŁäº§","Ġна Ñħод","Ġб аÑĢ","rac er","åĪĴ è¿ĩ","è¿IJåĬ¨ ä¸Ń","Ġtransport e","éĹľ å¿ĥ","Ġä ven","ĠMaster ing","odont ic","ŀáĢ Ĭ","- ant","N OW","ĠIn formatics","åīį ä¸Ģ天","度 é«ĺ","Ġ) )}Ċ","é£Ł äºĭ","ĠMon k","Ġstra ps","Pr ince","Ġдо ÑĪ","ĠRem arks","Ġsed ation","ĠаÑĢÑħи в","ĠCeleb ration","B AS","ĠA ry","Ġv ários","ÑģÑĤ Ñı","Ġins urg","åĪĩ 线","Ġdu el","ন ার","Ġblock ers","Ġdepend e","ç¦ģ åĮº","å¹¾ å¹´","Ġcoast s","è§Ĥä¼Ĺ çļĦ","åŁİ乡 建设","ĠLect urer","ç¾Ł åŁº",") H","* >",": G","ĠM HC","ä¸Ģ èĬĤ","Ġy y","å¤ĸ è²Į","IN I","伤 æĦŁ","åºĵ éĩĮ","aph ore","_T IME","ĠпомоÑī ÑĮ","Crit eria","Begin ning","Ġcon vol","Ġal de","åľ¨ å½ĵåīį","æīĭ æı¡","Ġemail ed","设置 æľī","Ġtherm ally","ĠÑĢабоÑĤа еÑĤ","ĠConsolid ated","ĠоÑĤноÑģÑı ÑĤÑģÑı","@ app","T ING","Ġo me","ĠR t","Ġا ÙģØª","Ġpre clinical","çĿĢ èī²","Ġbl ush","éĹ® ä¸ĸ","CO ME","ç¡®å®ļ 为","Ġlab elling","Ġpair wise","èĿ ¦","Ġfinger prints","ĠDies el","Mill is","ĠапÑĢе лÑı","M inn","n ose","Ġc em","çļĦ åŃ£èĬĤ","as ilkan","大 èĤĨ","Ġme ine","Ġif f","Ġ+ (","å¸Ĥ æĶ¿åįı","×Ļ× ij×","ĠPro ficiency","ÑĢи ÑĤÑĮ","ä¼ģä¸ļ 对","ĠPl ut","èIJ½ èĦļ","ä¸ĥ 彩",".A tt","ï½ ı","Ġtom u","湿 çĸ¹","âĿ ·","Ġfootprint s","èħĮ åζ","id ic","çĽ Ĥ","ide on","Ġsp ruce","ĠAd el","çª ©","åĬŀ äºĨ","ส ึà¸ģ","à¸Ħ ุ","ห าà¸ģ","åĹ ¬","åºĶç͍ ä¸Ń","ве диÑĤе","社åĮº çļĦ","Ġshot gun","æĥ§ æĢķ","Ġpancre atitis","Ġintermedi aries","{ :","en zo","Ġe jection","ĠD op","Ġtrans gress","äºĶ åĪĨéĴŁ","ãĢį (","_c opy","åĢŁ åĬ©äºİ","Comp ared","Ġabandon ing","ĠPE OPLE","ĠHaz el","Ġgegen über","Ġplag iarism","Ġb ry","st ern","ĠC PS","Ġde ven","æľī 帮åĬ©","ĠG ug","Ġì ½","ç»´ åħĭ","ĠFl ame","èµĽ 车","ĠÙĥ ÙĨ","Ġма ÑģÑĤеÑĢ","ĠاÙĦØŃ رب","åIJ¯åĬ¨ 仪å¼ı","ĠEnc ryption","ĠSERV ICE","< Node","ĠV ene","Ġpe ered","pos als","æµĭ å¾Ĺ","ÙĪØ§ÙĨ ات","ç²ĺ åľŁ","Ġà¸Ļ าà¸ĩ",". player","ot el","ठĪ","åıij åĶ®","Ġrel ocate","线 åŁİå¸Ĥ","ĠCon current","Ø· ÙĬ","ç»ĵæŀĦ è°ĥæķ´","ĠAug en","æ³ķå¾ĭ åħ³ç³»","ç͵影 éĻ¢","ÏĦε Ïģ","à°¿à° ¤","Ġll ama","稻 çͰ","-no ise","Ġdiber ikan","D ow","Ġa it","ĠN ico","è¦ģ éĿł","ach al","Ġdef iance","ts on","à¸Ħ à¸Ħล","ĠProt ecting","ÙĪØ¨ Ùĩ","å¯ĨåĪĩ èģĶç³»","Ġarchae ologists","Ġpneumonia e","z ano","Ġe ater","un ches","Ġch atter","ap ac","çŃī ç´ļ","åĽŀ 声","åIJį æ°Ķ","az aki","ĠWh itt","åIJĥ è´§","OL ED","åħ¨éĥ¨ éĥ½","ç»ķ è¿ĩ","ëĵ¤ ìĹIJê²Į","æĸ© æĿĢ","绩æķĪ èĢĥæł¸","Month ly","é¶ ´","' \".","/ client","Ġan esthetic","(\" *","Ġм г","Ġpred ation","atur an","Ġmut agen","mat ism","Ġvit esse","Ġ×Ľ× ŀ×","Ġdenomin ations","貨 å¹£","ä¸İæĹ¶ä¿± è¿Ľ","B EGIN","S ar","Ġc ages","Ġe tym","å®ĥ æľī","Ġí ĶĮ","Ġस à¥įव","ĠSS R","ĠTown send","çļĦ è¿ĺæĺ¯","Ġpr imal","Ġб акÑĤеÑĢи","æ¿ ¤","Ġ×ij× Ľ","æľĿ åIJij","[] >","éĴ¢ ç»ĵæŀĦ","æķ¸ åѸ","ĠSem inary","Ġmamm ary","Aw esome","Ġteor ÃŃa","ĠAmend ments","- media","/ is","I OR","Ġw icket","ĠR otation","èĢĮ åĬªåĬĽ","з онÑĤа","å·¥ä½ľ åİŁçIJĨ","éº ¾","åİĨ ç»ĥ","æį¢ ä¸Ĭ","æ»ij åĿĹ","Pre v","ĠHel ps","ÙĦا ÙĬا","å·®å¼Ĥ åĮĸ","çīµ è¿ŀ","è¿Ļéĥ¨ åī§","ĠCHAR ACTER","Ġcomorbid ities","Ġd izer","人 å½±","éĩį åĽŀ","Ġsk irts","Ġins besondere","åĩł å¼ł","Ġé x","ĠÙĨ ÙĬ","Ġ? >ĊĊ","èĢĥè¯ķ æĪIJ绩","é¼» æ¶ķ","war f","ĠNS F","ĠвÑĭпол н","Psych ology",". Res","K ING","R UN","ul man","æķ° å¹´","Ġacc use","Ġel as","Ġson ic","Ġпод веÑĢ","ĠÑĩе лÑĥ","ĠBac illus","Ġfinan zi","çļĦ èĥĮå½±","ĠC zy","ink l",".m at","ç®Ģ éĻĭ","çĸij åķı","Ġguard ing","zn ym","Ġpropag ating","à¹Ģà¸Ķ ิม","ราภļ","è¾½ éĺĶ","Ġsediment ation","Ġwszyst kie","a er","IJ ׾","an throp","ĠT eg","ig al","Ġmen ores","è¶Ĭ éķ¿","æĸ¹å¼ı æĺ¯","ÙĨد گاÙĨ","amm u","-h ours",".w ait","Ġoblig atory","éĴ» è¿Ľ","æķµ 人","K nown","ĠS ick","æľī åģ¿","Ġad icional","ĠاÙĦ ÙĪÙĦاÙĬات","çŃī æľīåħ³","Ġbl ister","åŃĹ æł·","Ġbi ographical","oj en","-qu arters","ĉ ĠĠĠĠ","Ġ ire","ĠP orsche","ä¸į 大äºİ","åľ¨ ç͍","çľĭ ç͵影","ж ноÑģÑĤÑĮ","å̼ æĺ¯","æŀĹ åĩ¡","èĩªå·±çļĦ åĬĽéĩı","Att ack","ï¼Ŀ ï¼Ŀ","cf g","ĠÑĨи к","æľīåĬĽ åľ°","ĠEB ITDA","Ġapprent ice","ĠMER CHANTABILITY","P ow","çļĦ éĥ½","çļĦ èī²å½©","ĠN U","ä¸Ń çĶŁ","Ġco ached","äºļ å½ĵ","δ η","Ġìķ ł","éŃĤ éŃĦ","ĠEp ile","ĠPR ACT","æĹº åŃ£","ĠCru c","Ġsail or","åĬ¨ 人çļĦ","ä¸İ å°ı","çł ·","Ġsc ree","让 人们","æ¸ħ æ¸ħæ¥ļæ¥ļ","çľ¼ éĥ¨","-pro duced","éĢļ常 ä¼ļ","Ġdivers es","èĬ¬ åħ°","ма Ñħ","é¦Ļ æ²¹","Ġج اء","à¸Ħว à¸ļà¸Ħ","èģĺ ä»»","èī¾ ä¼¦","åıĤè°ĭ éķ¿","Ġh s","ne u","她 éĥ½","Ġche min","els ka","Ġcour te","Ġpred atory","sec ured","伯 æł¼","èĤĿ èĤ¾","Ġcomputation ally","Û²Û° Û±","= \\)","É «","Ñ ĵ","Ġv yd","è¿Ľ æ°Ķ","é«ĺ ç´łè´¨","ç¾İ çϽ","ke i","á» ħ","án cer","Ġdeal ership","ĠBre ath","umbers ome","欢è¿İ 大家","ĠMid night","ĠCEO s","Ġdread ed","ĠC reed","ter ror","ĠN ost","Ġra ped","éŁ³ 符","Ġart istry","Ġid iopathic","ноÑģÑĤ ÑĢан","-am ino","Ġuncont roll","ĠÑĢекомендÑĥ еÑĤÑģÑı","G race","Ġt ÅĻÃŃ","ä¸Ģ å°ģ","ĠK ib","ä½ł éĢĻ","åīį åįĬ","äºĶ ä½į","æĬķ åIJij","éϤ æķ°","çIJĥ èĽĭçϽ","ĠاÙĦÙħ ÙĤاÙĦ","Ġlas ci","èĥĨ æ±ģ","åĨľæ°ij çļĦ","Ġprosec uted","Ġkur z","Ġextr insic","ion ate","ĠH N","ç¬ij åĵŃ","Ġwind y","å®ģ å¸Ĥ","ÑĢов ка","æĶ¾åľ¨ å¿ĥä¸Ĭ","çĬ¯ç½ª åĪĨåŃIJ","ĠнапÑĢав лениÑı","ĠĠĠĠĠĠĠĠ ĊĊ","ĠB ER","æĽ´ æĸ¹ä¾¿","Ġdec ays","ä¸ĵ åijĺ","è´¹ çİĩ","af x",".c ard","åĩĢ åľŁ","çĽ¸ä¿¡ èĩªå·±","ĠÑģп ек","Ġprz em","dat um","纯粹 çļĦ","% e","\\ operatorname","_ return","Ġ ����","ĠS utherland","ĠC aus","ç͍ å®ŀéĻħè¡ĮåĬ¨","ĠLe ón","é¢Ħ åºĶåĬĽ","ĠSher idan","Ġbull ish","ĠìŰ ê²°","Ġleng uaje","ĠtÄĽ ch","ĠS VM","ate k","ĠF rey","ä»ĸ æĺ¯ä¸Ģ个","说 ä¸įåĩº","åħŃ ä¸ĥ","Ġborder line","ĠвоÑģ п","ĠÑĤÑĢÑĥ дов","Ġdischarg ing","elm Ã¤ÃŁ","Ġf aux","Ġy olk","ĠL oud","çģ« åħī","åºĶ该 æľī","å·´ 马","è¯Ĺ ä¸Ń","ìĤ¬ 를","cr ime","Ġtrabal h","Ġreplic ates","à®¾à®Ł à¯įà®Ł","ĠÙĪØ² ÙĨ","/ al","Ġ à¹Ģม","ill iam","Ġun ethical","Ġdis dain","æŃ£ åĪĻ","ĠUn limited","满 æ´²","éħ¸ 碱","Ġgig abytes","çĪ·çĪ· 奶奶",". Org","T rip","Ġt ám","åı¯ æĺ¯ä¸Ģ","ä¹Ł è§īå¾Ĺ","Ġent h","ĠGu ess","rel igious","ĠÙĥ ÙĨت","éĢ£ æİ¥","eed back","ĠYoung er","ç¾ŀ è¾±","kow o","x A","le ague","ĠC out","åΰ æĿ¥çļĦ","ä¸ĭ 设","Ġref rigeration","æŀĹ åľ°","Ġnull a","Ġhon oured","æ°ijæĹı æĸĩåĮĸ","ĠConf idential","èĪĴéĢĤ çļĦ","ĠNich olson","Ġs org","Ġis Valid","Ġk itten","Ġse conde","ear ning","Ġso ient","æĸ° çīĪ","Ġrest raints","èĤ¡ æľ¬","Ġarg on","åįķä½į 为","ä»İèĢĮ è¾¾åΰ","給 她","äºĭä¸ļ éĥ¨","uen cias","ĠTu ple","ĠAqu inas","¶ ģ","in ox","æ¯Ķ æĭŁ","ä är","Ġdist racting","ĠZ er","åıĸ æĿIJ","æĹ¶éĹ´ åİ»","æĪĺ èΰ","çļĦ人 åľ¨","-f uel","åħ« åįĥ","Ġস à¦ķল","è·ij æĿ¥","Ġindustrial ized","Art ikel","Certain ly","$ /","çļĦ 表éĿ¢","èµ· èĪŀ","å¹² åĬ²","Ġdiv ider","ми ÑĢа","Ġcit oy","Ġfig s","èĪĴ çķħ","ĠпÑĢе де","-dim ethyl","Ġmonst rous","Ġwh im","æ³ķ åľĭ","Ġso al","åģ µ","าภį","Ġes as","ç¥ŀ åĨľ","å¸Ī 妹","ention ally","ĠUS ING","ĠPar ade","Ùı ر","åIJ¾ å°Ķ","uw en","è¿Ŀ约 éĩij","Z F","at ype","Ġin co","ĠS ES","od ot","åĬ¨ å¼¹","Ġsub net","Ġ: )Ċ","æ® ¼","any ahu","主è¦ģ é¢Ĩ导","åıĮ è¯Ń","æ¯į çĮª","éĤ£ä¹Ī 好","Ġmas hed","ĠBr une","Ġattract iveness","ðĿIJ º","æĮº æĭĶ","Ġconven ed","ĠAlf onso","ĠобÑĬек ÑĤа","Ġaston ished","ĠÐŁÐ¾Ð¿ иÑģ","ĠB ose","åΰ è¿Ļ个","-s hell","att ach","Ġ}ĊĊ ĊĊ","åı³ èĦļ","é²ľ ç¾İ","ĠBal anced","è¡° å¼±","ल à¥Ģ","Ġkl asy","ĠDIR ECT","O v","ĠI CS","Ġco efic","ç»Ħ å§Ķä¼ļ","ĠZ oe","ãģĻ ãģIJ","Ġid iosync","æĭħ å¿ĥçļĦ","âijł âij¡","/pro file","Ġlever aged","ENS ION","è¿Ļäºĭ åĦ¿","æĹłç§ģ å¥īçĮ®","Ġsow ohl","C HE","ĠM enge","Ġby e","ge o","ä¸įæĺ¯ åIJĹ","æĬķ 篮","æĮī çIJĨ","ĠاÙĦÙħ Ùĥتب","èĢģå¸Ī 说","Ġsusp icions","åħ¬åħ± åĪ©çĽĬ","Ġfacilit ator","çĵ¶ ä¸Ń","Ġrepro ducible","èı² åĪ©","ĠDaniel le","Ġenorm ously","缺çĤ¹ æĺ¯","b ags","ĠA VA","ant ry","ĠY ue","交 æıĽ","à° ¶","é¡¹çĽ® åĴĮ","äºĴ åĪ©","帮 çĿĢ","Fig s","Ġcz yt","Ġthir teenth","æĥħ æĵį","ä¿Ŀ æľī","æīĵ åľ¨","li ber","æŀĹ èĤ¯","Ġredu cir","ED MF","Cl ip","Ġtotal mente","è¯Ĺ çļĦ","Le ader","Ġroad way","Ġsn aps","ÐĴ ÑĤ","Ġwa arbij","ç¼ĺ çͱ","åĿł èIJ½","+ :","Ġp óź","ri osis","ĠK rit","cl i","ĠSe at","Ġé rt","ĠCH ANGE","Ġhint ed","meas ured","qv ist","on et","Ġst ares","Ġgl ared","ç²¾ æ°Ķ","Ġbi ologists","ĠС ÑĢ","èĥĮ 离","ĠWest on","çļĦé«ĺ éĢŁ","ĠSS H","ĠпÑĢиÑĩи нÑĭ",". Resource","åľ¨ è¿Ļ次","к нÑĥ","åĽŀ çļĦ","åįĹ æŀģ","lo ed","-re peat","åĵŃ ç¬ij","Ġrub y","ĠAdjust ment","ĠNerv ous","quarter ed","Ġcál culo","Ġoblast i","ĠSt en","Ġapp are","åĩł å¹´åīį","li we","ED ER","ä»ħ éĻIJäºİ","à¯ģà® ³","bin om","Ġwithdraw ing","- termin","Ġh Pa","ä½ľ çŃĶ","ä¹ĭ æľ¯","áĢ ®","ä¸ĥ ä¸ĥ","ĠPre heat","æŃĮ é¢Ĥ","Ġkil o","Ġuns upervised","马åħĭæĢĿ æģ©æł¼æĸ¯","大åĬĽ æİ¨è¿Ľ","Ġriv ol","q c","ure en","per c","ys er","amb iente","Ġне ÑĦ","ç´§ 缩","غ از","é¡¶ ä¸Ĭ","'] ;ĊĊ","éĹŃ å¡ŀ","gu id","Ġscram ble","EDMF unc","en an","é gal","ater ally","éĢļ åĪĻ","åıĬ åºĶç͍","Ġph ishing","æŀĦ æĥ³","-m ax","æľĥ ä¸įæľĥ","å¾ģ æĸĩ","æĽ´å¤ļ 人","èĴĻ çī¹","Ġarte facts","ĠAless andro","Į ĵ","åı¯ 她","é«ĺ ç¨ĭ","sp inal","ва нием","åIJĥ çĤ¹","à¸Ī ิ","å« ¦","Ġê° ĸ","Ġwid en","ĠFull EDMFunc","Ġamazing ly","à¸ģัà¸ļ à¸ģาร","ĠLag rangian","ocom plete","-rank ed","A cknowledg","Ġb ât","Ġpro cur","ĠV od","æĬĬ éĴ±","Ġdec rypt","å¦Ĥæŀľ éľĢè¦ģ","å¾· è¡Į","zi ako","éģĶ æĪIJ","Ġsek arang","ĠlÃł m","éķ¿æĹ¶éĹ´ çļĦ","Ġسر طاÙĨ","润æ»ij æ²¹","ä¸Ń éķ¿æľŁ","æ³ ĵ","Ġev ils","稳 稳","Ġме ж","Ġhair y","CL UDE","ĠÚ¯ ÙĦ","ãģĪ ãģ¾ãģĻ","utter stock","ä¹Ķ æľ¨","ĠPra ha","æĸ°åĨł çĸ«æĥħ","ÅĦst w","ĠÙĪØ± زش","- empty",". Any","z ki","ä¸Ģ 缮","ä¸įæĺ¯ 为äºĨ","é¢Ħ ä¹ł","é£ŀ åİ»","èĩªçĦ¶ çݯå¢ĥ","ĠÐIJ нд","olic ies","å¤ļå°ij 个","ç͵åŃIJ ä¿¡æģ¯","æĨ Ķ","ãĤ¢ ãĤ¯","ĠBra gg","Ġtriple t","Ġangl isy","Ġlamin ated","( CH","[ lower","Ġn garan","æķ°æį® ä¸Ńå¿ĥ","Get ter","ev olution","ä¸ĭéĻį åΰ","çĬ¯ç½ª è¡Į为","æģĴ æĺŁ","Ġalarm ed","ou in","Ġin mate","art ifact","表 ä¸ŃçļĦ","me asures","arent a","ĠApp earance","éĿŀ常 å¤ļ","Ġkin ematic","Ġâĸ ¶","ĠRES UM","Tok ens","ĠвÑĢа Ñĩ","é ter","ĠUn c","ĠMe ad","Ġcreat inine","Ġpri zed","çĩ İ","çİ© åĦ¿","èįĴ åĶIJ","ĠÚ©ÙĨ ترÙĦ","ĠпеÑĢв ÑĭÑħ","ĠاÙĦÙħر أة","Deg ree","é¡¿äºĨ é¡¿","( search","he en","Ġl ame","Ġv ii","ĠB MP","æĹ ³","æľī æĪij","åľ° çĽĺ","ec ia","ãģĮ ãģĤ","ĠEl k","Ġobserv ance","Inter active","软件 çļĦ","ĠBarn ett","ÅĪ uje","VIR ON","ĠAlej andro","^ .","t ro","ĠN issan","ah s","æĹł æĤĶ","ĠCl int","æºIJ åľ°","ಠ¶","amb iguous","Ġang st","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","rat os","åĬŀåħ¬å®¤ 主任","czy Äĩ","纪å§Ķ çĽijå§Ķ","ĠBM J","- used",". yml","W ere","l abor","ĠG ros","ĠK omb","ym ing","-g ly","æľĿ çļĦ","ĠSer iously","OL S","ĠOut s","ков ой","Ġmultipl iers",") V","res i","Ġha vet","è¿ĩ æĪij","å¾Ī æ·±","IT CH","-p rem","ĠС ол","ĠÙĦÙĦ س","ĠIsland er","èī²å½© çļĦ","ĠÃĸ sterreich","æµ·åįĹ çľģ","( unsigned","ar ono","ĠM TV","Ġinter ne","aj n","____ _ĊĊ","ĠLong itudinal","(G L","oter ic","ĠبÛĮÙħ ار","ĠFUN CTION","æĺ¯ åIJĮ","æī Ī","ok k","ov ement","AN SW","åıĭ 好çļĦ","Ġge ologic","夫 åIJĽ","ĠMont eneg","Ġ×ª× Ĺ","ĠFer reira","ĠдейÑģÑĤви й","b uster","ĠI w","ä¸Ģ åıij","åĴĮ çľģ","åij »","对 æľªæĿ¥","åıĹ é¨ĵ","æĪĺ ç¥ŀ","éģĵè·¯ 交éĢļå®īåħ¨","车è¾Ĩ çļĦ","缸åıį çļĦ","ĠBart lett","ĠBB Q","åĺŁ åĺŁ","ĠÙħÙĨØ· ÙĤØ©","Ġполож ение","ĠÑħÑĥд оже","in yl","Ġd unes","æĸ Ľ","ĠF oley","ĠW uhan","Ġper ched","åĽ½ åIJĽ","å®¶ éŨåı£","æ³ķ çŃī","æĸ° æĶ¿","Ġdem arc","ว าม","ãĤĴ ç͍","Ġfinal ized","ä½Ľ åĥı","ĠÐĵ ÑĢа","Ġcrack ers","Ġoat meal","Ġexhilar ating","= np","f ÃŃ","ĠP ett","ĠB ö","** }","æīĵ å®Į","èĬĤ åζ","Ġз еÑĢ","ú car","ĠPre ferences","æī§è¡Į å®ĺ","ĠPerson ally","Ġenvelop es","ĠLepid optera","å±Ĭä¸ī ä¸Ńåħ¨ä¼ļ","ĠR iding","è¿ĺ è¡Į","æıIJ éĢŁ","ON ES","kt et","Ġиз ÑĥÑĩа","çī¹åĪ« 好","æĺ¾ç¤º çļĦ","éħ¶ çļĦ","Ġs add","el or","ad justed","ä¸Ģ æĽ²","Ġà ŀ","Ġrel iant","å°Ĩ æĿ¥çļĦ","æ¡ ¿","ĠÙĦ ÙĥÙĦ","Ġس اÛĮر","ç´§ è·Ł","Ġsitu ação","Ġnatural es","åį° åζ","Ġmer id","(). __","Ġgar rison","rach ten","Ġhect ometers","Ġincarcer ated","b ble","} z","at ine","ĠK uz","ï¼ī -","æł¹ çļĦ","Ġع ص","que le","å¿ħé¡» 以","åħ¶ä¸Ń ä¹ĭä¸Ģ","log ging","UL O","ĠConse il","ì³ IJ","Ġm oor","ĠE re","ĠN LR","æĪij 以åīį","大 åIJ¼","ä¸ī å°º","ĠTo oth","amb i","ĠÙĨ Ù쨱","AM I","ĠAnal yses","ĠобÑĢазова ние","ĠProc urement","Ġnatür lich","çļĦ ä¿¡å¿ĥ","Ġinv oking","æĪĺ æľº","åİ¿ å¿Ĺ","Ġpast ures","两个 åŃ©åŃIJ","ĠAN T","åı¸æ³ķ å±Ģ","å°½åı¯èĥ½ åľ°","Ġinteress ante","Ġziek te","ut nya","æľī åĽĽ","Ġab l","ge geven","ä¸İ æľ¬","åŃ¦ä¹ł æĸ¹æ³ķ","ENT ITY","æĢİä¹Īæł· äºĨ","रà¥įठ®","- added","N in","Ġv yr","å°ı ä¸ī","被 çĽĹ","Ġcar ve","è§ģ æŃ¤","Ġна й","使ç͍ çļĦæĺ¯","Ġpract ised","ло е","ĠHay den","ĠопеÑĢа ÑĨии",") ãĢģãĢĬ","X S","n ol","Ġw elt","ä» ®","ä¸Ģ éĶ®","Ġpro g","ĠL AT","Ġat las","è¿Ļ åī¯","Ġph ased","ाठĪ","ĠFin ch","Ġmis deme","Ġirrit ating","飲 é£Ł","åµĮåħ¥ å¼ı","Blo om","Ġrozwo ju","H ans","h g","et Code","âĢĿ ï¼ģ","ä¸ī èģĶ","è´Ń åħ¥","à¸Ĥ à¸ĵะ","æ³ķå¾ĭ æı´åĬ©","_m at","Ġâĸ ª","åįķä¸Ģ çļĦ","E dition","Ġc pu","Ġb itten","Ġin experienced","et ro","ur ic","Ġв Ñĸ","Ġmicro processor","åı¯æĺ¯ ä»ĸ","ä¸įçŁ¥éģĵ èĩªå·±","ĠDist inguished","æįŁå®³ èµĶåģ¿","_RE QUEST","çĸ¤ çĹķ","Z M","ä¸Ģ ä¸įå°ıå¿ĥ","å¤ļ ä¸ĩåħĥ","éĤ£ åı¥","åıª åĽł","Ùĥ Ùİ","ars ch","Ġscre wed","ĠاÙĦØŃ ÙĥÙĪÙħ","ĠÙĦÙĦ ÙĨ","碰 ä¸Ĭ","åIJĦ个 çݯèĬĤ","çļĦåľ° çĤ¹","level s","pattern s",", ......","r j","Ġf umes","ow ano","åΰ ä¸Ģèµ·","çħ§ çĿĢ","åĸľ è¿İ","ih i","é¼ĵ åĭµ","åĪĽå»º å·¥ä½ľ","Ġби о","Stat istical","Ġìĸ¸ ìĸ´","K ent","Ĺ ×Ļ×Ŀ","âĢ ¹","ĠAl arm","æīĵ åĪĨ","æĶ¶ è§Ĩ","Ġprof ond","ĠبÙĩ تر","-F our","Ġcomponent es","éĶĢåĶ® éĩı","Ġliqu ef","ÙĬÙħ ÙĬ","Ġpetition ers","åĿŁ å¢ĵ","\" ।","Ġd ps","ĠC ada","ĠK all","å·¥ çļĦ","ç±» èį¯çī©","åı· 为","ĠاÙĦÙħ عد","Ġcond enser","ĠPol o","ä¹ĭéĹ´ åŃĺåľ¨","Ġdraw ers","can vas","ìĭľ ê°Ħ","åĤ» çĵľ","Ġfres co","ĠCONCLUS IONS","ĠT rie","ä¸į ä»İ","Ġch ic","Ġpr er","Ġinter related","ä»Ģä¹Ī éĥ½æ²¡æľī","æŁ¥ çIJĨ","ĠAP PE","Ġol ives","Ġgluc ocortic","éĸ¢ éĢ£","Ġ________ _","ĠAuf gabe","é»ĺé»ĺ çļĦ","à§įদ à§įর","Ġinterchange ably","P ra","ĠB orders","ĠB ootstrap","ĠH are","ĠSch iff","Ġbi ochemistry","arr er","Ġber ry","ÙĦا Ùĥ",".res ize","\\+\\ _\\+","ĠngOn Init","= <","H CO","N z","Ġa es","Ġse ams","å¦Ĥæŀľ ä¸įèĥ½","åıijçĶŁ çİĩ","éĻįä½İ æĪIJæľ¬","лек ÑĤÑĢо","æİ¥è¿ij äºİ","Ġmehr ere","Ġjew ellery","ĠÙĪØ¹ ÙĦÙī","Ġangi ography","Ġg ird","人 ä¼ļ","Ġgener ality","ĠPr ima","Ġcoll ide","çĥĪ æĹ¥","Ġdark ened","Ġ×IJ ×ķ×ŀר","ä¹Ļ éħ°","Image View","ĠTax onomy","лÑĭ м","Ġdys plasia","Ġjew els","ĠнаблÑİ Ð´Ð°","Ġstab bed","Ġneurotrans mitter","سط س","ĠL ark","ĠHow ell","Ġза пÑĥ","empt ive","Ġdim ethyl","gu ess","纵 è§Ĥ","åĭĴ æĸ¯","ĠBern ie","ĠпоÑĤ ÑĢеби","ĠâĶ ľ","Ġtv Ã¥","Ġwart oÅĽci","Ġlaat ste","çļĦ å®£ä¼ł","ĠP us","form in","åĨį 审","åIJ¬ ä¸įæĩĤ","æľįåĬ¡ æ°´å¹³","-c oding","à¥įठŃ","ĠPre face","just ice","ĠÐĹ Ð´ÐµÑģÑĮ","μο ÏĤ","çά èµ·æĿ¥","ĠNiger ians","ĠInit iatives","ĠÑĢай он","================================================================ ========","S ant","n ights","Ġw ody","Ġn Äĥm","åħ¨ é¢Ŀ","Ġfl aming","ãģŁ ãĤī","è¿Ļä¸Ģ æĹ¶æľŁ","ç§» é»ĺ","ĠComp iled","ä¹Łæľī æīĢ","Ġuns pecified","Ġdw ind","æģ¢å¤į åΰ","Ġapart heid","Ġdil at","orden atuak","angg ap","Ġlapar oscopic",".Tab Index","F est","ig as","Ġdo el","Ġп олÑĮзÑĥ","çŃī åįķä½į","ï¼ģ ï¼ģâĢĿĊĊ","å®ī åį±","åįĬ åĪĨ","ç¦ı å¾·","ĠAng us","Number Of","Ġsz em","ĠContract or","Ġunle ash","B erg","X t","_ command","ar ren","ĠS ich","群 èIJ½","Cl one","æĬ¢ 夺","ĠAud rey","ç»§æī¿ äºĨ","Ġpac ient","Ġcrown s","prov ide","Ġimpe cc","ĠÑģказа ÑĤÑĮ","ĠI CM","se gment","Ġk ebutuhan","å¤ļ åıij","æ±Ĥ çĶŁ","士 åįĴ","æīįèĥ½ 使","β ο","ĠUN ITED","post ed","åĽĽä¸ª æĸ¹éĿ¢","( NO","_ ALL","ĠD ome","åıª è¦ĭ","çĬ¶ è¯Ń","Is n","åĨ¬ èĩ³","çļĦå½±åĵį åĬĽ","à¹Īาà¸Ļ ัà¹īà¸Ļ","ocl ass","Ġtyped ef","â Ĵ","Ġs agen","ĠA rag","Ġy ks","ph ans","з Ñĸ","è·¯ åŃIJ","Ġпо ÑĢа","ĠEd itions","æĿĢ æĪ®","åį±éĻ© æĢ§","Ġ$$ Ċ","Ġserial ize","ÑģÑĤÑĥп лениÑı","p ause","ä¸Ģ æ°Ķ","è° ¤","æľĢ å¼Ģå§ĭ","Ġjust ices","ç§ij å°Ķ","ĠSc outs","è¸ ī","Ġ×©× ł×Ļ×Ŀ","Ġreflex es","ç²¾ç¥ŀæĸĩæĺİ å»ºè®¾","L ISH","ä½ ĥ","ĠO ch","ze h","ĠApp end","åİ¿ 人æ°ijæĶ¿åºľ","ĠÙĥ Ø«","Ġব ির","ĠÑĤеле ÑĦон","Ġpyt est","ä¸Ģ æĻĥ","ie i","ci Äħ","Ġн омеÑĢ","å¸Ĥ ä¸Ń","ол Ñİ","Ġر ابط","å·´ æĭī","ĠTrans former","elle tt","ান à§ĭ","ĠUk rain","Ġlig aments","æī¹åĩĨ çļĦ","ãĥį ãĥĥãĥĪ","ké nt","ĠSpot light","niejs ze","ĠBurg ess","Ġhypothalam us","Ġt b","ĠF iona","Ġle aching","ij os","ан г","DP I","ĠÄį lov","Ġkill ers","Ġcommission ing","Ġhosp ice","Ko ordenatuak","Ġjul io","Ġ ðĿľ","ĠP LEASE","ĠE usk","ä¼ł æĿ¥äºĨ","Ġrest a","Ġsi ete","èŀ ¨","æ¿Ģ è¿Ľ","åı¦ä¸Ģ åĢĭ","ĠìĻ ķ","Ġapt itude","Ġlign in","Ġun ifying","çĶŁ åľ¨","د ÛĮد","好 åķ¦","æĥ³ ä½ł","åĪĻ çͱ","èįī çļĦ","ব à§ĩন","Ġgrand daughter","ache v","åıªèĥ½ 说","éĢļ常 åľ¨","ئ ات","Ġtak ich","ள à¯Ī","تÙĬ جة","à®® à¯įப","Ġgri ps","åĬ´ åĥį","g oto","h aupt","ĠL ec","ise cond","Ġreg el","åıĬ åIJĦ","åıª 管","æ¯Ķ æ¯Ķ","æĬĬ æĪijçļĦ","ven irs","ล à¹īà¸Ńม","Ġза ÑĤ","审 å®ļ","_f ilename","Ġaltern ativa","cast s","ª× ŀש","ков о","ç¦ħ å¸Ī","åºŁå¼ĥ çī©","ol aryng","ĠB out","ä¹ĭ 计","没 说","Ġhum ankind","åĨĽ ä¸Ń","ĠRep ublik","Ġadjust s","zie h","ĠExp end","Ġsick le","çŃ¾è®¢ çļĦ","Ġmagnet ization","Ġinqu ired","Ġslugg ish","d onald","x v","it ty","Ġp rou","å°± çŃīäºİ","ä¹Ł å¤ļ","ov ar","Ġz ape","Ġbi oge","Ġdoc ente","Be ck","________________ ______________","à¥ĭ ऽ","ĠCard iology","ãĤĤãģ® ãģ§ãģĻ","ĠKrist en","ĠÑĸ н","Ġhistó rico","Ġimplic a","Ġinic iativa","J oint","k raft","ĠH ike","åľ¨ éĢĻ裡","Ġclass ifiers","çĸ Ŀ","åıĪ æĥ³","ĠEx eter","书 缮","äºī æĸĹ","cont acts","ä¹Ŀ æ±Ł","åºĹ å®¶","ÐŁ еÑĢ","æ®Ĭ ä¸įçŁ¥","Ġcatch y","æĸĩæĺİ å®ŀè·µ","èħIJ æľ½","-lim iting","il idad","ä¸Ģ æĹłæīĢ","ä¸Ģ æĢĶ","Ġus ia","Ġbusiness men","Ġcr umbs","åĭķ åĬĽ","绿 åı¶","unk er","Ġrapid ement","Ġrain water","åĩŃ ç©º","ĠTor ino","ĠShel by","ĠE rm","Ġse ura","Ġro gue","åij¨ çļĦ","马 æ¡¶",".get Message","exp and","inte gr","ÃŃc ÃŃch","ä¸Ģ大 æĹ©","ã썿ĢĿ ãģĨ","Ġpunct ure","ĠPhen omen","O i","_ option","c ic","m berg","Ġbe kerja","主 å¼µ","Ġб еÑĤ","-t alk","emp uan","has il","Ġsuit case","åĦĺ 管","ĠÑħол од","- na","Ġs cler","st va","æµ ľ","å¹´ å¤ľ","gh um","æĹ¥ æ¶Īæģ¯","Ġfr équ","åĩºçݰ éĹ®é¢ĺ","æĸĩä»¶ åĴĮ","Ġmatch up","ĠRa ise","çĻº 表","íĮ ħ","ĠWool f","ysty rene","ĠR ai","ÑĢ Ð¾ÐºÐ°","å¤ļ ç±³","Ġ+ #","ĠAn ast","æ±Ĥ å©ļ","æĢ» èĢĮè¨Ģä¹ĭ","arn o","ä¸ŃåĽ½ 社ä¼ļç§ijåѦ","èĬ± å²Ĺ","bi ological","åħģ 許","Last Name","าà¸Ĭ à¸Ļ","åĵ¥ä¼¦ æ¯Ķäºļ","Ġst ump","row ed","ĠX YZ","att ia","åĨĽ ç͍","ĠÙĩ Ùī","èĶ »","Ġbur ge","æĤī å°¼","Ġec lectic","æ¼ı æĸĹ","ĠActive Record","Ġnest led","Ġsquad ron","consult é","ÙħÙĤاÙĦ Ùĩ","le on","ĠE hr","ĠF ilipp","se lection","ĠK ish","Ġpre tt","ç¥ŀ éŃĤ","æĢ» ä¸įèĥ½","Ġvol umen","Ġر ÙĪØ¯","Ġconcent ric","Ġinsp ectors","Ġmedium s","Ġbull s","Ġrepublic an","實éļĽ ä¸Ĭ","Ġpamph let","st al","un ia","ĠP ew","æĪij æŃ£åľ¨","大 æĢĴ","å°± å¤ŁäºĨ","Ġ{ /*","åľ° 說","便 æIJº","Ġben ches","UT ES","umb uhan","ÐŁ еÑĢе","λλ α","cc al","é«ĺ 产","建 åįİ","常 ä½ı","羣 æĥ³","æĭ¿ åĩºäºĨ","æ²ī å¯Ĥ","ĠDec o","â̲ )","æ¸IJ åıĺ","exp ressed","缩 åĩı","åļ ı",".find All","åľĺ é«Ķ","prop ylene","è°ħ è§£","Ġn M","Ġre define","ĠM if","æ°´ åĬ¡","Ġx u","Ġد ائ","åĿĩ åºĶ","Ġ×ij× ĸ","Ġple ural","ĠìĿ´ 루","Ġontwikk eling","ĠBev ölker","Z B","v ars","Ġme adows","æŃ¤ è¨Ģ","åıį èħIJè´¥","å¢ŀ åİĭ","AL ES","åı¶ 天","æĽ² åŃIJ","師 çζ","Ġê³ ³","çĤ¸ èį¯","Ġprz eb","×IJ ×Ļ","_set tings","d ifference","st el","ĠB rowning","Ġcre ación","ç¬ij åĺ»åĺ»","Ġexc ursions","Ġmol é","/ th","Z C","ie ÅĦ","æķĻ å§Ķ","éŨ ä¸Ĭ","æĮģ ä¹ĭ以","é£İ å°ļ","èİ ħ","over ning","Ġsuper markets","Ġprofess ores","Ġspecial ties","ĠPart e","gy z","æŃ£å¸¸ è¿IJè¡Į","umer ate","Ġsyn apses","Ġhabit antes","ĠSign als","赫 å°Ķ","Ġتر Ùĥ","' Am","ĠE ch","åΰ éģĶ","á genes","æł¡ 对","Ġum bil","é¹ ¦","ãģ¦ãģĦ ãģªãģĦ","森æŀĹ åħ¬åĽŃ","Ġprod uto","à¸ŀร à¹īà¸Ńม","èĺĭ æŀľ","( status",". InputStream",": b","B ERS","ess on","), [","Ġar ty","æľº æĪ¿","×Ļ× ŀ×Ļ×Ŀ","Ġsc o","Re vised","Ġinf e","èİ· æī¹","Ġaccount ants","Ġqui eter","Ġcampaign ing","éĽĨä¸Ń äºİ","áĢº áĤ","Ġvine yard","Ġkas ag","arend ra","F ern","ĠC rest","æľī æĺİæĺ¾","ĠU ppsala","对 身ä½ĵ","æµ· æ·Ģ","Ġtest es","çłĶ åѦ","ĠPr at","Ġcond izioni","ĠоÑĤ Ñģ","è¸ µ","OP E","è´¦ åįķ","หà¸Ļ à¹Īวย","åIJĮåѦ们 çļĦ","æĿijæ°ij 们","æĹłæķ° 次","éĵĥ 声","em ment","äºĨ åĩºä¾Ĩ","Ġqu arry","ĠCal cutta","ĠØ® ÙĪØ§ÙĨ","ĠMart a","çĶľ ç¾İ","gr é","æĬĽ åĩº","å¼Ĺ åħ°","Ġ×Ķ×¢ ×ķ׾×Ŀ","ĠInform al","im ide","ĠC ri","ĠK ond","Ġz it","ec al","主è¦ģ åİŁåĽł","ese hen","(t rain","_n on","宫 çļĦ","imb ledon","Ġ×Ĺ ×Ļ×Ļ×Ŀ","åħ¬å®ī éĥ¨","bat is","CRE MENT","ĠпÑĢогÑĢам м","Ġmistaken ly","Vict oria","C ourses","p ail","大 çĹħ","é«ĺ çĥŃ","Ġà ¦","æĸ¹æ³ķ 论","bl adder","ä»»ä½ķ æĹ¶åĢĻ","积æŀģ åľ°","åįĸ çļĦ","ĠRad ar","Ġont ological","åĵ¼ åĵ¼","Ġunderm ining","ĠBrew er","Republic an","é½IJå¿ĥ åįıåĬĽ",") i","ĠW D","ä½ľ åĵį","Ġdis abling","è· ¤","Ñĩ ке","æĹł åĬŁ","æĻĤ çļĦ","Ġnov iembre","èĨľ çļĦ","ĠSam son","Ġrul ings","ä¸īè§Ĵ æ´²","C AM","} ',","ĠS rin","ak ings","大 æ²³","对 æķ´ä¸ª","å¹´ å¹¼","她 便","ä½İ éĢŁ","èĭı å®ģ","åĢĴ åľ°","Ġgraph ically","Ġú til","Ġru pees","çī§ åľº","anth us","Ġvine yards","( Context","Ġh ires","ä¸į ä¸ĭåİ»","äºĨ 声","Ġnew found","Ġsupp ressor","èĢĥ åīį","me ier","ÏĢ ÎŃ","Ġcaus as","vi amente","Ġcontra ind","áĥĿáĥ ľ","ĠدرÛĮ اÙģØª",", U","_ term","b ole","w arning","ud get","Ġcl ases","ä½ł ä»Ĭ天","éħį éŁ³","追 æĿĢ","åĭķ æīĭ","Ðļ Т","ଠ¨","Ġscreen ings","Ġáĥ Ĺ","Where as","V PN","a uthors","ĠF aces","çĶŁ çĶ£","Ñı ÑĢ","说 åΰåºķ","å¼Ģ è£Ĥ","åħ¥ èĤ¡","çĹ ¿","æĶ¶ è´§","ç±» æİ¨","çĮ ĸ","æĿİ äºij","-M ed","Ġಠ¦","Ġrepet itions","Çİ o","ĠCant on","Ġethn ographic","Ġcler ical","æ¯ĭ 庸","ĠCoh ort","æī«é»ij éϤæģ¶","Ġt ast","çļĦ å§¿æĢģ","ĠH alle","èĩª 以为","æĪij们 è¿ĺæĺ¯","ç¾İ 满","ĠNot Found","ç»ĵæŀĦ ä¸İ","æīįèĥ½ åľ¨","ĠÙ¾ اسخ","ĠOut reach","åįģåĪĨ éĩįè¦ģ","ĠëĮĢ ìĥģ","ä¾į 女","ĠпÑģи Ñħи","åľ£è¯ŀ èĬĤ","äºĨåı£ æ°£","d rug","er ic","ä¸Ģ éĹ®","Ġk ét","åı¯ è´µ","ĠK irst","Ġا Ùĩ","æĶ¶ ç´§","æħ µ","Ġد ÙĨداÙĨ","主è¦ģ 表çݰ为","è¡£ è¢ĸ","稳 åİĭ","Ġfa ible","Ġmodern a","Ġ×ij ׾×","UI Kit","éģ¥ è¿ľçļĦ","ĠTal ks","ĠReturn ing","rup al","ç¾ħ æĸ¯","-pe er","Ġl ze","un y","ĠP OW","ä¸Ĭ 好","Ñĩ ÑĤ","Ġz im","èİ« æµĭ","ĠGr ü","리 ìĬ¤","Ġcolon el","æľīä»Ģä¹Ī äºĭ","wi ata","Ġaer odynamic","Ġvra iment","Ġculmin ation","/ form","ĠF RE","æľī æĻĤ","Ġk ho","ä»ĸ æĿ¥","æ® ĥ","交 æĦŁ","ä¸ŃåĽ½ æĶ¿åºľ","åįĹ å¼Ģ","åij¼ 声","ĠMat lab","à±įà° ª","ĠاÙĦص ÙĨ","èŁ ¾","檢 測","輸 åĩº","Tok yo","ĠCrow ley","Ġb ends","ĠAl ley","竳 çļĦ","ĠÑĤ веÑĢ","Ġrad ially","ĠBar oque","çĺ¦ èĤī","ĠDown s","Ġcontr ôle","è§ĴèIJ½ éĩĮ","Ġpoc zÄħt","Ġphysic ists","Ġতà§Ī রি","( add","b aby","ا ÙĥÙĦ","Ġcon ex","ĠCh op","ink en","Ġinv aders","è´¨ éĹ®","ĠSp inal","ç»´ åIJ¾å°Ķ","åºĹ 主","Ġsav vy","ĠAD S","*** ĊĊ","ĠÑĢекомен да","á¿ĸ ÏĤ","_ body","z ure","re ys","Ġs ø","Ġde xt","ĠL age","对 ä¸ĢäºĽ","Ñĩ ено","ĠSp rach","è¡Ģ ç¼ĺ","ling u","enc a","èµĦæºIJ åħ±äº«","upp orted","γ Ïī","Ġ×ij ×Ļ","ä¸Ĭä¸ĭ åĬŁå¤«","éĨ« 師","Ġllev ar","ĠÑģогла Ñģно","( models","st elle","ĠS EL","ĠA AI","ĠH arcourt","ĠV EGF","æĭ Ĺ","ĠSt ain","éĢļ ç͍çļĦ","ĠPl anned","ĠNot withstanding","鼨 ä¸Ń","Ġdim inu","Ġze it","Art igo","å¾Ĺåĩº ç»ĵ论","Ġexped itions","ĠSort ing","lip id","g ui","í į¼","Ġpo zy","Ġsim ile","åIJ¬ åIJİ","és zet","å·´ æĸ¯","Ġnov as","ä¼ļè®® çļĦ","奥 çī¹","Ġsubt ly","è¡° èIJ½","ĠBot anical","Ġíĺķ íĥľ","bard ziej","å®ī举 å°¼",". access","Z w","Å Ĩ","对 æĪij们çļĦ","éĩij é»Ħ","Ġwater y","åıĤ åĨĽ","æ½ ¢","Ġparticip antes","label ed","ĠÐŃ ÑĤа","Ġê²ĥ ìŀħëĭĪëĭ¤","æĮª ç͍","Ġlibert ad","Ġhypert ensive","çĶŁ æĬ½","ĠK ow","æ³ķ åѦéĻ¢","å¾Ĺ å¿«","Ġexp anse","åĮ» çĻĤ","add ad","Ġtotal ing","ĠØ´ رÙĪØ¹","Ġин ÑĤенÑģив","Ġprox ies","ä¸Ģ对 ä¸Ģ","æĸ¹æĸ¹éĿ¢ éĿ¢","* }Ċ","Ġt aman","ri ção","ĠN FC","Ġr ere","Ġz az","æĥħ ä¸įèĩªç¦ģ","Ñħ ал","Ġâ «","åģļ åĩĨå¤ĩ","Ġinf ek","æĬĹ çĻĮ","Ġreflect ance","ĠاÙĦع رض","ĠOff set","å°Ĭ èĢħ","å¿ł å¿ĥ","Ġjak ie","леÑĤ и","Power ed","ĠVander bilt",", O","b aren","Ġf x","Ġis omer","Ġpo lem","å·¥ä½ľ ä¸Ĭ","èĬĤ 度","Com pletion","IS ON","ĠAm bro","缴æİ¥ åľ¨","Ġpsych otic","é£Łåĵģ èį¯åĵģ","ĠDies er","带头 人","ĠоÑĤноÑģи ÑĤÑģÑı","d ostÄĻp","Ġa ç","ĠD ose","å¾Ī å¥ĩæĢª","Ġsom m","è les","Ġnature za","gor it","èĤº åĬ¨èĦī","Ġtherm ostat","×ŀ× ¡×¤×¨","Ġ---- .","Ġsuperconduct ing","æ±Łæ³½ æ°ij","_ ct","f ake","Ġb aja","om bre","ä¸į å±Ī","äºĨ åĽŀåİ»","ĠSt or","è¿ĩ ä¸Ģ次","æĹ¶éĹ´ éķ¿","/h ow","Ġdeb ilitating","殿 åłĤ","Ġcircul ate","Ġisot opic","Ġвод ой","Ġs ire","Ġb w","ĠRe ceptor","Ġpe kerja","æľĪ åŃIJ","æ°Ķ åĸĺ","Ġconf ounding","ros ive","å°į ä»ĸ","ĠFin ished","Ġwall paper","à¤Ĥ à¤Ĺ","ĠÙħØ´ اÙĩ","ĠConserv atives","Ġinter iors","ank ed","åħ± æĢ§","ä¼ĺ 缺çĤ¹","æĢİä¹Ī åĨĻ","ĠIN DU","Ġclient e","ëĿ¼ ìĿ´","空æ°Ķ è´¨éĩı","è¡Ĺéģĵ åĬŀäºĭå¤Ħ","ĠS SC","Ġper itoneal","æĸĩ éĢī","äºĨä¸Ģ åIJį","åĽ¢ ä¼Ļ","_P R","ĠоÑĤвеÑĤ ÑģÑĤвен","ĠFP GA","Rom ans","ĠClare ndon","Ġanter iores","Ġprzyk ÅĤad","e conomics","Ġa uster","Ġp uesto","as ome","st att","ĠD ile","Ġnot wend","å¸Ĥ æķĻèĤ²å±Ģ","ER ING","æĿİ å¤©","ä¼¼ æĺ¯","ÙĪÙĤ Ùģ","Ġdysfunction al","使 ãģĦ","ts y","é£İ åĴĮ","-in teg","æĹ¢ å®ļ","æīįèĥ½ 羣æŃ£","éĢī项 ä¸Ń","æķ°ç»Ħ ä¸Ń","Ġpon er","ĠChamber lain","itä ts","輩 åŃIJ","ĠмоÑī ноÑģÑĤÑĮ","ĠEntreprene ur","Ġжидко ÑģÑĤи","ĠD end","Ġhe fty","æĹ¶ æīį","Ġinter vie","ä mp","by gg","sk ého","å²Ľ çļĦ","Ġко ÑĢи","Trans actions","é£Ľ æ©Ł","å¾Īå°ij æľī","igt ausend","_pro file","Sing leton","ãģ¨ãĤĤ ãģ«","Ġeig ene","Ġtoug hest","es cap","å¤ļ è§ģ","ç»ĵ 转","ĠSe le","dis patch","éļIJ ç§ĺ","çݰ代 社ä¼ļ","(p oint","Be autiful","ëŁ ½","Under stand","Ġ×ª× ł×","以å¾Ģ çļĦ","Ġtras form","åĨłçĬ¶ åĬ¨èĦī","Ġsensit ivities","Ġh amp","ä¸Ģ åıį","æĺ¯ æľ¬","ä¾ Ĺ","å®¶ 裡","æ¯ı ä¸ĢåĢĭ","Ġpower house","ä½İ æĶ¶åħ¥","Ġintrodu ctions","wer king","Ġnan os","uld ade","ì¸ ¡","thumb nail","俨 çĦ¶","ĠC IP","æĬ ľ","-s itu","Ġfore closure","å®Ŀ å¦Ī","θ ο","Comp act","ĠRock efeller","Ġfavour ites","/ =","Ġs ilt","çļĦ è¯į","缮 ä¸į","Ġent rar","å±± 人","ĠPl ast","端 èµ·","è½® èι","ĠÑĤа н","Ġcivil isation","ÑĢова нии","-k il","Ġovert urned","Ġmason ry","ĠпÑĢоÑĤи во","i Å¡","ĠH AL","ä¸Ĭ ãģ®","çŃī èħ°","ĠAr x","客 å®¶","èĭ¥ éĿŀ","ÙĬÙĨ ÙĬØ©","çľī å¿ĥ","ÏĥÏĦ ική","ÑģÑģи и","ä¸Ńå°ı åѦçĶŁ","象å¾ģ çĿĢ","ä¼ĺèī¯ ä¼łç»Ł","ĠÑģÑĥм мÑĭ","/ ui","M J","S ounds","d aily","çļĦ æĸ¹éĴĪ","un ek","åı¯ è§Ĥ","ç¾ Ķ","åħ³ åı£","quest a","Ġdin am","ĠPass ing","åĴ¨è¯¢ æľįåĬ¡","à¦¾à¦ľ ার","Ġinterrupt ions","Ġterd iri","Ġhurd le","# print","g rant","ĠP RI","æĪij ä¸Ģ个","Ġun ten","åħ¶ ä¸ī","åIJį 稱","Ġdisc ut","Äį ÃŃslo","(s olution","raf ish","Ġва ÑĪ","ÙĪØ² Ùĩ","æ¸Ĺ åĩº","ĠÑģам ого","è·ª ä¸ĭ","Ġcraw led","ĠRhe in","ĠVolks wagen","æķĻ è¯²","Ġcommun es","第ä¸Ģ æľŁ","è¿ĺæĺ¯ 个","Ġmar co","ä¿ĥè¿Ľ ä½ľç͍","})\\ ]","olk ien","Ġrelativ istic","Ġпомога еÑĤ","c odeline","it iva","Ġf ern","ill ac","åĴĮ å¿ĥçIJĨ","Ġar du","产 äºİ",".s ign","Ġbi ologist","ĠPer uvian","éķĩ ä¸Ĭ","Im mun","Class ifier","ĠCle aring","ĠPlant ing","Ġminimal ist","ĠCover ed","Ġprost hetic","为ä¸Ģ ä½ĵçļĦ","Ġ무 ìĹĩ","GRAP HY","Ġquir ky","ĠÑģопÑĢов ож","è±Į è±Ĩ","? \",","k ých","ĠW and",".s lf","é¢Ĩ 头","éľĢè¦ģ ç͍","ÏĢ ÏīÏĤ","Ġbro od","èµ°äºĨ åĩºæĿ¥","ì¹ ł","ĠBeg riff","x z","æľī åĪ«","æĪij ä¸Ģ个人","ÙĪ Ø§Ùħ","ĠSt d","äºĨä¸Ģ 座","ĠÙĬ Ùħ","}) _{","è´¡çĮ® åĬĽéĩı","Ġprotest ing","âĻ Ģ","ĠглÑĥ бок","M and","_ us","am ins","æĺ¯ åħ¨","ĠH abits","æŃ£ 交","Ġmen urut","], \"",".C heck","Ġscient ifique","æŁı æĭī","Ġmetaph ysics","è©ķ ä¼°","Ġgau che","ĠStream ing","ĠÑģвеÑĤ а","Ġepist emic","st ice","ĠG ry","ä¸İ åīį","eb u","Ġgl a","çļĦä¸Ģ éĥ¨","ä½Ĩæĺ¯ è¿Ļ","çĤº ä»Ģ麽","åŃĺåľ¨ éĹ®é¢ĺ","part ner","Att endance","ekt ion",".y aml","ĠEug en","iatr ists","ĠcientÃŃfic a","Ġì» ¤","Ġmalign ancies","ĠØ£ÙĬض اÙĭ","ĠÑĤол Ñīи","Ä ĺ","Ġc att","Ġc umbersome","ig or","ari ables","Ġrem orse","Ġge val","æ²ī æ²ī","å¨ģ æµ·","ĠÑı к","測 å®ļ","æķĻ室 éĩĮ","ĠKy iv","ĠÙħÛĮØ´ ÙĪÙĨد","ulk ner","ĠDispon ÃŃvel",". An","u ously","ä¸į æ¼ı","åĴĮ åįİ","ä¸Ĭ 讲","Ġset Up","Ġmult iv","åIJ« éĩıçļĦ","Ġpit chers","Ġdict ator","ĠAF TER","Ġl át","æľī æĦŁ","æķ ĺ","ru kt","æľ¬ å½ĵ","Ġstr ony","æ¯ı 亩","Ġgrow led","ĠâĨ Ĺ","æ¼Ķ åĵ¡","对äºİ æĪij们","ç¿» å¼Ģ","Ġperspect iva","اØŃ ب","Ġboy cott","Ġઠ°","ĠWin chester","call back","çİ©æĦı åĦ¿","% /","B esk","_ month","ĉ color","ĠP OT","oc ultural","Ġob sz","Ġب ÛĮر","amp aign","è¨Ģ è¾ŀ","å¾® ç²Ĵ","aken ing","ëŀ ľ","鼶åĶ® åķĨ","abol ismo","Ġenvis aged","émat iques","ĠFranken stein","ur angi","ĠP EM","åľ¨ æ°´ä¸Ń","æĹ¶ ä»»","Ġ' Ċ","? ...","W inner","h ap","Ġ ith","al ance","ä¸į éĩįè¦ģ","ĠH af","ĠW ies","大 åıĺ","ep a","çŃī å·®","æľĢ ç®ĢåįķçļĦ","Ġ\\( +","Ġcle ft","Ġver be","çĺ ª","Ġbes oins","缸äºĴ åħ³ç³»","ĠHaw thorne","ĠNeed ed","å·¥åķĨ æĪ·","ĠجÙĩ اÙĨÛĮ","æ¶Īè²» èĢħ","N il","r ush","ra ut","ä¸ĭ æľī","ÑĤи ем","æ²³ ä¸Ń","_s ession","ÙİÙij Ø©","ĠØ«ÙĦاث Ø©","al to","ou z","Ġ[ `","æ¯ı æĿ¡","ĠRes idence","ãģĹ ãĤĪãģĨ","ĠâĪ £","èģļ é¤IJ","ĠRad iol","æĬĢèĥ½ çļĦ","Ġ×Ľ× ŀ×Ķ","rior ity","ĠMidd les","ĠCorrespond ence","m als","Ġby li","ä¸İ ç¾İåĽ½","AS ON",".get Logger","æľĿ å¤ķ",".A ct","ĠDi ocese","Ġfra il","Ġtro va","Ġcov eted","å¦ĸ ç²¾","éªĤ éģĵ","Ġauc une","Ġdisob edience","Ġindist inguishable","Ġ ợ","en arios","st uff","rom ycin","д оÑĢ","س د","Ġra j","çı ı","Ġaf ores","åľ£ æ¯į","Ġice berg","ÑģÑĤви ем","Ġнов ого","é§ ħ","èĤĨ èĻIJ","ĠинÑĦоÑĢма ÑĨиÑİ","Ġpleas antly","ا گر","ĠD ura","ĠN ASCAR","Ġsu cks","è¿Ľ éĢĢ","æŃ£ 绣","ä¿¡ çļĦ","Ġmet ri","ĠApr ès","ĠInter state","Ġgest ión","jen o","p icture","æĺ¯ 第ä¸Ģ","ä¸į çŃīäºİ","Ġr arity","éĩį éĩįçļĦ","Ġfil ings","å¤ı 天çļĦ","ı s","ãĥĪ ãĥ©","Õ¡Õ¶ Õ¡Õ¯","Ġcommercial s","Ġ×ł× §","ĠÑģоб иÑĢа","Ġtwe ede","/ \"Ċ","C oun","I ce","_ In","Ġp apa","ä¸į èĭŁ","æľī å¤ļç§į","Ġи мÑĥ","Ġwater ed","Ġmi embros","Ġborder Radius","ĠSupport s","浩 çī¹","èĢģå¹´ 人çļĦ","ä¾¿å®ľ çļĦ","ĠBah amas","Ġìĺģ ìĸ´","ĠTerrit ories","Ġfondament ale","Ġsacr ificial",": v","X O","Ġt ại","ĠB oll","ĠJ ans","ust en","Ġso ff","und ering","Ïģ εί","Ġneg ativity","缴æİ¥ ä»İ","MM A","鼨 çļĦ","æĦŁè§ī åΰäºĨ","ĠâĨĴ Ċ","Ñģа Ñħ","à¹ĥà¸Ĭ à¹Ī","Ġdecom posed","-em ployed","Ġ``` Ċ","æµĵéĥģ çļĦ","( as","ĠP WM","åı¯ åĪ©ç͍","Ġsp rite","Ġinter loc","Ġoff re","éĢī äºĨ","å¦Ĥæŀľ ç͍","å© ķ","礼 æľį","Ass ets","át ék","奴 æīį","ãģĿãģĨ ãģ§ãģĻ","Ġzosta ÅĤa","M ate","o ises","ï¼Į (","Ġto im","ĠF ury","ang un","ass ay","å¿ĥ è£ı","Ġund erv","Ġна лиÑĩие","Ġchang ement","not ification","ç»Ħç»ĩ å½¢å¼ı","Äĩ i","Ġhom ogeneity","ĠìĹ ħ","è¯ģåΏ åħ¬åı¸","ĠHon olulu","天çĦ¶ çļĦ","à´¿à´ ¯","温æŁĶ çļĦ","Ġverte brate","ĠاÙĤتص ادÛĮ","æĺ¯ åħ¨åĽ½","éĩį ç½®","Ġco leg","ãĢĭ ;","ym oon","-m ot","Ġleft overs","åį° åº¦çļĦ","鼷 æĸ¯","ĠCourt ney","ĠDi rac","Ġμ l","表达 èĥ½åĬĽ","ĠاÙĦÙĤ اÙĨÙĪÙĨ","-N ine","ĠProtocol s","Ñĥб еÑĢ","ĠпÑĢоÑĨеÑģÑģ ов","åľ¨ å¤ļ","æĿ¥ å¤ĦçIJĨ","cc ia","主 é£Ł","æľĪ èµ·","ั ล","è£ħ çĿĢ","è© Ń","Or ig","ĠTHE Y","æ¾ ¹","ä¼´ å¥ı","اÙģ Ø±","対 å¿ľ","Ġcoex ist","ĠC asp","å°± å½ĵ","对 è¿Ļç§į","Ñħ ан","Ġdet ta","Ġback ups","æĭī æī¯","po z","éĽª çļĦ","ä»ģ ä¹ī","uest ra","æľīçĤ¹ åĥı","Ġnit ro","å¹´åīį å·²åĽŀçŃĶ","Ġunderw ear","in vasive","Ġe tymology","Ġth alam","iqu ant","çŃĶ åį·","à´ ±",".C O","Ġber arti","ä¸įå°ij çļĦ","æĢĿèĢĥ åĴĮ","Ġdecom pose","ĠÏĢÏģο Ïĥ","à¹Ģศ ษ","Ġnauczy ci","ä¸į æĢ¥","ign a","åIJĮ 为","â̦ âĢĿĊ","ran et","/m y","ãģª ãģĬ","åħ¶ä»ĸ åľ°æĸ¹","åıªæĺ¯ æĥ³","ader ie","å·¥ä¸ļ çĶŁäº§","ĠÑģк ла","ĠProp agation","ĠÑĩаÑģÑĤ ноÑģÑĤи","à ¿","人 è¦ģ","ĠঠIJ","г еÑĤи","Ġserv o","Ġدر س","æĿ¡ä»¶ ä¸ĭçļĦ","çϼ åĭķ","麻 å°Ĩ","اÙĤ ÙĦ","Ġalphabet ical","Ġperc orso","ĠWarsz awa","Ġhym ns","N early","ĠT oby","ä»ĸ å¦Ī","å¹´ ç´Ģ","ä¸ĭ éĻIJ","æµģ åħī","åı¤ èij£",".C lick","äºĨè§£ çļĦ","åħ¸ æķħ","以ä¸ĭ æľīæľŁå¾ĴåĪij","Ġwild fires","sl ash","Ġaz imuth","åĬłå¿« äºĨ","éľį å°Ķ","Tom orrow","Ġë°° ìĹ´","fluid ic","ly a","è¯ ĥ","Ġhas te","ĠSt rict","ne ck","Ġк ÓĢ","Ġes erc","Ġdur ations","线ä¸Ĭ 线ä¸ĭ","Ġered et","b uff","ĠS int","Ġun ordered","ib aba","Ġman oe","æıIJ åįķ","ж ÑĥÑĤ","pre ter","çĶļ æĺ¯","BI LE","é«ĺä¸Ń æķ°åѦ","Ġviv re","ĠDiscover ing","ĠмеÑģÑı ÑĨа","ĠPOL ICY","ĠÐĵеÑĢ Ð¼Ð°","Ġcio è",". ba","ì į¨","ĠJ ury","Ġ\" ]","æ³ķ åŃIJ","çĸ Ĭ","ĠDe ployment","ä¹ī å·¥","çĥŃ å¤ĦçIJĨ","åįķä½į åĴĮ个人","ĠÏĦ á½°","æĺ¯åIJ¦ éľĢè¦ģ","ĠìĿ´ 를","çļĦæĸ¹æ³ķ æĺ¯","Ġdegener ate","ĠFung i",". »ĊĊ","ĠR CA","Ġ$ ĊĊ","ĠNew ark","Ġhard wood","ĠIN PUT","Ġhab lar","åºĶç͍ åΰ","Ġpret reatment","建çŃij ä¸ļ","æĭĶ åĩº","Ġoverse es","Ġ×ķ×ľ× Ķ×","ĠPrevent ing","注è§Ĩ çĿĢ","ĠMultip lying","_ ac","ä¸Ĭ 大åѦ","对 大","ä½ł æķ¢","æľ¬ ä½į","Ġev ade","à´ ķàµįà´ķ","ift ung","åĿ¦ çϽ","Ġguarantee ing","èĪī è¡Į","ĠQU AL","Ġrapport o","indust ry","/ us","A IR","S ac","Ġres urgence","Ġac uity","Ġد ÙĨ","ла ÑĢÑĥ",".s uccess","款 è§Ħå®ļ","ห า","κ ή","æĽ¾ æľī","off ee","æ¹ĸ åĮº","Ġfol ly","ĠConf licts","auc er","Ġmock ing","ĠÃģ l","æĬµæĬ¼ æĿĥ","ĠмеÑģÑı ÑĨ","Ġempt ied","/ acs","D t","z ko","ĠP he","Ġun necessarily","å°ı å±ĭ","Ġmod ifiers","ĠÙĪ Ø®","-l act","Ġkg f","Start ed","anas ia","D ashboard","Ġp izz","ĠF arn","Ġk ang","å°± å¾Ģ","ual itas","Ġind em","ĠÙģ Ø±Ùħ","æĴ ¸","ÑģÑĤа ве","é¡» çŁ¥","éħ¸ çĹĽ","Ġré el","Ġsolid ified","ĠOb tain","饰 åĵģ","Ġimmun oglobulin","ĠMos que","Ġmultic enter","工伤 ä¿ĿéĻ©","ĠнаÑģÑĤоÑı Ñīее","/ Object","r innings","ä¸Ģ å¹ķ","Ġz ain","èĤ² ãģ¦","温度 为","çħ® çĨŁ","ĠинÑĤе гÑĢа","","æĿ¥ æĦĪ","è¿ĺ æķ¢","ï¼ī +","èĢģ æ±ī","ns ics","Ġfamiliar ize","Ġnav bar","åŁºæľ¬ä¸Ĭ æĺ¯","Ġacet one","Ġabsor ber","ĠدÙĬ سÙħبر","ĠDanger ous","ç©Ĩæĸ¯ æŀĹ",". Integer","d ra","Ġst igmat","Ġu c","=\" {","请 ä¸įè¦ģ","Ġза ÑĢа","Ġap abila","vis ions","ĠFe uer","岩 æµĨ","Ġкон ÑĦ","çļĦ好 åĿı","Ġcig ar","ĠSpr inkle","Ġantidepress ants","i ard","åľ¨ ä»Ĭ天","ä¸Ĭ æī¬","ult ures","å¤į éĢīæ¡Ĩ","-d en","满 天","é¦ĸ å®¶","æĸĩåĮĸ ä¸İ","Ñĩи ка",".f ull","(m m","mat i","ĠEarth quake","åºĨ åħ¸","ĠBer k","éªij 车","Ġà¦ī à¦ł","Ġà¶ ¸","some one","ĠJess ie","æĢĿæĥ³æĶ¿æ²» å·¥ä½ľ","respons ive","ĠStru ggle","j unt","el os","ul am","un cia","ĠW EEK","åħ¥ åĽŃ","éĩij æĸ¯","aw ar","Ùĥ ÙĦØ©","ว ี","ви га","ä»»ä½ķ äºĭæĥħ","å½Ĵ 宿",".B ody","çļĦæĸ¹å¼ı è¿Ľè¡Į","Ġabsent ee","ĠëıĻ ìķĪ","âĪĻ âĪĻ","æĵĤ åı°","×Ļ׾×ĵ ×Ļ×Ŀ","Ġeconóm ico","P VC","Ġst alled","ĠP ek","ie use","çī¹ æĭī","åŀ Ľ","è¿Ļç§į æĥħåĨµä¸ĭ","yt et","ê³¼ íķĻ","ĠCa uchy","ĠUnivers itas","è´¢åĬ¡ çĬ¶åĨµ","æŁIJç§į æĦıä¹īä¸Ĭ","ĠBio informatics","` .ĊĊ","er er","Ġre te","Ġex hort","ark i","ĠHe ading","tt ed","aj ärvi","缴æİ¥ ç͍","Ġarch aic","æķ°åŃĹ ç»ıæµİ","æĶ¯éĥ¨ 书记","ç¥Ń åı¸","Ġnaj le","Ġmej ores","Ġsubm its","ĠнапÑĢÑı жение","Ġadsorb ed","@ RequestMapping","ĠM ales","ĠK ier","Ġwill s","Ġte atro","åIJĮ éģĵ","æį º","åĽł çĹħ","çİĭ 室","éĢĻ æĻĤåĢĻ","çīĮ çħ§","ব া","Ġsett les","-T wo","Att ention","×Ļ׳ ×ķ×ļ","ĠTob ias","Ġecon ó","I AM","¨ ìĸ´","Ġ à¸Ķà¹īวย","å°ı éĿĴ","èĢĮ å¼ķèµ·","å¦Ĥ å±±","ãģ® ãģĵãģ¨","Ġcor als","åıĸ åħ¶","æĿ¡ 缮","å¸ĥ æĸĻ","éł ¼",".C lear","bl ich","να ÏĤ","æīĵéĢł çļĦ","ÑĢован нÑĭе","Ġmuc ous","ĠExam ining","Ġconced e","Prob ability","ĠÐŁÐµÑĢевод Ñĩик","- entry","ĺ ×Ļ","Ġd j","ic ill","Ġan astom","Ġind ia","ä cht","Ġseg ue","æľī人 认为","éĶĢåĶ® 人åijĺ","æ¯ı个人 éĥ½æľī","ĠدÙĪØ± اÙĨ","ategor ized","ĠÑĤÑĢебÑĥ еÑĤ","Ġگزار Ø´","为 好","ĠY E","ç¾ £","Ġgra ffiti","ĠInd us","Ġб оли","Ġоб о","ä»»ä½ķ 人éĥ½","Ġcapac idade","path s","Ġ×Ķ×ŀ× ª×","ĠNeuro psych","ĠMas cul","Ġhonor ary","Ġà¦īপ র","an ov","Ġb fs","uc lease","æ·± èĢķ","ĠاÙĦÙħ ختÙĦÙģ","Ġant ipsych","ĠDes arrollo","Ġоб ÑĥÑģ","Ġdistrib utive","IM AGE","Ġgrand ma","æ·¡ æ¼ł","Ġtemp érature","æĵ¦ äºĨ","à¸Ħร à¸Ńà¸ļ","èľĤ çªĿ","ĠProp ag","ĠLaure l","Ġbang sa","Ġingen ious","ĠCumm ings","åĩºä¸į ç©·","对 åħ¶ä»ĸ","Ġdem e","Ġaut opsy","Ġschedul er","åįij å¾®","ĠнеобÑħодимо ÑģÑĤи","éĿĴå²Ľ å¸Ĥ","ĠInvol vement",") arg","< _","å¿ ¿","è¿Ļ èĬĤ课","åħ¬ ç§ģ","æļĹ æ·¡","éĵ¶è¡Į 贷款","мо е","åľ¨æŃ¤ æľŁéĹ´","ÙĪÙĦÙĪØ¬ ÙĬا","ëłĩ ê²Į","Ġaba ixo","_ div","p resa","Ġc air","çļĦ çIJĨæĥ³","æĿ¥ åĪĨæŀIJ","åĪĩ è®°","Ġmeng atakan","浪费 æĹ¶éĹ´","-go vernmental","åĩºåı° äºĨ","Ġuph olding","ĠиÑİ Ð½Ñı","âļ ł","= V","N ES","Ġn l","st asy","ad at","ĠW ATER","Ġ_ .","é¦ Ģ","Ġcr ÃŃtica","UT O","Ġod ors","Ġmis placed","ĠUnivers ité","ĠRu pert","ắ c",". ms","Ġc ed","ĠF j","ĠF iling","å®ļ æł¼","red en","Ġph age","åħĪ çŁ¥","Ġterm inates","Ġsem aine","èļ Ŀ","åĩĮ 天","ĠHand ler","Ġим ени","Invest ment","è½»èĢĮæĺĵ 举","D emon","ĠC GFloat","if ton","ĠV ince","ach sen","ÃŃ te","åıª 顾","çĥŃ çģ«","Ġস à§įà¦ķ","æī¿è½½ åĬĽ",". iter","Ġg ull","ĠC air","Ġit ch","ä»ĸ å¼Ģå§ĭ","Ġا عت","Ġع دة","ÑģÑĤа Ñı","Ú¯ ز","èĦļ æīĭ","ĠÑħ ÑĢони","atern ion","ï¼ħ ãĢĤ","Ġanx ieties","ĠJes uit","Ġweiter en","ĠYank ee","Ġial ah","$ )","( label","ĠM ethyl","Ġк ожи","强 硬","äºĨä¸Ģ æĿ¯","é² «","å±ĭ éĿ¢","ç¨Ģ æľī","ĠSmall er","èĬĿ 士","Ġ{} \".","ä»İ å·¦","éĢī åĮº","Ġdon key","Ġqual ifies","ÐŁ ÑĢ","æ½ľ ç§»é»ĺ","look ing","Ġinstruct ive","Ġgrat is","ĠGran ada","Ġagon ists","Ġdissatisf ied","ç»ļ 丽","{item ize","( Exception","N oun","çļĦ åı«","åīį ç¨ĭ","ĠAd missions","èµ° ä¸ĭåİ»","ν ή","å¸ĥ 满","åij¼ åĸĬ","Ġax on","Ġgenes is","ï s","ĠSpect roscopy","æ´ĭ溢 çĿĢ","Ġ( ,","ĠV ag","ä¿¡ å°ģ","Ġcur iously","çľ¼ çĿģ","éĵ¶ å·Ŀ","èĹı æĹı","Ġmid day","Ġmemb aca","-T V","Ġpoll ination","ĠLiber ia","ĠSimpl ified","S pect","é ticos","ast al","ĠV itt","ä½ł çļĦ人","她 ä»İ","ç¾İ åij³çļĦ","-c ourse","æķħ æĦıçļĦ","Ġband a","mes h","æĶ¹éĿ© åıijå±ķ","åį§ åºĬ","ĠBir ch","Ġpollut ant","ĠболÑĮÑĪ Ð¾Ðµ","Ġসর à§įব","ĠDw ight","ĠDud ley",". Execute","h ore","Ġf ris","ol fo","ĠC GI","Ġbe gg","Ġper v","ĠSt ati","ons umsi","æĹł æįŁ","ĠSTAT EMENT","ç¼ĵåĨ² åĮº","Ġajud ar","- land","/ es","p w","çļĦ ä¿¡åı·","ĠA AC","çĤ¹ çĿĽ","äºĶ éĩij","Ġfil tr","EC s","ino za","è¡Į为 èĥ½åĬĽ","ĠMer ch","_st mt","Ġпоп иÑģ","ç¾İæľ¯ åѦéĻ¢","ĠShel ter","ĠDefic ient","ĠSyll abus","D AR","Ġl icht","ĠW ink","ĠIn vention","è¿Ļ个 æĸ¹æ³ķ","Ġreal ist","ãģ¨ åIJĮãģĺ","çļ® ä¹¦","ุ à¸Ľ","å°į äºĨ","ĠAfter wards","设置 åľ¨","åħ¨éĿ¢ 建æĪIJ","ĠMicro bial","ĠAtt endance","Ġconform ational","Ġ×Ķ×IJ× ĵ×Ŀ","æĶ»åĿļ æĪĺ","ç§ī æĮģ","ĠزÛĮر ا","ĠÑįкÑģп лÑĥа","K ol","q m","Ġa ku","ĠM ok","ĠF ake","Ġk ary","Ġpo ÅĽ","项 ç¨İé¢Ŀ","èħ ĵ","Ġbi ocom","éĥ¨åĪĨ ç»ĦæĪIJ","åĢĴ éĢĢ","Ġpen gh","æķ´ä¸ª ä¸ĸçķĮ","Ġequilib rio","ìĶ ¨","æĸĩ åĩŃ","Ġwhen ce","åºĶ éĤĢ","ç¾İ è²Į","ี à¸ŀ","ä¿® ç¼®","Ġred ress","å¾Ħ åIJij","ĠBre nda","num ara","Ġpré par","å·« å¸Ī","ĠÑĥÑĢав нений","ĠпоÑĢÑıд ке","Ġphilanthrop ic","Ġpéd agog","L IB","Ġm ely","ĠAl ive","ли к","IN CT","åĨħ容 æĺ¯","åĿIJ ä¸Ĭ","ĠInter im","Ġsn apping","éľĩ æħij","å®ĩ èĪª","Ðķ Ðł","Ġба лан","ĠAuf gaben","Ġطر اØŃÛĮ","ĠCHE M","_ limit","ĠN ess","Ġsp ar","åĻ İ","ĠIm mediate","Ġfr antic","Ġপ দ","Ġaltern ately","Ġré flex","年代 ä¸ŃæľŁ","Ġzw ier","richt en","ĠبØŃ ÙĬرÙĩ","Ġvigil ance","å¢ŀæ·» äºĨ","沦 为","B ASE","ĉ S","ĠL ift","ä¸Ĭ 楼","æİ¥ æīĭ","头 ä¸ĬçļĦ","åħŃ çº§","æĦıè§ģ 建议",".append Child","ĠBow man","ĠиÑģÑĤо ÑĢиÑı","à´¿à´ ķàµįà´ķ","ĠдвÑĥ мÑı","ĠVeget able","为 æıIJé«ĺ","ä»ĸ çĶļèĩ³","æĹ¥ åİĨ","ĠÙĪ Ø¢","velop e","ÏĦ Ïİ","áĥ Ĺ","ä¿¡æģ¯ æľįåĬ¡","AB B","è¿Ļä¹Ī 好","幸 äºı","اØŃ ÙĬØ©","ĠBrother hood","ĠÑģÑĤаÑĤÑĮ е","abh äng","ĠAlic ia","æłªå¼ı ä¼ļ社","; \"> .ĊĊ","O cean","m ah","ĠI BD","ĠC KD","ĠL oy","av u","con cat","Ġsp anned","å±Ĥ åĩºä¸įç©·","èĦ¸ éĥ¨","Ġbom ber","åįłæľī çİĩ","ĠBound aries","骨质 çĸıæĿ¾","N PC","Ġs iano","Ġm asing","ĠL ors","æĢ§ åİŁåĪĻ","ла Ñı","å¯Ĵ åĨ·çļĦ","纸 å·¾","Ġdiss olving","Ġfol genden","ĠCam den","ĠSche mes","èĭ¥å¹² éĹ®é¢ĺçļĦ","ë¹Ħ ìĬ¤","ĠëĬ IJ","ĠÑģопÑĢоÑĤив ление","alak ip","h ner","ä¸Ģ éĸĭå§ĭ","ب ÛĮÙĨ","æ¶Ī æĿĢ","严 å¯Ĵ","å¹²éĥ¨ çļĦ","Ġign ite","ä¸ģ é¦Ļ","алÑĮ нÑĭÑħ","Ġcro issance","è´¢æĶ¿ éĥ¨éŨ","ĠEC B","財 åĭĻ","Ġdeterior ated","Ġros emary","ĠI CA","åľ ©","Ġv árias","ا Úº","ĠV LAN","ãĥ ¶","EN DS","ĠCont acts","alt res","Ġroot ing","Ġrev oked","ä¹± ä¸ĥåħ«ç³Ł","éĺħ读 çIJĨè§£","stra ÃŁe","HD L","Ġeleg ans","nipp et","æľŁæľ« èĢĥè¯ķ","Ġbroker age","èĬ¹ èıľ","ucceed ed","R d","Ġs ockets","æĺ¯ ä¼ļ","av ÃŃa","Ġdis illusion","ĠCh anged","Ġro y","åı¯ä»¥ åıijçݰ","Ġcor nea","ĠÑĢа ÑģÑĤи","де б","ĠEurope a","åī§ åĽ¢","Ġqual itÃł","åģı çα","æĦĪ æĿ¥æĦĪ","åĿ¦ è¯ļ","ĠCook e","ĠMid lands","夸 å¥ĸ","Ġrefres hed","ĠPun kt","Ġdisgust ing","ĠÑĦÑĢан ÑĨÑĥ","ĠC atar","ig ing","ĠRe creational","æĶ¹ æĢ§","Ġcost o","亿 ä¸ĩ","ĠÐł и","Ġalcohol ism","ĠBul k","Ġکار بر","ówn o","Ġà¦Ĩপন ি","ĠаÑĤ моÑģ","v ival","er in","or bit","ر ØŃ","æĶ¶ éŁ³","æł¹ æ²»","æĹ¶éĹ´ å¤įæĿĤ度","éĿĴ éľīç´ł","ç³»ç»Ł ä¸ŃçļĦ","ĠMe adows","ÑĦ ÑĢи","ĠGe ol","æĮīçħ§ è§Ħå®ļ","Ġstri ped","å¼Ĺ éĩĮ","Ġunders erved","C AL","Å ŀ","at asi","ĠD Y","Ġph énom","ask i","ĠTrans cription","Ġseg urança","åijĬè¯ī äºĨ","æĬ¬ éłŃ","çļĦçī¹çĤ¹ æĺ¯","Ġpúblic os","ĠØ¢ÙħÙĪØ²Ø´ ÛĮ","ĠMOS FET","ĠFör der","m ml","æĸ¹ æł¼","åģ ½","åIJĮ åIJį","å¿« å¿«","Re venue","çļ® éŀĭ","omb onana","Index Of","æł¸å¿ĥ ç«ŀäºīåĬĽ","ĠNorm andy","Ġabbrevi ations","ain ting","Ġres umes","ĠV E","Ġpre print","åIJĦ åľ°åĮº","Ġза ви","Ġcast les","алÑĮ но","çİĦ å®Ĺ","Ġepid ermis","Ġзд ÑĢав","Ġt ess","ar ita","Ġim par","ÙĪ ÙĬÙĨ","车 ä¼ģ","åı« 她","Ġcontact o","建çŃij çī©çļĦ","ĠÐĶ Ð¾Ð±Ð°","h oud","j ans","ĠB AC","éĤ ı","vis iae","Ġ×©× Ĺ","æĬĸ åĬ¨","Ġmerc iful","Ġим Ñı","Ġrů zn","Ġintrus ive","Ġমাধà§įযম à§ĩ","ĠP act","ä¸į 说è¯Ŀ","ĠE MA","th reshold","Ġj auh","Ġsub divided","ĠEx clusive","åĪĩ ãĤĬ","oph ones","Ùİ Ø§ÙĨ","Ġnom inees","Ġž ád","ĠPath way","Ġvibr ational","à¹Ħà¸Ł à¸Ł","ĠÙ쨱Ùĩ ÙĨÚ¯ÛĮ","ãĤ¸ãĤ§ ãĤ¯ãĥĪ","åĵŃç¬ij ä¸įå¾Ĺ","H arvard","Ġc arts","ä¸į çIJĨè§£","Ġr ite","ä¹ĭ ä½į","ung i","æľĿ æ°Ķ","纸 å¸ģ","ĠاÙĦÙĥ ÙĪÙĬÙĥبات","LA Y","ĠKom ment","Ġmetaph orical","Ġunsatisf actory","à¹Ģหลà¹Īาà¸Ļ ีà¹ī","C BD","N ap","Ġw issenschaft","Ġb anners","ĠG ins","Ġdo ÅĽwiad","åIJİ ç»§","_{ (","ung g","èĮ ²","ä»ĸ们 没æľī","æ¸ħ æ¸ħ","ย าว","ho fer","bl r","æ·±åħ¥ 贯彻èIJ½å®ŀ","འĶ","-J an","Ġintro spection","ĠMarian ne","ä¸Ģ模 ä¸Ģæł·","ĠS le","id l","ak kan","ä¹ĭ çζ","Ġ< ĊĊ","ĠCh au","éĤ£ åı¥è¯Ŀ","çļĦä¸Ģ åľº","ĠVal ve","ĠEr referentziak","-B e","ä»ĵ ä½į","ä¿¡ç͍ 社","ì¶ ©","ê¹ Ģ","Ġaden osine","n ative","w ares","ä¸Ģ ä¼Ĺ","ä¸Ĭ å°Ĩ","èĢĮ 论","Ġ\\ %","duct ed","æĹłè®º åľ¨","æĥł å·ŀ","ĠгÑĢÑĥп п","-hydro x","v ang","ĉ db","Ġs ÃŃmbol","Ġb ik","Ġm alle","åıij æķ£","ĠSt ato","Ġi ombonana","ب ÙĪØ¨","æĹł åĩł","pro per","Ġac ima","ox ox","åύ çŃī","ç»Ĩ å°ı","ĠÑģÑĤа ÑĤÑĥ","ظ Ø©","comp ared","Ġjudg ements","dest ination","ĠSax on","^^^^ ^^^^","d ur","ĠC CR","ĠM SS","为 åīįæıIJ","è¦ģ å®ŀçݰ","ric hes","æ¨ µ","ĠEx amine","éŨ æ´¾","ĠQu elle","éĻ· 害","Ġformal ism","LO Y","Ġdigit ale","à¸ķัว à¹Ģà¸Ńà¸ĩ","วà¹ĪาภĪะ","æį§ çĿĢ","Ġë§Įëĵ¤ ìĸ´","S HA","ĉ default","Ġt rophic","æĪij æĸ¹","ä¹Ł å¹¶ä¸į","åĨħ è¡£","éĴ ´","空 äºĨ","ва лиÑģÑĮ","اÛĮ ع","åıĤä¸İ çļĦ","Ġcircum vent","èĢIJ çģ«","éĥ½ä¼ļ 被","诺 夫","èį· åı¶","inst agram","Ġroz m","å±ł å®°","ä»Ĩ 人","à¸ķัà¹īà¸ĩ à¹ģà¸ķà¹Ī","J udge","§ ׾","ä¼ İ","åľ° é»ŀ","天 æ²³","便 å¼Ģå§ĭ","端 çĤ¹","æĿĢ èĻ«","æīĺ 马æĸ¯","BC D","\\, =\\,","ĠEX AM","àµģà´ ¨àµįà´¨","Ġpneum atic","Ġá½ ħ","Ġosm otic","Ġtranscend ental","Ġëĭ¤ìĿĮ ê³¼","ĠÐĿикол а","Ġcaracté rist","ĠM anc","ĠH ul","Ġj uego","Ġcar ies","-l arge","ĠSc rabble","alt et","èĥ¶ çīĩ","缸åºĶ åľ°","\\ mid","in j","Ġex cl","å°± å·²ç¶ĵ","åĪĨ æķ°çļĦ","ĠWe apon","çĸ ½","åħĪ ç§¦","оÑĢ ÑĤа","æĸŃ ç»Ń","AN DS","å±ħ 士","ãģĵ ãģ¡ãĤī","ĠCourt esy","èĢĹ æĹ¶","大éĥ¨åĪĨ çļĦ","ĠEC ONOM","ĠÑĢи Ñģк","enschaft en","Ġchuck le","åķª åķª","ĠдоÑģÑĤи га","ĠScar let","Ġstrom al","Ġl ily","ve get","äºĨ è¿Ľåİ»","ĠR n","ell us","å¹´ 齡","åºĶ èĢĥèĻij","Ġpe e","ĠAn agram","è£ħ ä¸Ĭ","çģ« çĤ¬","EC O","åħħ çĽĪ","ç¶ ±","票 ä»·","æĥĬ 天","çĥŁ çļĦ","Ġغ ذا","ìĿ¼ ìĹIJ","Ġcategor ization","Ġnah il","çĽijæĬ¤ 人","Ġmisf ortune","Ġo phthalm","н ок","ĠD us","Ġk ettle","èĢĮ å¤į","Ġinv itations","Ġк нÑı","è¡Ģ éĩı","ÑĢÑĥ ÑİÑĤÑģÑı","à´ ¦","è² ŀ","Ġste els","èįī æľ¬","ç»Īäºİ åľ¨","Ġdisp erse","éĽ¾ æ°Ķ","Ġdik et","ç»Ĵ æ¯Ľ","Ġimpe achment","ĠToul ouse","Ġnex us","S old","e is","ot is","ä¸Ń æ·»åĬł","ä»İ åħ¶","ж ного","Ġrun t","κ λη","åħ« æĪĴ","Ġexc ite","ä¸ĥ 大","Ġcheck er","å²ģ çļĦæĹ¶åĢĻ","ĠÐļ ÑĢаÑģ","Ġà¦Ĩ ন","HP V","Ġdent ists","Ko ordin","ĠοÏĢο ί","( ST","M ilitary","ĠM SN","è§£ å¯Ĩ","-l oving","ÙĨد ر","Ġforg iving","Ġнов Ñĭй","ĠBot swana","ĠLion el","ĠW nt","ĠN ahr","ä¹ĭ æŃĮ","æİ¨ åΰ","åħ« è§Ĵ","æĤ¨ åľ¨","ë¦ Ń","ÉĻ m","Ġtu vo","Ġaccord ed","ĠزÛĮ اد","ĠدÙĪÙĦ ت","å«£ çĦ¶","Ġcler ks","E QU","R obin","ĉ in","Ġc inc","çļĦ æııè¿°","st ars","ĠS lim","ow ay","个 é¡¹çĽ®","cl ampsia","æĸ° éĥİ","åĪĹ ä¼ł","çij Ļ","绿 çģ¯","Ġoptim izer","cycl ing","огÑĢа ÑĦ","Ġglob ale","åįļçī© é¤¨","othy roidism","OO OO","溯 æºIJ","Ġabras ive","Ġpalav ras","Ġintox ication","K am","Ġqu aint","av oir","æŀľ 羣","ìĿ µ","ran j","åΰäºĨ ä¸Ģ个","Event Handler","ĠبÙĨ اء","itarian ism","ĠCrist ina","Ġinex plic","Ġtread mill","ĠOphthalm ol","Ġnahil alakip","= âĪij","ĠT weet","est anden","ip iko","åIJİ è¢«","èĢĮ æĦŁåΰ","Ġob es","两 é¢Ĺ","Ġcar oten","åħī æĿŁ","-m i","ä¿® çļĦ","not ice","å°¼ åı¤","Ġন à§ĩà¦ĩ","Ġpra ising","Ġδ Ïį","Ġpoison ed","emperature n","ĠPatri ot","ĠÙĬÙĤ ÙĪÙħ","_ ->","ï¼ ¤","Ġl anc","ĠT yson","ĠF U","lic tion","å°ı èĬĤ","rit u","å±ŀ å®ŀ","Ġid ols","¡× Ŀ","Ġsemb l","éĹªçĥģ çĿĢ","Ġt ind","Ġà ±","\"> &","_d own","Ġeth ically","çŀª çĿĢ","Ti O","Ġsare bbe","/ create","\\ log","j or","çļĦ ä¼ĺç§Ģ","ĠA ar","ĠB arg","ĠL argest","Ġu id","sp in","Ñİ Ð´Ð°","ÑĤи ÑĤе","ĠØ£ غ","缸åħ³ å·¥ä½ľ","ĠIS S","ìķ Ķ","icks on","Ġü bers","à¤ķ à¥ĩ","Ġreform ing","åĨ¥ æĥ³","Ġاب رÙĬÙĦ","Ġcomed ian","L ith","b ite","z um","at Äĥ","æĺ¯ æĢİæł·çļĦ","æľī æĥħ","æĶ ľ","д ки","Ġspec s","Ġer h","åįĬ æķ°","ĠCont oh","Ġপ ড়","Ľ× ł×¡","éĿ¢å¯¹ çĿĢ","Names pace","Ġoverl aps","天空 ä¸Ń","ĠÑģемÑĮ и","æŁij æ©ĺ","Ġоднов ÑĢеменно","Ġa cht","ĠC SP","åºĶ äºĪ","æµģ è¿ĩ","}} )","è°Ī åıĬ","éľĩ é©ļ","ĠпÑĢед ÑĭдÑĥ","ĠRam irez","åŁĶ 寨","ĠÑħаÑĢакÑĤеÑĢиÑģÑĤи ки","Ġporn ography","in ib","Ġи деÑĤ","Ġinf lection","Ġinte lect","rag s","ðĿij Ģ","ãģı ãģ¨","éľĢæ±Ĥ éĩı","Ġtransform ational","Ġcro oked","Ġaccompl ishing","Ġbol st",".Rep ository","' article","ĠR ails","èĩªå·± èĥ½","è°ĥ çļĦ","å·²ç»ı 没æľī","ĠPr asad","Ġap ologies","pa ul","/b ase","-com pliance","ãĥ¡ ãĥ¼ãĤ¸","á± ®","Ġhyg ien","èŃ¦ç¤º æķĻèĤ²","rvats ki","-indust rial","ä¸į 注æĦı","åĴĮ åĪĺ","Ġz oon","å¸Ĥ åĨħ","ef a","她 å°±æĺ¯","ä¼ģä¸ļ ä¸Ń","éĺŁ åľ¨","á» §","ä¹Łæĺ¯ è¿Ļæł·","åĨ· æĪĺ","æĽ¾ 被","åı¸æ³ķ è§£éĩĬ","comb ined","ĠÑģеÑĢ Ð´Ðµ","Ġfen ó","~ \\","ĠM ás","缸 约","åύ çī©",".S plit","à´ ¨àµįà´","åĿĩ çͱ","æŃ¢ åĴ³","Ġconc ussion","æ±ī ä¸Ń","æ·±åħ¥ 人å¿ĥ","ió d","Ġத à¯Ĭ","ifi é","ĠRodrig o","( read","st ro","ĠT urns","od ers","åĽ½ ãģ®","å¾Ī å¼Ģå¿ĥ","äºĮ éĥİ","åIJij æĹ¥","Th inking","æĶ¾ èĤĨ","Ġ) :Ċ","è¯ij 为","ç²¾åĩĨ æī¶è´«","ÙıÙĪÙĨ Ùİ","( ret","ĠS ime","ä¸ī 两","ç¾İ åĽ¢","è´¨ åŃIJ","éľĢè¦ģ èĢĥèĻij","rac hen","ĠGe omet","éĤ£ä¹Ī å°±","æ¯ı天 çļĦ","enz iale","Ġoverwhel m",". backgroundColor","C MS","F t","G RE","P ID","ĠS ä","éģ Ľ","ÏĦ ζ","åĮħ éĩĮ","ĠCont ain","å¾ģ åħĨ","Ġparticip ar","Ġred shift","Ġmer k","ç§ĭ æ°´","än ner","è°± åĨĻ","Ġbio availability","Ġà¦ī দà§įà¦","Ġcannab in","ĠINTER NATIONAL","ĠHein z","ĠاÙĦإسÙĦاÙħ ÙĬØ©","f em","Ġe czema","ĠF on","ĠG ina","Ñĭ ÑĢ","ph s","è¿Ľè¡Į è°ĥæŁ¥","éĽĨ ä½ĵçļĦ","Ġcou pons","åģľ æĶ¾","çīĽ å¸Ĥ","å¼± èĢħ","é«Ķ çļĦ","èĩ³å°ij è¦ģ","leq slant","åµĮ å¥Ĺ","alloc ate","ĠÑģÑĤÑĥд енÑĤов","- medium","M d","_ Id","ŀ ×Ļת","Ġl äng","ä¸į 大çļĦ","ver d","æĹ¶ éļĶ","ĠV os","å°ı åģ·","Ġmod ulator","gen re","éĿŀ常 ãģ«","æ¿Ģ æĺĤ","à¹Ĥ à¸ļ","åıªè¦ģ æĪij们","äºİæĺ¯ å°±","ĠAv g","æĬ¬ çľ¼","Est at","ãģĺ ãģ¦","ĠпÑĢоиз ведениÑı","Ġdisplace ments","Ð ĥ","Ġl att","ä»ĸ åıijçݰ","å¾Ĺ æĦıçļĦ","Ġstud ie","Ġra yon","Ġfl ocks","车 çªĹ","_p ass","ĠPresident e","Ġwarm ly","æī« åľ°","ãĤ¤ãĥ³ ãĤ¿","мÑı ÑĤи","Ġthirst y","Occup ation","[â̦ ]","< '","ä¸Ń ãģ®","Ġbut cher","éķ¿ åº¦çļĦ","ade k","ĠZ i","Ġcar ácter","å®ĥ æĺ¯ä¸Ģç§į","çϽ 人","ko z","æģĭ æĥħ","èģ½ è¦ĭ","Ġlur king","Ġzv ý","I j","Ġc edar","ou cester","ce go","ĠD ove","ä»ĸ è¿Ļ个","ä¿ ł","åģ ķ","å½¢ æħĭ","li Å¡","æķĻèĤ² åĩºçīĪ社","ÙĦÙĬ د","ende ley","trans l","ÙħÙĪ Ø§Ø·","赤 åŃĹ","çķĻä¸ĭ æĿ¥çļĦ","ĠìłĦ ì²´","Ġendomet rial","\" s","- ranking","Y outh","ut zt","ad ah","op ers","Ġ\" ","Ġ( )ĊĊ","é on","д ова","Ġden omination","OS H","è½® æľº","åĶIJ åĥ§","Ġswe ats","æīĭæľº çļĦ","Ġcircum ferential","èı² èı²","ĠUnter st","Ġberk embang","Ġprogen y","ãģĭãĤĤãģĹãĤĮ ãģ¾ãģĽãĤĵ","ĠBolshe vik","O tt","Æ Ĵ","ol lection","Ġde lect","Ġund ocumented","å¼Ģ åħĥ","æĸĩ åºĵ","ert i","cent ric","çĹħ èıĮ","çİĭ æ°ı","æĿ¿ æĿIJ","åĸĦ æģ¶","Pl ug","èİ·å¾Ĺ æĦŁ","input s","èĻļ å¿ĥ","ĠGreen berg","Na N","ĠErgeb nisse","Ġutens ils","åĵ½ åĴ½","/ add","C andidate","S hel","d imensional","ĠC rab","ag land","ä½Ĩ è¿Ļ个","åĪļ ä»İ",".f e","ĠDis advantages","ÑĤив ное","伯 伯","mu ir","Ġyellow ish","Ġdeform ity","Ġamyg dala","ain ya","Ġpl ais","ä¹Ł åıĺå¾Ĺ","æĢ§ æĪĸ","ç© İ","Ġclass ific","Ġconsider ar",".s ervices","æľ¨ åħ°","De an","_f ront","Ac ross","æĪij们åı¯ä»¥ çľĭåΰ","Ġvitam ina","æģ°å½ĵ çļĦ","Ġacquaint ances","Ġh ø","Ġis omorphism","á h","æŃ¤ 书","ull o","æĦŁè§ī å¾Ī","Ġbott leneck","Be hind","æľ± å¾·","ÙĦÙĥ ترÙĪÙĨ","ãĢĭï¼Į ãĢĬ","çĤĴ èĤ¡","æĸij æĸĵ","æĢľ æĤ¯","å·· éģĵ","Ġforc ibly","N ig","æĸ¹ åĿĹ","æĪij们 åħļ","val ho","认 åĩº","çİĭ æĸĩ","æ¦ Ķ","åıijçĶŁ æĹ¶","æĭĽ å¼ı","({ '","ĠIncre ases","Ġwhis pering","ĠPump kin","Ġsubmar ines","ĠG EO","éĿ¢ 带","any thing","ĠDe i","åıĸ èĢĮ","Ġза ÑĢÑı","æ³¢ 士","ĠاÙĦع راÙĤ","Ġboard ed","ĠSal on","ĠLog istic","åĽŀçŃĶ éĹ®é¢ĺ","رÙĪ Ø¨","åįģäºĮ äºĶ","å°ĺ åľŁ","æį· å¾Ħ","ĠEll es","祥 åĴĮ","Ġdent istry",", sizeof","at ians","ĠG ret","ĠJ edi","ĠK nee","æķ° çϾä¸ĩ","Ġback log","åıijå±ķ ä¸İ","Ġcost a","èĭ¥ èĥ½","_d epth","ë¶Ģ ë¶Ħ","éļ¾éģĵ æĺ¯","Ġপà§įরত ি","er ning","qu il","åIJį æĢĿ","é£İ 顺","æ¯ı æĻļ","hes ion","åį¡ æĸ¯","ল া","çīĽ æİĴ","App arently","æijĩ æ»ļ","uten berg","acci o","ĠÑĤеÑĢÑĢи ÑĤоÑĢи","Ġdát um","ch rist","ess or","ĠN icht","é«ĺ éĽħ","aj at","åħĥ ç¥ŀ","è®° ä½ıäºĨ","è¿ŀ çݯ","onal do","ÙĦا ÙĪÙĩ","Ġrub ble","ĠÎļ λιÏĦικÏĮÏĤ","ĠPoly gon","Ġescol as","( on","- CO",". OK","M os","é quence","对 å®ĥ","ĠঠĿ","çŃī æľįåĬ¡","ĠAr un","ĠAs ians","转 åŁºåĽł","ç²¾ æĺİ","Ġredu cer","é£ŀ åΰ","ĠÕ Ĭ","æľĢåIJİ çͱ","×ij ×Ļר","ĠTur bo","Ġgest ured","çļĦåŁºæľ¬ åİŁåĪĻ","ĠHor iz","elijk heid","Ġprés ident","ĠBL ACK","æĥħæ³ģ ä¸ĭ",") _,",", max","B iblical","IJ ר","ĠS amm","Ġent reg","éĿŀ åIJĮ","管çIJĨ è§Ħå®ļ","Ġum p","çͰ å¾Ħ",".T ry","Ġnotice ably","Ġow ls","grav ity","èĤĭ 骨","Ġemanc ipation","- formed","K udos","V y","b road","n omin","ild ed","ä¸ī çŃī","æīĭ æŀª","å¤ĸ å¸ģ","çī¹ å¼Ĥ","uc ia","Ġpublic ado","ts ky","ness es","åįĹ æľĿ","èĭ¥ æľīæīĢæĢĿ","Ġnecess idade","Ñĺ ан","éģ¿ éĻ©","Ġ] ];","flu oro","Ġdomin ion","èį¡ æ¼¾","Ġdiscl oses","Ġسب ÙĬÙĦ","Ġencont ra","Ġeing es","ä¸Ń西 åĮ»","Ġgiov ani","f ighting","Ġ à¸Ĺำ","Ġo Ê»","åŃIJ æĺ¯","ĠCh il","æķĻ æĪij","ĠÙĩ زار","Ġlo is","Ġhom ens","ĠWil ly","Ġмом енÑĤа","phin x","Ġprzep rowad","( By","_ run","_ images","z ee","ly wood","âĢĿ -","天 ä¸ĬçļĦ","Ġoper able","ĠпÑĢи веÑģÑĤи","à¹Ħ หà¸Ļ","ÏĮ με","å¤ļå°ij 次","ç¦ģ 令","Ġcyt ometry","ìĿĮ ìĿĦ","ä¸Ģä»¶ äºĭæĥħ","ĠCher yl","relations hips","-dess us","Ġa ryl","re is","ĠF AT","ere a","Ġem akume","åĪĻ å°Ĩ","ãģĦ ãģ¯","Ġnon verbal","çŁ³ åĿĹ","Ġbad ania","ãĤ¹ ãĥļ","æ¡Į æ¤ħ","ĠTH ER","è·Į åĢĴ","Ġê·¸ ëŀĺ","رÛĮ Ùĩ","áĥĶáĥ Ľ","adv anced","ĠкоÑĢ ÑĢе","麻çĥ¦ äºĨ","Ġtriumph s","Ġexcav ations","Ġге огÑĢаÑĦи","ĠPharise es","ĠS ized","ie ga","ĠV argas","Ġz usamm","åIJį æĽ°","åı£ 渴","Ġcor rig","ĠZ eng","æİĴ çIJĥ","ä¸ĢäºĽ å°ı",".A ction","Ġfront line","Ġcarb ide","ev idence","æĬ¢ åįł","åIJIJ èķĥ","ĠWood y","è¿ĽæŃ¥ çļĦ","ĠLat itude","å¾Īæľī è¶£","çijŁ çijŁ","ĠED TA","Ġredirect ed","ĠÑįлем енÑĤ","Ġgust s","Sha res","Ġransom ware","ĠP ueblo","ä»ĸ å°±ä¼ļ","Cl in","åŁºæľ¬ åĬŁ","ÙĪØ¯ ÛĮ","åĪ¶åº¦ 建设","SE O","èģĶç³» åľ¨ä¸Ģèµ·","ĠPort able","Ġesp ÃŃ","à¸ī ัà¸Ļ","ophy te","ä»ĬåĽŀ ãģ¯","ĠbÄĽ hem","ĉ is","ter en","为 å¸Ī","ĠK iev","ĠSt all","ĠDe ux","ç§ij æķĻ","æķĻèĤ² ä¸İ","AC ION","为äºĨ å®ŀçݰ","=' \"","ĠGeneral ized","Ġmeng ambil","çļĦå¿ĥ ä¸Ń","Ġsit io","ĠпÑĢодÑĥк ÑĤÑĭ",": \\\\","j ie","ver a","erm ont","Th or","å®ĥ ä¸įä»ħ","å¸ĥ éĩĮ","æĿĢ æİī","-B Y","èĭ±åĽ½ 人","Ġpeng gunaan","ëIJĺ ì§Ģ","ĠдейÑģÑĤви е","ĠÙĪÙĥ ذÙĦÙĥ","( at","Ġp ijn","Ġfl aps","åķ ®","被 éªĹ",".s rc","è¿ŀ 绵","Ad vertising","Ġten ir","Ġsequ estration","Ġauf ge","åIJ¬åΰ è¿Ļè¯Ŀ","ĠGal actic","Ġadvers aries","intern o","âĸij âĸij","C BA","ã ³","åİ» åIJĥ","Ġob dob","ĠZ he","Ġnie z","ĠAL J","?âĢĻ âĢĻ","loc als","Ġساز ÛĮ","Ġangl ais","ĠкомпÑĮÑİ ÑĤеÑĢ","D w","m inton","Ġd unk","çĶ ¥","Ġwith hold","ist ique","æĪIJ 群","ov ia","Ġher al","éľ ¹","è§£ æķij","西 西","Ġav alia","Ġ×Ķ× Ł","Ïģ κ","AC G","Ġ×IJ×ķת ×Ŀ","ĠзанÑı ÑĤиÑı","ĠErgeb nis","Ġincompet ent","---------------+ ---------------+","E i","æĪ »","Ġun important","ä»ĸ è·Ł","ĠK ita","èĩª è´£","èĢħ ãģĮ","ç¥ŀ ç¶ĵ","åĽĽ 个人","ĠMe al","鼶 ä»¶çļĦ","Ġbott len","åĵŃ å£°","Ġdoubt less","Ġven ir","ĠпеÑĢв Ñĭе","Dig its","غÙĨ اط","ĠMere ka","< (","B ucket","ĸ ন","Ġn enh","æĭ ¡","好 äºĽ","Ùģ Ø§ÙĤ","广 度","æłij 人","æĽ¾ 说","ĠVer izon","Ġax ons","Ġদ à§ĩà¦ĵ","Ġappreci ating","Ġlect urers","çĽĨ æł½","Ġî nt","ĠJah res","Ġhelm ets","B alt","_ host","ÑĤ ова","ue va","Ġz in","ॠ°","Ùĥ ÙĬÙĨ","ĠÙĨ زد","Ġregular ization","Ġrés z","Ġহয় à§ĩ","_ST ATUS","Ġomin ous","ĠاÙĦÙħختÙĦÙģ Ø©","+ g","_ dev","x g","ĠT res","ip end","ĠK aj","å°Ĩ æĪij","Ġph ần","å¿« åľ°","ĠSu itable","ĠCont rolling","ĠNe j","ening katan","Ġoriginal en","App l","Request Body","à¸ľ ิว","å¦ĸ éŃĶ","ĠìļĶ ìĨĮ","ĠпÑĢинима ÑĤÑĮ","éļıå¤Ħ åı¯è§ģ","= _","G ary","r ÃŃan","Ġc ân","ch ien","Ġan orexia","ĠD OT","ĠD ienst","per form","èĢĮ ä¸İ","åıĪ ä¸įèĥ½","è¿IJ ç͍çļĦ","ral tar","Ġа налоги","ĠPer ipheral","ĠProgram m","Ġauf grund","Ġta as","èĤĿ 硬åĮĸ","深度 åŃ¦ä¹ł","Ġsingular ity","M ul","_ dec","Ġb aker","Ġп ли","ĠÙģ ÙĦا","èĬ± åºı","åĨ³ èĥľ","-m at","çģ« ä¸Ĭ","èŀį åªĴä½ĵ","Ġе л","å¤ľ æĻ¯","ë¡ľ ìĦľ","ı k","Ġast rology","Ġú j","ugg estion","Dem ocratic","Elect rical","Ġclamp ing","Ġacom pañ","^ i","ĠI MM","ie ber","ĠLe ather","éĢļè¿ĩ åľ¨","è½® æ¤ħ","under standing","Ïİ ÏĤ",".y ear","Ġunsett ling","ĠBritt any","# >","ĺ ר","æĹ¶ å¿ħé¡»","å°± æ¯Ķè¾ĥ","les h","ĠRes ervation","çĶŁæ´» åŀĥåľ¾","ок ой","以ä¸ĭ åĩłç§į","èģĶç³» æĪij们","ĠCH F","ĠاÙĦب د","Ġми неÑĢа","çĵ¦ å°Ķ","Ġcere visiae","ĠاÙĦاÙĨ ت",".Is NullOr","Ġjov ens","q b","Ġp ung","iv ät","her son","ठī","Ġmon astic","转 åĢº","为äºĨ è§£åĨ³","è¯į ç»Ħ","Ġopportun istic","ãĤĬ è¿Ķ","ĠSl ug","åħļåijĺ çļĦ","好好 åľ°","å¯ĵ è¨Ģ","Ġdeliber ation","ĠdziaÅĤ ania","F ed","W rap","o ie","åı ¼","ĠS cheduling","ĠT ape","ag uchi","ĠF TC","Ġk eter","åĴĮ åıijå±ķçļĦ","com pleted","ĠTe atro","Ġpost ulated","Ġve le","åĪ· åĪ·","ĠMont réal","çīµ æī¯","Ġarbit rator","icz ne","Ġarte an","ĠForecast ing","Ġполож ениÑı","Ġíıī ê°Ģ","ĵ ¨","ĠT rom","ĠP DE","åĴĮ æ°Ķ","计 ç¨İ","åIJij åĨħ","ale ur","Ġke V","åĨ³ ä¸į","çĶļèĩ³ æľī","ativ amente","Ġparl ament","-load ed","Ġpari etal","f ailure","人 åij½","å¾Ī æĸ¹ä¾¿","áĥ Ļ","ĠBe irut","Ġcontent ment","Ġrespect fully","AD I","Ġmicro array","ĠRelig ions","ĠEnc oding","Sam uel","ÙĴÙħ Ùı","åĬ¨ ä¸įåĬ¨","de code","Ġz usätz","Ġlong temps","any ol","æĹ© çŁ¥éģĵ","åį¡ çī¹","追 æį§","mod ium","Ġog ran","Ġli ens","ç«Ļåľ¨ éĤ£éĩĮ","Inst agram","................................................................ ........................................................","íĬ ¹","িল à§ĩন","á¿ ·",".To Int",".con cat","Ġarist ocratic","ĠÑĩеÑĤ веÑĢ","çļĦ çľ¼åħī","ĠH ire","Ġsub po","Ġline a","fic as","Ġ` /","sequ ential","å¤ľ 空","zie Äĩ","eger i","åłĨ æĶ¾","Rel ation","Ġspr áv","eff ects","Ġmobil ize","ĠÑĦак ÑĤи","/lib s","ĠÑģÑĤоÑĢон Ñĥ","ĠмÑĥзÑĭ ка","ĠباÙĦØ¥ ضاÙ쨩",". Instance","\\ cap","ĠF AR","cl ar","æĸ° 款","Ã¥ t","ĠÙĤ ÙĦب","è¡ĮåĬ¨ çļĦ","رÙĪ Ø·","νο ι","ä¹¾ æ·¨","Ġdismiss ing","Ġ×¨× ¦","çĢ ¾","ĠManufact urer","ĠAw esome","g is","çļĦ 设å¤ĩ","Ġcons oles","ÑĤе ÑĢеÑģ","Ġstand by","失 ä¿¡","èĤ¡ æģ¯","Ġа меÑĢикан","æ²³ è°·","ĠGe ophysical","æķĻåѦ 楼","ÙIJ ر","奥 æĸ¯åį¡","åĴĮè°IJ çļĦ","Ġdost ÄĻp","Tri angle","Ġwyn ik","ĠEpidem iol","ĠGriffith s","ĠA man","Ġpl c","åѦ æľŁçļĦ","Ġsur m","Ġcal iber","Ġrest raining","å·® çķ°","çĽ¸ä¿¡ æĪij","ĠTw entieth","ĠART ICLE","áĢŃá̝áĢ ĦáĢºáĢ","ĠسرÙħ اÛĮÙĩ","ĠG SM","ook y","å°Ĩ å®ĥ们","è§£ æĥij","Ġз Ñĥ","第ä¸Ģ 大","ÑĢÑĥ жи","é¡¿ é¥Ń","Man chester","æļĸ åĴĮ","Ġspot ting","য় à§ĩ","Ġnod al","ÑĴ е","çľĭå¾Ĺ åĩºæĿ¥","Z s","Ġm ute","ab ord","ت ج","åĬ¨ æ¤įçī©","å°Ĩ è¿Ļ","å·¥ä½ľ éĿ¢","åıĸ èĪį","ĠSh adows","gg en","Ġposs ui","reg ional","æıIJä¾Ľ åķĨ","èĨ º","ÑĢова ÑĤÑĮÑģÑı","åıij表 åľ¨","Ġunders ide","k ia","å°Ĩ 使","éĢģ åΰäºĨ","亦 ç§°","orph ic","---------------------------------------------------------------- ----------------",". float","_ real","per ate","åħ¨ éĿł","æİĴ ç»ĥ","å±ħ ä¸Ń","ĠCons istency","Ġanim aux","ĠFun ny","FL D","Ġتر Ú©ÛĮ","Ġharmon ics","Ġdeterior ating","Ġdispon ibles","divid ers","ĠíĹ Ī","O ral","et imes","æ¯Ķ 以åīį","Ġpor cent","ste ht","å®Ĺ å¸Ī","Ġpict orial","Ġanim ais","ĠÑģи лÑĮно","ł×Ļ ×Ļף","Ġਠ®","Ġmö chte","èĥ¡æ¤Ĵ ç²ī","Z V","z ünd","æĹ¶ æĹ¥","rand e","-n umbers","æ´Ľ æĸ¯","èĤ¡ç¥¨ çļĦ","Mon ochromatic","IZ ED","çŀª 大äºĨ","ĠFeder ico","ĠLingu istic","Ġerad ication",". activity","F reedom","k ken","Ġl or","ver mel","ĠG arten","ĠLe a","text rm","åı· åĴĮ","Ġaff ords","Ġس اÛĮت","Ġر ÙĤÙħ","åĹ ļ","Ø£ ت","Ġemp ath","Number matics","å¿ħè¦ģ æĿ¡ä»¶","Ġguess es","Ġjur isprudence","Gu ess","à¦Ń াব","ĠTrib al","à¹Ģà¸Ĭ ิà¸ĩ","dep ending","âŃIJ âŃIJ","W ARD","z j","Ġc ependant","Ġv á»ģ","ä¸į è¯Ĩ","ph otos","Ġbl inking","à° ¹","åı· 楼","Ġnucle ation","æģĴ å®ļ","æľºæ¢° 设å¤ĩ","iko ak","Ġsaved InstanceState","inos aur","çļĦçݯå¢ĥ ä¸Ń","ĠBerm uda","H ell","ĠT c","ĠB ANK","Ġal mac","Ġso ar","说 æĸĩ","Ġinter connect","here al","und os","èµ° è¿ĩçļĦ","Ġproject ive","æ¯Ľ åĪ©","ĠCam pos","ç«ĭåĪ» å°±","cap ac","Ġdével opper","ĠÑģвеÑĤ ло","Ġlinen o","ĠOrdin ance","E J","s ocket","Ġde ceived","op ies","Ùĥ اÙħ","à¹ģ วà¸Ķ","Ġquant ization","ĠCommun ists","Ġta al","Ġagree able","Ġsar coma","Ġà¤Ĩ हà¥ĩ","ĠíķĻ êµIJ","å°ıå¿ĥ翼翼 åľ°","ĠÙĤدر ت","R ick","n ip","ĠL ua","大 åĪĢ","æľ¬ 级","éĺ² çģ¾","çϾ ä½Ļ","åIJ« æ°´éĩı","Ñĺ ал","è¿Ļä¹Ī å¤ļçļĦ","è¸ ŀ","ĠBar rel","ĠRec her","Ġreform ed","æĦĽ çļĦ","Every body","åħ¬çĽĬ æĢ§","طر ØŃ","ĠRecip rocal","v iz","ä¿Ŀ åŃĺåľ¨","ä¼ł ç»Ļ","ĠAs ync","Un iform","ĠVol k","éĩİ æĪĺ","çŃĶæ¡Ī è§£æŀIJ","å°ĸ 端","æľīä»Ģä¹Ī ç͍","à¥ģ म","Ġਠħ","Ġhyd rate","Ġinters ecting","æĩĴ æĥ°","ä¼łè¾¾ äºĨ","_ Name","çļĦ åį°è±¡","ĠA in","ঠĻà§įà¦ķ","å¤ļ ä¸ĢçĤ¹","Ġи ноÑģÑĤÑĢан","转 å½ķ","èIJ½ å¹ķ","ĠCol ombo","idd y","èĭı æł¼åħ°","ĠTrans c","åħ·ä½ĵ è¦ģæ±Ĥ","Ġber d","åıĤåĬł è¿ĩ","Ġsatisf actor","Ġkn elt","æĺ¯ä¸į ä¸Ģæł·çļĦ","éĹ² èģĬ","èĢģ头 åŃIJ","ov as","她 被","ç³»ç»Ł å·¥ç¨ĭ","ka an","×ķת ×ķ","èĪĴ å±ķ","å·¥èīº åĵģ","tra ditional","é«ĺè´¨éĩı çļĦ","yk le","ĠÕ° Õ¥Õ¿","æĦŁè¦º åΰ","Ġescal ate","Ġpobl ació","缴è§Ĵ ä¸īè§Ĵå½¢","ç«Ļ起身 æĿ¥","M ak","çļĦ å¾®ç¬ij","ĠC age","ĠF argo","Ġrem pl","Ġز ÙĨاÙĨ","Ġanc illary","æĸĩæľ¬ æ¡Ĩ","ç¯Ħ åĽ²","ĠSlav ic","al gebra","ĠÙĪ ÙĤاÙĦ","Ġmust er","Ġvo ort","Pre ferred","æĿ¥åΰ è¿ĻéĩĮ","èĢģæĿ¿ å¨ĺ","Ġkl ar","Ġë³´ ê³ł","åľ°ä¸ĭ 室","æİł è¿ĩ","Ġchol era",". ')Ċ","/ media","Ġe arl","ĠM ura","ĠN ij","éĥ½ çĿ£","åĽĽ æĿ¡","ĠX OR","ID ER","è¯Ħ æµĭ","Ġbi ographies","Äį uje","æ¼Ķ çļĦ","Ġmicro biology","çĽĺ æĹĭ","è¡Įä¸ļ ä¸Ń","åĸĿ çĿĢ","å¿«éĢŁ å¢ŀéķ¿","Ġspokes woman","ĠÕĢ Õ¡Õµ","ĠBalk ans","P ars","Ġt ernary","çļĦ æĸĹäºī","ĠE O","ity a","ĠJ ays","åĽ½ ç¨İ","å¼Ģ æŀª","éĹ® åΰ","Ġequ id","é¢Ħ æĦŁ","åħħ è£ķ","Ġcaus ative","Ġе в","_c all","(m at","Ġprop ane",".R ef","æģ© æĸ¯","æķĮ æĸ¹","å¡« æĸĻ","æŁĶ æĥħ","Ġoccup ant","-E ast","ĠTrend ing","ĠTaiwan ese","Ġfaç ade","游åĩ» éĺŁ","åĶł åı¨","en ade","ent ious","åľ¨ ç½ij绾","åĩº åħ¶","æĮĩ æ¨Ļ","Ġgr inning","Ġant ar","åı³ è¾¹çļĦ","Ùİ ØŃ","沿 ç͍","ĠNOT ES","Ġà¸Ļ าย","ĠGreg or","f inding","Ġt igers","çļĦ ä½ĵ积","以 éĻį","Ġpos ibilidad","æ·± åij¼åIJ¸","骨 çĽĨ","çŃij åŁº","ĠPal o","Ġbirth days","DP E","æĹĹ è¢į","ÙĤØ· Ø©","Ġسب تÙħبر","Custom ers","Ġnour ishment","Ġoko ÅĤo","èĩªè¨Ģ èĩªè¯Ń","ĠTreas urer","ĠL SU","ĠL ankan","oc arp","ub ishi","è§ģ æĪij","ั à¸Ĺ","社ä¼ļ æ²»å®ī","èIJ½ 實","æĸ¹åIJij ä¸Ĭ","åĬ³åĬ¨ çĶŁäº§çİĩ","æĪ° åł´","踪 å½±","åľ¨ä»ĸ çľĭæĿ¥","寡 å¦ĩ","奥æŀĹåĮ¹ åħĭ","Ġstolet ÃŃ","? a","c ab","ol ut","ĠC aj","ort o","ĠG rac","Ġun married","ä»ĸ ä¸įä¼ļ","Ġcl own","Ġpre condition","éĥ½ åĸľæ¬¢","æ°Ķ äºĨ","å¤Ħ äºĭ","åijĬ è¾ŀ","inc s","æĹ© äºĽ","Ġter utama","Ġdistrib ución","ĠØŃ اÙĦت","è·ij éģĵ","ĠÙħر ØŃ","æĺ¯å¯¹ çļĦ","_CO MM","h ancing","Ġb urs","ĠJ OURNAL","æľĢ åŁºæľ¬","åı¯ä»¥ æıIJä¾Ľ","ull ende","è§Ĥ çľĭäºĨ","æĬĢæľ¯ ä¸Ĭ","å¾Į ãģ«","Ñģи н","-h ospital","稳 éĩį","ĠBo one","åIJ¯ è¶ħ","Ġnos es","/w idget","Ġrefriger ant","Ġপরà§įয নà§įত","ad to","æīĢ æĥ³","St orm","æ£ Ł","Ġopt ically","马 è¹Ħ","å·²ç»ı ä¸įæĺ¯","-c ig","ĠBe ans","ĠHist oire","иÑģ ал","çĶ³è¯· 表","ä¸į好 äºĨ","}= -\\","åı¯èĥ½ä¼ļ 导èĩ´","ä¸ij éĻĭ","两ä½į æķ°","×ķ×ŀ ×ķת","ĠVic ente","ĠÑĦоÑĢми ÑĢованиÑı","奢ä¾Ī åĵģ","-net work","\" As","e va","x u","Ġf red","çļĦ å°ijå¹´","æĺ¯ åħ·æľī","åľ¨ åįİ","ĠG TP","交 ç»ĻäºĨ","ĠÑĩ ÑĢез","ุ ร","å®īè£ħ äºĨ","High light","Ġà¦Ĺà§įর হ","\\ xi","ĉ Name","Ġh á»ĩ","ig ten","ort y","Ġus ka","è¿ĺ 为","ĠPro be","Ġins ults","att end","ĠÙĦ Ùģ","Ġcoll age","ĠÐļ ÑĥÑĢ","cz nego","Ġsn atched","Ġric ord","à¸Ĺัà¹īà¸ĩ หมà¸Ķ","Ġâľ Ķ","ĠSadd am","éͦæłĩ èµĽ","Ġ ÑģÑĤоÑı","act orial","å¾Ĺ éĿŀ常","Ġz god","×ķ× ¡×ĺ","ÑĢе е","Ġpot encia","bo Box","æ©Ł åζ","ĠExp ense","ç¬¬åĽĽ æĿ¡","å¯ĨåĪĩ åħ³æ³¨","大ãģį ãģı","ĠBeweg ung","C ER","m oral","çļĦ æĿĥåĬĽ","Ġre i","åľ¨ çłĶç©¶","Ġr ÄĻ","ĠSt arr","å®ļ 罪","Ġfe ito","Ġcur ator","Ġbo ils","ä¸Ģå®ļ æľĥ","åħĪçĶŁ 说","мо на","Ġram ach","æĭĮ åĮĢ","Ġllam ado","-but yl","it ore","Ġb n","## Ċ","以 西","çĶŁ 计","æĿ¥ çĿĢ","ach s","Ġent w","ĠZ ab","æĸ½ ç͍","人çļĦ çĶŁæ´»","åįĬ æŃ¥","ĠGr ö","Ġstick er","Ġmoder ated","ãĤ« ãĥ¼","á±ļ á±","ноги е","Ġ urn","Ġt ame","ĠI EP","ĠP ren","ĠP CM","ĠD odd","Ġpract ising","rac iones","红 åħī","éĻ© äºĽ","ĠPol ly","Ġber asal","ĠTom atoes","ذÙĩ ب","Bo ost","äng t","Ġë² ł","åįĹåĮĹ æľĿ","-play ing","ĠÙĬؤ دÙĬ","à¸Ħวà¸ļà¸Ħ ุม","åĴĮ å®¶éķ¿","å¦Ĥ éľĢ","æĢ» éĩıçļĦ","_s amples","æī¬ 声","éĽĦ ä¼Ł","æİ¨è¿Ľ ä¼ļ","èĤ¥ æ²ĥ","unic ode","è¾ħ èѦ","ĠHen rik","ä¼ļ计 æĬ¥è¡¨","ĠÑĢабоÑĩи Ñħ","ĠC ites","åľ¨ ä¸ĸ","Ġsa it","æľ¬ åŃ¦æľŁ","强 壮","ü tt","ç½Ĺ å¾·","Ġsem e","Ġfavor ably","Ġpow st","Ġwrong doing","çļĦäºĭæĥħ äºĨ","ĠJud as","Ġìĭľ ìĬ¤íħľ","ĠLind en","Ġinterpre ts",":n il","Ġsulph ate","Ġcardiomy opathy","åľ¨ ä»ĸ们çļĦ","好 åIJ¬","Ú© ÙĪ","ĠPl umbing","AC M","ĠEr fol","ĠاÙĦÙĥ رÙĬÙħ","Ġnephe ws","ĠÔµÖĢÖĩ Õ¡Õ¶","{ },","} R","ĠB EGIN","ä¸į èĤ²","og els","ĠU UID","æĬĬ åŃ©åŃIJ","ত ম","irl o","æł¹æľ¬ 没","Ġtag ging","åĮºåĪ« äºİ","ĠMcC oy","à¹Ģà¸Ī à¸Ļ","Ġì¹ ľ","Ġ[- ]","ĠGlob es","Ġdécouv rir","ot ically","ä¸į çĶļ","è¦ģ é«ĺ","æľ¬ åIJĪåIJĮ","社ä¼ļ å·¥ä½ľ","ç»Ŀ ä¸ĸ","å·¨ æĺŁ","à§Ģ à¦ķà§įষ","Ġstock ing","èIJ½å®ŀ æĥħåĨµ","ĠMa ver","Ġroyal ties","Bas ically","Ġдви жение","Ġreass ure","ĠSerial izable","Capt ion","-equ ipped","Ġsymb iotic","ĠS OM","du izend","Ġpart en","Ġro am","ob server","æĪij们 ä»Ĭ天","Ġdef iant","Ġب ÙĬÙĥÙĪÙĨ","西 游记","Ġsuccess ively","Ġphot ore","å°į æĪij","ĠØ® اک","åį· äºĮ","ĠMill i","Ġkn itted","ëĤĺ ëĬĶ","æľµ æľµ","篮 åŃIJ","ĠSom ali","ĠðĿij ¦","è½° åĬ¨","æī¿åĮħ 人","ĠMedic ina","Ġmenc ari","s age","Ġp ai","Ġg óc","ĠL ek","Ġne aring","ĠV ass","åIJį åī¯","Ch ord",".j ackson","æŀ¶ çļĦ","-F riendly","Ġliquid ation","Ġvac ations","íļ ¨","ĠMi racle","Ġ\"@ /","liwo ÅĽÄĩ","ureth ane","( Name","Ġc ine","iv in","Ġim ágenes","éĤ£ é¢Ĺ","Ġ. ----","ÑĢи ÑģÑĤа","æł¡ 级","éĻĦ åŃIJ","dom in","ĠVer fü","ĠDem ographic","缼 å¤ı","æ¯ı天 éĥ½åľ¨","lem ish","绿èī² åıijå±ķ","Ġgel den","Week ly","Ц Ðĺ","Ġcombinator ial","Ġa ches","çļĦ åIJ¸æĶ¶","ig ations","ÑĤ наÑı","ä¹Ł ç͍","Ġag g","æĽ´ åºĶ该","Ġlong ed","åIJ¬ æĩĤ","Ġlog rar","Ġbit map","ĠÙħÛĮ ÙĦ","èĮĥåĽ´ 为","áŀ »","è¯Ńè¨Ģ åѦ","Ġsales man","ĠÄij o","ĠON LINE","ĠMel an","Ġintim idation","ĠSubst anti","ĠÑĢегÑĥ лÑıÑĢ","Ġa e","Ġth a","st alk","un od","å¹´ æĺ¥","ó ch","ĠÎ ¨","ĠCon nie","Ġav an","Ġer os","Ġgu ise","ä¸Ģå®ļ æľī","Ġ×IJ ×Ļ׳×ķ","Ñģка ÑĤÑĮ","åIJİæĿ¥ åıĪ","åIJIJ åĩº","Hist oire","Ġpom p","ноÑģÑĤ Ñıми","ਾਠ°","à§ĩষ à§įà¦Ł","ĠSlov ak","Ġeuropé enne","C arb","] ãĢģ","re peat","Ġn ello","Ġg arb","Ġab it","æīĢ çŁ¥","ج رة","å§Ķ å©ī","çĶ· åŃIJçļĦ","List Node","éĻĪ æĹ§","atur ik","æķ£ åıijåĩº",">< !--","åıĤåĬł çļĦ","ĠSk ype","Õ«Õ ¾","夸 大","Ġlact ation","ĠSaw yer","à¦Ĺà§ģল ি","对è§Ĵ 线","Ġle ash","ĠO ceans","å°Ĩ ä¸İ","è´¨ æľ´","li hat","ĠMar ino","hel ia","æĪIJ为 ä¸ŃåĽ½","å°Ħ åĩº","Col lections","ĠÙħÛĮ ر","رÙĬ س","ĠInc idence","çļĨ 为","Ġпа мÑıÑĤи","ĠFoot notes","amer icana","Ġprod otto","Ġnh au","ĠSuggest ed","ä¼ĺå¼Ĥ æĪIJ绩","ĠاØŃتÙħ اÙĦ","Ġëľ »","= UTF","Ġa ry","çŃī ä¸Ģä¸ĭ","åı¯ä»¥ ä¸İ","çϽ æĹ¥","gg io","AT M","å¹² æİī","OR G","满 头","æī¾ 寻","ĠPer i","èĥ½å¤Ł 让","ç»ĵåIJĪ çļĦ","}}\\ ).","Ġপà§įর দ","Ġfu ente","ĠFranc isc","Emer gency","çļĦ åħ³æ³¨","ĠB SD","se f","Ġu ro","éĥ½ åŁİ","åıĺ ç͵ç«Ļ","èĮ ±","æĿ¿ åĩ³","تÙħ ر","ĠTerm ination","驾驶 人","åĭ¾ ç»ĵ","Ġprofes ionales","& S","W TO","Ġm osa","ĠM age","ne b","ĠAn and","åĨĻ å¥½","(s ql","ল à§įল","inf ected","Ġclim bs","Ġসম à§įà¦Ń","nut ÃŃ","zon ych","ĠÅ¡k oly","- hat","y at","ĠH ens","åĴĮ çIJĨè§£","ind ra","å°± 象","èĥ Ń","第ä¸Ģ åį·","ç´§ çĽ¯","è¡ĮæĿİ ç®±","Contract s","r ón","w oven","èĩª ä½ľ","Ġbl ob","Ġpresent es","IT ES","à¥įठĽ","æħ¢ äºĨ","Ġtou red","çĽĸ åŃIJ","æĢİ麼 樣","ĠTarget ing","तà¥įत र","å¿ħä¸įåı¯ å°ij","Gate way","B or","¢ ×ĵ","ĠK ernel","好 ç͍","Ġag it","Ġcomm iss","èİ Ĩ","ç»´ ç³»","Ġcompon ente",".j pa","èģļ ä¹Ļçĥ¯","ĠVari ance","Õ«Õ ¬","Ġlocom otive","Ġmemoir s","Ġಪ à³įರ","Gram mar","éĸ² 覧","ĠArx ivat","st ances","ä¸Ģ æķ´","ĠD ian","åĽ½éĻħ å¸Ĥåľº","çѾ åΰ","ĠMich a","ãĥ³ãĥ ij","/ www","ad b","ĠR ég","ĠO G","èĩ ĵ","Ġ_ Ċ","Ġob rig","æ°´ éģĵ","×ķ× ©×ij","ull ary","å¹² 线","Ġsil icate","Ġma iores","Ġple in","ĠOff shore","建议 大家","ĠпеÑĢе вод","å¥Ķ èħ¾","Ġdiper lukan","{ ,","in ches","æľī åķ¥","ĠG uth","ä¸Ń åı¶","ĠAr tic","é£İ 声","ĠInd icator","ĠÙĨ ج","_l ock","å±Ĭ 满","lem ma","åħ¼ åħ·","ĠSign aling","Ġspin ner","ĠDor is","ĠTool kit","ĠPare to","ÐłÐµ ÑĪение",". tr","à Ŀ","çļĦ 幸ç¦ı","åįģ æĹ¥","é¥ Ĵ","éĿŀ常 大çļĦ","顺 å¾·","èĪŀ å¼Ĭ","身ä¸Ĭ ä¸ĭ","Ġkon nten","æĤ¬ 念","Ġpremature ly","à¹Ģหล ืà¸Ń","Ġi Å¡","Ġsol che","ĠCon ce","Ġreal idade","æīĢ以 ä»ĸ们","Ġtruth ful","èIJ¨ å°Ķ","Ġnu cl","à¸Ħร ัว","×ķ×ij ×Ļ×Ŀ","à¹Ģà¸ķ ิม","ĠCollect or","èĴ² åħ¬èĭ±","rx js","àŃĩ à¬","Ġengra ved","N Ps","[ end","p ap","Ġof ic","Ġyou re","大 åĨĻ","ä¸ĭ 身","м ÑĭÑħ","Ġpublic ação","Ġdev ido","çŁ³ ç¢ij","Ġл ÑĮ","ç¾İåĽ½ æĶ¿åºľ","æ°£ æ°Ľ","/p age","à±įà° ®","áĥĶáĥ łáĥ","Ġ। ĊĊ","é«ĺè´¨éĩı åıijå±ķçļĦ","ÑĥÑİÑīи Ñħ","×Ļ×ĺ ת","èĦĸ é¢Ī","Z d","_ prime","Ġc err","对 她çļĦ","Ñı з","æīĢ æıIJä¾ĽçļĦ","转 磩","Ġpost up","æ¯į å©´",".h ome","æĴij çĿĢ","èIJ¥åħ» çī©è´¨","广大 人æ°ij群ä¼Ĺ","ĠInher itance","çĮķ çĮ´",") g","- reading","Ġd oth","st ores","Ġre ared","èĩ Ĩ","åī ¤","åĪĨ éĩİ","ung k","æ¶Ī èĤ¿","æķĻèĤ² åѦ","ส à¸łà¸²à¸ŀ","æĻ® é²ģ","è½® åΰ","ç»§ç»Ń 说éģĵ","èĶ º","Ġast ounding","Fl ip","åħ¬å¼Ģ æĭĽèģĺ","éį Ĭ","ĠLE ARNING","Mo ore","Har ris","ĠпоÑĤен ÑĨи","å¤ļ ç³ĸ","å®ī ä¹IJ","ĠZ ig","èIJ½ æĹ¥","Ġleg ality","Ġpur ported","Ġemb roidery","ĠRet rieve","оз Ñĭ","åıªæĺ¯ 个","Ġtab la","è̏ èĤ©","ĠP arr","ac an","ĠE LL","ĠO EM","ĠJ ail","Ġz ÃŃsk","Ġcount able","è¿Ļ个 ä¸ĸçķĮä¸Ĭ","Ġna ïve","èĢĮä¸Ķ ä¹Ł","å¥ĩ å¹»","åĨ³å®ļ çĿĢ","ä½ĵèĤ² éĶ»çĤ¼","æįIJ çĮ®","姨 å¨ĺ","ĠVeter an","à§įযান à§įড","å¯¦åľ¨ æĺ¯","W IN","are lli","igh am","ĠSch les","æŃ¦ å°Ĩ","æĸ¹åIJij åıijå±ķ",".E nd","ëį ¸","gener ally","ĠInj uries","Ġparen ch","Pret ty","éļıçĿĢæĹ¶éĹ´çļĦ æİ¨ç§»","ĠK ry","ت ÙĪØ±","èĩªå·± åĸľæ¬¢çļĦ","St d","æ¯ı åĪĨéĴŁ","-f ounded","ĠGo ose","éĿŀ常 æĦŁè°¢","å¯Ł çľĭ","èĩªçͱ åŁº","ĠPan cre","Ġton er","缸ç»ĵåIJĪ çļĦ","ĠSchul en","िन à¥įद","Ġc oke","ä¸Ģ ç²Ĵ","ĠD ining","æ°´ åĬĽ","æĮĩ æķ¸","Ġser ão","Ġcour ant","éĻĪ å®¶","ĠDef ined","ä¹ĭä¸Ģ æĺ¯","Ġশ à§ĩষ","Ġbiod iesel","俯 çŀ°","-contain ed","ĠOdys seus","ĠJacqu eline","åĴ¬çīĻåĪĩ 齿","h orm","z uf","Ġf and","le ch","Ġfor kl","缸 åħ¬","ĠPro pri","交 å¾ħ","åħĪ ä»İ","ä¹ī åĭĩ","éĢł çļĦ","ม à¸Ń","ĠAg n","Ñij л","å°ıåѦ æķ°åѦ","ĠNi ño","ĠBring ing","âĺĨ âĺĨ","ĠاÙĩÙħ ÛĮت","J ika","T ick","Ġst ell","Ġcl ung","æĪIJ åħ¨","åīį ç¼Ģ","产 éĩıçļĦ","St amp","me ida","临 èµ°","æĿ¾ ä¸ĭ","Ġcapital ized","ĠðŁ ¤","Ġfle a","ĠSO UTH","Ġintens ify","asm ussen","wij l","ĠPron úncia","gru ppe","Ġg ros","Ġper der","Ġman oro","çłĶç©¶ å·¥ä½ľ","é«Ķ èĤ²","Ġhyd rological","Ġchlor o","á»į i","adequ ate","ĠHob bes","ữ ng","Ġê±´ ê°ķ","ĠD aph","Ġch ất","èĩª å¹¼","å¾Ĺ ä¸Ĭæĺ¯","çĿĢ ä»Ģä¹Ī","ĠRes urrection","اد ÙĨ","æŃ¦ åύçļĦ","å·¦ ä¼ł","éĢIJ 个","秦 æ±ī","éĹŃ åĺ´","ศ à¸´à¸¥à¸Ľ","Ġresist ive","Ġtrib un","Ġcher ries","éĵĥ èĸ¯","çīĽä»Ķ 裤","ειο θε",". board","Ġb oh","ĠM outh","ĠH ath","åĩº èĩªå·±çļĦ","æľ¬ ä¾ĭ","ĠPro duce","西 å¤ı","离 è°±","èİ· çĽĬ","IL S","ĠDem okrat","Ġpoll inators","ĠÐĿе за","ĠsÅĤ u","ĠизвеÑģÑĤ но","! \",",". users","ä¸Ģ å·´æİĮ","å¹´ æĬ¥","æīĭ å·¥ä¸ļ","她 è§īå¾Ĺ","Ġthen ce","æĬĬ äºĭæĥħ","ç²¾ çĽIJ","ĠÙĨ ÛĮر","do ch","èĽ °","ĠVal erie","å¹²åĩĢ åĩĢ","æĥ© æĪĴ","Ġimposs ibility","Ġk ül","per ms","é» Ŀ","ĠSe en","Ġsom os","ĠÑĥ би","Ġsum ming","ä¹Łä¸į å¿ħ","Ġevent os","ĠIN IT","ĠÑĤе ло","sub t","æijĩ æĽ³","èĻ« çĹħ","BS D","èµĶ ä»ĺ","ĠShare Point","Ġmaj esty","Ġneu rologic","èĩ³åħ³ éĩįè¦ģçļĦ","VIRON MENT","d ic","h dl","her r","ĠH SP","Ġun comp","ĠAn fang","Ġx yl","oci ón","IN ATION","åįĹ æ´ĭ","Ġfour teenth","ĠFl ux","Ġmis ura","ĠìĿ´ ìķ¼","Inter ceptor","Sp here","åŁºå±Ĥ ç»Ħç»ĩ","èĩªä¸» åŃ¦ä¹ł","Ġà®ļ à¯Ĩய","Ġdich otomy","Ġcolleg iate","- next","C ED","l apping","ĠC FO","ĠK ata","ó a","ãģĻ ãģĻ","Ġsever ed","اÙħ بر","Ġ×IJ ׾×ķ","Ġży cie","Ġfeder ally","J F","k ernel","Ġf ools","ä¸į è¿Ľ","ĠG FP","ä¸Ń å°±","ang ling","ä¸İ åİŁ","ĠSp ur","ĠоÑĤ клон","æ´Ĺ ç¢Ĺ","ĠCommun ion","Ġerr atic","éłŃ çļĦ","à¸ľ ู","थ ा","Servlet Response","Ġjuven iles","ĠHubb ard","Ġassembl ages","c ognitive","f itting","h man","} \".","Ġ ãĢį","Ġc zerw","ĠT one","ĠN ominated","Ġle aps","å¿ĥ æĢ¥","ĠPr ussian","çĶļèĩ³ ä¼ļ","-e ven","Ġjoy ous","Ġmanifest o","Ġaccommod ated","áºŃ y","Ġactu ators","ĠAPPL ICATION","ĠMagist rate","åIJ¸è¡Ģ 鬼","m ess","ä¸į å®īåħ¨","é«ĺ é«ĺçļĦ","æŃ¤ é¢ĺ","ä¸įèĥ½ 被","ĠØŃ ÙħÙĦ","Ġâ̦ .","ĠRem oving","Ġflash light","Ġsuffer ers","Ġwithdraw als","Ġfox es","Ġudaler ria","Tur key","Individual s","& quot","l au","è¿Ļ 便æĺ¯","aus en","éª ĭ","ts ég","Ġwater front","-t emporal","è¶Ĭ æĥ³","-g arde","_s ymbol","åı¦ä¸Ģ 端","æľīä»Ģä¹Ī åĮºåĪ«","çļĦ主 å¼ł","ĠЧ еÑĢ","çϼçĶŁ äºĨ","_CH ECK","' et","çļĦ èĥĮæĻ¯","ĠA riel","Ġha irc","cre ating","az ers","çļĦ主 æµģ","éϽ åħī","Ġextravag ant","Ġétudi ants","ĠاتÙģ Ø§ÙĤ","ĠP eking","人 æķ°çļĦ","为 é¢ĺ","ens ä","ern et","Ġste amed","Ġimm ortality","à¸Ĥ à¹īà¸Ńà¸ĩ","æĸĩä»¶ ç²¾ç¥ŀ","Ġtim id","_l ayer","Ġस à¥Į","Ġprá ce","ĠÎŃÏĩ ει","Ġn áv","ĠS ve","ĠS OD","æľī éģĵ","ne au","å¾Ĺ 天","Ġrec al","å¹¶ èİ·å¾Ĺ","è¿Ļ个 æľºä¼ļ","åįĩ ç´ļ","-p aid","izz ata","Est imated","pie j","Ġlicense e","Ġsegu inte","äºĨ好 å¤ļ","Wal ter","ĠкÑĢи ÑĤи","ĠTherap ist","un ordered","ä¸į ç´Ĭ","æľī åij³","è¡Į åĨĽ","åĬĽ æīĢèĥ½","ĠAr qu","Ġmon ocytes","ä½ķ 人","hes da","æĬ¥ çŃĶ","_p ush","ĠNet anyahu","ét iques","ä¹ĭéĹ´çļĦ èģĶç³»","Ġri j","è¯ģåΏ å¸Ĥåľº","Ġwal nuts","×ķ×ŀ× ĵ","Ġë°Ķ ë¡ľ","Siyent ipiko","ĠÑĥÑĢ Ð¾ÐºÐ°","ä¸Ģç«Ļ å¼ı","Ġe her","ĠS OP","ä¸Ń äºļ","л ог","ÙĪ ÙĬات","ĠSt unden","å¾Ī æŃ£å¸¸","éĶ ¢","常 温","Ġins et","å¸ĥ 线","Ġens embles","第äºĮ æī¹","vey ard","ä¸įåΰ çļĦ","ĠاÙĦس Ùħ","çģĮ è¾ĵ","åĪijäºĭ è¯ī讼æ³ķ","ĠÐ´Ð¾Ð¼Ð°ÑĽÐ¸Ð½ ÑģÑĤава","ĠDES C","Ġsect eur","Ġsigh s","ĠÑĢÑı да","l um","Ġx e","çϽ åħī","нов е","åģľ å·¥","×¨× Ŀ","第åįģ 竳","ĠGru po","èĪªç©º èĪªå¤©","Ġì² ł","ĠAu ft","Ġku at","ĠREP LY","ĠHua wei","åĪĩåħ¥ çĤ¹","Ġtachy cardia","G ent","T n","s orted","çŃī å¤Ħ","Ġval ori","Ġfl utter","Ġsl ump","é¦ĸ åĪĽ","ä¸ĬçļĦ 讲è¯Ŀ","Ġر شتÙĩ","ä½Ļ 弦","Ġir responsible","注åĨĮ åķĨæłĩ","æ¯Ľæ³½ä¸ľ æĢĿæĥ³","Ġrecurs ively","ê°ľ ìĿĺ","ĠJa ime","c ord","Ġand rogen","un ky","ĠG inh","Ġr uff","rom ic","here inafter","Ġcor rente","è¶ħ é«ĺ","å¤ĦçIJĨ åIJİ","éĩįçĤ¹ åħ³æ³¨","ĠDay light","Ġing Ã¥r","Ġult raf","ĠOff ices","æķ°åѦ 模åŀĭ","Ġdetect ives","è¿Ł éĴĿ","ĠRos emary","ì¡° íļĮ","Ġtorn a","ĠÐłÐµ ÑĦеÑĢен","Ġreduct ase","ĠгоÑĢи зонÑĤа","çļĦ çĶŁçī©","æķĻ çļĩ","Ġsc our","raw d","ĠFl oating","表示 äºĨ","顺 æĹ¶éĴĪ","åĪĨéĴŁ åĨħ","Ġcomposition al","áĢºáĢ ¸áĢ","åĤ» äºĨ","éľī èıĮ","otox in","ĠThy roid","priv ile","ĠLazar us","c ust","Ġon da","å®¶ æķĻ","çİĭ æĻĵ","Ġsw immers","Ġপ াà¦ĵ","pi as","èľ ¥","鼻 æ±ł","Ġay at","溫 æŁĶ","ÖĦ Õ«","ĠQuiz let","ĠÑģледÑĥÑİÑīи м","Ġcapit ale","Ġscrat ches","ĠBrow ne","s eller","ภº","ĠF az","çŃī åľ¨","ÑĢа но","åıª ä¸įéģİ","ç»Ļ å°ı","à¸ģ à¹Īà¸Ń","Ġsens ibility","ĠÑģÑĤа ла","æıIJåĩº éĹ®é¢ĺ","æıŃ çīĮ","é«ĺæł¡ æ¯ķä¸ļçĶŁ","END ER","åħ»æĪIJ èī¯å¥½çļĦ","Ġrall ies","渾 身","f arm","y nd","ì ¢","Ġb rib","ic orn","ĠV ide",".com mit","æķĪæŀľ å¦ĤåĽ¾","æĹ§ åĿĢ","Ġহ ার","ä¹ĭéĹ´çļĦ çŁĽçĽ¾","à¦Ĥ র","Ġส ามารà¸ĸ","éĢĻäºĽ 人","èIJ¥ä¸ļ é¢Ŀ","ĠpolÃŃ ticos","æľ¬é¢ĺ åĪĨæŀIJ","ĠسÛĮ اسÛĮ","ç½ķ è§ģçļĦ",". round","ar ous","om u","ĠM SP","ä¸į å¤ļçļĦ","Ġk aki","og na","Ġu omini","л ла","缴 è§Ĩ","ä¹ł æĢ§","å¹² å¹²åĩĢåĩĢ","ĠPr inter","}\\) ;","亲 åIJ»","cont rolled","æĿĢ æ°Ķ","Ġbacter i","Ġcatal ysis","çľ¼åīį ä¸Ģ亮","ĠPleist ocene","og u","ä½ľ åĽ¾","ж ей","ä»Ĭ天 æĻļä¸Ĭ","秦 天","Ġkom pet","Ġmac OS","è¿Ļå°± éľĢè¦ģ","ä¸Ģ大 æī¹","äºĨ好 åĩł","= h","M unisipyo","c ans","ä»ĸ 竣çĦ¶","ov able","èĢĮ å½Ĵ","æĹ¥ æĹ¥","ç§į ç±»çļĦ","çŁ¥ åİ¿","ien iem","other mia","该 æŃ»çļĦ","Ġfil le","éĢļè¿ĩ ä¸İ","Ġcurrent Node","èī¯ çŁ¥","uj eme","åĪĽæĸ° åŀĭ","åħ±åIJĮ å¯Įè£ķ","raz ioa","gra f","ĠProv incia","æŀª æĶ¯","姨 å¦Ī","dist ribution","ĠPeters en","Ġnatu ur","ĠRais ing","\\ h","st ations","éĥ¨ å°ļ书","åįģ è¿Ľåζ","åħĥ å®ĩå®Ļ","åĽŃ èīº","medi ation","-P al","بر اÛĮ","æŃ£å¼ı å¼Ģå§ĭ","ĠÚĨ Ø´Ùħ","Ġham ster","gener ational","ĠBab ies","åĩºå¸Ń äºĨ","Ġ×Ķר ×ij","ĠÑĤÑĢ ÐµÐ½Ð¸","- Control","ĠB TC","oc aine","Ġun consciously","åīį 人","æ°´ 墨","ĠCl othing","åŁİ 建","ä¹° 车","Ġespec ie","En hanced","log out",".T YPE","à§Ģ তà§ĩ",".R est",".J oin","ç¾½ ç»Ĵ","æĹ¥å¸¸ å·¥ä½ľ","å¹´é¾Ħ 段","static method","ológ icas","Ġал ког","Ġobliv ious","S ARS","_ pr","v ian","ĠR ess","ĠE ly","lic ting","éĤ Ĥ","Ġman os","Ġequ atorial","ä½ķ åľ¨","Ġdiv ul","æĸĩåĮĸ æ´»åĬ¨","çķĻ æĥħ","ĠTra b","Ġviol encia","df df","wer pen","èĥĥ çĤİ","ĠKn ock","Wait ing","Ġsinus oidal","Ġbrew ery","æľ¬èģĮ å·¥ä½ľ","r ini","ĠB IO","ĠF UND","对 ä»ĸ说","ä¸ĭ åŃ¦æľŁ","Ġdev iate","Ġwid gets","áĥĿáĥ ¡","Ġreproduc ibility","D av","N ag","ĠS ne","ä¸Ģ æĪ·","Ġle opard","ust i","åѦ è¿ĩ","ä½Ĩ 缮åīį","ç»ĵ æĪIJ","æīĵ æĬĺ","交 çͱ","西 èĴĻ","ĠOr bit","à´ Ļàµįà´","ĠÙħÙĨ اطÙĤ","unt i","éĤ£ä¹Ī ç®Ģåįķ","_f ree","×ķר ×Ļ×Ķ","æł¹æľ¬ ä¸įæĺ¯","ĠÐĺ менно","}) +","æīĩ å½¢","Ġovar ies","Ġhydrochlor ide","ĠSubstanti ivi","ĠT olerance","ä¼ļ éģĩåΰ","ĠV ida","Ġп ÑĭÑĤа","Ġinter cultural","åħ³ åį¡","à¸Ļ าà¸Ļ","rem o","Pl aintiff",")\\ \\","ĠFe in",".print f","ä¹³ æ±ģ","ãĥ¼ãĥ ģ","ĠоÑĤде лÑĮнÑĭÑħ","à¸Ľà¸£à¸°à¹Ģà¸Ĺศ à¹Ħà¸Ĺย","ĠBT U","F ear","M m","y uan","Ġin quis","åĪ ª","Ġcon notations","çº £","å°ı äºĨ","èĩªå·± æľī","ĠBl um","bo ats","å¾Ĺåΰ ä¸Ģ个","ç¬Ķ ä¸ĭ","Ġcapac ité","溶 è¡Ģ","çĭĤ æļ´","ĠPers istent","è¿Ł ç¼ĵ","Ġdrought s","Ġwart o","è·¨å¢ĥ ç͵åķĨ","üs se","ĠVij ay","Ġs ito","Ġm ids","大 伯","ż eli","ĠÐŁ олÑĮ","ä¸Ńå¿ĥ å°ıåѦ","Ġnetwork ed","è´´ 身","åħ© 種","伸 éķ¿","à¸ľ ม","æĢ»ç»ĵ ç»ıéªĮ","},\\ ]ĊĊ","ĠPs y","Ġperce p","ĠWalt ers","ĠвклÑİÑĩа еÑĤ","ĠSt av","Ġ} _{","éļ ½","Ġв аÑĢ","ari i","ond yl","æīĵ åĢĴ","ü cht","åĮĹ è·¯","çĤº 主",".c an","Ġber upa","ÙĬد ÙĬ","Rec ording","Ġdur ée","à¹Ģà¸Ĥ à¸ķ","Ġperf ected","cred entials","ĠиÑģÑģледова ний","Ġvzd ÄĽl","\" She","- Res","Ġa compan","Ġf ittings","ol ong","éĿ¢ éľ²","æĬĬ æİ§","ÑģÑĤа ÑĢ","ãĢij **ĊĊ","纪 å®ŀ","Ġfem ur","ĠGen ius","ç»Łè®¡ æķ°æį®","erse ys","ĠBur mese","Ġmargin ally","iti é","ĠDocument ary","Ġobey ed","à¶Ń à·Ĭ","ĠStef ano","C es","P d","ĠI H","ul on","éĹ °","EN AME","Ġinc ense","Ñĩи нÑĭ","iny in","æµ® èºģ","Supp lier","Ġpes os","ĠEsc ola","åıijè¾¾ çļĦ","èıľåįķ ä¸ŃéĢīæĭ©","ÑģÑĥ лÑĮÑĤа","æīİå®ŀ å¼Ģå±ķ","åı¯æĮģç»Ń åıijå±ķçļĦ","Ġeer ie","ĠDion ys","Ġunim agin","M un","Ġ ers","ĠT iber","ĠG é","ĠO val","Ġen fin","ÑĨ веÑĤ","æ·± ä¿¡","å¼ķ è¨Ģ","æİĴ å°¿","çīĪ éĿ¢","æŁIJ 项","ä¹Łä¸į ç®Ĺ","Ġdat um","ĠNe arest","оди на","æĬĵ åΰ","缩 æĶ¾","æĽ¿ ä½ł","Ġapopt otic","R IS","Y ork","çļĦ å®īæİĴ","ĠT olkien","ad am","ĠB em","åľ¨ 京","Ġ* )Ċ","ÑĢе ÑĪ","ĠZ ack","交 åıĭ","å¿« åΰ","Ïĥ θ",".E mail","alah an","ĠÑĢаÑģÑģка за","j it","ĠI AU","é poque","Ġrel a","ни ли","ĠAd jective","ĠPer imeter","èŀį 为ä¸Ģä½ĵ","è¯ī 说","çĹĩ çĭĢ","æ²»çĸĹ åIJİ","è·³ 绳","åł± éģĵ","Ġnam n","Ġabund antly","à¸ģระ à¸Ĺ","Jun ior","Ġmuff ins","ĠWrit ings","Ġp oke","ed u","ar os","çļĦ å¦Īå¦Ī","ĠS ok","Ġg ia","Ġrem in","ĠAct in","ĠNe olithic","OM S","-F eb","Ġà¦ķর à§ĩà¦Ľà§ĩন","iy ama","æľĢé«ĺ æ³ķéĻ¢","èħĶ åĨħ","_ex ec","graph s","ụ c","Ġসà¦Ĥ শ","hh hh","Ġcuc umbers","Ù¡ Ù","ĠFebru ar","äºļåİĨ 山大","J et","Ġd ab","çļĦ åıĤæķ°","et Åij","ĠR owe","se i","èĢ Ļ","art z","ell um","åΰ å°¾","建 æ¡£","çϽ èĬį","ä½Ļ çĶŁ","注æĦı åΰäºĨ","ðĿij ¢","Ġlimit less","Mod ules","èĤ¥ 大","Ġtu ples","ë° Ģ","ĠIr win","ĠVoc ational","Ġuitge geven","ĠS hat","ĠG au","æĥ Ĩ","ĠV enn","ks z","ï¼ī ï¼ĽĊĊ","ven e","ç«ĭ åĬŁ","ĠاÙĦÙħ Ùı","bb b","Ġfall out","å°į ä»ĺ","Ġmoment ary","æĢ§èĥ½ åĴĮ",".pro cess","ĠEV ENT","Ġpione ered","Mn O","Ġvys ok","Ġgemeins am","Ġ( );Ċ","ĠL ateral","ĠN AV","ä»ĸ 羣çļĦ","aus anne","... \"Ċ","è¿ŀ æĿĨ","åĩı å̼","ĠÙĬ ÙĪÙĦ","æ¯Ĵ çļĦ","ç»ĵåIJĪ å®ŀéĻħ","æĦıè¯Ĩ åľ°","ĠNon fiction","ĠÙ쨱 ز","åͤ èµ·","g uns","at ius","el k","ĠC orm","ĠB owie","æĹł çĹĩçĬ¶","æĪĸ 被","ç¥ŀ è¯Ĩ","å¦Ĥæŀľ æĥ³è¦ģ","Ġca uliflower","Ġcentral ity","åĬ³åĬ¨ çļĦ","ĠÙħت ÙĪØ³Ø·","ĠDam on","æĥ¨ åı«","æĸĩèīº å¤įåħ´","æĶ¶è´¹ æłĩåĩĨ","ÄįnÃŃ ch","Ġorb iting","Ġbund led","M ixed","p icker","Ġto ppings","åľ¨ äºĨä¸Ģèµ·","ak on","Ġat rium","å¤ļ å¾Ĺ","ĠFl our","ĠVer w","åŁ¹è®Ń åĴĮ","Ġmel atonin","缣 åıĭ","æĢİä¹ĪåĬŀ åij¢","Ġbark ing","ĠìĪ «","æĬĹåĩ» çĸ«æĥħ",". De","o fer","ĠS ato","op rot","๠IJ","ä»ĸ çļĦ人","æĹ© äºĨ","Ġsw aps","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ","Ġপ à§Ĥরà§įব","第ä¸ī 产ä¸ļ","mi ÅŁ","æ´ģ çϽ","èĤ¾ çĹħ","Ġর à§Ł","-pr one","-inf rared","( command","P X","åľ¨ ä¸ŃåĽ½çļĦ","Ġqu ae","天 éĻħ","æĹł å½¢çļĦ","é»Ħ åŁĶ","è´¢ è¿IJ","Ġexec utions","à§Ģ à¦ķ","åĨ° åĩī","Ġimag em","åijĬè¯ī ä»ĸ们","Ġnarr ated","Ġì§Ģ ìłķ","纪念 ç¢ij","ĠPack ers","è¿Ļ项 å·¥ä½ľ","Ġkans sa","ĠокÑĤÑı бÑĢÑı","c aps","Ġm iaÅĤ","ĠB irk","以 ä¾Ľ","èĩª ä½ĵ","Ġsp ies","ĠZ ac","第ä¸Ģ å®¶","اÙħ ج","å¿ħé¡» åħ·å¤ĩ","çĹĩ çļĦ","ĠاÙĦس عÙĪØ¯","éĽĦ åİļ","ĠÚ¯ ÙģØªÙĩ","Gen etic","Ġvibr ating","ÙĥاÙģ Ø¦","ĠÙħÙ쨵 ÙĦÙĩ","ĉ Scanner","Ġin hom","对 åķĬ","对 çݯå¢ĥ","å¹´ èĸª","è¨ Ł","ĠAl buquerque","other m","Ġquant itÃł","Ġdomestic ated","âĹı âĹı","ĠNaz ionale","Ġmoi ety","æľĪä¸Ĭ æĹ¬","B X","_ ON","Ġs ok","Ġc áncer","ĠE MT","Ġor chard","oc ode","Ñĩ нÑĭми","èĵ ĵ","οÏħ Ïĥ","é¹ ī","ÐĴ ÐIJ","åħ¨éĥ¨ çļĦ","ÑĨиÑı ми","ĠKen yan","_g ame","ĠDiagn ostics","Ban ay","追溯 åΰ","çļĦ çī¹èī²","ĠA pa","ĠB rack","åľ¨ 为","ä¸Ń çĤ¹","è¦ģ æī¾","ä¼ļ éĢīæĭ©","ft p","产 éĶĢ","Ġco ax","() }","-b odied","ä¸ĩ ä¼Ĺ","éļ¾ æ°ij","Ġcle ars","è¿ŀ æĹ¥","Ġé g","ĠOr ang","ĠAm t","åħŃ ä¸ªæľĪ","èι çļĦ","èµ¶ å¾Ģ","ä¸įäºĨ äºĨ","Ġreson ated","æĮĩ示 ç²¾ç¥ŀ","ĠÐĶа ÑĤа","еви Ñĩ","ãĥĹãĥŃãĤ°ãĥ© ãĥł","k tr","¹ áĢ","in ä","ion age","ĠS OS","op atra","ä¸Ĭ åĴĮ","### Ċ","å¿ĥ ä¸ĭ","äºĮ å±Ĥ","ES I","-f itting","ĠEn zyme","ĠIm agination","ä¸Ģèά 认为","CT T","lim at","å®ŀè·µ ç»ıéªĮ","åħħåĪĨ åľ°","ãģ¨ãģĹãģ¦ ãģ¯","/de crease","ut ant","åľ¨ åIJĦ个","Ġsh ovel","å°ı éĽª","ier enden","ung ere","西 åij¨","éĹ®é¢ĺ è¿Ľè¡Į","ãģ« ãģĹãģ¦","ĠâĢ¢ #","éĩĩ æĶ¶","æ²¹ çĥŁ","ðĿij £","çļĦ大 äºĭ","è°ģ ä¹Ł","Em ployment","ĠMal d","ĠDO UBLE","ç²Ĺ çķ¥","åįļ士 åIJİ","çī¹åĪ¥ æĺ¯","ï¬ĥ ï¬ĥ","Ġth ine","ĠP unch","Ġu ber","ä¼ļ æ¯Ķè¾ĥ","èĢĮ åıĺåĮĸ","fl äche","Ġshort ness","Ad ams","æī§ æķĻ","符åIJĪ æĿ¡ä»¶çļĦ","ãĤ· ãĤ¢","èŀº æĿĨ","Ġpag klas","rin os","丸 åŃIJ","ocard ial","Ġsteril ization","Autor itate","åħĪéĶĭ模èĮĥ ä½ľç͍","ol os","Ġch iar","èĩª éĢĤåºĶ","ign ing","çŃī éĩįè¦ģ","åģļ äºĽ","yn ku","太 éģİ","äºĨä¸Ģ 段æĹ¶éĹ´","Ġconn u","缸åħ³ çŁ¥è¯Ĩ","é½IJ é²ģ","Ïĩ ν","æ¡Ĩ åĽ¾","ÃŃn io","ĠRam on","åıijçĶŁäºĨ åıĺåĮĸ","ÑĢован нÑĭй","ĠÑĤка ни","ĠвÑĭбÑĢа ÑĤÑĮ","Ġpagklas ipika","D y","k Pa","} B","åľ¨ å¿ĥä¸Ń","è¿Ļ 帮","æĸ¹ ãĤĴ","æķ° ä¸İ","é£İ éĽª","-t own","aster xml","ä¼ĺåĬ¿ åĴĮ","çѹ 建","-W orld","Ut ilities","пеÑĢ Ð²Ñĭе","layout s","ĠØŃس اب","Ġinconven ient","T odos","n ig","ĠL ille","ä¿ Ł","bb c","ano ia","Ġmess engers","ĠاÙĦد ÙĥتÙĪØ±","ĠBig gest","Click ed","å½ĵåľ° æĹ¶éĹ´","MC s","æŁ¥çľĭ æĸĩ竳","Ġcra ve","ül és","à¸ģรรม à¸ģาร","Ġà¦ľà§Ģব ন","Ġsilhou ette","Ġsupernat ant","çļĦ æŀĹ","äºĨ è¿Ļä¸Ģ","Ġch ats","ä¸Ĭ æľĪ","æĪij们 å¸ĮæľĽ","å¹² åķ¥","ĠÙĥ ÙĪØ±","در سة","æļĤ æĹ¶çļĦ","ĠVict ims","Ġlymph ocyte","åİī害 äºĨ","åŁºçĿ£ å¾Ĵ","@@ @@","ĠкÑĥÑĢ Ñģ","lyss es","ä¸ĭå®ļ åĨ³å¿ĥ","v ict","Ġc ork","ĠR U","å°± åıĪ","ru zione","Ġgra der","ĠSc oring","Ġcounter ed","Ġmedi ocre","charg ing","ç¥ł åłĤ","Ġt ints","ĠC CTV","é£İ 度","æĶ¹ åIJį","To One","è¿Ļæĺ¯ æĪijçļĦ","ĠÑĢе волÑİ","Ġত à§ĭম","æĮº 身","Ġgang lion","ĠCrom well","ĠGinh adi","Ġp aws","çļĦ æ¯Ķéĩį","ĠE aton","ĠThe or","Ġhe ns","we go","äºĶ æĹ¥","éĴ± äºĨ","座 ä¸Ĭ","å°±ä¼ļ åıijçݰ","åŁ¹åħ» äºĨ","表达 çļĦ","мÑĥ м","驱 使","Ġdar über","ĠFar aday","è¦ı ç¯Ħ","åĪĹ表 ä¸Ń","çĭŃ ä¹ī","âĸł âĸł","z zi","Ġan unci","ĠD ETAIL","å°ı èįī","ĠÙħ ÙĬ","α λλ","arr ative","ĠThat cher","oper atively","Ġপ à§ĥ","æĶ» åŁİ","yth mia","ç¥ĸ å¸Ī","ĠWhite head","æİĮæı¡ çļĦ","æĮĩæĮ¥ å®ĺ","Cond itions","åĴ¯ åĴ¯","Ġblij ven",", âĢĺ","A rist","s olve","Ġs sh","ĠM oj","ĠIn voice","Ġ& #","ĠÙĪ Ø§Ø³","èĦļ ä¸ĭçļĦ","Ġpun ches","ĠMov ements","Ġ׾×Ķ× ©×","Ġinaug urated","ĠпÑĢог ÑĢе","gradu ates","åij» åIJŁ","Ġf c","çļĦ èĭ±éĽĦ","Ġqu ase","ĠY eh","æ°´ æ»´","带 æĪij","å¿« åİ»","Ġé r","Ġden ounced","çĵ ®","ĠIS I","ĠÑģÑĤа ло","dis p","å·Ŀ åİ¿","ÑĩеÑģк омÑĥ","ника м","ĠBon k","éĤĢ请 äºĨ","ãģĻãĤĭãģĵãģ¨ ãģ§","Ġdiplom ats","ĠElev ated","Ġh ops","Ġh ugs","å¤ļ äºij","Ġass ent","èĦ ĺ","Ġsl ant","æĬ¥ æ¡Ī","Ġhere of","积 æ·Ģ","åį´ å¾Ī","à§ģ à¦ģ","Not ifications","Ġম াস","ĠMac Arthur","Supp ly","Ġpued an","ĠKl assen","ĠParticip ate","éĴī åŃIJ","Ġcaution ed","Ġmaneu vers","V or","] !=","qu eda","ist ischen","é vel","hen ic","å®¶ çļĦ人","æŃ£ éģĵ","ĠUn employment","ĠÙĦ Ùħا","print ln","Ġid Åij","LE VEL","à« Ĥ","ĠØ· ÛĮ","ìĤ¬ ìĿĺ","ä¸ĭåĪĹ åħ³äºİ","Any thing","åĬĿ 导","ifl ora","P ipe","Ġv antage","ĠP tole","ĠD EG","åľ¨ åħ¨å¸Ĥ","为 æĬĵæīĭ","Ġв ов","az in","ร à¸ĵà¹Į","ä¹IJ ä¹IJ","à¸ķ า","ĠпÑĢи д","ĠMon a","(c nt","éĺħ è§Ī","ĠBen ny","Inter pret","èħ° 带","Ġà¦ľ নà§įম","oli opsida","èݲ åŃIJ","Ġgan ze","Ġenvol v","B uilt","J H","_ load","h ound","ĠD AL","ĠK ie","天 å¸Ŀ","çľ¼ èī²","amp hetamine","çĹħ äºĨ",".s ystem","åİĭ ä½İ","åĨĻ ç»Ļ","à¸ļ ล","è¿ľ 端","ĠAm it","Ġquant ifying","åĮºåŁŁ æĢ§","Ġtele f","Ġregist ro","Ġ×Ķ×Ļ ×Ļת×Ķ","à¸£à¸¹à¸Ľ à¹ģà¸ļà¸ļ","ĠChes apeake","ðŁĹ ijĊĊ","al let","çļĦ ä»ĭç»į","ĠC ependant","å°ı ãģķ","åĬŁ èĩ£","ãģĻ ãģİ","Ġgen et","ðĿ ĸ","ç¦ı çͰ","éĺ¿ åħĭ","ĠTrans forming","Ġsn orted","æijĩ 篮","ĠErk rank","Ġved ere","ãģĵãĤĮ ãĤīãģ®","Õ¡ÖĢÕ ¿","Obs ervation","ĠëĦ ĺ","обÑĢа зи","Ġdiscrimin ant","ĠHerz eg","G MAT","l id","ĠT ick","Ġal m","ĠR ä","éĤ£ 座","èģĶ éĺ²","rict ional","Ġcontinu ación","âĨĴ âĪŀ","æĭĸ éŀĭ","代çłģ å¦Ĥä¸ĭ","Ч а","ĠEdge Insets","Ġheap q","_class es","ĠD ancing","æĪij ä¸įè¦ģ","Pro position","rop a","åľĨ åľĨ","fin ance","ĠÑģÑĤои моÑģÑĤи","Ġমত à§ĭ","جÙĬÙĦ ات","- rock","Ġm anger","Ġre naissance","ra pping","人 åij¢","Ùħ ÙĪØ¯","ï¼ī ãĢį","}\\ ;","Ġmicro controller","(f rame","ĠAS A","æľĪ份 çļĦ","ĠEmer ald","Ïģά ÏĨ","èĤ¡æĿĥ æĬķèµĦ","Ġdun geon","æİ¡ åıĸ","èĩªåı¤ 以æĿ¥","åĺĪ æĿĤ","Ġp ascal","Ġv ede","ĠN ATIONAL","å¿ĥ 声","æĶ¶ å¤į","Ú© ارÛĮ","太 çϽ","转 è§Ĵ","è´£ ç¼ĸ","çŁ³ éŨ","ĠCO D","ĠFran ch","è¿· ä½ł","uer ak","Ġdil ated","main ly","çļĦç¥ŀ ç§ĺ","Ġmé dec","ĠJung le","ĠGael ic","ĠD LL","ĠN arendra","ار س","Ġver fü","å¦Ī åĴª","Ùİ Ùģ","Ġelement i","à· ĵ","èļ ĵ","ÐĶ Ðµ","çļĦä¸Ģ项 éĩįè¦ģ","Τ ο","çļĦ çĶ»","ks on","Ġam élior","ä¸ī å¹´çļĦ","ж нÑĭй","ж ением","-m ounted","Ġver oor","ĠPol es","ä¸ĥ ä¸Ģ","ÚĨ ار","Ġconsequ ential","éħįç½® æĸĩä»¶","vy Å¡","hist orical","åģļçļĦ å°±æĺ¯","èŁ ł","ĠMom ents","Ġtestim on","Ġì¤ij ìĭ¬","ĠPs ic","Ġkr wi","ĠWonder ful",": string","S CH","Ġbe eld","åŃ º","ä¸į åıĬæĹ¶","âĢľ .ĊĊ","ĠH urt","é«ĺ ç¥ĸ","使 é¦Ĩ","itt s","ĠSupp liers","åĬªåĬĽ æıIJé«ĺ","Be at","ĠMot iv","Ġlit urgical","Ġmanip ulations","å; æ¶²","á̱á̬áĢ ĦáĢºáĢ",". ie","Ġm uda","ir ani","Ġk ak","Ġse jak","Ġconf ine","é¢Ĩ çĿĢ","Ġsqu id","ãĥ¼ãĥ ĭ","åĪĨéħį çļĦ","Ö¸ Ö","Ġtrail ers","ĠRecogn ize","èµĭäºĪ äºĨ","æ¸ħçĥŃ è§£æ¯Ĵ","ĠGastroenter ology","èĶ· èĸĩ","ĠÑĦевÑĢа лÑı","Ġcovari ates","\\ $","Ġv org","ap ai","Ñĥ ка","æĭ Ī","æİ¨ åĬĽ","æĴŃ éŁ³","åį· çĥŁ","æĬĺ ç®Ĺ","Supp lement","Ġveloc idade","åĬłå¿« 建设","ä¹Ķ 丹","Ġmanip ul","Ġत à¥įय","Ġبت ÙĪØ§ÙĨ","U W","u ft","Ġh indi","Ġdis emb","ne utral","Ġunder lie","Ġо ÑĦоÑĢм","äºĮ 鼶","羣 çα","Ġsl ur","Ġrisk ing","é¦Ļ èıľ","Ġfamil ie","è£Ĥ éļĻ","å¹¾ åĪĨ","æ¯Ķä¾ĭ çļĦ","åĪĨæĶ¯ æľºæŀĦ","广éĺĶ çļĦ","æīĵéĩı çĿĢ","Ġwors ened","ĠC ottage","ĠD omen","大 æĬµ","ĠJ ep","Ġen list","åı¯ éĢīæĭ©","è¿ĩ æĪ·","me ister","æķĻèĤ² åѦéĻ¢","Ġа г","ä¸ĥ 天","à¸Ĥ à¸Ń","ĠAir bnb","æĪijä¸į æķ¢","Ġcab ins","Ġди ÑĢек","Ġproportion ate","Ġmu u","ĠGott es","à¹Ģà¸ĭ ล","< s","u let","Ġm ies","ĠT ire","ĠD SS","æĸĩ æĺĮ","Ġob en","ĠHe ap","ĠJe ong","Ġhyp o","Ġmol est","(B uild","Ġà¦ĩ সলাম","ĠRail ways","ç³ĸå°¿çĹħ æĤ£èĢħ","สิà¹Īà¸ĩ à¸Ĺีà¹Ī","ĉ h","Ġal p","ĠH ose","å°ı åĮºçļĦ","éĥ½ åįģåĪĨ","ä¸İ çݰ代","Ġpo hy","hat ic","Ġcol span","Ġac eler","اÙĦ بÙĦد","è§ģ è¿ĩçļĦ","Ġcamp ground","ä½³ çļĦ","è§Ħ模 åĴĮ","ĠJo achim","Ġresist encia","ĠSie gel","íķĺë©´ ìĦľ","Isa iah","Y n","Ġd rizzle","Ġre aring","åľ¨ æŀĹ","Ġk oj","ger i","åĨ² åĩº","-P resident","乡éķĩ ä¼ģä¸ļ","Ġfet ched","åĴļ åĴļ","S ERVER","_ conf","Ġin oltre","im ension","/s erver","Ġeduc ativo","ä¸ĢäºĽ éĹ®é¢ĺ","ä¸ĵä¸ļ æĬĢèĥ½","æģ© æĢ¨","àŃ Ł","ĠBuck ley","S ig","v art","it ans","Ġd au","çļĦ æĶ¹éĿ©","ĠD FT","ĠE do","ĠW oll","ess els","è¿Ļ è¾¹çļĦ","ä»ĸ çľĭåΰ","Ġsp ire","æĢ§ è´«è¡Ģ","ĠWe egy","Ġent ier","ĠLe isure","æŀģ 强çļĦ","çļĦä¸Ģ åijĺ","ä¹° ä¸ľè¥¿","éķĩ æ±Ł","à¹Īาภª","æī© 容","ĠConst ellation",".de ep","æ¯ıä¸Ģ 项","åIJĦ级 æĶ¿åºľ","ĠобÑĬ ема","ëŀĺ ìĬ¤","ĠÐłÐ° ÑģÑģ","Ġobten ir","ĠBhar at","Ġkitt ens","% CI","y ama","¤ ×Ļ׾×ķ","ĠL aden","éĢ ħ","åѦ ä¸Ģåģļ","天 åĽ½","éĹ´ çĽĺ","ĠZ Z","rem aining","åºĶ该 å¦Ĥä½ķ","اص د","Ġпов е","Ġ×ij×IJ× ŀצ","à¸łà¸²à¸© าà¸Ń","B eta","w ahl","ĠG d","åĴĮ æµ·","Ġcan v","åī Į","åĪĹ çļĦ","æĸ¹å¼ı æĿ¥","ĠÐľ ожно","ĠCam el","à¹Ģล ีà¹īยà¸ĩ","åĪĩå®ŀ æĬĬ","ĠPok er","Prem ium","Ġreop ening","循åºı æ¸IJè¿Ľ","ĠT eb","ĠL ager","ak rish","ĠG ris","Ġdis ent","æĸ° æĿij","æĹł éĻħ","Ġeff etti","Ġent icing","éĵ Ģ","-f ont","åĪĿ åѦèĢħ","ĠÐĵ и","Ġble ach","_T EXT","Ġciv ile","ĠFif teen","æ¸ħæĺİ èĬĤ","-sid lakang","consum er","å¿ĥ缮 ä¸Ń","ĠÑĤÑĢанÑģп оÑĢÑĤ","Ġsh ady","д ение","ç»ĵ 交","æ±Ĥ æķij","è¡Ģ 迹","çĶŁäº§ è¿ĩç¨ĭ","à¹ģ à¸ģรม","ĠVol cano","ĠPal azzo","åºĶå½ĵ æĺ¯","Na OH","ATH ER","ĠTah un","å¤įæĹ¦ 大åѦ","A ustin","C annot","Y m","g z","çļĦ æ¸ħ","Ġn atal","ĠS amb","end om","æķĻèĤ² åŁºåľ°","Ġза мен","âĢĺ (","list en","身ä½ĵ çĬ¶åĨµ","Ġìĸ »","大大 æıIJé«ĺ","ĠSteel ers","ĠLoren z","Ġreciproc ity","k T","Ġa illeurs","al do","ĠC ame","Ġk ah","Ġu ch","åıį å·®","gg y","Ġmen em","主è¦ģ éĢļè¿ĩ","Pl ants","ÙĦÙĬ ب","Ġsecret ions","ĠÑģп иÑģок","çģ° åº¦","èªį çβ","åĵªéĩĮ æľī","ĠCR P","éĥģ éĥģ","Ġenerg ized","è¯ĬæĸŃ ä¸º","Ġradi ographic","ç´Ģ éĮĦ","奢 åįİ","ĠÐłÐµÑĦеÑĢен ÑĨе","Ġst rad","Ġcon vent","cl in","天 ä¸Ģ","sp un","Ġ> >ĊĊ","ä½İ æĪIJæľ¬","请 ä»ĸ","Ġkey boards","æĻº åºĵ","inn i","ĠÑĦ еÑĢ","åĪĢ çļĦ","Out line","à¹ĥห à¹īà¸Ļ","ä¹Į æĭī","Ġcontra ceptive","Ġconstitu encies","\\! =\\!","ĠVenez uel","m oid","èĢħ æľī","æĮĩ æĺİäºĨ","Ġmet ropolis","该 ç³»ç»Ł","ĠDef ender","Ùij ا","Pre paration","åĭ¤ ä¿Ń","Ġপà§įর à¦ļ","flu orescence","ĠCrit ique","ç͵åĬ¨ åĬ¿","Ġirrit able","Ġcombust ible","丧失 äºĨ","ç¼ĸè¯ij åύ","ĠP OR","Ġit ertools","Ġk irk","ap ital","对 被","éĤ£ åı¯","Ġset embre","太 ä½İ","äºļ åĨĽ","æķĻåѦ 模å¼ı","软 ç»Ħç»ĩ","æĮīçħ§ åĽ½å®¶","Ġri par","表æĺİ äºĨ","ĠÑĢÑĥ ками","ĠоÑģнов ной","ĠAqu atic","Ġtrop ics","ĠPie ces","ాల à±ģ","ĠاÙĦÙĨس اء","- chief","ļ àµįà´ļ","Ġb act","Ġin izi","ĠM ora","ĠCh iesa","åĮĸ ç®Ģ","Ġreg urg","å¦Ĥæŀľ åı¯ä»¥","çļĦä¸Ģ ç»Ħ","æĺ¥ åħī","Ġquant ité","Ġdé cor","æīĺ ç¦ı","Ġcarb oxylic","Ġing res","Ġsubt ree","éĢĨ å¢ĥ","çŀ§ çĿĢ","ĠاÙĦØ« اÙĨÙĬØ©","ĠÐĴа Ñģи","Ġdetr iment","/met abolismo","Ðŀп ÑĢеде","ĠMatth ias","B os","Ġm ensch","ou i","Ġal bo","人 çŃī","ĠK ung","å°± éĢĻæ¨£","èĢĮ éĢłæĪIJ","-m aker","è¿ŀ è¡£è£Ļ","ĠQu and","æĺ¥ æĹ¥","Ġroom mate","åĶ® åįĸ","-se a","Be en","ĠಠĹ","çĭĤ å¦Ħ","ĠPRO VID","clock wise","å¯ĨéĽĨ åŀĭ","ĠH ate","个 åŃĹ","Ġman or","éĤ£ä¹Ī å¤ļçļĦ","åĩĨå¤ĩ éĩij","纷 åijĪ","ÐIJ д","ĠMa ori","èѰ åĵ¡","opol ys",". Exploring","B j","Ġd ando","id one","ol st","Ġpresent i","AS ED","çļĦ大 èĦij","acc ia","æĬĵ æįķ","ô nia","åĸĩ åĺĽ","Ġreleg ated","\" N","Ġp ector","el ike","ĠM itar","大 æīĵ","Ġme zzo","Ġar ched","Ġmin ha","ä¸įæĺ¯ åĽłä¸º","ĠÑĥ ÑģÑĤ","èģĶ ç¤¾","è¶Ĭ ä½İ","çŁ³ çªŁ","Wh ole","åºĹ åĨħ","ĠAv atar","æŀ¯ èIJİ","ĠâĬ ķ","Quant um","Ġconscient ious","S s","j ach","Ġp one","ĠCh ou","cul ating","Ġdisc erning","åħ¸ éĽħ","Ġkon st","大å°ı 为","Ġsky rock","éģ© æĩī","æķ°åŃĹåĮĸ 转åŀĭ","è±ģ åħį","ĠStras bourg","E conomics","K m","x or","ou ple","ĠT ulsa","ra ised","Ġex uber","art an","olog ischen","sp ÄĽ","ä¸ŃåĽ½ 大éĻĨ","apt ion","åį´ ä¸įèĥ½","ĠاÙĦت Ùĩاب","ĠÑģо еди","') )ĊĊ","Ġmal attia","ëıĦ ìĿĺ","æĸ°éĹ» åıijå¸ĥä¼ļ","Ġprincip ali","ĠTy ph","riber y","Ġunm anned","触åıij åύ","ĠReprint ed","ĠSovere ign","Q G","qu oting","ĠB alkan","ä¸į æħİ","åĴĮ æī§è¡Į","å¼Ģ çıŃ","强 çĽĹ","Ġsignific ado","reg istration","éĢģ åĩº","çģµ çļĦ","åħ¬åħ± 交éĢļ","Ġä hn","atan abe","计ç®Ĺæľº ç½ij绾","ç¾ŀ æ¶©","à¸ŀัà¸Ĵà¸Ļ า","percent age","ĠHiro shima","< table","B AR","Ġb ÅĤ","os cel","ä¸į ä¸į","åĩº éĶħ","çľĭ ä¸Ĭ","天 主æķĻ","ج ÙĪ","oms ky","æ²Ĵ éĮ¯","OL A","æ»ŀ çķĻ","Ġaccompan iment","ĠвÑĭÑĢа Ñīи","[ size","m og","Ġc zym","ĠI p","ä¸Ģ 串","ĠE ber","se h","å¼Ģ åIJİ","åºĶ åĮħæĭ¬","åħī åľĪ","è¾¹ åĿ¡","æĿĥ åĬ¿","Ġsw apping","è´Ł æŀģ","?âĢĿ âĢľ","æ¯Ľ æ¯Ľ","ĠPhys iological","hol tz","comp ound","Ġbond age","æĿ¯ ä¸Ń","Ġкон ÑĨа","ĠGrand pa","Ġíĺ Ī","Ġjurisd ictional","Ġà¤ħ स","ĠPredict ive","Ġresh ape","Ġextrac urricular","趨 åĭ¢","y zed","ä¸į ä¸ĭæĿ¥","ĠL ime","che lle","Ġsol enoid","ane ity","çĥŃ è®®","欢 ç¬ij","Ġpast i","ãĤĤ ãģ¡","Ġdim ana","æĮij èµ·","æĹħ游 èĢħ","åħĪè¿Ľ éĽĨä½ĵ","æ¦ľ é¦ĸ","ĠнаÑĩа ло","School s","Interest ing","awat ts","âĮ ª","éµ ¬","ĠKauf man","à¸Ĭุม à¸Ĭà¸Ļ","I an","L ed","ĩ Į","Ġal gal","为 该","对 éĤ£äºĽ","çī¹ éĩĮ","ç´ł æľī","å·²ç»ı å®Įåħ¨","à¸ļ าย","Ġcolor ation","ĠEl f","ব à§ĩষ","Pr inciples","UN CTION","Ġmac ros",".in ternal","æĪ° çķ¥","åĬłå¯Ĩ è´§å¸ģ","Ġwp rowad","Ġnghi á»ĩm","è° Ł","æ°Ķ åľº","æ´Ĺ å®Į","ĠGood win","Ġrib bons","èĥľåĪ© åı¬å¼Ģ","ÑĢован ной","ĠавгÑĥ ÑģÑĤа","Ġberl aku","Ġleth arg",") N","çļĦ çĹĽèĭ¦","ce le","ĠM undo","ä¸į èĩªè§ī","âĢľ (","ĠO ro","Ġun checked","ĠV ad","度 æķ°","çģ ij","Ġimp ot","less on","ä¸ŃçļĦ éĩįè¦ģ","aut ics","Ġorig em","ĠAm p","Ġج ÙĦÙĪ","Ġব à§ģ","Ġresid ences","ä½ĵèĤ² é¦Ĩ","Ġinse parable","oblast s","Ġcorres ponde","_ HE","çļĦ åijĺå·¥","为 éĺ²æŃ¢","ĠEx actly","åħī çģ¯","ĠPl ata","ÄĻ ci","Ġа з","åİ¿ 令","Ġkind er","å°į æŃ¤","Ġcapac it","Ġsleep s","æĺł åħ¥","ál va","Ġfunc iona","ãģ» ãģĨ","entre prise","Ġvulgar is","' ann","_ history","Ġa ko","Ġrem ake","Ġدر د","rec ords","sal ary","E UR","_ CELL","ĠR iy","Ġch itosan","åĮĸ åѸ","åĬł å°Ķ","éķ¿ åģĩ","ŀ× ľ×","ú l","Ġmit ral","ĠпÑĢ Ð¾Ðº","RA FT","è´¦ ç°¿","Ġà° ²","Ġ기 ë¡Ŀ","Ġצ ר×Ļ×ļ","ĠElim ination","çŀ© 缮çļĦ","R ounding","í ĵ¨","al ty","Ġin continence","os ols","Ġcur va","ä»ĸ们 两个","IN ARY","è¿Ļ个 äºĭæĥħ","红 æĺŁ","å¿Ĺ æĪIJ","Ġang ka","ĠMy ths","uz zi","åĶIJ å®ĭ","Inter action","ç´« èī²çļĦ","ç»Łè®¡ åĪĨæŀIJ","éģĵè·¯ 交éĢļ","пол не","Ġpenet rated","Ġmock ed","Ġfortun ately","à¸ļุ à¸Ħà¸Ħล","Ġì²ĺ 리","ĠScr atch","W d","Ġro ared","å¼Ģ åΰ","Ġsub routine","ix en","æĸ¯ é¡¿","åħħåĪĨ è°ĥåĬ¨","å®ŀéªĮ å°ıåѦ","æĹħ游 èµĦæºIJ","Åij d","ìłķ ë³´","h j","Ġt RNA","an é","ad just","大 åłĤ","äºİ ä¸ĸ","她 æľī","çݯ è·¯","éº Ŀ","ĠGu ests","äºĴ æĦŁ","Ġsitu aciones","è´Ń éĶĢ","ä¸Ģ次 çļĦ","\\, +\\,","Ġrenew ables","/ per","Y O","l aces","Ġf oci","æĹ¶ äºĭ","ä»ĸ æĢ»æĺ¯","èĢĮ éĤ£","æĥħ çIJĨ","ĠSp arta","åĨĽ åľ¨","An a","What s","ĠÙĨ ÙĪÙģ","var o","ĠÙħع ظÙħ","Man ual","ä¸Ń央 æĶ¿åºľ","orb ent","Ġá ¸","ĠÄij ến","å·¡ æĬļ","ĠLag range","b ac","Ġb ary","çļĦ ä¸ī个","对 ä¸įåIJĮ","å¸Ĥ çĽĪ","Ġsc off","ins en","é£İ éĢŁ","Re gex","åѦçĶŁ è¿ĺ","оÑĤ вÑĢа","åĿĩ å·²","à¯įà® £","åįł åΰ","ĠاÙĦس ب","æıĴ åĽ¾","ĠUnivers itat","à¦¿à§Ł া","Ġsucc es","Ġsag te","ĠLanc ashire","Ġmultim ed","Wal let","Ġchir urg","æļĤè¡Į åĬŀæ³ķ","é¢Ħ示 çĿĢ","f inger","t ell","è¿Ļ ä¸īç§į","æīĭ å¿ĥ","ä»İ ä»ĸçļĦ","Ġset Id","å·® äºĨ","Ġestim ators","Ġprefer entially","伸 缴","ç²Ĺ ç»Ĩ","å°ıç¼ĸ 为大家","Ġfren zy","Ġquadril ateral","ĠOverse as","ic ie","ĠA go","us zt","æĶ¿ å±Ģ","Ch rom","ž dy","å¯Ĵ æ°Ķ","æĬ½ è°ĥ","ä¸Ĭä¸ĭ 游","Ġske letons","ĠFab er","Ġrelie ving","ĠDok ument","Ġsuperim posed","Ġár bol","@ p","g cd","ult iple","Ġup beat","ล à¹īาà¸Ļ","åİĨ ä»»","Ø® Ø´","åŁºæľ¬ åİŁçIJĨ","ç§» èĩ³","Se an","ĠAut obi","ĠТ ÑĥÑĢ","çĶĺ èĶĹ","æĢ§è´¨ åĴĮ","Ġmedi ab","ãĤ¦ ãĥ³","opoly mer","an chor","Ġo asis","ect l","绣 å¸ħ","Ġس ÛĴ","大家 éĥ½æĺ¯","ĠØ´ ع","Ġaccept ability","Ġinn umerable","ä»İèĢĮ æıIJé«ĺ","PC I","cor rh","Ġ기 ì¤Ģ","èŀįåħ¥ åΰ","Ġstagn ation","Ġдев ÑıÑĤÑĮ","нÑĨиклопеди Ñı","çļĦ å®ŀéªĮ","Ġe ps","ĠT owers","Ġha irst","Ġme x","Ġtr is","å°Ĩ éĤ£","èIJ ¼","ĠZ d","ĠInd epend",".\" )ĊĊ","éľĢè¦ģ 使ç͍","çĥŃ ç͵","Ġtemper ed","En umerator","ек ÑģÑĤ","jo ined","çļĦéĩįè¦ģ æĦıä¹ī","æijĺ èĩª","tra vel","ĠCir cles","ä¸Ńå¹´ 人","consider ed","æī¿ç§Ł 人","çļĦ æĪIJåĪĨ","ĠF os","ĠG atsby","Ġâ ģ","но Ñĺ","åıĹ èĭ¦","ç®Ĺ ä¸įä¸Ĭ","acter ia","ger a","áĢ Ŀ","EC S","Ġprefer ring","Ġesc ap","Ġrect um","ĠAp oll","_st at","ãĥŁ ãĥ³","Relations hip","Ġeinzel nen","G ard","ĠT uhan","Ġv iet","åĽ¾ è°±","ĠठŁ","éļı æĹ¶éĹ´","ĠCol lected","çģ« çĤ®","-p henyl","åģľ æľº","ceed ing","çĶ³è¯· çļĦ","è¯ij æľ¬","ĠCare ers","ĠRh ythm","ĠÙ쨱 ÙĨسا","ĠиÑģÑĤо Ñĩник","Ġhypoc risy","---|--- Ċ","/ include","T weet","Ġ( ).","Ġad iab","Ġme c","own ership","aj ari","ĠCl iffs","ι β","åĩĨå¤ĩ 好çļĦ","临åºĬ çĹĩçĬ¶","Ġtire lessly","/ about","= utf","ĠH ortic","Ġhe arth","Ġdis claimer","form en","æľĢ ç®Ģåįķ","æ°´ çĶŁ","åįĩ æľ¬","Ġparent hesis","礼 åłĤ","èģĮä¸ļ çĹħ","夹 åħ·","Ġsyst ém","Tor onto","ĠпÑĢоп оÑĢ","P olitics","m ix","æīĢ çļĦ","Ġ' .'","Ġcre v","Ġб аÑĢа","Ñīи ми","vo je","ĠJu vent","èĹ¥ çī©","Ġsid elines","ĠFro zen","à¹ģวà¸Ķ ลà¹īà¸Ńม","/ store","L eb","ac etic","åľ¨ åºĬä¸Ĭ","Ġ& :","Ġо па","ãĢĭ ï¼Ľ","Ġstr atum","åŁºæľ¬ éĥ½æĺ¯","à¸ģาร ศึà¸ģษา","åĨ³å®ļ æĢ§","ét abl",".B uilder","å°±æľī åı¯èĥ½","ĠÐĹа ÑĤем","ĠUl ster","opa edic",".Serial izable","ĠCONST RAINT","ĠMongol ian","( Arrays","/ products","= head","B urn","ĠT rick","ad jective","ĠM erg","ä¸į å¤ĸ","ç¾ ļ","表 åĴĮ","æĺİ çĽ®","mer c","ä¸ŃåĽ½ å¸Ĥåľº","for cer","åĪĿ æģĭ","Ġব িà¦ķ","Ġrelig ios","åĨ° åĨ»","çļĦéĹ®é¢ĺ æĺ¯","ĠCur iosity","æĪIJéķ¿ ä¸º","à±įà° ķ","æ¸Ķ æ°ij","ĠVeget ables","Ġlut te","éĥ¨ é¦ĸ","ÑĤе ÑĪе","æ·± å±±","åıĤ æĶ¿","-A m","Ġprecip it","ĠRew ard","ĠBoh r","ĠGradu ally","' homme","u q","he ated","ĠB ust","æĿ¥ æİ§åζ","åħ¨ èģĮ","éķ¿ éķ¿","ten ces","ç¦ ª","Ġgl ide","è¾¹ éĺ²","ä¾ĭ è¡Į","Ġdeb emos","ĠMark er","-M ail","çĶŁåij½ ä¸Ń","磨 åIJĪ","ĠNeb en","Ap ache","Ġhoe veel","Ġkv adrat","ĠStir ling","Ġdeport ation","Ġerkl ärt","Ġreconcil ed","à¹Ģศรษà¸IJ à¸ģิà¸Ī","W onder","ĠH abs","åĨħ æĪĺ","tern o","æĶ¯ æĬ¤","be au","ä½Ĩæĺ¯ ä½ł","åı¤ éģĵ","Ġsoft ening","éĿĻ çļĦ","ÙĬر ات","æķĪæŀľ 好","Ġcomment ator","ÙĤد اÙħ","Ġ________ ____","çIJĨè´¢ 产åĵģ","Ġstew ard","æķŀ å¼Ģ","é§ķ é§Ľ","Ġantidepress ant","Ġpedig ree","ĠGó mez","ĠTalm ud","S prite","u vi","} g","ve ction","å̼ ä¸İ","ĠAP PRO","raz ier","æ³° åĭĴ","ç½ijç«Ļ ä¸Ĭ","ĠتØŃ ÙĦÛĮÙĦ","ĠDownload ed","Ġà¦ĸ à§ģব","åĪĩéϤ æľ¯","Ġevoc ative","ĠRép ublique","_ le","l Ãł","Ġa ix","ĠH á","æľī å½¢","Ġz yg","ç»ĵ çķĮ","ç¦ º","ĠPr é","åĽŃ çļĦ","ref lect","é¡¶ å³°","ç¹ģ è¡į","Ġaccum ulator","Õ«Õ ´","汪 汪","Ġcosm opolitan","ĠColumn s","Ġencaps ulates","Ġhaul ed","æĥ¦ è®°","Õ ¼","ä½ł åķĬ","æ³ķ æĭī","Ġdet ract","chn ik","è¾¹ èµ°","äºĨä¸Ģ 项","游 人","ĠApp ellate","Ġter apia","Ġge h","身ä½ĵ ç´łè´¨","è¡ĮåĬ¨ èµ·æĿ¥","Ġин ÑĦ","uv ial","Sp read","ät ter","prot ective","çĥĺ çĦĻ","æİº æĿĤ","Ġpreoccup ied","Ġretrospect ively","out ines","ä»ĸ ä¸įçŁ¥éģĵ","ge ar","ep ers","æľº çͲ","ĠPro st","æģ¯ èĤī","To ast","ä¼ļè®® 强è°ĥ","Ġsie ht","å¢ĵ èij¬","inde er","ĠاÙĦÙħÙĪ Ø§Ø¯","ĠMuss olini","غÙĨاط ÙĬس","Ġb umper","ark et","-m atch","失 羣","Ġda un","çģµ èĬĿ","UT R","稳 ä½ı","ব রà§įত","}( {\\","à¹Ĥ à¸Ńà¸ģาส","港 èĤ¡","ĠFig ura","æĽ¸ ç±į","Ġdispos ing","ĠAp J","大éĥ¨åĪĨ 人","Ġlig t","èµĦæł¼ è¯ģ书","خص ÙĪØµ","åİĮ å̦","ĠÔ± ÖĢÕ","Ġapare ce","Ġultr ason",": w","Ġm over","ĠC n","ĠM ott","ĠD ementia","ĠTh reshold","ĠY og","æĸĩ èģĶ","åĪ© åύ","群 人","温 çĥŃ","ho ea","Ġ×ľ× ĺ","ä»»ä½ķ ä¸Ģç§į","å·¨ åĵį","è¡ĮæĶ¿ 审æī¹","Ġsyn ov","æ·±åħ¥ åΰ","ÑĢÑı з","åĭĴ ç´¢","Ġ×ĵ ר×ļ","ãģ§ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ","Ġìļ´ ìĺģ","çĽİ çĦ¶","Ġaanv ullende","+ K","st w","Ġim un","ov ali","ä¿¡ å¥ī","äºĶ 天","è½» ç¬ij","å·ŀ åĪºåı²","-h ist","UR ING","اس ات","!! !Ċ","â̳ ,","çļĦ好 å¥ĩ","Ġcri ar","Compar ator","Ġa uteurs","re ar","Ġs ól","ce k","ä¸į åĿĩ","ä½ľ å®¶çļĦ","éĩį ç£ħ","æĦı æ°Ķ","æķĻèĤ² å®¶","ست Ú¯ÛĮ","åĸĦ å¾ħ","æľĿ 代","ĠØŃ سÙĨ","Ġflow chart","ÙĬر ا","Ġvirtual ization","ĠCON F","ĠвеÑĢ ÑĪи","ĠET Fs","Ġ×¤× ¡","ĠCasc ade","Ġerfol gre","ERIC AN","( raw","d ynamics","ĠS ai","ter ious","åĪĨ ç«ĭ","é«ĺ è·Łéŀĭ","åĨħ éļľ","åijĺ å¤ĸ","aw ak","ĠRes idual","pect ing","åĨ² çł´","ĠWord sworth","ãĥĸ ãĥ«","Z y","ä¸Ĭ 书","åĪĨ 寸","ign et","åħ¬ 竳","让 对æĸ¹","åĨĽ æł¡","æķĻèĤ² æķ´é¡¿","红 æĸij","æīĢæľī çļĦ人","ĠÙĪØ§ÙĦ سÙĬ","ï½ İ","inder ella","Ġпод клÑİ","çŁ¿ åºĬ","çµIJ è«ĸ","软件 å¼Ģåıij","à¸Īะ à¹Ģà¸Ľà¹ĩà¸Ļ","-J un","æ»ĭ éĺ´","åįķçĭ¬ çļĦ","ĠAccred itation",". or","l ite","ĠP iot","ĠU B","åıij èĦ¾æ°Ķ","Ġrec ounted","ç»Ļ 人çļĦ","Ġне га","ĠSu isse","ĠMe adow","éĢģ èĩ³","åĩı 产","éĤ£ä¹Ī æĪij们","Ġbreak up","ä¸ĵä¸ļ å§Ķåijĺä¼ļ","ĠBook er","Ġ×©× Ľ","Bl ocks","èĤ¯å®ļ ä¸įä¼ļ","ĠHand el","Ge ography","구 매","-dis able","ĠPrevent ive","Ġสำ หรัà¸ļ","ĠоÑĦи ÑĨиалÑĮ","á ł","sc aler","è¿Ľè¡Į æĵįä½ľ","stand s","马 èĻİ","ĠPost ers","MS G","ĠìŀĪ ìĹĪ","ĠâĹ ¦","Ġcha ired","ĠÑģамо е","å¤Ħå¤Ħ éķ¿","ĠAgg regate","ĠKuh n","红åįģåŃĹ ä¼ļ","ct er","к ÑĥÑİ","Ġam o","-f ilm","rad i","Ġbeg itu","Ġbi ologic","ĠCal v","Un able","鼨 åIJİ","ĠGener ative","æľŁå¾ħ çļĦ","Ġng uyên","Car ol","äºĨåĩł ä¸ĭ","ĠJu ice","ĠKin etic","F ilename","P ending","ä¸Ģ ç¼ķ","ĠL ES","以 å¤ĩ","Ġar throp","è´¨ æ£Ģ","uss ing","士 å¤ļ","ек оÑĤоÑĢÑĭе","çİ°åľº çļĦ","ÙĬÙħ اÙĨ","áĥĺáĥ ĵ","Ġdzie cko","à«Ģ àª","ĠBert rand","Bit map","ĠобÑĬек ÑĤов","มà¸Ļ ุษยà¹Į","Ġeconóm ica","Ġпам ÑıÑĤ","ĠBerks hire","æijĴ å¼ĥ","S entence","Ġs yd","Ġde arly","ä»ĸ åı¯","ov ala","è´Ń è¿Ľ","ÉĻ d","ков ÑĭÑħ","Ġwave forms","æIJľ çĭIJ","Ġsuffer ings","×ķש ×Ļ×Ŀ","ĠRA ID","Ġhust le",". book","@ Service","Z G","m otor","Ġle kar","缸 è¾ħ","导 轨","ĠSh ack","å¿« äºĨ","ç»Ŀ ä¸įèĥ½","à¸ģาร ศึà¸ģษ","γ ο","but tons","Ðŀ ÐĿ","Äģ b","èĢĥèĻij äºĨ","দ িন","Ġcup board","-x l","Ġlev ied","Ġconoc imientos","Ġconna issance","Ġantit rust","ĠÐľÐµ ждÑĥ","ãĤĴæĮģ ãģ¤","Ġeman ating","ĠGentle man","ĠDart mouth","Ġpyro lysis","U CTION","z am","es ophageal","ĠC GRect","ag ents","æľ¬ è½®","Ġpar sec","ĠEm ilia","Ġod w","SE Q","ÃŁ t","æģ© çα","年代 以æĿ¥","çļĦ主 导","Ġë© ´","å¿«æį· éĶ®","Ġthunder storms","离åIJĪ åύ","( class","ภĨ","ĠI thaca","åĴĮ é»Ħ","ĠK osten","Ġcl and","ĠâĢľ Ċ","åIJĮ æ²»","æĹ¥ ãģ«","ÑĢи Ñĺа","å¸Ī çĶŁçļĦ","ĠX er","° /","è¦ģæ±Ĥ åѦçĶŁ","Ġaspect o","åįĸ æİī","æĮģç»Ń çļĦ","Date Format","Am endment","æij¸ æİĴ","å½ĵåľ° 人","颤 åĬ¨","以æŃ¤ 为","ĠпÑĢом е","-cut ting","} !","ĉ ĠĠĠĠĠĠĠĠĠĠĠ","ĠP uzzles","ers i","ä¸į éĢĤåºĶ","ĠH ertz","Ġk rist","å¹´ ä»ħ","Ġso ir","ä¸ŃåĽ½ ä¼ģä¸ļ","æŃ» æ´»","Ġج ÙĪÙĨ","æĹ¢ åı¯","æĢĿæĥ³ 认è¯Ĩ","Ġlic z","íĮ ¨","Ġtransm embrane","Ġsketch ing","ĠBAS IC","Ġcarp ets","ĠMist akes","enceg ah","M erge","N ik","n out","Ġf b","ut ively","ĠC ui","å¾Ĺ è¦ģ","Ġtwo fold","æīĵ æ³ķ","Ġreal isation","ز اÙĦ","éħį é¢Ŀ","第ä¸Ģ 书记","Ġsem plic","CH ECK","-e lectron","è·Ŀ ä»Ĭ","æľīä¸Ģ 座","à§įয াà¦ķ","Ġsubs ystems","çīµ æīĭ","richt ung","Ġmim ics","Ġدست گاÙĩ","ĠIllust rator","&&&& &&&&","ĠíķĺëĤĺ ëĭĺ","ĠëͰëĿ¼ ìĦľ","ĠHuss ain","Ġdisappro val","Ġhemisp heres","( al","( student","/ utils","ĸ ×ķר","ou che","Ġon boarding","ome gran","Ñĩ Ñij","æŃ¤ çĶŁ","没æľī éĹ®é¢ĺ","ron omic","æŃ¥ 驣","便 æľī","çłĶç©¶ æĸ¹åIJij","à¸Ķ าว","IS I","(s orted","Ġblock er","Ġcompl iments","ÙĪÙħ ÙĨ","å¿ĺ åį´","ีย à¹Į","ãĥ¼ãĥ Ĭ","Ġinteract ed",".Data Annotations","ĠMu ir","屬 æĢ§","Ġtraged ies","裸 éľ²","Ġcyst eine","Catal og","fact ors","ysk land","Ġmyel oma",") }ĊĊ","d zy","Ġa ching","re ibt","ĠS ING","Ġj aki","ä¹Ł æĽ´åĬł","ne uro","å°ı 溪","ä¸İ å¤ĸ","åĨħ éĻĨ","被 人们","ج ار","-m asing","LL LL","æŃ¦ èѦ","ĠPost greSQL","App s","ç͍æĪ· ä½ĵéªĮ","好åĥı åľ¨","isp iele","åIJIJ è¡Ģ","Ġincl ine","ĠPur itan","rä gt","Ġгла з","h ooks","he ta","ĠS ohn","Ġst uk","èĥ½ ä¸į","çĿĢ åĺ´","天 åı°","Ġcomm encing","éĢŁåº¦ 为","Ġmess ed","();ĊĊ Ċ","control s","Ñīие ÑģÑı","Ġìĺģíĸ¥ ìĿĦ","ĠConver gence","G es","i ples","Ġch aper","Ġinter no","ä¸Ģ个 æľī","èĦ ¹","Ġdep recated","ĠÑĥ меÑĢ","åİ¿ 人","Ġদ à¦¿à§Łà§ĩ","Ġпом ожеÑĤ","Ġப à¯Ĭ","ëĭĪ ê¹Į","æ¢Ĺ å¡ŀ","Ġinterven ed","Ġamen able","ĠپاÛĮ اÙĨ","ĠпоÑĢÑıд ка","å°ijåħĪ éĺŁ","w ak","ĠA vec","ĠF arb","ä¸Ĭ å½ĵ","ठĩ","天 æĺİ","Ġass ures","ink e","导 éĢļ","ĠCl arks","å¦Ĥæŀľ 羣çļĦ","(' %","èĭ± çī¹å°Ķ","Ġdat ang","Ġsem plice","ĠGener ating","深度 èŀįåIJĪ","强åζ æī§è¡Į","Dem ografia","Ġvé ritable","æ² ¢","èĥ½ åIJĥ","ov ent","å®¶ å¢ĥ","ä½Ĩ ä¸İ","ä¿¡ ç®±","æłĩ éħį","Ġhand written","Ġ% %","å¼Ģå§ĭ åľ¨","ðŁĶ ¥","Ġë²Ī 째","Ġperpetu ate","/ ext","I OD","ĉ map","Ġs owing","ĠT ricks","Ġpl ank","ree ce","å¾Ī ç¾İ","ider ed","åıį å°į","Ġwater falls","åľŁ å±Ĥ","è§£éĩĬ 说","ÙĬÙħ ÙĬدÙĬا","è´Ńä¹° äºĨ","è¯ķéªĮ åĮº","æĶ¹åĸĦ äºĨ","Ġhal ten","ĠTarget ed","ĠTrad itions","æIJĸ äºĨ","Ġsulfur ic","Ġcram ps","Ġajud a","S rc","n umer","is Empty","çļĦ éĺ³åħī","im ia","æľĢ åħ³éĶ®","å±± 寨","IN PUT","æĤ£ ä¸Ĭ","æĿ¾ æķ£","اع ر","Ġprec inct","२ ०","ĠнаÑĥ к","Walk er","forder ungen","Z oom","Ġh itch","ĠG wen","大 ç´Ħ","ĠJ apon","Ġpre text","è· Ħ","ä½ĵ è£ģ","/s ervice","Ġpost pone","è´¹ çŃī","è¿ŀ äºij","å¾Ģ å¤į","оÑĤ но","端 åºĦ","ত à§ģন","script size","/d ocument","Ġhydro gel","ĠاÙĦØ´ عر","Ġmature d","Ġsept um","ĠкÑĢи ÑĤеÑĢи","Ġsolic itor","a ide","ĠA ram","Ġconsider ado","æĬĢæľ¯ è¿ĽæŃ¥","ĠÑģов Ñģем","Ġko ÅĽci","Av ailability","æįŁå®³ çļĦ","ĠÕ° Õ¡Õµ","æ°´çħİ æľį","ĠDISC USSION","M ig","d re","{ figure","Ġl ighthouse","Ġde utsche","çľ Ī","und ance","两 份","æİ¥ åΰäºĨ","æĢ» éĺŁ","Ġopp oses","æĺŁ éĻħ","æľĿ ä»ĸ","çĶļ ä¹Ī","å¿Ļ äºİ","åī©ä½Ļ çļĦ","ĠKre is","ôn io","Ġfamili as","b ones","Ġas ign","åİ» æīĵ","oth èque","éĺ² é£İ","å¾ĭ 師","é¢Ħ ä¼°","Ġза м","ĠÙĨ ÙħÙĪ","cz na","Ġhost ilities","è̳ çļĦ","æ¢ģ åIJ¯è¶ħ","é©» æīİ","Ġgj enn","å¤ı令 èIJ¥","Ġn ya","ĠG uns","èĩ´ è¿ľ","éł ij","elt emperaturen","éĵ¶ èī²","Ġpet abits","æĬ¢ åħĪ","ìĺ ¬","ĠGer hard","Ġkann st","å®ŀä¹ł çĶŁ","ĠDEF IN","çĺŁ çĸ«","ĠDaven port","; C","N omin","R aj","_ change","os em","Ġun loading","Ġب ÙĪÙĦ","两 æŀģ","管çIJĨ å¤Ħ","ĠMar j","UR ITY","antic ipated","çļĦä½ľç͍ ä¸ĭ","FA Q","ĠHan over","OUR CES","ĠоÑĢгани заÑĨиÑı","ĠиÑģполÑĮзова нием","Ġexert ion","Ġbout ique","Administ razioa","ä¸İä¼Ĺ ä¸įåIJĮ","Ġhorr ified","ĠdÄĽt ÃŃ","Ġì£ ½","P urch","os et","ãĢģ \"","ĠF lickr","Ġcons ac","ues a","åŁº ç«Ļ","Ġpower fully","Ġpop rzez","AN U","ĠPr äs","ny t","å½¢å¼ı åĴĮ","Ġcorrespond ed","/p kg","çĨĬ çĨĬ","ĠØ« ÙħاÙĨ","s alt","Ġd ime","åIJ Ĩ","ä¸Ģ åİ»","个 ä¸įåģľ","æĹ¥ 产","äºĮ çŃī","èį ¤","å½¢ èĢĮ","ä»ĸ们 æĬĬ","Ġsuper flu","ĠEd dy","§× ¡","俱 åħ¨","Ùħس اعدة","ASC II","Ġdissip ated","çļĦ æķĪçİĩ","åľ¨ æ³ķå¾ĭ","æĪij å¿ħé¡»","对 æĪĺ","еÑĤ ÑĢа","没æľī ä¸ĢçĤ¹","Ġvis itation","æĿİ çİī","æĿŁ æīĭ","ĠCommun ism","æĭĨ åĪĨ","Ġdys lexia","ç³ķ çĤ¹","Ġдан нÑĭм","åĭĿ åĪ©","Ġcaf eteria","æĺ¯ ä¸ī","qu ets","ĠB AT","ari ed","ç¾ ¿","å·¥ä½ľ å®ŀéĻħ","æ°Ķ èĻļ","ĠQ T","æĶ¹ åĨĻ","à¹Ĥ à¸Ħ","Ġcement ed","Ġrenov ated","Ġparadox ical","ĠM inds","大 åĸĿ","ĠاÙĦ ار","åħ¨ ãģ¦","-f ast","Ġrun away","à¸Ħ à¸ĵะ","col are","ĠDef initely","åŃĶ çļĦ","из вод","TER N","主è¦ģæĺ¯ åĽłä¸º","å¼Ĥ常 çļĦ","ĠÑıзÑĭ ке","æ°ijæĶ¿ å±Ģ","ĠÄĩ wic","Ġéx ito","大 ä¸įäºĨ","对 æīĢ","æĦı æĮĩ","Ġdef y","ĠAn geb","éŨ ä¸ĭ","Ġmar ina","ĠLaw yer","èĢĮæĺ¯ è¦ģ","ĠExp anding","ĠاÙĨ رÚĺÛĮ","Co ach","};ĊĊ Ċ","atie ve","å°½éĩı ä¸įè¦ģ","Ġgang lia","ĠDomin ion","ĠSPEC IAL","G UI","] n","ĠN erve","è¿Ļ å¹¶ä¸įæĺ¯","åĽŀ æĥ³èµ·","г ом","åĪ« æīŃ","Ġmet as","Ġfr ivol","Ġоб ÑģлÑĥжи","Ġnov os","à§ĭ স","èµ°äºĨ è¿ĽæĿ¥","Ġà¦ıà¦ķ à¦ľà¦¨","olin ergic","ĠP IL","人 ä¸ĸ","ç͍ å®Į","è¿ĩ å¿«","çϽ 头","rid or","è´¹ åĴĮ","ä¸Ģ次 åıĪä¸Ģ次","çĿ¡ 覺","仪 çļĦ","Ġgeb e","ĠвозÑĢа ÑģÑĤ","Ġfurnish ings","ĠD ependency","ĠE ing","ak nya","ãĤ ħ","Ġra pt","åĨį æĹł","Ġfind ViewById","Ġlevel ed","éĤ£ä¹Ī è¿Ļ个","Ġaccept or","_f n","æĵįä½ľ æĸ¹æ³ķ","Ġhost el","Äĥ r","ker as","Ġblind ly","olt Ãł","ĠÚĨÛĮ ست","亲æľĭ 好åıĭ","举个 ä¾ĭåŃIJ","; #","im ize","ĠK ast","ä½ľ åĪĻ","Ġra cks","Ġк ÑĢоме","社ä¼ļ åĴĮ","ĠSam oa","raz ole","gu ides","ĠоÑģ нованиÑı","ĠÑĤо Ñĩно","èĬĤ缮 ä¸Ń","del ay","æ°¢ æ°Ķ","鼷éĶĭ ç²¾ç¥ŀ","黯 çĦ¶","-po inter","on yl","en zyme","at itude","çļĦ ä¼Łå¤§","Ġto lu","ne ut","Ġtra z","社ä¼ļ çĶŁæ´»","çģ« æŁ´","IT IVE","ĠAss emb","æĪijçļĦ æľĭåıĭ","Ñĩа еÑĤ","Ġsac rament","æĪijæĺ¯ 个","æĪĴ å¤ĩ","íĥ Ħ","MO s","Ġwarrant ies","Ġapr on","ĠвÑĤоÑĢ Ð¾Ð³Ð¾","Ġmang rove","Suggest ed","- create","p ending","Ġto asted","Ġv är","Ġj är","й да","eb p","Ġठ¡","éĢĻ åı¥è©±","åĨľ 夫","Ġл ока","å®Ī ä¿¡","Ġinstall er","(d ist","æĸ°éĹ» æĬ¥éģĵ","çβ äºĨ","Ġdegrad ing","æĸ°åħ´ 产ä¸ļ","Ġscram bling","ĠÕ« ÖĢ","ĠLithuan ian","å¸ĤåľºçĽij管 å±Ģ","= -\\","Ġm ÅĤ","ĠD ab","hen ko","Ġо ÑħÑĢа","æ· ħ","ç¥ŀ åºĻ","ĠSh ira","Ġsom ente","æĶ¯ 书","áĢ °","าร ะ","ĠС ам","伤 åijĺ","å·´ 赫","App lying","Ġinterpret ers","aks ud","CS I","Foot ball","ÙĬÙĥ ا","Ġ׼׾ ׾","hd ys","ĠPatri arch","ĠиÑİ Ð»Ñı","mk dir","ä¸Ģ æµģçļĦ","ĠL ETTER","ä¸İ ç»ıæµİ","天 çĮ«","å®ĥ æĺ¯ä¸Ģ个","ĠTr istan","rodu ce","第äºĮ 大","æ©Ł éĸ¢","Ġnas cent","à¸Ĥà¸Ńà¸ĩ à¸Ħุà¸ĵ","è¨Ń ç«ĭ","çĻ»è®° çļĦ","è¿Ī åIJij","ãĥ¢ ãĥĩ","Ġমà§ģ à¦ĸ","ĠShar pe","ĠBun ny","Ġgriev ance","Agric ultural","c old","in ente","ï¼ ´","ĠS ÅĤ","ĠT ant","ĠC attle","Ġch an","Ġà Ĩ","å¹³ éĿ¢çļĦ","ĠAr chie","æī¾ æĿ¥","éĻį æ°´éĩı","Ġste aming","Ġpredict ability","Ġmount s","Ġnie j","ĠاÙĦعÙĦÙħ اء","ĠHav ana","Ġfath om","Ġprofiss ional","Ġtekan an","C ot","q f","ĠL MS","ĠO kin","以 ä¸Ģ个","ĠCh anc","ö der","ØŃ ص","å¼ķ 诱","æĿİ æ°ı","Å¡ a","lig ere","Key word","Ġcontroll able","inst ant","ï¬ģ c","èĥģ è¿«","Ġproc rast","ĠCab in","ãģĹãĤĩãģĨ ãģĭ","-ref lection","ĠHitch cock","/ item","B eth","l ots","Ġs idel","Ġr f","Ġnot oriously","è¿Ļ åĴĮ","个 çľģ","åѦ åĮº","du ra","交 åĩº","è dia","Ġsw am","اÙħ Ø©","Ġ×ľ× §×ij","ç«¥ å¿ĥ","¨× Ľ×ĸ","ĠEss a","ĠìŀĪ ê³ł","ĠÙĪÛĮ Úĺ","ĠGrim m","Ġخر ÛĮد","re th","Ġd uch","çļĦ å·¦","ĠT ile","ĠJ U","ä¹Ł åħ·æľī","cess ive","ä¸ī 楼","åıijå±ķ æĪIJ为","Ġfin ans","ем ое","æĥħåĨµ åıĬ","Ġuser Name","Ġinform ação","Ġseg uro","èķ Ļ","Ġble iben","ĠSand s","ĠاÙĦر ØŃ","ĠOS HA","Exper iment","ĠÙĪÙĬÙĥ ÙĬÙħÙĬدÙĬا","ĠÑĤемпеÑĢа ÑĤÑĥÑĢ","( In","Ġl ago","ĠP Y","pe ating","çIJĨ æŁ¥","ck el","és us","inter view","uit a","ĠSum mers","اÙģ Ùĩ","sl ot","æĹłè®º æĺ¯åľ¨","áĥĶáĥ Ĺ","éĢĽ éĢĽ","à¹Ģà¸ķ à¹ĩม","Ġযঠĸন","Drop down","åĤ¢ ä¼Ļ","ĠT AG","ĠB ain","ĠN egeri","ĠRe leases","ĠRe conciliation","Ġem uls","ise en","è§£ ä½ĵ","ĠÙĪ ÙĬع","vent y","ä¸ŃçļĦ æīĢæľī","ä¾Ŀ æĵļ","æ²¹ éŨ","èµµ äºij","ба в","第äºĶ æĿ¡","é¥®é£Ł ä¹łæĥ¯","æĥħå½¢ ä¹ĭä¸ĢçļĦ","éĹ¯ åħ¥","æĶ¿åĬ¡ æľįåĬ¡","Ġà¦Ĩল à§įলাহ","Ġbl asp","è§£ çļĦ",".S ql","ĠÑĤ ÑĥÑĤ","æį¢ æĿ¥","è¾ĵåħ¥ 端","Ġphenomen ological","ç·¨ éĽĨ","بÙĨ اء","ä¾µæĿĥ è¡Į为","Ġات جاÙĩ","---|---|--- Ċ","éĤĤ éĢħ","Ġmediab estanden","# 'Ċ","M emo","W alking","çļĦ åĨ³å¿ĥ","ä¸Ń ç͍","ĠIN CLUD","åı¶ åĩ¡","Ġscient ifiques","ãģij ãģ©","Cross ref","Ġfort night","總 çµIJ","åĴĸåķ¡ é¦Ĩ","ì° ©","ãģĵãĤĮ ãĤĴ","Ġplac ental","ĠO U","大 ä¸ī","ت ÙĪÙĨ","å°ı åºĹ","Ġcross origin","ĠMod ification","Ass ociated","Ġlie gen","Ġprz es","Ġà¤Ń à¥Ĥ","áĥĶáĥij áĥĺ","ĠProvis ional","æĨ¤ æĢĴ","éĺ»åĩ» æĪĺ","ĠS yd","ĠT BI","æī ±","åİ» åΰ","ç¥ŀ æĢģ","书 æŀ¶","ç´ł æķ°","Ġobserv ar","ĠInf rared","Ġci ud","ãĥķ ãĥĪ","Arch ae","ĠнаÑģ лед","ĠGuy ana","ĠRas ul","çļĦæĥħæ³ģ ä¸ĭ","ĠCzechoslov akia","n osis","Ġcom ida","ĠG OST","Ġв пе","æĸ° åĵģç§į","éĿĴ è¡£","Ġartic ular","Ġeconom ia","za ÅĤ",".find ViewById","ðŁĮ ¸","ĠRodrig ues","usp ended","Ġê°Ģì§Ģ ê³ł","ĠE ind","... +","Ġquest ões","Ġfil thy","èģĶ æİ¥","åı¯èĥ½ ä¼ļæľī","Sh ot","åĨ¬ çĵľ","æĺİç¡® æıIJåĩº","Att achment","G ov","çļĦ æĸĩ","as ikan","Ġse crete","天 èĿİ","ck ed","Ġam put","缸 åĮ¹éħį","代 åĬŀ","Ġس اعت","Ġsoft ball","_s ample","Ġmer asa","Ġcapt ains","ĠVer onica","ĠUp grade","Ġল à§ĭà¦ķ","ĠNex us","ĉ exit","ĠI ps","Ġv itt","è¯ ½","ĠR AS","åĴĮ åij¨","au ce","Ġprot racted","红 çĥ§","ĠاÙĦÙħ ÙĬاÙĩ","ĠHer aus","omy el","ائ دة","Ġно ÑıбÑĢÑı","à¹ĥà¸Ĭ à¹īà¸ĩ","Ġexempl ifies","çĩŁ æ¥Ń","ĠCass andra","Ġpeque ño","Fa ith","ר×Ļ׼ ×Ķ","s ms","Ġp ard","Ġem pathetic","åIJĦ ç»Ħ","-s izing","çα ä¸ĬäºĨ","Ġmen arik","Ġelse if","ĠâĪ ĩ","è¯Ĺ 人çļĦ","ĠØ® اÙĨÙĩ","åľ°åĮº åĴĮ","ä½³ 人","å¦Ĥæŀľä½ł æĺ¯","ĠSearch ing","åĴ¬ äºĨ","çĮľ åΰ","æŃĩ å°Ķ","sur vey","ĠMcK ay","æĹłå¤Ħ ä¸įåľ¨","鹦 é¹ī","> C","a ith","b ond","Ġy oke","ä¸į èĢģ","为 éģ¿åħį","ang ol","对 å³Ļ","åħ³ çħ§","管 äºĭ","åı¯èĥ½ è¦ģ","å¾Ģ éĩĮ","Ġinc ongru","æľīäºĽ äºĭæĥħ","ç· »","éĩĩåıĸ çļĦ","ĠAbs orption","Ġwo es","åĩ¹ åĩ¸","ĠобÑıза н","Ġrisult ati","Ġ à¸ĭ","an ha","çļĦ éĺ¿","ä¸į ä¿Ĺ","ĠF AS","æĪij çľĭè§ģ","èĢĮ è¦ģ","(\" ,\"","/s q","å®īåħ¨ ä¿Ŀéļľ","IV ATE","åģļäºĨ ä»Ģä¹Ī","çά åΰ","ĠлиÑĤеÑĢа ÑĤÑĥÑĢÑĭ","ĠÚ¯ÛĮر د","ĠдоÑĪ ÐºÐ¾Ð»ÑĮ",") \\({}_{","- validation",". te","l ád","ĠL one","æĪij们 éĥ½çŁ¥éģĵ","ĠSe ah","Ķ× ľ×ļ","ç»´ 稳","CC SS","Ġ×ķ× Ľ","ĠConf irm","su ite","Ġਠ¤","æī©å¤§ åΰ","' ins","B ungtod","{ }\\","Ġst ør","大 å¨ĺ","Ġout ages","sk ar","Ġbound less","绳 ç´¢","Ġcontradict s","Ġpersec uted","à¹Ģà¸ģีà¹Īยว à¸Ĥà¹īà¸Ńà¸ĩ","Ġzod at","ειοθε ÏĦήθηκε","R ic","Ġp thread","Ġto c","ĠC umberland","ĠD X","åľ¨ åѦçĶŁ","åİŁ æĺ¯","æĶ¶ åIJ¬","Ġmon olithic","åĭ »","str at","Ex ceptions","оÑĤ воÑĢ","Ġrecomm and","ĠHar rington","Ġblock age","ĠÑģа йÑĤе","Table Cell","Ġstock ed","ĠExpl an","Õ¡Õ¶ Õ£",".x label","Ġcamb iar","Ġsuperv ising","' imper","ä¸Ģ æ¦Ĥ","Ġle cz","ç͍ æĪ¿","åľ° åĽŀçŃĶ","Ġpr une","é«ĺ å®Ĺ","åħ¶ ç»ĵæŀľ","ateg orical","ma a","äºĨä¸Ģ å±Ĥ","Ġdisc ursive","Cl ay","Ġtool box","Ġhyp hen","Ġcooper ating","å§¿ åĭ¢","å¾Īæľī æĦıæĢĿ","ĠSpect ral","Ġrealiz ada","ĠFocus ing","mac ro","Ġcruc ifix","ĠاÙĦÙħسÙĦÙħ ÙĬÙĨ","F IELD","S ociety","Ġd rib","ent an","ĠT CR","ĠF landers","ĠW ider",".g raph","ÙĪÙĨ Ø©","ãĤĦ ãģ£ãģ¦","object ive","ç·ļ ä¸Ĭ","á»Ļ i","Ġharmon ies","ĠArist ot","ro dy","ä¿ ¾","ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĊ","ĠAn ime","ĠTo ys","ĠAm az","Ġvan af","çĶŁäº§ æķĪçİĩ","Ġneg oci","æľĿ é®®","çİ© ä¹IJ","ðĿij ĭ","Ġforce ful","ÑĪи ÑģÑĮ","磨 éļ¾","Ġvalu ations","è¾ħ é£Ł","æĪĺæĸĹ ä¸Ń","ĠÐŁÐ¾ Ñĩ","Ġclot ting","Previous ly","à§ĩà¦ĸ ানà§ĩ","ĠDivid end","ocia zione","[max n","Ġcupc akes","çļĦ åįĬ","un ce","åĬł åΰ","Ġra isins","Ġrep el","ÅĤ os","ĠRet ention","åħĪçĶŁ åľ¨","Ġgi ugno","Error Message","çĭĤ çĥŃ",".sh ields","Ident ifying","äm ä","Ġtheat res","Ġunbear able","F uel","b ahn","it ie","èĢĮ éĻį","å¹¶ åħ¥","Ġpar v","Ġsl ä","Ġdev out","ĠAll geme","Ġlog out","åĽŃ éķ¿","åĨ· ç¬ijéģĵ","ĠMc Connell","åIJĽ èĩ£","á± ¤","Present er","Ġpige ons","Ġkomb in","Ġসাধ ারণ","Ġhaci endo","F an","_ OF","z ony","at iven","Ġc ravings","Ġse eding","Ġr r","å°Ĩ çͱ","Ġfact ored","val or","è¿ľ 举","å©ļ äºĭ","Des de","Ġà¦ı ল","æĶ¶åħ¥ åĴĮ","ĠCorpor ations","ؤ ÙĪÙĦ","문 ìłľ","èŃī æĵļ","ĠÙĨÙħ ÙĪÙĨÙĩ","inherit doc",". ip","ä¸į è«ĸ","ĠO WN","Ġpl ást","å¤ļ æĸ¹éĿ¢","éĩį éĢ¢","und ry","ли заÑĨии","Ġна гÑĢе","Ġdown right","ç´ł æıı","cur ve","ç¼ĸ åζçļĦ","é±¼ åĦ¿","Ġau près","åĢį å¢ŀ","æĶ¶èİ· äºĨ","ĠLiving ston","ä¸ĢèĦ¸ çļĦ","Ġìĭ¤ ìłľ","Ġinstinct ively","Ġà°ª à±įà°°","R K","Ġa vert","al et","ĠB ao","ĠÑģ ко","Ġdist ressing","å¿ħ èĥľ","-t ask","ż enia","éĢīæĭ© åĴĮ","è·ij çļĦ",".L ast","åħ³éĶ® æĬĢæľ¯","å°ĩ åħ¶","æ·±åħ¥ åľ°","弯 磩","ogl io","æ¶Ł 漪","un st","Ġpl ankton","Ġcont ar","éĥ¨ ä»¶çļĦ","è¿ĺ åħ·æľī","æłĩ é«ĺ","Ġintern acionais","Not ify","ÙIJ Ùħ","Ġdry ness","æĺ¯ä¸į å¤Ł","ĠIns ects","æĬĬæı¡ 好","ĠоÑĢгани зова","ĠCF D","Ø®ÙĦ ص","ĠعÙħ ÙĪÙħÛĮ","ĠSpani ards","ic et","ĠC IO","ĠL j","oc er","в ей","ä¹ĭ èĬ±","Ġdes apare","Ġinter connection","Ġpo eta","åıį éĿ©åij½","ĠAb st","ç»Ī æĹ¥","æ·· æ··","Ġinstit uciones","é½IJ èģļ","ĠاÙĦÙĥ Ùĩ","èĻķ æĸ¼","Ġtrem or","require ments","ĠìķĬ ëĬĶ","ĠEspañ ol","( of","_ raw","z man","ĠH ort","pr ung","马 éĵĥèĸ¯","æł¹æį® æĿĥåĪ©è¦ģæ±Ĥ","é¾Ļ 骨","çļĦäºĭ äºĨ","UT IONS","Ġ×Ķ×ŀ× ľ×","贷款 çļĦ","à¹Ģà¸Ĺ à¸ŀ","çļĦæ°Ķ åĢĻ","ĠCos mic","ĉĉĉĉĉĉĉĉ ĉĉĉ","ĠSoul s","Ġneutroph il","ĠÎijÏģÏĩ ειοθεÏĦήθηκε","Ġdiket ahui","Ġb enda","ĠB AB","Ġla h","Ġ+ :+","Ġsc ant","any e","Ġsl ov","ĠOr well","pre cision","Ġsuper position","缴æİ¥ æĬķèµĦ","Ġbring en","èįī åĽ¾","Ġsubt itle","NE Y","éļIJæĤ£ æİĴæŁ¥","ĠEver ton","ĠIMP ORT","ĠScan ning","ĠLah ore","Ġcondol ences","B ry","ĠI AS","ys cale","ï¼Ī ï¼īãĢĤ","Ġpers ön","ĠAr rest","æ²» 好","æľĥ 被","å¼Ĥ æŀĦ","ãĥ¼ ãĤ¶","_d irectory",".f asterxml","以为 çĦ¶","å¿ĹæĦ¿æľįåĬ¡ æ´»åĬ¨","ĠMons ieur",") },","in voice","Ġp w","ĠI OC","æĺ¯ æĮīçħ§","ĠW ij","éķ¿ é£İ","Ġtrans vers","ern acle","ãģ« ãģı","éľĢè¦ģ éĢļè¿ĩ","æ²¹ ç®±","Ùij ÙĦ","æľ± éĽĢ","éĽĻ çľ¼","ĠCoord inates","Ġparc els","ĠSchwe iz","èĢģé¾Ħ åĮĸ","C n","Ġw ither","ĠS ON","ĠL Y","人 æŃ»äº¡","op is","ne un","ll ib","ĠRe bel","äºĮ 个","ĠPl anner",".p i","ä¸Ńå¿ĥ 主任","éĺ¿ åĵ¥","éĢĢ æ¬¾","çļĦæīĭ æľ¯","風 æĻ¯","ifferent iated","ᣠģ","Ġbid irectional","º C","ĠÑĤÑĭÑģÑı Ñĩи","ĠглÑĥ би","Ġexperi ência","åĮĸåIJĪ çī©çļĦ","Ġin icio","ent imes","Ġl p","ĠF aso","åİ» æĥ³","Ġchild ish","Ġprof issionais","çīĩ ä¸Ĭ","ãģĮ ãĤĵ","\") ).","Ġà° ľ","ĠLib re","ĠоÑĤно ÑĪений","ĠRefuge e","лим пи","T ill","t frac","Å »","åĴĮ ç»ıéªĮ","Ġdi oc","åıijå±ķ äºĨ","ε κ","ðĿij Ķ","Ġnie ces","åĺī åºĨ","Ġпом оÑĩÑĮ","èµł ä¸İ","ä¸Ģä¹Ŀ åħ«","æĶĿ å½±","Ġsprink led","Ġwag ons","ĠP avel","ver ified","ĠG im","为 ä¸Ĭ","ass ociation","con serv","ä¹ĭ çIJĨ","使 她","IC s","åĪĴ å®ļ","æ²³ éĩĮ","ĠPM S","彩 礼","ĠÐĺ Ñħ","Reg ression","Õ¥Õ ¼","Ġ׾×Ķ× Ĵ","Ġanalog ues","åį¸ è½½","Cent re","k ont","Ġ( ±","ul ai","Ġwe pt","以 举","å¹´ éĩĮ","Ġdis continuous","åģ Į","äºĮ è¯Ŀ","Ġت تÙħ","Ġmil dew","AL D","inn on","ãģĤ ãģ¾ãĤĬ","秦 é£İ","è¿İ çĿĢ","ĠMach inery","Ġconvey ance","ĠÑģÑĤÑĢа ÑĤе","ĠÑģам ой","à§ĥত à§įয","áln ÄĽ","çļĦ æĢĿè·¯","est ens","ĠQ Q","ä½ķ çŃī","}, {\"","Ġ׾ ׾×IJ","Ġmoder ator","Ġchat bot","Ġبت ÙĥÙĪÙĨ","Lo an","ắ t","B ee","P airs","w arming","Ġo ÅĽ","Ġf MRI","ĠL oth","ge an","á rt","Ġinv it","èIJ ¸","Ġuse Selector","ä¸ĵ 访","ĠSp o","Ãł s","λ αν","Ġbes ie","Ġrend ition","第äºĶ çϾ","çļĦæĥħ èĬĤ","ĠPredict ing","_ insert","om be","ä¸į èĩ´","ĠW ORD","ĠL ender","Ġop is","管çIJĨ åѦ","Ġ` <","èIJ½å®ŀ 好","fall en","極 çĤº","\\% \\)","åĹħ è§ī","ĠMarse ille","f ell","Ġ xt","ĠC ES","ĠC PT","Ġde h","ĠN ieder","åĴĮ 交æµģ","ĠPh rases","çļĦä¸Ģ å¼ł","ĠPr ä","Ġpa ix","è·Ł ä½łè¯´","ĠاÙĦÙħ ÙĪØ³","Ġпо нÑıÑĤÑĮ","Ġprop iedades","ĠÑĨ ена","æııè¿° çļĦ","å¿§ æĦģ","åİŁå§ĭ çļĦ","িস à§įà¦Ł","Strateg ic","ä¹Ł åĽłä¸º","Ġ_ \"","å¤ĸ å±Ĥ","é© ķ","ĠSp urs","-w orth","-P aul","麻 é»Ħ","åĿļæĮģ ä¸ĭåİ»","ä¸īåįģ äºĶ","论æĸĩ éĽĨ","ulin um","ÑĨион наÑı","å±ħä½ı åľ¨","ĠAchie ving","NAS DAQ","N ome","Ġp ihak","ä¸Ģ éĢļ","th in","rit eria","å·¥ä½ľ åľ¨","ho pper","ä¸Ńå¿ĥ åĴĮ","楼 éģĵ","ÙIJ ب","X n","om us","Ġcomp action","ä¹ĭ æĻĤ","å¦Ĥ 梦","åħ¶ æĹ¶","æį® æĬ¥éģĵ","æŃ¥ åŃIJ","æºIJ 代çłģ","of a","éłŃ é«®","δ ιο","åĢį æĦŁ","ç͵影 çļĦ","ĠاÙĦج رÙħ","çĿĢä¸Ģ å¼ł","à¸ļริ à¸ģาร","ĠконÑģÑĤÑĢÑĥк ÑĨии","( items","ä»ĸ 身ä¸Ĭ","èĥ½ 以","æº Ł","ãģ® äºº","éŁ³ çļĦ","é¾Ļ èϾ","éĿł èĩªå·±","Ġdefault dict","éĴ¢ åİĤ","æļ´ è·Į","ĠÄij ầu","è¶ĬæĿ¥è¶Ĭ å°ij","Ġsug ary","/ qu","N uclear","ĉ size","im ab","åĩº èµ°","ä¼ļ åıĹåΰ","ks et","ĠSh ri","åĩĨ èĢĥè¯ģ","å¼ķ çĪĨ","å®īåħ¨ éĹ®é¢ĺ","享 æľīçļĦ","éģĹ çĹĩ","è½´ 对称","-dis ciplinary","ĠBeck y","ĠGeb ä","è²¢ çį»","á ´","ĠA CA","ĠM aw","åľ¨ éĿ¢å¯¹","ast ric","å¹´ æĪijåĽ½","ord es","Ġtr asc","Ġ$ \"","Ġcomm uting","Ġrec ite","Ġب ات","Ġmon op","ĠAs he","Ġsom it","à° ģ","Ġtext ured","ush a","Ġair tight","à¸Ĥ ัà¹īà¸Ļ","æĵįä½ľ è§Ħç¨ĭ","-A nal","ĠBlack burn","Dis abled","Ġpou žÃŃ","-cont act","_w indow","è½° 鸣",".as List","íĸĪ ìĬµëĭĪëĭ¤","ĠMidd leton","Ġ_âĢľ _",". created","P IN","T et","in ha","ĠG ogh","åı¯ çĩĥ","èĥ½ å¹²","å¹´ èİ·","ï¼ģ ãĢijĊĊ","ĠEx clusion","ç»´ ä¹Ł","å®Ī åľ¨","Ġন à§Ł","è®°å½ķ äºĨ","ìŀIJ ìĿ¸","å¼· çļĦ","ĠмеÑĤ ÑĢов","è§Ģ é»ŀ","ĠBrand t","ĠOcc idental","åIJ¼ éģĵ","ĠÑĤÑĢеб ова","obl astic","Cour tesy","ĠHerzeg ovina",". ylabel","K id","Ġm umbled","æľī åı¯èĥ½æĺ¯","Ġor ifice","éĹ ĸ","å¿ĥ å¦Ĥ","Ġent fer","Ġка ÑĤа","Ġré uss","Ġcarbon yl","ä¸ĢåĢĭ åĢĭ","Ġ׼ ×ķ׾","pers ons","Ġaffirm ing","-load er","åıĸèĢĮ 代ä¹ĭ","+ M","- verbal","E gg","im ing","ass ignment","Ġz aman","å®ĥ åĮħæĭ¬","åħļ æ´¾","}\\) _","ura mente","å®īåħ¨ 带","ä¸Ģèµ· åIJĥ","Ġfa res","اس بة","åŃĻ æĿĥ","ä¸Ń央 ç͵è§Ĩåı°","uns aturated","å¾Īå¿« å°±ä¼ļ","ĠاÙĦÙģ ÙĨ","hab it","B K","} S","or nal","åľ¨ åIJİéĿ¢","ĠG aines","Ġle ben","ĠU CS","éĩį ä¸Ńä¹ĭéĩį","Ġна знаÑĩа","æĸŃ ç»Ŀ","Ġcheck box","Ġtax ing","Ġprop hes","Ġtas a","ĠType Error","èµŀ èªī","ĠGlobal ization","ä¹łæĥ¯ äºİ","ðŁij ĩ","UIT ableView","æŃ£æĸ¹ å½¢çļĦ","Ġanalges ic","ĠProvis ions","P ractical","Q V","æĪIJ 大","ĠY ates","ÑĢе л","åĨį åĪ©ç͍","åİĭ ä¸ĭ","ÄĽ n","éĥ¨åĪĨ æĺ¯","Tr ig","Ùij ر","è¿· 宫","ĠMart ian","ĠпеÑĢ Ñģона","Conf lict","ĠENGL ISH","Ġp iet","ĠC CP","ĠW imbledon","Ġj ú","çŁ¥ ä¹ĭ","ann is","-m iddle","ÑģÑĤа ÑĤи","Äħ pi","åIJ¸ æ°´","-g as","ç¾İåĽ½ åĴĮ","Ġmis chief","Ġtang ential","ĠPRO T","Ġbatt led","interest ing","ĠSho es","Ġgad get","Ġoverhe ating","Ġhypers ensitivity","ĠMÄģ ori","I b","g k","ë Ĥ¬","ra bb","对 åĽ½å®¶","ä¸ĭ 课","é« »","ен ом","ont aneous","ĠQ oS","该 ç±»","çļ® æ¯Ľ","ĠPar amount","Ġdé mar","æ©Ł åł´","鼻 æµģ","èµĦæł¼ èĢĥè¯ķ","ĠPed ag","ĠÔ ¼","ĠFa ulkner","Ġவ à¯ĩ","ĠMonteneg ro","> x","in ne","ag d","Ġby stand","ä¸Ģ个 女人","计 çļĦ","éĩij é»Ħèī²","на пÑĢимеÑĢ","Ķ× ¨","ET ERS","èĹı çļĦ","ائ Ùģ","Ġharm ing","åıĬæĹ¶ çļĦ","-cent ral","ç·Ĭ ç·Ĭ","éĽĩ ç͍","佩æĪ´ åı£ç½©","椰 åŃIJ","à¹ģà¸Ķ à¸ĩ","ĠKimber ly","B run","W ASHINGTON","Ġs na","Ġm ặt","ä¸į 說","æľī æ°´","èĩª éĩį","æŃ£ æĢģ","æ¸ħ 羣","Ġت رب","Ġos iÄħ","Ðŀ У","è̳ 鸣","ÑĤив наÑı","Ġaccum ulates","Di abetes","Inf obox","Ġscu ole","Ġξε ÏĦα","Ġpro ximate","ĠK U","ci endo","Ġro aming","ç½ ¹","á o","åĽŀ ä¾ĨäºĨ","åĨľ èĢķ","Ġant rop","欢 声","ç´§ éļı","é¾Ļ æ³ī","Le ading","AA F","ĠMus ée","æĮĩ导 åijĺ","abs ence","SO URCE","[ root","Ì ¯","ð ĵ","åı ±","åĴĮ åĪ«äºº","åīį 天","ĠZ IP","çĦ¶åIJİ åįķåĩ»","è¡¥ æ°´","ĠвÑĭ глÑı","Ġsen ate","磨 ç»ĥ","èĥĨ 碱","Ġunknown s","çĩĥæĸĻ çĶµæ±ł","ĠجغراÙģ ÙĬا","Ġpráct icas","R ECT","ĠA spect","天 亮","åŃĺ æ¡£","(' <","èIJ½ éŃĦ","ĠاÙĦÙħ ب","Ġput ih","Ġbegin nen","Ġvo ie","æĺİæĺ¾ åľ°","ĠBro ker","Ġexpert ly","ç¡®ä¿Ŀ äºĨ","åĿ¦ è¨Ģ","à±įà° ļ","ĠHa em","ຠĻ","ĠAmb iente","itic us","éͤ çĤ¼","纪å§Ķ 书记","ĠتارÛĮ Ø®ÛĮ","åĨ·åĵ¼ ä¸Ģ声",", num","ĠC âu","æĹ Į","Ġk wal","av ar","Ġwill ow","Ġи денÑĤи","æŃ¤ çķª","Ġdet al","çϽ ç¾Ĭ","çĭ¬ åįł","ĠAng er","Äģ ng","ĠÑħ оде","Ñį кономи","-ch anger","ÏĦο Ïį","éģĵçIJĨ çļĦ","Ġprost hesis","ĠرÙĪ ØŃ","Ġott obre","Ġtö rt","åįķçīĩ æľº",": ss","æľī å¿Ĺ","è¿ĩ ä»ĸ","ä¸İ æ°´","èµ· çģ«","×Ļ× ŀ×ķת","Ġest uary","该 æŃ»",")) **","å¦Īå¦Ī 说","Te ach","ాఠ®","æĢİ麼 辦","us se","ĠR oses","ä¸ĭ 人","ä¹Ł 表示","ĠRe ino","Ġhand ker","Ġdep reci","Ġprot otyping","è¶Ĭ åĨ¬","å®ŀéĻħ åĩºåıij","å°¼ 西äºļ","è´´ çĿĢ","è®¤çľŁ 贯彻èIJ½å®ŀ","pol is","ĠпоÑĤ ок","åIJµ éĹ¹","Ġcosm ology","èķ´åIJ« çĿĢ","( =","z ka","on el","ro ids","il inear","os yl","å° ·","èĢĮ éĤ£äºĽ","ä¸İ æĸ°","ç½ij çĬ¶","è¡Ģ æ°Ķ","Ġpress ured","大家 æĹı","ä¸įè¿ĩ æĿ¥","ÉĻ k","file Name","è®¤çľŁ åģļ好","Ġ}} \">Ċ","Dim ension","( pt","åľ¨ ä»ĬåIJİçļĦ","å°± åħ¶","å§ £","ĠÙħ ÙĪÙĦ","å¹¶ 表示","ĠEx ponential","Ġhand out","åģ¥åº· æķĻèĤ²","ĠMal one","ĠMill igram","Ġ×ĸ ×ŀף","jud ice","客è§Ĥ ä¸Ĭ","ĠBeck ett","onc é","âłĢ âłĢ","H aw","S ter","an nte","Ġin m","ĠM azz","а ÑĨии","ĠW EST","Ġout p","å¤ļ 边形","å¸Ĥ 缴","-b ye","è¿Ļ个 大","太 éĩį","(' [","è¿ŀ çĿĢ","çĶŁäº§ ä¸Ń","åįĥ æĸ¤","Ġиз д","_n ormal","åĩºçīĪ äºĨ","ĠBel grade","ãģij ãģªãģĦ","ðĿIJ ¾","Ġפ ת","殺 äºĨ","ાઠ®","hyd ration","Ġà¶ ¯","竣工 éªĮæĶ¶","мÑĸ н","Ġregroup ing","ĠдиÑĦ ÑĦеÑĢен",". ');Ċ","l ij","ĠV la","Ġgra vy","èģĮ 人åijĺ","å¾® æľº","IT EM","æĭī åįĩ","LL A","审 讯","èĭ¦ å¿ĥ","ĠÙĪØ§ÙĦ ص","æĶ¿çŃĸ æĢ§","ĠÙ¾ ذÛĮر","Ġdownload able","女åĦ¿ çļĦ","ĠWild er","}/ ${","æħĮ äºĨ","ØŁ Ċ","æī©å±ķ åΰ","åķŁ åĭķ","Ġcruc ified","ĠElig ible","åħħæĸ¥ çĿĢ","ĠÕ¢Õ¡Õ¼ Õ¡ÖĢÕ¡Õ¶","Ġterl alu","= http","åľ¨ è§Ħå®ļ","åľ¨ çľĭåΰ","og ie","ÙĬ ÙĪÙħ","å¾Ī å°ıçļĦ","ty le","æĽ´ éļ¾","Ġcre ams","çİ°åľ¨ çľĭæĿ¥","ĠÕ ®","CT A","è´µ éĩį","ĠÐŀ ÑĤе","ĠSun ni","ĠÑģоб а","ÈĽ ii","Ġ×Ķ×IJ× Ŀ","ĠSIM BAD","Ġper cutaneous","èĢħ ãģ¯","Ġimp erson","Ġа гÑĢе","AD R","Ġdistrib u","Ġrevolution izing","ĠSleep ing","Ġcuid ados",". ss","< form","ct ure","ost omy","fl ush","اÙĦ ÙĦÙĩ","Ġmat ric","Ġappro fond","Ġsl oping","ision es","ĠÑĥ ника","ĠShe ldon","éĥ¨åĪĨ åľ°åĮº","ĠDo ors","Ġ×ľ× ł×ķ","ĠInst alling","ç¬¬åĽĽ 个","ĠìĿ¸ íĦ°","à¤ľà¤¼ ार","* A","- alt","C ognitive","V oor","ie ces","对 åĨ³","ï¼ļ #","د Ùī","ä¸ī æĿ¿","Ġbl asted","ç² ¼","åįģ 竳","çϽ èĮ¶","Ġব ার","ĠEst as","ĠNeed less","åĪijäºĭ è¯ī讼","-z A","Ġreun ited","ĠProble me","è¾Ľäº¥ éĿ©åij½","+ Y","Ġw iser","çļĦ æłĩå¿Ĺ","id ÅĤ","ĠP ax","Ġj umper","ä»ĸ å¸ĮæľĽ","ĠHe idi","åķ ĵ","ÑĤе п","ار Ùģ","åı° ä¸ĬçļĦ","ĠSch rö","Cont acts","为äºĨ æĸ¹ä¾¿","iam ine","çģµ åĬ¨","寻 è§ħ","å¹´è½» æĹ¶","kk ue","æŃī æĦı","jekt iv","ĠRic ci","Ġë³Ģ ê²½","Ġpermett re","e ils","f ri","h un","Ġd rows","çļĦ é»Ħ","ent anyl","Ġr ichtig","-s ong","оÑĢ Ð¼Ð°","ä½ķ æ³ģ","åŃ¦ä¹ł éĽĨ","Ġbelie vable","å®£ä¼ł 贯彻","åĬłå¿« åıijå±ķ","Ġknock down","Ġnap ÅĻÃŃklad","Ġneces ita","Ġ׾×ŀ× ¢","('/ ',","g rey","v orm","ch ina","个 åįķä½į","ĠIt o","æķ° 以","å¾Ī è¿ij","æĽ´ åIJį为","Ġche ered","าร ยà¹Į","Ġcou plings","Wh ilst","è¿Ļæĺ¯ çͱäºİ","Ġcommand ment","pret ty","Ġspect rophot","×ĵ ר","-F ive","Ġconv oy","Ġসম স","ĠMeg abytes","jonal itet","Ġcommut ative",": mm","c ookie","g ian","§ ר","Ġg Ã¥r","对 æµģ","Ġche ers","èĩ´ 以","ĠâĪ ĥ","åħį åıĹ","çĦ¶åIJİ æĺ¯","ãĥ© ãĤ¤ãĥ³","æ¡Į åīį","éģĩåΰ è¿ĩ","ĠÎij ν","åĩºä¾Ĩ äºĨ","ĠBan ana","ameth asone","Ġyak ni","Ġintuit ively","Ġclums y","Ġsatisfactor ily","( ID",". play","ĠU G","å¤ļ å¤Ħ","ĠâĢĶ Ċ","Ġoriginal e","ĠÙĤ ÛĮÙħ",".E quals","ç½ijåıĭ 们","Ġмог Ñĥ","Develop er","ĠDip tera","* ....","ĠC rem","ĠF oto","çŁ ¾","主 æĹ¨","еÑĤ ÑĮ","è¶Ĭ å¿«","举 ä¸ĸ","ĠEd s","éĤĦ 好","æ³ķå¾ĭ æľįåĬ¡","ĠÑħ а","ĠBen in","ä¹Łæľī åı¯èĥ½","æ²Ĵæľī ä»»ä½ķ","éĢĨ è¡Į","ĠIg E","çĭŃ éļĺ","ĠاÙĦÙģÙĦÙĥ Ùī",". offer","ĠG AL","åĮĸ èĦĵ","æĮĩ çļĦ","Ġgeneral izations","é£ŀ æľºçļĦ","å¯Į 人","ί κ","Ġprevent able","èĥľ åľ°","帮åĬ© ä¼ģä¸ļ","ÐIJ ÑĢ","æīĢ示 为","Ġimmun ological",".ex ception","ĠContin ent","åįĴ ä¸Ń","à¸ģระ à¸ļวà¸Ļà¸ģาร","ĠëIJĺ ëĬĶ","Ġ׼×IJ× Ł","嫦 娥","_ it","j ent","on u","ĠT EN","od et","ĠThe e","ĠK ud","èĩªå·± ä¸į","åħĪ ç¥ĸ","éħį è§Ĵ","çľģ éĴ±","Ġbro om","Õ¡Õ »","Ġfi end","ĠاÙĦØ® ارج","ĠмеÑĤ одов","å½¼æŃ¤ çļĦ","uno ang","åĿį å¡Į","ĠصÙĪØª ÙĬÙĩ","B US","è´ °","åѦ èĢħçļĦ","Ġman i","ck et","ä¸ī ä¸ĸ","çϽ è¯Ŀ","ÑĤа ÑĨиÑı","åĨĽ æĶ¿","ĠÐIJ к","çī¹å¾ģ æĺ¯","ìĪĺ ê°Ģ","å°İ æ¼Ķ","Ġhorse back","Ess ay","ÑĶ ÑĤÑĮÑģÑı","ĠÙĥÙĬÙģ ÙĬØ©","-auth ored","el h","ĠF Åij","ä¹ĭ å¾Ĵ","ä¸İ åĪĨæŀIJ","缸 åħ³ç³»","rent issage","Ġbook ings","社ä¼ļ 稳å®ļ","鼨 åŃ£","åĵĪ éĩĮ","ĠبÙĩ ترÛĮÙĨ","ç¨ĭåºı åĴĮ","è·³ è¿ĩ","Äĥ o","Ðĺ Т","ÃŃt ott","Õ¡Õ´ Õ¢","è̽ æIJģ","ĠKyr gyz","; amp","y ed","ĉ D","os il","д лÑı","ĠCh att","å®ī éŨ","ร à¹īà¸Ńà¸Ļ","çϾ å®ĺ","oph aryngeal","ĠпÑĢи боÑĢ","é¾Ļ å±±","Ġrob ber","Ġ׼ ×ijר","ĠÚĨ Ú¯ÙĪÙĨÙĩ","æī° åĬ¨","ĠتØŃ دÙĬد","Ġobj ekt","ãĤŃ ãĥ¥","rä fte","Ġenforce able",".Test Case","ĠEstim ating","Õ¸ÖĢÕ ®","ĠÙĪØªØ³ جÙĬÙĦات","åįİå°Ķ è¡Ĺ","n avigation","q w","re ceived","ĠM me","ç«ĭ ãģ¡","ĠSp oon","ude au","Ġprop ried","Ġο á¼","

$","å¶ º","Ġداد ÙĨ","ĠHimal ayan","æ®Ĩ å°½","Ġeloqu ent","ANSW ER","( bytes","es cent","äºİ æĺ¯åľ¨","mer n","Ġна Ñĺ","æ¥ ¹","ĠSch oen","æĿİ æĻĵ","åĵª 天","Ġsett embre","ĠìŀĪ ìĹĪëĭ¤","ĠUr l","ÙĪÙģ Ø§Ø©","Ġinoc ulation","Ġorch id","< V","H ubble","I ENCE","Ð ģ","Ġa ik","Ġc ah","äºĨ å°ı","ĠSt arts","çľĭ åĩºäºĨ","åİŁ åŃIJçļĦ","åĮĹ å¸Ī大","_d esc","Ġcomplex ion","è¡Į为 åĴĮ","Ġcounter feit","æĸľ éĿ¢","ĠBra vo","rif uge","à§ĩশ ন","Ġreinforce ments","K et","Ġ à¸Ńาà¸Ī","od ds","ä»ĸ å·²","èĥ½ æī¾åΰ","Ġam a","产 åѦ","æī¾ æŃ»","ĠNov o","ðĿij Ļ","_f actor","Ġom issions","综åIJĪ ä½ĵ","Ġbud dies","Ġjed em","å¥ī åij½","ĠHa as","ĠGer ry","Ġfee ble","åıĽ ä¹±","èĻļæĭŁ æľº","रà¥įठ£","Ġclip board","Ġwitch craft","ÙIJÙĬÙĨ Ùİ","vermel ding","çļĦ çī©åĵģ","Ġre connect","ĠA mino","ĠI EC","ĠB land","๠ĵ","é«ĺ ä¸ŃçļĦ","åĬŁ åºķ","èĩªå·±çļĦ å·¥ä½ľ","Ġس اÙĦÙħ","Ġlo oph","ĠComp assion",".A t","kan ia","Ġsong writer","δ ÎŃ","åĮ»çĶŁ 说","èįĴ éĩİ","Ġneutral ize","ĠUI View","éĻªåIJĮ ä¸ĭ",", --","^ ĊĊ","äº ĺ","od ie","Ġch rys","æĹ¶ 说","ä¸İ æĸĩåĮĸ","-b in","min or","ä¹ī ä¹Į","çĽĬ å¤Ħ","ĠHere in","-C an","SU MMARY","ĠWil mington","ா஠®à¯į","ł×Ļ ×Ļ×Ŀ","ĠCOMM ENT","R Q","un ik","å¹ Į","åĩº ä¸Ģç§į","-f ifth","Ġlight ed","è¶ħ 人","оп ÑĢоÑģ","设æĸ½ çļĦ","িন à§įদ","ĠTri angles","æĪijåĴĮ ä½ł","ĠNin eteenth","ĠÑĢÑĭ б","Ġtá» Ń","Ġphyt oplankton","w ap","Ġr b","Ġra gged","ÑĢе да","ç»ĥ åħµ","IL ITIES","Ġempty ing","ä½ĵèĤ² è¿IJåĬ¨","Ġretro fit","Ġkep ala","ĠColon ies","Ġdesp ised","Ġvest ibular","fg fg","вÑĢоп ей","ĠØŃر کت","ĠIv ory","Ġzar ówno",". level","> S","f le","om inal","Ġsc anners","any i","Re ceiver","æĸ¯ æīĺ","об ла","缸åħ³ èģĶ","Ġcoll oidal","ÙĬÙĩ Ùħ","Sp onsored","æķ°éĩı åĴĮ","ä¸ĵå®¶ åѦèĢħ","å¯Ħ è¯Ń","ĠRock ies","ç´ļ çļĦ","Ġattain able","à¦¾à¦Ł ির","ĠT v","Ġle eft","we chsel","ĠÙħ اÙĦÛĮ","Ġprodu cir","Pro to","åIJ¬ ä»ĸ","çĽĬ çĶŁ","ĠС п","æł¸å¿ĥ æĬĢæľ¯","Ġaward ing","éģµ ä»İ","Ġminister ial","Ġwszyst ko","Ir ish","Ġneurotrans mitters","ĠGUID E","sever al","L it","Ġe psilon","æĺ Ļ","ĠM DA","ay n","ant ara","åIJĪ çļĦ","å±ķ çı¾","eng ah","åĪ« æıIJ","åħļ ç»Ħç»ĩçļĦ","éħĴ åIJİ","Ġе лекÑĤÑĢи","ĠÐľ екÑģи","Act a","Ġà¦Ĺ à§ģর","iÄĻ Äĩ","ĠDol phins",". args","Ġb ist","Ġ' '.","Cont inuous","ĠMin n","Ġка киÑħ","ĠMc Lean","Ġ×Ļ ×¨","Ġesc orted","ä¿ĿéĻ© åIJĪåIJĮ","Ġtransl ators","ĠاÙĦج غراÙģ","íĺ Ī","εÏģ μαν","Mor ning","Ġsebag ian","Ġantif ungal","ĠUIC olor","Ġginh ulagway","> $","L ag","s ia","Ġp ared","çļĦ éĺ¶æ®µ","ĠA β","olog ues","æĹ¥ åĴĮ","Ġbl aze","Ġна ÑĪего","ĠÙĬ ÙĥÙĨ","» )","éĿĻ è°§","Ġma estro","ĠSol ic","Ġconfig ura","åı¯èĥ½ä¼ļ åĩºçݰ","icz nych","Account ing","Ġenumer ated","ĠEvangel ical","d q","ä¸į ä¸Ģ樣","çľ¼ åľĪ","Ġmus k","-p article","Ġgen naio","Ġap oy","bre eding","ender ita","Ġdest ino","ĠAtt rib","çĶĺ éľ²","ĠMcC le","াà¦ĩ ন","Ġbol est","ĠاÙĦز ر","Ġnurt ured","Transaction al","对 人类","é¢ Ķ","å¦Ĥ åIJĮä¸Ģ","æı ĸ","è§£ éļ¾","Ġsol es","Ø´ ÙĬ","è§Ĥ çĤ¹çļĦ","宣 读","Ġfall acy","Ġdog ma","çģ¯ çļĦ","ĠEX PL","éłIJ è¨Ī","åİ¿å§Ķ 常å§Ķ","ĠìķĬ ìĿĢ","éĹ®åį· è°ĥæŁ¥","ĠHumb oldt","- \\(\\","G az","ĠP rit","п енно","å¤ĸ ç͍","é¦ į","åıį æ´¾","æĬ¥ 导",":: ~","çĶŁæ´» çݯå¢ĥ","ä¹° åįķ","åĿIJ èIJ½","Ġaw oke","浪 è²»","Ġpun ishing","Ġideal ized","浩 浩","Ġesp acios","à¸ŃาภĬ","ĠSales force","ĠÑĪкол а","Ġbehand eling","T aken","ri ott","ass essment","Ġpr ud","Ġtr ang","ob od","åħ¨ æĸ¹ä½įçļĦ","up i","ĠMed io","大家 éĥ½åľ¨","Ġap ric","_P AR","éĥ½ä¸į æĥ³","ĠÎļ Ïħ","ĠFem t","ØŃر اÙģ","Ġпок ол","Ġпоме ÑīениÑı","âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ","ĠWitt genstein","à¹ĥà¸Ĭà¹īà¸ĩ าà¸Ļ","Ä ļ","èĩª æĺ¯","åĪĨ éĻ¢","è¿Ľ æĿ¥äºĨ","ç£ ļ","ĠEr ich","Ġelectro c","oa uth","اÙĪ ÙĬ","Ġмог ли","Ġshar per","ĠPron ouns","à§ĭধ ন","f ek","w rapper","Ġd ank","Ġal ten","ĠL ur","Ġا ÙĬ","é«ĺ åį±","erm aid","缸 ä¸Ģèĩ´","ĠMon de",".f ire",".M igration","ĠFind s","ĠKar achi","Ġmild er","轻微 çļĦ","Ġشب Ú©Ùĩ","Ġdém ocrat","-prep ared","st wo","ĠR ory","ĠN ir","ort ex","Ġme gal","Ġcomm ended","èĩªå·± 被","Th ousands","åĮĹ éŃı",")) \\","宣 æ³Ħ","-h it","Ġevalu ación","ĠSpe akers","åŀĤ ä½ĵ","å¥Ķ èµ°","ĠÑģÑĢед ней","াà¦ĩ ল","çļĦæ°Ķ åĬ¿","Command s","Mc G","omi ast","Pop up","çļĦç¬ij æĦı","ĠNavig ator","wyd d","Ġp ik","ĠØ Ľ","ĠU ILabel","æĹł èĥ½ä¸º","åĽŀ æļĸ","ĠSch rift","大åѦ åѦæĬ¥","ä»Ĭ天 æĪij","-e lected","ãģĤ ãģĴ","ĠEst os","ãĤĪãģĨ ãģ§ãģĻ","é«ĺä¸Ń çĶŁ","Ġпи ÑĪе","ĠSz cz","羨 äºĨ","è¡Ŀ çªģ","Ġtek nik","% BF","Z U","\\ eta","Ġ ว","ĠS ears","ä¸į è§ĦèĮĥ","end ale","Ġen unci","å¹³ å®ļ","ä¸ĩ åŃĹ","å¿ħ ç»ı","éĽĦ æĢ§","ĠCap it","else y","Ġнап иÑģа","æĺİæĺİ æĺ¯","ä¹°åįĸ åIJĪåIJĮ","Õ¸ÖĤÕ©ÕµÕ¸ÖĤÕ¶ Õ¨","ĠcientÃŃfic o","Ġvég ét","Ġ**************************************************************** ********",". assign","j än","ag as","ĠK W","so hn","åıĬ æľīåħ³","ç»ĵ äºĨ","Ġstr on","åķĨ åľĪ","åħ± éĢļ","顺 ä»İ","Ġma ig","_l oc","éĦ §","ĠGP IO","ĠSudan ese","åIJĦæĹı 人æ°ij","Ġcerebell um","' aff","ï¼ ®","Ġre ch","ĠI c","为 æİ¨åĬ¨","ä½ĵ æł¼","åīį æĸ¹çļĦ","ms ub","Ġtake aways",".S end","ã es","Con v","ais en","è´¨éĩı éĹ®é¢ĺ","omin ant","严éĩį åIJİæŀľ","Ġtent o","ĠRy der","Ġó r","æŁ¿ åŃIJ","X E","l aughter","ĠP aren","ĠB rou","Ġal meno","å¾Ĺ æĿ¥","æľ¬ é¡¹çĽ®","াঠĻà§įà¦Ĺ","/m odel","ãĤĴ ä¸İ","Ġappropri ateness","ç¿» éĺħ","Ġmis chie","-D ec","_P ORT","Ġpiss ed","åĽ½èµĦ å§Ķ","Ġdrap ed","f resh","v ale","ĠC ER","ĠP ran","模 æĢģ","Ġang ina","à¥įठ¬","ĠÙĬ ÙĪØ¬Ø¯","Ġج دÙĬد","Ġfear less","çĭĤ å¥Ķ","ĠWood ward","èįĴ æ¼ł","ĠGar age","ä¿Ŀåģ¥ åĵģ","åľ¨éĤ£ åĦ¿","ĠÑģодеÑĢ Ð¶Ð¸","ĠÙħÙĤ اÙĪÙħ","éĵ¿ é͵","( See","G li","U ma","Ġ( :","ä¸į åIJĪéĢĤ","ä¸Ń åIJĦ","Ġint ently","ä¹ĭ åıĪ","ten e","-s how","å̼ æĹ¶","çİĭ å¿Ĺ","na ud","ĠInt ell","å·´ æ¯Ķ","æ±ĩ 款","Ġcat ers","æ´ŀ åºŃ","å¼· ãģĦ","åIJ¯åĬ¨ äºĨ","éģĭ è¡Į","ĠHon estly","Õ¾ Õ¥Õ¬","沪 æ·±","Ġstan ov","àŃį ର","าà¸ĺ ิ","ĠÙĩÙĨÚ¯ اÙħ","= list","a W","Ġre writing","ut ivo","ĠR ays","以 大","åıĹ éĤĢ","fic it","Ġdefin ir","缮çļĦ çļĦ","лÑĥ й","éłŃ ä¸Ĭ","Ġuns ustainable","ĠDur ant","Ġква ли","ográ fica","ĠMif flin","æĺ¯ éĶĻ误çļĦ","åį ŀ","Ġor o","ĠIn ference","æĺİ å¤©çļĦ","pro fen","å·² è¾¾åΰ","Ġexam en","åħī 头","ส มาà¸","IJ× ij","åģľ èį¯","alt ro","Dis covery","iej ÄĻt","ì¹ĺ ëĬĶ","Õ¡ÖĢ Õ¡Õ¯","Ġthumb nail","/res ources","Ġjel ent","Ġmultiplic ative","-tra umatic","à¸ķะ วัà¸Ļ","Ġauster ity","_ âĢľ","Ġt â","ĠE leg","æĪIJ å¥Ĺ","å¾Ī æĹ©","ĠAn f","uk ung","å¤ĩ å¿ĺ","ä¸Ķ åľ¨","è¿ľ æĻ¯","èѦ è§ī","ê² Ģ","Up dates","åħ¼ å¤ĩ","ç«Ļåľ¨ åİŁåľ°","Ġbil ayer","ÑĥÑİÑīи е","Ġà¦ħন à§įত","Rob ot","Ġlept in","Ġlän kar","ĠСк олÑĮко",". day","и мо","åIJĪ æĪIJçļĦ","Ġam ps","å¸Ĥ 人大常å§Ķä¼ļ","г ова","ĠSe pty","åħī çݯ","éϤ æİī","-p ack","Å¡ ek","ĠNe ue","伤 人","Man aged","åĩºä¸Ģ åī¯","first Name","ĠMcC ull","Ġepidem ics","Ġjard in","ab r","oc last","ĠJ oule","çĤ¹ å·¦åı³","the llo","ĠFr üh","æĮij äºĨ","ĠоÑģ ве","bert ura","ĠBol t","Ġpunt i","ĠبÛĮشتر ÛĮ","ĠZion ist","ĠINV EST","ä¸į å¦Ļ","æĪij åIJ§","ĠO bi","æ± ´","èĢĮ æŃ¤æĹ¶","eth ics","ãģ® ãģ§ãģ¯ãģªãģĦ","æµ· 滨","bl a","Ġbud ou","ĠLong er","欣 è³ŀ","ĠاÙĦÙħد ÙĬÙĨØ©","ĠCaucas us","iret roviral","h ur","j um","Ġp ener","ĠW issen","and ong","åĴ «","ĠK ör","ĠRe ese","è¿ĺ ä¸İ","да ли","Ġbar rage","ĠAc oust","ä¸įæĸŃ åĬłå¼º","Ġdoor step","å¢ŀéķ¿ éĢŁåº¦","åĪĽéĢł æĿ¡ä»¶","åıĮæĸ¹ å½ĵäºĭ人","赤 裸","ĠSar as","主æĮģ åı¬å¼Ģ","ξ ε","Ġvý sled","Ġìĥģ íĻ©","Ġdepr ive","ĠRah ul","creens hot","C t","åľ¨ åħĪ","天 æ°£","åįģ åĢį","ox id","empt yset","Ġmis guided","Ġcir cled","Ġov ulation",". Config","P ract","ĉ auto","ä¼ļ è®©ä½ł","ร à¸Ńà¸ļ","èĮ ģ","Ġна пи","åıĭ çα","åı² çļĦ","yle m","не ÑģÑĤи","Ġpersonal ize","Ø« ÛĮر","IL ABLE","éķ· æĻĤéĸĵ","Ġillust rious","ĠاÙĦÙĨ جÙĪÙħ",".Ent ry","_de code","Ġaten ção","ĠíķĦìļĶ íķľ","ĠMeteor ological","ĠÑģим в","éķ¿æķĪ æľºåζ","ĠÑģмеÑĢ ÑĤи","ĠEmpower ment","ĠSepty embre","_ zero","he ment","el ed","Ġse jarah","主 ä¸ļ","é«ĺ æķĪçİĩ","Ġfl irt","æ¯ı é¢ĺ","Ġterm ini","çļĦä¸Ģ å¥Ĺ","Ġinform at","åĵĪ å¯Ĩ","Ġrein vest","Ġleng ua","Ġphilanthrop y","Ġtrz eba","T EM","Ġh ib","un ki","Ġv b","Ġr x","è¦ģ æĿ¥","åıij äºİ","天 æ°´","åľº åĿĩ","ç»Ļ 对æĸ¹","ä¸Ģèά ä¸į","μ οÏħ","Ġìķ Ķ","è°ĥçłĶ ç»Ħ","ĠAcadem ies","~ {}","Ġa cyl","Ġd arn","ÑĨ ÑıÑĤÑĮ","è·¯ éĢĶ","Ġchar ms","ಠļ","ä½İ é¢ij","æŀĹ å®¶","Ú¯ ÙĪÛĮ","Ġе й","æĤ¨ æĺ¯","åıĤåĬł å·¥ä½ľ","Ġpet abytes","è°IJ æĮ¯","ạ o","Ġaest hetically","注å°Ħ æ¶²","ĠاÙĦÙħÙĪ Ø¶ÙĪØ¹","Ġdislik ed","ä¸Ģ ç·Ĵ","ĠG aw","iz ability","åΰ è¿Ļ","èĢĮ å½¢æĪIJ","åı¯ä»¥ åĩıå°ij","ĠAn kara","交 åī²","çϽ 骨","ãģ¾ ãģ£ãģŁ","Ġens ued","ä¸ĥ æĹ¥","æĽ¿ æĪij","éĿĴå¹´ 人","å´ĩ 祯","ä¸ŃåĽ½äººæ°ij è§£æĶ¾åĨĽ","abc def","Ġgraft s","J ard","al ate","Ġb iliary","om or","è® ·","op an","åIJİ ç¼Ģ","Ġoff end","ins urance","空 æĹ·","çİĭ ä¹ĭ","ج ÙĦÙĬز","AS Y","ä¿® çŃij","éĺ³ èĻļ","伤 çĹħ","èĩªçĦ¶ ä¿ĿæĬ¤åĮº","Ġsepar able","åIJĦç§į åIJĦæł·","ĠAnt iqu","Ġsn ork","ائ ÙĬÙĦ","ĠاÙĦس ÙĨØ©","ал к","å·¥ä½ľäººåijĺ çļĦ","çIJ³ çIJħ","หม ืà¹Īà¸Ļ","Ġlou ng","ĠCharg es","Columb ia","H amilton","P ier","Ġs ẽ","ot an","ĠP adding","ĠH askell","æĿ¥ æıIJé«ĺ","交 æĪĺ","ij ärvi","ä½Ĩæĺ¯ è¿Ļ个","Ġbi otic","ç©¿ çĿĢä¸Ģ","èĦļ å°ĸ","Ġfre i","财产 çļĦ","æ¶Īéĺ² æķijæı´","Ġenthusi astically","ĠCop a","Ġwicht ige","t our","Ġ á»į","ĠC TC","ĠD ice","对 åĨħ","æĪIJ è¿Ļæł·","éª ¥","Ġrequ iere","ãģ« åĩº","åĬŀ 主任","çīĩ ä¸Ń","ĠاÙĦÙħ رÙĬ","Ġbi ogas","åģ¥åº· åĴĮ","æ²»çĸĹ æķĪæŀľ","æĹħ游 æĻ¯åĮº","麦 å½ĵ","Ġtod avÃŃa","ãĤĤãģ® ãģĮ","Ġcomun idade","Ġê³¼ íķĻ","åĩĦ åĩī","Ġ(... )","äºĭ實 ä¸Ĭ","ĠVenet ian","Ġre interpret","am ol","ĠD au","ak os","ä¹ī è¯Ĭ","Ġrep ose",".d ot","åı¶ æŀ«","OC R","Ġclean se","ìľ¼ ëĤĺ","çĽij管 éĥ¨éŨ","াà¦ĩ à¦Ł","Ġ[+ ]","Ġorth opedic","Ġanch oring","ĠIsabel le","; ,","_ record","çļĦ éģĵå¾·","Ġcon texte","ĠF ountain","ĠW onders","ĠJ el","ÙĨ Ùħ","ob en","æĸ° å©ļ","æĺĵ ç»ı","ait a","缮åīį æĪijåĽ½","åºĬ ä½į","以åIJİ åĨį","rab les","åºŁ æĹ§","à¥įय तà¥ĩ","Ġreform ers","à¸Ħวาม à¸Ħิà¸Ķ","Ġফ ল","zet ek","纬 度","ĠSpar ks","Ġà¦ķারণ à§ĩ","ĠرÙĬ اض","ĠPolytechn ic","( def","_ container","u pl","in ov","çļĦ åıĸå̼èĮĥåĽ´","Ġl ick","Ġre position","Ġg aya","ĠD ating","å¿ »","åľ° æĥ³","ä½Ĩ åıªè¦ģ","pos ición","And rea","íķĺ 볤","×ij ×ķ×ĵ×Ķ","Ġ×ij ׾","न à¥įद","ãģĭãĤī ãģªãģĦ","å®¶éĩĮ 人","åĩĨç¡® æĬĬæı¡","-ph osphate","åľ¨çº¿ éĺħ读","ĠÑģам Ñĭй","ĠRein forcement","ĠToxic ology","@ extends","L K","ĠS ack","Ġv ad","Ġus aha","ĠU top","ĠSt arter","ä½ĵ ä½į","æĽ´ 强çļĦ","Ġmet e","app elijke","Ġder og","åĵª åIJĴ","ĠWar ay","Ġrandom ness","zo om","Ġkin ematics","ä¸įè¶³ ä¹ĭå¤Ħ","çļĦåľ° ä¸ĭ","اسÙħ اء","Ġchant ing","ĠÅĽwie cie","B UT","P acific","ĠC is","ĠP ROM","av as","Ġdis son","æĶ¾ 纵","å°±æĺ¯ å°Ĩ","ĠWor st","æĭī èµ·","Ġпо ÑģÑĤÑĥпа","èĭı å·ŀå¸Ĥ","bb ox","Ġcat cher","Ġtrain ings","æ½ľ åħ¥","Ġspl ice","åħ»èĢģ æľįåĬ¡","é«ĺçŃī éĻ¢æł¡","Ġastron omer","ĠRot ary","ĠHD MI","áĥ£áĥ ļáĥĺ","Ġpermett ant","åĽĽéĿ¢ åħ«æĸ¹","æľīéĴĪ对 æĢ§åľ°","= [\"","b ibli","Ġd n","ĠO ra","大 é¢ĺ","åĴĮ èĥ½åĬĽ","Ġpre processing","使 ä½ł",".c os","ĠاÙĦØ¥ ÙħاÙħ","ĠÅĽ wiata","Ġcomun itÃł","Ġê°Ĵ ìĿĦ","Inject able","Ġlineback er","ĠSatur days","p ig","ch ang","od iment","ĠD IN","åĩ Ī","以 éĢĤåºĶ","ØŃ ÙĪ","è¾¹ 说","èĮ¶ é¦Ĩ","Ġanaly te","éĥ½ä¸į ç͍","æij© 羯","æ¡Īä»¶ ä¸Ń",", â̦ĊĊ","\\ boldsymbol","w f","Ġw yr","çļĦ åĵģè´¨","ĠC oconut","com pute","Ġco er","æĪij们 è¿Ļ个","æĮĩ æķ°çļĦ","à¸ķ ัà¸Ķ","hel le","é£İéĻ© è¯Ħä¼°","ĠPet ition","othe rapeutic","Ġком менÑĤа","ĠImp ression","çĩķ 麦","强度 åĴĮ","çļĦ身 åŃIJ","ãĥĭ ãĤ¢","âĬ ķ","ĠNag y","Ġinterfer es","iett ivo","Ġrédu ire","or ning","Ġle iden","åĴĮ 马","å¹´ ä¸ĭåįĬå¹´","Ġlos ers","åĤ £","ç»Ĩ 鼨","群 å±±","ĠاÙĦع د","Ġtransport ers","Ġ׾ ׾×","å¾Ī好 åIJĥ","ĠRos ie","ä¸Ĭå¸Ĥ çļĦ","Ġfran cs","ĠCrime a","Ġবল া","ĠSequ ential","Ġparan oid","Jess ica","D ash","_ now","Ġ ál","Ġb ordered","Ġ( &","Ñĩ ена","Ġset enta","åŁº åĿij","ke h","æĬ¤ éĢģ","çģ« é¾Ļ","è¿ĺæĺ¯ ä¸į","æķij åĽ½","Ġ×IJ× Ĺ","æĶ¿æ²» çļĦ","Ġpi ÄĻÄĩ","社åĮº åį«çĶŁ","Ġassert ive","çѹ æİª","ĠTax ation","Vis itor","Ġabol ish","é´ ¨","Ġcarn ival","embed ded","à¦Ĥর à§ĩà¦ľ","D ip","re asonable","åľ¨ æĹģè¾¹","Ġam ar","Ġsc olaire","å±± 头","ĠSch ae","è½» åŀĭ","Ġut g","Ġconvert ir","ĠNational s","æ°¸ ä¹IJ","Ġprop ósito","å·¨ èŁ¹","Ġcit rate","å·Ŀ èĬİ","ét rica","é¼» çĤİ","åĪĨéħį åΰ","ä¸įéĶĻ çļĦéĢīæĭ©","è«ĭ åķı","ĠBox ing","ä¸ŃèᝠæĿIJ","Ġfart hest","-Se ven","ãģĻãģĻ ãĤģ","C p","S ymptoms","c ru","ary ana","åĪĨ æijĬ","æĻ Ķ","å¤ĸ åIJij","建 è¨Ģ","è´¨ çĤ¹","æķĪ åºĶçļĦ","說 éģİ","Ġhom o","Ġcross roads","温度 çļĦ","Ġsou Äįas","Ġdir itto","ctu ation","让åѦçĶŁ åľ¨","Ġdisreg arded","ĠmÃ¥n aden","æĺ¯ å¾Īéļ¾","od iazep","ĠB ore","èĩª æĭĶ","ents itatea","Ġ} ));Ċ","Ġover stated","Ġtrans pose","ç½ij è´·","-t imes","ì§ ľ","Ġillust rator","ĠSam my","æ°ı çļĦ","ัà¸ļ สà¸Ļ","Ù¾ ر","好好 åŃ¦ä¹ł","ĠTal bot","ĠÑĦÑĥнк ÑĨиÑİ","Ġsuffix es","áĢĦáĢºáĢ ¸áĢ","Ġreop ened","Ġsmugg ling","= |","s ime","ĠP ius","ĠR IP","è¦ģ æĢİä¹Ī","Ġdo et","pos p","èĢĥ åħ¥","åĨĻ çħ§","Ġland fills","Sh ang","å¨ģ æľĽ","र à¥ĩ","ĠÐij ог","å®Ł é¨ĵ","ĠBan co","è¾² æĿij","ब à¥įर","رش Ùģ","Ġbrace let","Ġbord ering","B TC","çļĦ ä¼ĺçĤ¹","om ot","em otional","ir ting","åĬ ¾","大 åѦçĶŁçļĦ","è¦ģ 积æŀģ","ç»ĵ çĤ¹çļĦ","An at","ç§° è°ĵ","Al cohol",".g enerate","Ġvarious ly","åĪļ ä¸Ģ","éĤ£ä¹Ī 容æĺĵ","Ñģи он","Ġrecogn ises","è¿Ļä¹Ī 好çļĦ","æĤ£èĢħ åľ¨","Ġcomment aries","æīįæĺ¯ æľĢ","Direct ed","ĠгÑĢÑĥ д","踩 çĿĢ","à¸ģระ à¸Ī","ĠÑī о","Ġ매 ìļ°","Ġmož né","Ġpalav ra","åĨĹ ä½Ļ","( abs","Ġbe vat","ĠG ideon","ĠK lo","Ġsp ieg","çŃī éĥ½æĺ¯","Ġser enity","å¾Ģ åĽŀ","çIJĥ æĺŁ","éĽĦ å¿ĥ","Ġве дÑĥ","âĦ Ŀ","ĠTen ant","ĠCompar isons","Ġпоз д","ĠìķĮ ê³ł","Ġë¹ ł","ĠÑģÑĤÑĢÑĥк ÑĤÑĥÑĢ","Õ¡Õ¦ Õ´","ĠاÙĦأش خاص","j ac","k ova","Ġin p","ä¸Ģ æŀĿ","ter re","ĠD ank","大 éĥ¨","в ÑĪего","ĠCl ive","Ġreal istically",")) ))Ċ","åĵª ä½į","Ġacqu a","èĢIJ ä¹ħ","Ġoz nac","Sp ark","ĠDE BUG","Ġ×Ĵ ×ķר","develop mental","ĠBild ungs","ĠNas ional","Ġfisher man","Ġfluor ine","ĠIntrodu ce","Ġtreacher ous","Ġstren uous","\" :ĊĊ","m Ã¥l","or izontal","Ġhis sed","缮 ä¸Ń","Ġatt ends","ç«ĭ æŁ±","ĠWh ale","åħ» æ´»","ä½Ĩæĺ¯ å®ĥ","æĿİ åħĭ","Ġtotal ed","åĽ½éĻħ ç»ıæµİ","ç¿» èѝ","ൠ¾","ĠпеÑĢ Ð²Ð¾","èĤ¡ä¸ľ çļĦ","ĠIg lesia","Ġlid ÃŃ","Ġexplor ations","ĠÑĢеги она","Lead ership","# \"","Ġ igen","æľī è¯Ŀ","ä¸ī åı£","å·® ä»·","çł´ éϤ","Ġmor bid","å¨ģ æŃ¦","æľīä¸Ģ åıª","缩 å½±","åĸ· æ¶Ĥ","ega ard","Ġshell fish","çĥŃæĥħ çļĦ","à±ģà° Ĺ","ç»Łæ²» éĺ¶çº§","à¸Ĺร ัà¸ŀ","èĢ» è¾±","liter al","Ġaster oids","ĠCaucas ian","ĠSout heastern","æĸŁ éħĮ","d V","ĠW ahr","th am","è¦ģ ä»ĸ","è¦ģ è¾¾åΰ","å°± èµ°äºĨ","д Ñı","åIJĮ æµİ","åĨħ åIJ«","ç®Ĺ åij½","me able","é¦ĸ è¯Ĺ","éĴŁ æĥħ","Ġball ad","CR M","Ġà¦ķর à§ĩà¦Ľà§ĩ","å¿«éĢŁ åľ°","Ġdelet ions","æĪĺ士 们","Ġherb icides","arke it","çļ±çĿĢ çľī头",", System","c oder","é«ĺ æĬĢæľ¯","ов Ñĭй","åı¯ä»¥ åıĤèĢĥ","æĬ¥ äºĨ","ĠX XXXX","åį´ æ²¡","ä¾Ŀ ç¨Ģ","åĬ³ åĬĽ","Ġten ha","çĭ¬ç«ĭ æĢĿèĢĥ","æĸ¹ç¨ĭ çļĦ","Ġti ếp","Ġescrib ir","ëĪ Ħ",". ab","is ie","ĠM utation","Ġat ing","ĠV ive","Ġdem asi","ä½łä»¬ 两个","Pl ain","åķĨä¸ļ åĮĸ","éªĹ åıĸ","Ġlegisl atures","à°¿à° ¯","Ġinterrupt s","Ġhö g","Ġ×IJ×Ĺר ×Ļ×Ŀ","å°ıæķ° çĤ¹","à¸łà¸± à¸ĵà¸ijà¹Į","éģ´ éĢī","N ING","ĉ delete","çļĦ çĽ¸å¯¹","æĬ ¨","ap ort","éĩį ä¿®","å·² å©ļ","çİī 佩","ç»§ç»Ń ä¿ĿæĮģ","æľīä¸Ģ é¢Ĺ","容æĺĵ åĩºçݰ","Õ¶ Õ¤","ĠIncre d","lett a","éķ¿å¤§ åIJİ","Sec ure","åľ° æĶ¯","to xic","ร à¸Ńà¸ĩ","æľ¯ ä¸Ń","æ¸ħ æĶ¿åºľ","/s ystem","Ġpop olazione","åİĭ çĹĽ",".get All","æĬĹ äºī","ĠØ« بت","å¤ī æĽ´","Ġহয় à§ĩà¦Ľà§ĩ","Ġপার à§ĩন","ĠZimm erman","Ġtoh oto","P OP","Ġ ull","ĠS CO","大 ä½ľ","åĴĮ 责任","åĽ½ 强","Ġ< %","Ġover th","éĩı 表","Ġind épend","管çIJĨ ä¸Ń","ç´ł é£Ł","æ±Ł åŁİ","åħ´ 缼","-g al","Ñĸ Ñı","ãģı ãĤĮ","综åIJĪ åĪĨæŀIJ","è®°å½ķ ä¸ĭæĿ¥","ĠNO K","स à¥įथ","Sl ider","ayan an","Ġkonk urs","ĠDÃŃ az","M argin","k urs","Ġo j","Ġp added","Ġv istas","ĠB UND","ว à¹Ģà¸ķà¸Ńรà¹Į","rac ji","åĪĹ åĩºäºĨ","æĭī äºĨ","Ġamount ing","ĠоÑĤ д","Ġrepe aled","Set ter","/p nas","ĠNa ams","Ġestud antes","è¡į å°Ħ","Ġunser en","åºIJ å±±","D OS","E mitter","H at","n ir","al oh","Ġh oud","Ġn ymph","end ering","Ġrel ays","uc ers","Ġser ine","æ·± å¤ĦçļĦ","-d a","ãĤĴ çŁ¥","-c atching",".l ayer","ĠØŃ ÙĪØ²Ùĩ","-re act","CM D","ĠÑģво ем","Enter prise","ĠSpace X","Ġcod ice","ĠUtil ization","Ġenlight ening","Ol iver","çϾç§ij åħ¨ä¹¦","compan ies","ĠBET WEEN","ĠGOV ERN","* \\","B J","j ons","ol ian","ow els","ĠB EFORE","ä¹ĭ é«ĺ","管 æķĻ","åIJij åħ¶","æīį ä¸įä¼ļ","åıªæĺ¯ åĽłä¸º","CO PY","ç͍æĪ· åľ¨","å®īæİĴ åľ¨","横 å¹ħ","Ġgly cemic","亿ç¾İåħĥ çļĦ","Ġanat om","construct ed","ãģŁãģł ãģĹ","I st","P atch","el on","im ura","ĠE ust","ĠF uchs","åĩº éĸĢ","ĠK j","Ġdo e","æľĪ 度","å¹³ åİ¿","带 åİ»","åį´ ä¸įæĺ¯","产åĵģ æĪIJæľ¬","çĭ¬ è§Ĵ","Ġsil enced","×§ ×ij","Ġinsect icides","åľ°è´¨ çģ¾å®³","åģ¥èº« æĪ¿","ĠÑĥÑģи ли","Ġglimps es","× ł×Ķ","ä¸Ĭ åĽ¾","ä¸Ĭ çģ«","ap agos","éĥ½ æĬĬ","Ġpass ports","Ġrev olve","åħ¨éĿ¢ æıIJé«ĺ","è§ĦåĪĴ çļĦ","社ä¼ļ主ä¹ī 建设","ãĥ¬ ãĥĵ","åłª æ¯Ķ","è¿Ļåľº æ¯ĶèµĽ","- neg","d st","l h","æŀ ·","_{ -\\","}\\ }\\)","çŃĶ ãģĪ","è¿Ļç§į åģļæ³ķ","ç¼ĸ èijĹ","ĠSc rum","ç«ŀ ä»·","лен наÑı","ç»Ŀ对 ä¸įæĺ¯","ãĤı ãģĭãĤĬ","ä¸Ģæĸ¹éĿ¢ æĺ¯","对åºĶ äºİ","áģ ¼","álnÃŃ ch","le ver","ot ry","æĪij ä¸Ģå®ļä¼ļ","Ùħ ÙĨد","åIJİ å®«","æĸĩ æ³ķ","æĸ° çļĦä¸Ģå¹´","ä¸ī 段","Ġsign age","Ñİ ÑĢ","Ġна ÑĪиÑħ","æł¡ éķ·","åĽ½å®¶ ç¨İåĬ¡æĢ»å±Ģ","во но","ĠвÑĭ зÑĭваеÑĤ","Ġsou h","Ðij олÑĮ","å´© å¡Į","ĠMedic ines","Ġdik enal","à¦ı র","çĤ¹çĤ¹ æ»´æ»´","C ENT","E y","Ġs ard","us ca","Ġev idences","ç§į èįī","Ġsign ifie","åĨį ä»İ","å¢ŀ å¼·","ato ires","ulf ur","èĩªæĪij ä¿ĿæĬ¤","Û± Û·","èįĴ è°¬","Mon itoring","ĠسÙĨ Ùĩ","à¹Ģส ีà¹Īยà¸ĩ","Ġpuzz ling","обÑĢаз ова","Ġuw agÄĻ","ĠBent on","ĠнаÑħод ÑıÑĤÑģÑı","- ath","x es","Ġf isc","om ens","ra ud","ĠE rica","ä½ł ä¸Ģå®ļè¦ģ","è¿ĺ åı¯èĥ½","ah ua","å¹³ èĩº","à¸Ń ิ","ö sen","å¼ķ å¾Ĺ","é¢Ħ ä»ĺ","Ġprim ate","产åĵģ éĶĢåĶ®","ê che","Ġdeb uted","-w all","ĠAdd s","ĠAtl antis","Ġiz quier","èī³ ä¸½","Ġfluctu ate","Du plicate","ĠFat ty","Ġcresc imento","Ġonge veer","m apper","Ġw aff","ĠL OL","hen es","Ġ$ $$","ó logo","æĽ´ åħ·æľī","çķĻ æģĭ","ĠGl oucester","ĠSol vers","Ġtom to","壮 æĹı","ãĥŃ ãĥ¼","Ġposit iva","ĠGa ia","ç«ĸ èµ·","åı¸ä»¤ åijĺ","ocia ção","Ġoverc ame","Ġà¸Ķัà¸ĩ à¸Ļัà¹īà¸Ļ","c ursor","å¿ Ĵ","ĠJ adi","çŁ Ĺ","åĨħ çĶŁ","被 åΤ","åįķ 人","缸åħ³ 人åijĺ","term inal","ĠMod ify","çģ¯ å¡Ķ","Ðļ огда","纯 度","çļĦ身 é«Ķ",".Cont rollers","Ġर à¤¾à¤ľ","Ġpessim istic","Ġn esta","ĠM ä","ĠM ika","ä¸Ģ å¡Ĭ","ĠF aster","ĠF ortun","ast ies","ĠV ita","ĠWe is","Ġterm os","æĭī æĭī","Ġcapital ists","ãģį ãģŁãģĦ","çĻĮ ç»Ĩèĥŀ","åıĺæĪIJ ä¸Ģ个","Ġdar f","Ġкажд ом","Ġtiem pos","Õ¡Õ¾ Õ¸ÖĢ","(List Node","Ġrept ile","Ġc ùng","Ñĥ Ñģка","ĠV ocal","ĠÙĪ Ø§Ø¶","åıĪ éĩįæĸ°","èµ° ä¸ĭ","-t olerant","Ġdirect ement","à¹ģ สà¸ĩ","Ùİ Ø¬","åĨ² 泡","ĠMon et","Ġhab ÃŃan","áŀ ij","Ġê tes","Ġmitig ated","enf ant","å½Ŀ æĹı","Ġmuu alla","; B","= I","> b","åľ¨ åı¤ä»£","und ice","çĹħ åİĨ","-p oll","ä»ħ æĺ¯","Û± Û¶","({ \"","ãĤį ãĤĵ","æ©Ļ èī²","Ġrag azzi","Ġarist ocracy","Ġjäl keen","- ful","p ac","Ġm ote","Ġe agles","ich let","Ġdis arm","éķ¿ å¤Ħ","ÑĪ ÑĤи","å¸ĥ çļĦ","ST AND","Ġа кадеми","Ġvan uit","(d ocument","ĠعÙĨ Ùĩا","Ġregist rar","Ġtab oo","ĠÙ쨱 ا","OLD ER","conc ert","Ġscaff olding","Ġnak alista","ä¸įåı¯éģ¿åħį åľ°","Ġαá½IJ ÏĦ","Ġneoliber al","Y s","b auer","ĠB RL","ure a","we j","Ġrel azione","å¹¶ å¤Ħ","ĠCl air","Ġimportant i","æĵ Ģ","ĠâĪ ħ","Ġprob lé","ĠChe ap","åĶIJ 人","Ġjed no","ĠRen ault","ró ż","pril is","hend e","Ġupload ing","ĠWille m","Ġhe nd","ç͍ ä¹ĭ","å¤ļ å°Ķ","Ñĩ ениÑİ","........ .","Ġз елен","åĮĹ è¾°","Ġmus lim","Ġsi RNA","yt ocin","Ġinflu encer","ê° Ī","ĠÑĢе а","ÖĢ Õ¡","ĠWal let","åĬĿ éĺ»","asm uch","ĠDat abases","Ġشر ØŃ","Ġëĵ± ìĿĺ","à¹Ģà¸Ħล ืà¹Īà¸Ńà¸Ļ","Ġs eder","çļĦ 强大","ä¸Ĭ 头","Ġ{ {{","å®¶ æł¡","éĢł åģĩ","-d rug","æ¯Ķè¾ĥ ç®Ģåįķ","ĠLeg o","Log ical","ĠMel ville","éģ¥ éģ¥","ĠRequest s","Ġworsh ipped","ĠNaams vermelding","Ġt ali","åѦ æ³ķ","æŃ¤ æĸĩ","-m aterial","μ μα","âĪĴ Ċ","sk ých","group Id","è¿Ī åħĭå°Ķ","леÑĤ ним","-fl oor","à§ģত à§įব","Ġráp ido","ŀáĢĬ áĢº",") ])Ċ","M ER","R ational","ĠR unnable","Ġr umin","Ġsuper iors","æľĥ çļĦ","æĢ¥ èºģ","Ġе м","éľĩ 颤","ĠAir bus","ĠпÑĢед оÑĤвÑĢа","Ġком анд","æĺ¨ 天çļĦ","Ġhyper plasia","å·¥èīº æµģç¨ĭ","å¿ħçĦ¶ æĺ¯","第åħŃ æĿ¡","lat itude","æŃ¦åĪĻ å¤©","Ġd aran","Ġn ul","ĠF EM","ä¸Ń åŃIJ","ĠâĢ ĸ","æŃ£ æīĢè°ĵ","eg i","èĢģ 人们","ÑĤа лÑĮ","缸åħ³ æĶ¿çŃĸ","Ġca udal","Äģ h","Ñĺе ди","åīµ æ¥Ń","Ġproportional ity","Ġtermin us","ERV ICE","å¦Ĥæľī ä¾µæĿĥ","ĠHern ández","Ġদà§ģ à¦ĩ","ĠØ£ÙĩÙħ ÙĬØ©","é£Ļ åįĩ",". One","= en","= edge","^ (-","Ġ Ï","Ġn Ãło","åĴĮ ä¸ĵä¸ļ","对 对","als ki","çĹ Ī","-d d","-d rop","æĭī å¾·","ä½Ļ ä¸ĩåħĥ","ĠNov ak","éħĴåºĹ çļĦ","Ġdir ige",".sh ared","管è¾ĸ æĿĥ","Ġinjust ices","祷 åijĬ","ĠآسÛĮ ب","Ġperovsk ite",". weight","T IP","v irt","Ġb ie","ä¸į æĭĺ","ĠW V","Ġle zen","ĠÑģ де","èĢħ 对","åĽĽ ä¸ĭ","Ġbo asting","代表 éĺŁ","Ġbad ges","æĬ± è´Ł","èĤĮ èħ±","Ġrecept acle","Ġবিঠª","å°½éĩı åĩıå°ij","_L OC","ØŃد اث","å®ĹæķĻ ä¿¡ä»°","åĬĩ æĥħ","ĠFO OD","Ġbourgeois ie","ĠDeriv atives","= X","L iv","re gex","Ġf iddle","ing a","âĢĻ âĢĿ","å¾Ĺ æľī","ement o","åħ¥ åij³","è®° åıĻ","ĠPl ugin","ä»ĸçļĦ å¿ĥ","}} .","ëĭ¤ ê³ł","å½Ĵ ä¸Ģ","=\\ {","Ġ×©× ł×Ķ","Ġré volution","اش تÙĩ","æī¿æĭħ çļĦ","ç¾ŀ æĦ§","িস à§įত","失ä¸ļ ä¿ĿéĻ©","Ġphysic ochemical","vari ables","ĠоÑĢи енÑĤи","Ġiso forms","ĠجاÙĨ ب","P iece","ä¸į éĹ´æĸŃ","æĪij åij¢","åĴĮ éĩij","ult imate","ook ies","Ġdes igual","便 è¦ģ","Ġwater melon","ĠShe ar","åĨĻ æĺİ","Ġза бÑĭ","ĠС по","asc als","ĠDec ide","ãģ© ãģĵ","Ġphr asing","欺 åĩĮ","ĠIg M","Ġév én","ĠC SC","Ġme este","Ġdi ocese","Ġpublic ación","ä»ĸ们 æľī","太 åı¤","/s how","cur so","Ġturn overs","å¾ģ æĪĺ","ĠÕ Ĩ","اØŃ ت","rect angle","Ġoch ron","Ġ×ª× §","Ġviv ir","\\+ ::","Princ eton","ĠÕº Õ¡Õ¿","ĠاÙĦظ اÙĩ","Ġl ucht","Ġre chts","ĠK ali","å°± å¾Īéļ¾","ru cht","Ñı ÑĤие","ä¸İ åĽ½å®¶","æķ° ãĤĴ","Ġet ch","åĪ« åĨį","Ġexperiment ed","é»ŀ çļĦ","ĠMod ulation","Ab raham","Ġré v","Ġadm iring","ĠBon aparte","ĠмеÑĤ ÑĢ","Ġتأ Ø«ÙĬر","Ġsprint f","ĠVulner ability","Ġm unisipyo","am d","ĠW orm","Ġright ful","åŀĭ èĤĿçĤİ","amb ar","Ġsing leton","ĠÑĢе ÑĩÑĮ","æľīä¸Ģ 種","Ġautom ating","æļĹ éģĵ","ä¼´ çĿĢ","èĤº ç»ĵæł¸","å®ļä¹ī ä¸Ģ个","-ex pl","nis one","ĠCyr il","Ġparaly zed","/ im","çļĦ æĽ´","Ġan sch","ĠSt rick","Ġar beiten","Ġco aster","çļĦä¸Ģ ä»¶äºĭ","åĵį 亮","ä¿¡æģ¯ æĿ¥æºIJ","ĠAcc eleration","çļĦå°ı å§ijå¨ĺ","æĶ¹éĿ© åĪĽæĸ°","ä¼Ļ é£Ł","ĠìĿ´ ëıĻ","ç»ķ çĿĢ","íĺ ij","Ġundert akings","âĻ Ĥ","Ġë°© ìĭĿ","ĠSequ encing","orr hea","Liber al","Ġ×Ķר×IJש ×ķף","Voc abulary","Ġìī ½","T rib","s ar","Ġn ive","il ogue","çĶ ¬","ا ÙĬا","æĿ µ","åıĺ 身","æ±Ĥ 羣","æ¼ ķ","马 å°¾","Ø· ÙĪÙĦ","ãĥ« ãĥĪ","ĠBel mont","äºķ åĨĪ","ĠDO ES","åIJĬ éĶĢ","Ġspir ited","åŃ£èĬĤ æĢ§","ãģ¨ãģį ãģ«","èĵ¬åĭĥ åıijå±ķ","åĮĹæµ· éģĵ","被æī§è¡Į 人",", /","t oggle","ĉ array","Ġt are","Ġha re","Ġj oked","ĠK ä","so b","ew a","ä¸ī ä¸ĩ","Cont emporary","ä¸Ńå¿ĥ ç»Ħ","Ġfavor ing","/p ub","Ġশ হ","天ä¸ĭ ä¹ĭ","ĠFA ST","ĠÐłÐ°Ñģ ÑĩеÑĤ","Ġcerebell ar","E b","P n","al te","ï¼ Ĩ","ĠP PE","åĴĮ åħ¶å®ĥ","çľĭ äºĨä¸Ģä¸ĭ","天 éķ¿","Ġcre ed","转 äºĨ","æĬĢæľ¯ æľįåĬ¡","ĠSc aling","اس طة","Service Impl","ĠاÙĦد ÙĪØ±","Ġutter ances","Ġдоба влÑı","ãģĿãģĵ ãģ§","Ġjerk ed","ĠLös ung",", String","Ä §","Ġk ü","Ġи ми","åĨį æİ¥","Ġsl umped","ç²¾ æ·±","ĠRead s","Ġpassion ately","imp in","éļĬ éķ·","åįļçī© éĻ¢","Ġv ise","Ġas part","ä¸Ń ä¸ĸ纪","per i","æķ° æľĪ","åĽ½å®¶ éĩįçĤ¹","Ġside walks","å®ļä¹ī äºĨ","éļ» æľī","ĠÑĨе пи","ĠRespond ents","altern ative","çļĦ å¼Ģåıij","ä¸Ģ çĽı","è¡Į ãĤıãĤĮ","å¤ļ 以","ob ut","äºĮ éĺ¶","Ġк оÑĤ","ĠSh arks","oint ments","æ¯į 线","æľĥ åľ¨","è¿· éĽ¾","Û° Û°","Ġdistinguish able","ì¦ ĺ","Jul ia","Ġcomport amiento","ĠYam amoto","Ġrodz ic","Ġ×IJ×ł× ©×Ļ×Ŀ","ĠобоÑĢÑĥд ованиÑı","åIJįæĢĿ ä¹ī","ä¸Ń è¶ħ","ip hatic","Ġcl ipping","å®ŀ æĵį","主 å¸ħ","ĠÑģ не","åħ³äºİ å¼Ģå±ķ","ç»§ç»Ń 说","åĸĿ å®Į","ĠÙ¾ ÛĮر","ĠоÑģ лож","-ch air","Ġrespe ito","ð IJ","çļĦ åij½è¿IJ","Ġst ör","ĠR b","Ġus hered","ï¼ī ï¼ĽĊ","ãģ® åķıé¡Į","å±± åı£","Ġvol gens","Ġsw ipe","ura h","æĿ¿ åĴĮ","ĠÙĪØ§ÙĦ Ùħع","Ñģа ми","è£ģ åijĺ","Ġfrag ility","Ġlit urgy","ĠÑģол не","Ġsebelum nya","ĠØŃÙĪ Ø§ÙĦÙĬ","R v","t ops","Ġo str","Ġst itching","ä¸Ģ çѹ","Ġal lege","çĶŁ åŃIJ女","Ġpo op","ร à¸ĩ","Ġrep ressed",":: {","ĠRet reat","à§ģ স","оп ÑĢеде","缺 æ°´","è¼ķ æĺĵ","æŃ¦åύ è£ħå¤ĩ","ĠPoss ibly","å¥łå®ļäºĨ åŁºç¡Ģ","% b","- Class","/ os","B ir","S AN","al em","Ġper fe","lic ensed","æľ¬ æĿ¡ä¾ĭ","Ġass uring","Ġinv Ã¥nare","èĢģ å°ij","交 éĶĭ","ĠSh oot","Ġsuper star","Ġprote g","åĩºäºĨ éĹ®é¢ĺ","Ġmulti player","uv an","Ġthick ened","Ñīа ÑİÑĤ","-off ice","Ġíķĺ ì§Ģë§Į","Ġtorn ar","製 åĵģ","å¼§ å½¢","ĠÑģооÑĤвеÑĤ ÑģÑĤвÑĥеÑĤ","Ġmurm ur","Ng Module","A ber","c math","en ough","ï¼ ¯","çļĦ åĬ¨æĢģ","ĠC ID","æīį åĪļåĪļ","太 éķ¿","åĨĽ æĸ¹","åĨĽ èΰ","Ġка нализа","åį· åħ¥","Ġز ر","æļĤ æĹł","å¾Ī好 çľĭ","龸 主","óg icas","çIJ³ å¨ľ","ĠRon nie","ĠRud olph",".Buff eredReader",": What","oll is","Ġher man","å¸Ĥ åİ¿","St ructural","两 款","æĬ¤ æłı","ż enie","(m igrations","Ġsin on","æļĤ ä¸Ķ","æĻ¨ æĽ¦","ĠEss en","Ġsou venir","个æľĪ åīį","å·¡ èĪª","ĊĊĊĊ ĊĊ","Ġà¦ī à¦ļà§įà¦ļ","èĭį 穹","ĠоÑĤно ÑĪении","Ġdimin ishes","×ķ×Ĺ ×ķת","æĺĬ 天","Ġла боÑĢа","ĠاسÙħ Ùĩا","çIJĨ论ä¸İ å®ŀè·µ","[ q","\\ d","r ins","ĉ local","ig y","ä¸Ģ æ¡Ī","Ġsc ap","åįģ æĿ¥","头 åĴĮ","åıĹ ãģij","Ġد کتر","æĿ¾ é¼ł","çī¹åĪ« åĸľæ¬¢","Ġforce fully","è¸ ±","å¨ģ æħij","ãĤ¹ ãĥŀ","ä½³ ä½ľ","Ġoptim isation","æµ® éĽķ","ா஠±","Ne u","ĠиÑģп олни","Ġabund ances","ĠGraph ical","Ġpap al","Health y","Ġante cedent","ãĤ± ãĥ¼ãĤ·ãĥ§ãĥ³","_ attribute","re ward","Ġo int","Ġh indsight","em ade","ĠK eg","èĥ½ åģļ","Ġtr zy","eb an","ĠBra ve","াব ার","ĠìķĬ ê³ł","ัà¸ģษ ะ","×Ļ×Ĺ ×Ķ","ä¸Ģèµ·æĿ¥ çľĭçľĭåIJ§","મ ાàªĤ","èµ·éĩį æľº","iax ial","j ia","p ital","z é","ol ts","ä¸Ń éĵģ","ĠTh ore","è½ ¶","ä¸ŃçļĦ æķ°æį®","åħħ å̼","ĠPol k","ç¿» 天","Ġepis odic","ĠNav ajo","ãģĿãĤĮ ãģĮ","slags verk","$ x","S Z","z illa","he ar","éķ¿ å»Ĭ","Ġdep ictions","便 æĬĬ","ãĤĴ åıĸãĤĬ","λ ιο","Ġcolor less","ÙIJ ع","ĠÑĦи ÑĢ","ĠSeg undo","ĠJacob son","ĠGuard ians","é¡· åĪ»","å·§å¦Ļ åľ°","ĠGob ierno","Ġfoc ussed","ĠвеÑģÑĮ ма","F x","çļĦ è¿ĻäºĽ","çļĦ 羣æŃ£","大 éªĤ","). \\]ĊĊ","Ġ' ('","æıIJ è¦ģ","åħµ æ³ķ","ан ÑĤа","æīĺ å°¼","Trans former","Ġconver ging","ĠCare fully","nam ely","å¦Ĩ 容","åıijè¡Į 人","Ġster ling","_pro b","ĠاÙĨت ÙĤاÙĦ","ĠHom eless","elect ron","ĠÑĪи ÑĢи","ĠDVD s","s burg","Ġa fl","Ġpro medio","åѦ 鼷éĶĭ","åİŁ æłĩé¢ĺ","Ġس اخ","åħ³äºİ åį°åıij","åij¼ æ°Ķ","æŀ¶ åĬ¿","模å¼ı åĴĮ","第äºĶ å±Ĭ","ĠÃ¥ rs","Ġल ाà¤ĸ","ÅĦsk iej","ütz ung","Ġdang ling","è·Łåħļ èµ°","\" D","R b","_ --","re ter","ĠS auer","ol ateral","ĠC iting","ĠH NO","ine phrine","å°ı éģĵ","æ´ µ","çĦ ¼","ĠRe com","ann an","é¡¹çĽ® å®ŀæĸ½","ĠPol ignac","ĠAg ust","Ġà¦ķ াল","æ²»çĸĹ æĸ¹æ¡Ī","ĠST S","-H all","ĠPo ole","éĺŁä¼į çļĦ","Ġгов оÑĢ","ĠStruct ured","Ġnoss os","Ġsinter ing","ĠP EP","Ġnum érique","åĬĽ è¡Į","åĬł ç´§","æŃ£ å®Ĺ","åıĺ æķħ","à¹ĥ à¸ģล","Ġten ets","ÐĴ е","åľ¨åľ° ä¸ĭ","Ġroz hod","ä¾Ŀæ³ķ è¡ĮæĶ¿","ĠHon our","Ber ry","Ġcoven ants","æ·±åĪ»çļĦ åį°è±¡","G host","L ER","Ġn ay","æķĻèĤ² èµĦæºIJ","ä¸įåIJĮ ç±»åŀĭ","èᝠçIJĨ","ĠCommun icate","æĮĤ åı·","Ġcateg oria","çļĦæīĭ ä¸Ń","}\\, =\\,\\","Ġон лайн","ĠImmun ity","Character istics","åĴ§ åĴ§","çļĦ åĩº","äº µ","ot helial","为 广大","Ġj ihad","æĸ¹ 设æ³ķ","å¼Ģ åºĹ","ale za","Ġmed ico","éĢī æ´¾","Ġع اÙħÙĦ","è´£ æĢª","à¥įर à¥ĭ","Ġunfair ly","ĠCort ex","anj utnya","åιéĤ£ éĹ´","Ġpromulg ated","ĠValu ation","æ¯ĭ庸 ç½®çĸij",") ###","- sequence","Ġh iber","åľ¨ å¸Ĥåľº","Ġqu ÃŃ","Ġag rÃŃ","æ¶Ī éϤäºĨ","Ch arge","è¿ŀ ç»ĵ","çŁ³ åĪ»","ga an","ĠÙĨ ص","ĠTra its","临 ç»Ī","-C al","ĠSpec ify","ç²ĺ ç»ĵ","ĠDam ian","ĠSwitch ing","é¦ĸå¸Ń æī§è¡Įå®ĺ","Ġcamou flage","- raising","B esch","ĠT bsp","ub ro","lic es","ä¸İ å®ŀéĻħ","ĠEx act","éĿĴ èĹı","List ening","ĠIr regular","ĠÐĿа Ñģе","çī§ æ°ij","Ġ×ŀש ×Ķ","Ġréfé rence","+ |","G em","Ġn º","åľ¨ 两个","ĠF isch","ĠU rol","å°Ĩ éĢļè¿ĩ","æķħ åIJį","æ¯į çα","ĠоÑĤ пÑĢав",".c ity","ת ×Ķ","ä¸ĵä¸ļ åIJĪä½ľç¤¾","ĠØ¢ ÛĮا","å®ģ å¾·","اÙģ Ø¹","ĠOh m","ĠWild cats","åŁĭ èij¬","ĠLiber als","飵 å¾ĭ","य à¤Ĥ","å¾Īå°ij æľī人","१ ९","Ġtroubles hoot","Ġbubb ling","···· ··","ĠMoment um","Ġpenn ies","I v","ur so","å¾ ³","æĪij 第ä¸Ģ次","ä¹ĭ é¡ŀ","åĪ© æĸ¯","att rs","ç»ıèIJ¥ èĮĥåĽ´",".L ayout","Ġmal ice","âĶ ¤","ĠAS AP","Ġਠ¬","Ġanalyt ically","Ġgrat ification","ĠGib raltar","¨à¯įத à¯ģ","ĠPant her","? ##","ou ng","çļĦ æĬĢå·§","åľ¨ ä¸įæĸŃ","好 åѦ","åºĶ éĩĩåıĸ","Ġwork station","Ġbl inding","æĮĩ æ´¾","-s ervices","-l ined","For Key","ĠPal ma","âĶ ¬","Ġparallel ism","åĽŀæĿ¥ åIJİ","Part ly","Ġव à¥įय","ä¸ĢçϾ ä¸ĩ","ipo ises","Ġtá» ±","d ust","Ġr st","为 è§£åĨ³","åĴĮ å®ŀæĸ½","对 åIJĦç§į","Ġpar abolic","au ro","å¸Ī å¾Ĵ","ĠPl att","éĸ ¥","à¸Ĺ à¸Ńà¸ĩ","IT ATION","ĠPre ferably","Ġج ÙĦ","âij ¦","Per form","èĤĮ çĺ¤","鼻 åĬĽ","ĠÐĵ оÑģÑĥдаÑĢ","èĤĿ åĬŁèĥ½","åĩĮ éľĦ","ãĥ³ãĥ IJ","æĥ³è±¡ ä¸ŃçļĦ","geb iet","éģ® çĽĸ","Ġthé orie","Ġfict itious","ĠindÃŃgen as","ĠC orte","qu ired","å¼ ĭ","æĪij 说çļĦ","å¼Ģ è·¯","ç´ Ĺ","Ġmay onnaise","ç«ĭ ãģ¦","ON SE","man aged","çļĦ人 身","满 è½½","å¾· æĦıå¿Ĺ","\\( (","Äį ast","座ä½į ä¸Ĭ","-organ ized","Ġzeb rafish","ĠlỼ p","ĠC TR","åľ¨ æĿİ","ert ig","Ġob ej","-p ol","ĠMar ried","顾 åIJįæĢĿä¹ī","ĠMod ular","Ġpow sta","би ÑĢ","请æ±Ĥ æĿĥ","ĠIR R","pers ed","åĪĩæį¢ åΰ","ĠDivid ed","_trans form","Ġgehö ren","ẫ n","s ender","Ġd rier","Ġm ounds","å¸ ļ","Ġat end","ah un","Ùĥ Ùħا","éŁ³ 楽","IT IS","èĨ ©","çļ® çĸ¹","Ġau ft","ãģ¦ãģĦ ãģı","åİĭåĬĽ åĴĮ","Ġdia per","ĠдеÑĤ Ñıм","çŃī级 çļĦ","ä¸Ńåįİ人æ°ij åħ±åĴĮ","ĠScholars hips","ὸ ν","ĠOccup ation","Psych ological","Ġfilos ofia","â ½","Ġn ailed","ĠM ina","大 å«Ĥ","ob u","ĠRe levance","ç®Ĺ åŃIJ","认 å¾Ĺ","ä¸įè¦ģ 让","Ġorganiz ação",".B undle",".n odes","Ġble aching","èªį å®ļ","Ġpub s","çĶ£ æ¥Ń",", Skip","- App","s ell","Ġw raz","Ġw ÅĤad","est h","è¦ģ åIJij","Ġcomp ara","ert ime","Ġam ines","Ġset Name","Ġdep an","æĬĢæľ¯ æĶ¹éĢł","Ġпо ÑıÑģ","Ġinfl ows","log ged","ло ÑĤа","æĺ¯ä¸Ģ个 人","奶 çīĽ","èµĦæł¼ å®¡æŁ¥","ĠMC Q","ĠConvers ations","Ġconvention ally","æľīçĽĬ çļĦ","Ġinoc ulated","ĠOng oing","< Character","æľī ä¿¡å¿ĥ","èĥ½ ç»Ļ","Ġcl ans","åħ¬ çīĽ","ĠWe il","æĺ¯ä¸Ģ 缴","åľĨ åľĪ","Ġন à¦¿à§Ł","-le arn","à¶ ľ","æķĮ 对","ç²® æ²¹","ĠSav iour","ĠPersonal ized","æĻĴ 太éĺ³","ãĤª ãĥª","isto itu","èĬŃ èķ¾","as un","æĪij å½ĵçĦ¶","ä¸ĭ å¿ĥæĿ¥","æĪIJ èĻ«","没æľī åĨį","åIJĦ è·¯","ĠBl iss","éĺ¿ å¼¥éĻĢ","Ġarch ipelago","âij ´","åĨ¬ 天çļĦ","ĠText View","åķĨä¸ļ ç§ĺå¯Ĩ","åĮĸåѦ æĪIJåĪĨ","ĠاÙĦØ· عاÙħ","ĠMarg inal","}/ >Ċ",".O pt","Ġnic hes","Ġë°ľ ìłĦ","ĠTables poons","ĠSpart an","ĠIdi oma","X s","Ġn ip","æĪij åĪļæīį","Ġreg elmÃ¤ÃŁ","ç±» ä¸ĵä¸ļ","æ¸ħ æĸ°çļĦ","åĪĻ è®¤ä¸º","/m onth","ĠС Ðŀ","ĠCH RIST","Ġ׼ ׾×","ĠìłĦ 문","人ä¸İ èĩªçĦ¶","à¸ķà¸Ļ à¹Ģà¸Ńà¸ĩ","D j","ĠK abul","__ .","ĠCh aucer","Ġо ÑģÑĤанов","ton es","cz Äħ","Ġhar p","éŁ³ä¹IJ åѦéĻ¢","Ġbear ers","à¸ģระ à¸Ķ","Ġanx iously","ĠPAR A","Ġjul ka","Ġvrij gegeven","ĠT ren","为 éĿŀ","ĠJ OHN","对 è¿Ļä¸Ģ","de an","åĮĸ çĺĢ","ins ip","li ore","åij¨ åħ¨","Ġз в","Ġvo j","ĠبÙĩ د","Ġtro ppo","çĢ ı","Op in","Ġmars hes","ĠÑĤек ÑģÑĤ","-ban ay",")=> {Ċ","Ġw arl","em ics","ay ne","ĠB aking","ä¸Ĭ åºĬ","Ġun ambiguous","对 ç¾İåĽ½","äºĮ èĢħçļĦ","-t a","å·²ç»ı ä»İ","æľīä¸Ģ å¥Ĺ","à¸Ľà¸£à¸° à¹Ģà¸łà¸Ĺ","Ġformal dehyde","Ġesp èces","fin ally","ĠÎļ ο","ĠоÑĢгани зме","Ġ×Ķ׊׾","åĩºåħ· çļĦ","Ġcentrifug ation","èľķ åıĺ","ĠEuras ian","ĠBrune i","T ed","ĠRe peated","Ġinv is","åijĬ çϽ","-l aws","IT O","æĺ¥ èĬ±","uj ud","表çݰ åĩºæĿ¥çļĦ","ç»ıèIJ¥ æĢ§","ĠMet ast","Ġunf it","Ġfol ate","Ġয à§ĩমন","ĠGar ner","ycz aj","Ġ---|---|---|---|--- |---|---","Ġশর à§Ģর","Ġzape wn","ĠобÑĥÑģ лов","sime q","D LE","Ġs isi","as us","op ically","Ġman ter","å·² éªĮè¯ģ","项 çļĦ","åħ¬åı¸ 竳ç¨ĭ","æŀĹ ç«ĭ","org ung","ãģĮ ãģĦ","ç½Ĺ åħ°","Ġµ P","èĤ¯å®ļ äºĨ","EG A","èĺ Ĭ","\\% }={","Ġmobil ized","} italic","ä¸Ģ å¸Ĩ","ve u","oc cer","Ñĩ ÑĤо","åĮħ åİ¢","æ·± æµħ","éĢĻ äºĭ","ĠÑĦ ÑĢон","éļIJ å±ħ","建çŃij æĿIJæĸĻ","羣æĺ¯ 个","impl ies","ĠOUT PUT","íģ ´","Ġmidd el","Ġbottlen ecks",". validate","? -","el ige","åľ¨ åıijå±ķ","ä¹Ł å¾ĹåΰäºĨ","èĩª 个","äºĮ è¦ģ","èµ° é«ĺ","eter ies","Ïħ μα","ĠPass enger","à¹Ģล à¸Ĥ","Ġnaj bardziej","гов оÑĢ","Ġкни га","princ ipal","Tow ards","ĠGuerr ero","- ple","/ ****************************************************************","B ug","ou pe","çļĦ å½±åŃIJ","ĠP into","Ġpr iceless","è¿ĩ 人","ah ir","åĨħ çļ®","ĠÑĥ веÑĢ","Ġsi amo","Ġpay check","Ġbar racks","Ġist itu","çĽ¸å¯¹ åºĶ","éģĭ 輸","ĠRest ricted","ĠBarb ados","ĠLE AD","Ġpenn ed","Ġعبار ت","Ġciel o","D ocket","s ens","ĠT ight","Ġqu inoa","ĠZ em","Ġexpl ique","ĠX AF","-c oupled","Ġinc end","åħ¶ä¸Ń æľĢ","oe lectron","Ġdefault Value","åĪĢ åĪĥ","-L ife","å°±åĥı ä¸Ģ个","ĠBern oulli","ĠOpp en","Ġcres cent","Ġlawn s","Respons ibilities","Ġt ik","Ġo cho","Ġl ại","ĠC og","èĥ½ 满足","ä½ľ åģĩ","ĠV ater","身 çĤº","åģļ åĬŁ","åĽłä¸º å®ĥ们","åIJ¦ åĨ³","Ġиз лÑĥ","ĠPre fix","Ġhot spot","å¦ĤæŃ¤ ä¸ĢæĿ¥","بر اÙĬر","Ġvill es","ĠBur ial","贷款 åĪ©çİĩ","SD L","ริ à¸į","ĠмÑĥÑĪ ÐºÐ°","Ġë¸ Ķ",") L","/ time","ĠB ibl","ç͍ å¾Ĺ","Ġ[ :","éķ Ĥ","éķ¿ æ²»","Ġco iled","åĪĻ ä¸į","ĠLe it","à¸Ķ à¸Ńà¸ģ","ãģĮ è¦ĭ","ç»´ å¤ļ","çĶ» éĿ¢çļĦ","Ġpod m","Ġsou venirs","ãģ¡ ãĤĩ","ĠÑĦоÑĢ Ð¼","ĠBru ins","Ġinaug uration",", _ĊĊ","v end","Ġk lik","ä¸Ĭ æīĢ","åĪĨ è¯į","è¿ĩ æĹ¥åŃIJ","ä¹ĭ èĭ¦","æĪĸ 个人","ä»» åĩŃ","ĠEx terna","ØŃ ÙĨ","ĠSp iegel","Ġtri á»ĩu","æĢİä¹Ī å°±","ç®Ģ æĬ¥","ãĤ¤ ãĥ¡ãĥ¼ãĤ¸","ологи ÑĩеÑģкой","Ġà¦ıà¦ķ à¦Łà¦¾","Ġdele terious","Ġgon ad","Ġgarn ish","Ġnewcom er","omyc etes",") import","he il","pl aat","te aching","ount ains","Ġо зна","äºĮ 人çļĦ","the ory","åŁŁ çļĦ","å¯Ĩ 室","-w alled","-S em","ÙĬر ÙĪØ³","sl ag","ĠاÙĦب رÙĬ","othe lium",".pro perties","_H OST","cler otic","ĠìĦ¤ ì¹ĺ","-bal anced","ãģ»ãģ¨ ãĤĵãģ©","ãĤµãĥ¼ãĥĵ ãĤ¹","q h","Ġs own","Ġp iers","im ulation","ä¸Ģ å¼ı","ĠH eller","Ġk arma","ĠG elijk","og rad","ĠZ immer","ม ัà¸ģ","Ġfil hos","ĠAb rams","çĹĽ é£İ","Be ijing","Ġquart et",".Set Active","ĠклаÑģÑģ ов","- letter","ä¸į å¹³çŃī","Ġsa a","çī¹ éĤĢ","ethod ology","ç»ıæµİ åĪ©çĽĬ","Ġver wij","-p aying","äºļ éĩĮ","bb a","éĨĴ 缮","Ðľ Ðĺ","æĿĥåĪ© ä¹īåĬ¡","污æŁĵ çļĦ","Ġbreed ers","Et at","ĠMöglich keiten","Ġentspre chend","åħ¨å¿ĥåħ¨ æĦı为","Ġdelir ium","b art","i ou","ed ish","Ġin om","ell ung","建 åζ","Ġ) )Ċ","ĠCor ruption","Ġpolic emen","ĠpÅĻ ij","æ£ĭ çĽĺ","ä¸Ģä¹Ŀ åĽĽ","æĤĦæĤĦ çļĦ","Ġвод о","ĠاÙĦعÙĦ اج","(Build Context","- private","- educated","m om","äºĨ æķ´ä¸ª","éĢļ è¾¾","на Ñģ","æłĩ 示","é£İ åįİ","ä¸ĵ çıŃ","ä¸ļåĬ¡ æµģç¨ĭ","Ġন তà§ģন","æ¾³ éĸĢ","ĠFac ial","å²³ éĺ³","è¾ĥ好 åľ°","Ġdimensional ity","æľīèī² éĩijå±ŀ","Ġerk ennen","k Hz","iz acji","Ġpl azo","cl osures","è¡Į éģĵ","åĨ³ ä¸įèĥ½","åij¨ éģŃ",".l ink","äºij é£ŀ","ĠGet All","æĸĩåŃĹ çļĦ","Ġми ÑĢов","ĠHa ar","Mar ie","åµ ĺ","Ġalg um","Ġconve ctive","Ġpremi ères","ĠDh arma","ĠÑģоÑģ Ñĥд","Ġzeg t","* =","ĠV AN","å·¥ å»ł","æį® çĤ¹","Ġcent roid","ĠBl anche","ĠMe cca","åı¥ åŀĭ","åºĶç͍ é¢ĺ","æľºåħ³ çļĦ","Ġshoot ings","å±¥ 约","ĠFoot note","æĻ´ æľĹ","åĨħå¿ĥ æ·±å¤Ħ","åĴĮ社ä¼ļ åıijå±ķ","Af rique","Ġfebru ár","Ġsic urezza","@ Data","O y","r át","se ek","Ġex alted","ĠSt ochastic","è· »","车 åºĵ","................ .....","ãĤĴ ãģĶ","ext rem","-h ot",".e lement","ĠÙĪØ§ÙĦ Ø®","ĠAnt imicrobial",".G roup","ĠBur den","Ġrh omb","figure d","à¹Ģหมาะ สม","Ġà¦°à§Ł à§ĩà¦Ľà§ĩ",". change","Ġc erve","çļĦ æĬĢèĥ½","ĠM ane","ä¸Ĭ 没æľī","å®¶ æĶ¿","å°ı èι","Ġп ож","ob us","Ġgra pple","è¿Ľè¡Į æ²»çĸĹ","Ġна боÑĢ","éŁ ĭ","ĠÑĥ кÑĢеп","æĸŃ è¨Ģ","è¿ĺæĺ¯ æľīäºĽ","ĠMar quis","Ġmass ac","åĨ² åIJij",".f igure","Ġseg ala","Ġdeep ened","سÙħ اء","Part ner","æ£ĺ æīĭ","Ġeig entlich","Ġwrink led","ÑģÑĸ м","Ġbetek ent","Ġaand acht","G b","ĠS US","Ġv áºŃt","ä»ĸ åΰ","cre ator","åIJij 人æ°ijæ³ķéĻ¢","Ġvari ational","åıį 常","Ġimp ost","Ġens imm","Ġindust ria","åİĨåı² åĴĮ","åĨ³å®ļ äºİ","vo ices","ĠHuman itarian","é³ Ĺ","躬 身","Ġvoork omen","è¿·ç³Ĭ ç³Ĭ","H olly","Ġw oo","ĠL W","åĮĹ æ¬§","-m oney","ĠAb uja","éĢģ 礼","ĠÏĦ ÎŃ","asc hen","Ġinvestig adores","å°ļ ä¸Ķ","çݰ代 çļĦ","Ġamb as","Ġgar is","াস à§įথ","Ġprovision ing","Ġinflation ary","ró ci","ĠسÙĨ ÙĪØ§Øª","ிà®ķ ளà¯į","ĠìĹŃ íķł","Ġlun ches","ĠZag reb","Ġenrol ment","S ugar","Ġn oche","ĠT we","ain i","Ġch ce","ĠO ST","ä¸Ģ个 æĺŁæľŁ","Ġdef ences","Ġmet ody","级 以ä¸Ĭ","社ä¼ļ ç¦ıåĪ©","åĪĿ åĪĽ","la ught","Ġhom ozygous","ĠIr rigation","Ġà¦Ĺ বà§ĩষ","人åĬĽèµĦæºIJ åĴĮ社ä¼ļä¿Ŀéļľ","Ġга ÑĢан","Ġlumin ance","أس ÙĬس","ĠMcL aren","Marg aret","? >Ċ","å°ij éĺ³","ĠÑģÑĤ Ñĥп","æī¶ æīĭ","Att orney","ĠSEC OND","Ġning una","ĠженÑīи нÑĭ","ĠÙĨÙĪÙģ Ùħبر","' /",". Identity","if o","æĪij è¡Į","æĹł 罪","à¸Ļ าย","ĠAn arana","ÙĤ ÙĬØ©","Ġer en","ات ر","è´¢ æĬ¥","ás z","ocal orie","ä¸Ŀ çļĦ","ĠPlan ets","Ġfuel ing","িত à§įব","Ġrod zin","ĠSequ ences","Ġcherche urs","æłĢ åŃIJ","ĠIMD b","t ouch","at ia","çļĦ æ©Łæľĥ","ĠP ru","ĠP ari","ĠG ett","oci ative","æĢ» æĶ¶åħ¥","ĠAd jectives","论 æĸŃ","èᝠåѦ","æĻ¯ å¾·","å®Įåħ¨ 缸åIJĮ","åĪº æĿĢ","ÙĪÙħ تر","ĠSand wich","ĠRest rictions","ĠNE VER","Ġmaj d","ĠCre te","ĠتÙĨ ظ","ĠпÑĢимен ение","è§ģè¯ģ äºĨ","Ġcease fire","Ġgarant ir","pla ats","Ġимп ÑĥлÑĮ","(saved InstanceState","id imensional","Ġdis mal","å¤ļ æĿ¡","ĠاÙĦ اص","ix er","åĪ« åħĭ","Ġhum aine","管çIJĨ 人","Ġplant ar","ä¸įè¿ĩ åľ¨","æ²Ĵ 辦æ³ķ","iner ies","æ´ª æŃ¦","çĽĨ æĻ¯","è¿Ŀæ³ķ è¿Ŀè§Ħ","çĴ° ä¿Ŀ","äºĨåĩł åı¥","å¤ī æıĽ","ä½łèĥ½ ä¸įèĥ½","Ġvál to","Vict or","Decl aration","Ġg utter","ä¸Ģ æľĥåħĴ","ĠW inters","ĠN ÄĽ","ĠJ UD","Ġcont enders","å¤ļ ç»´","å¹¶ ç»ı","æĶ¶ ç¼´","容 许","Ġassoci ating","é¾Ļ åĩ¤","à¸Ī ัà¸ĩหวัà¸Ķ","Ġpersonal ised","æłĩåĩĨ å·®","è´¥ åĿı","ĠOrgan ized","ĠLiter atura","Ins ect","Ġincom prehens","Ġshar pen","ĠNak amura","è¿Ŀ约 责任","ĠоÑģнова нии","Ġphotoc atalytic","æĸŃè·¯ åύ","ĠMaurit ius","G allery","S ail","d ogs","k ick","ĠE FL","ĠO LED","ä¹Ł ç¡®å®ŀ","Ġsub divisions","æŃ¤ 为","åı° å¼ı","agn es","ست خدÙħ","Ġpay er","讲 ä¹ī","ãĥ¼ ãĤ¿ãĥ¼","ĠвÑĭ да","æĸĩ竳 ä¸Ń","Class ifications","Ġegg plant","Ðľ Ñĥ","(l ines","Ġá prilis","ĠAus bildung","ĠÑĢаÑģп олага","Ġnaj m","Ġнеп ÑĢе","楷 模","ĠоÑĤмеÑĤ иÑĤÑĮ","Ġindeterm inate","èİħ 临","C anc","R m","Ġn ug","ĠR ated","Ġex iled","ĠIn uit","ä½ł èĩªå·±çļĦ","çĿĢ è£ħ","Ġ' :","æ°´ åİ¿","Ġcol ossal","åIJį åŁİ","ĠAl ly","Ġvis ión","-m ails","ĠÙĦ ÙħÙĨ","ãĤĴ 示","æĻļ éĹ´","-h arm","åı¦å¤ĸ ä¸Ģç§į","ĠEX AMPLE","èŁ Ĩ","Circ uit","ר׼ ×Ļ×Ŀ","§ ×Ķ","il ik","ĠD are","ĠG én","æ°´ è·¯","å¦Ĥæŀľ å°Ĩ","Ġfac ie","ĠAt oms","èĢĮä¸Ķ è¦ģ","åŃ£ ç¯Ģ","å¯Ĵ æĦı","Test Method","ä¼Ĭ åĪ©","ĠIr land","æ¸IJæ¸IJ çļĦ","ས à¾","è¿Ļçķª è¯Ŀ","Ġplasm ids","G ordon","Ġp oni","ow ulf","æĺ¯ æīĢæľī","ä¸į åľ¨äºİ","个 æĢ§çļĦ","åģ İ","ob enz","Ġ\\( <\\)","Ġinvest s","å®īåħ¨ æ£ĢæŁ¥","Ġfr ase","é¥Ń ç¢Ĺ","ĠCong ratulations","横 è¡Į","mosp heric","веÑĢ Ñħ","ĠCy an","Ġbrown ed","åįķåħĥ çļĦ","èµĦæł¼ çļĦ","Ġheter ozygous","Ġreson ator","à¹Ģà¸Ķ à¹ĩà¸Ī","Ġ주 ìļĶ","ĠGard ening","åѦåijĺ 们","Ġmim icking","Ġcoraz ón","at ifs","åѦ æĹ¶","éĩı 产","া à§İ","éħ IJ","å¢ĥ åĨħçļĦ","ĠVer änder","ĠAnal ytic","/j ava","Ġcomfort s","Ġsac erd","Spec ifically","çŃĭ èĤī","éͦ è¡£","Ðij а","Ġobst etric","ĠÅĽ mier","ç½Ĺæĸ¯ ç¦ı","æł© æł©","/ Getty","Ġtrans plants","ç½ij åħ³","ä¹Ŀ äºĶ",".t rain","ĠBo let","ĠSE K","Exper ts","Ġmultic ast","Ġdic embre","ĠÚ©ÛĮ Ùģ","举èİŀ å¸Ĥ",".Reg ister","ĠÑĢеда к","k owe","Ġm imo","Ġto eg","ĠM ab","åı¯ä»¥ æľīæķĪ","ier ende","ни на","女 èģĮå·¥","ĠØ£ ÙĤÙĦ","ĠÙĨ ÛĴ","ç¨İ çļĦ","纳 åħ°","ä¸ī个 éĺ¶æ®µ","ĠговоÑĢи ÑĤÑĮ","ĠCondition ing","åŃIJãģ©ãĤĤ ãģŁãģ¡","( Base","s ong","ĠC URRENT","Ġj ut","Ġref inery","æĺ¯ä¸Ģ åĪĩ","åIJ« ç¬ij","-in creasing","Ġwer kt","æĦŁåΰ å¾Ī","\\, -\\,","มห าวิà¸Ĺยาลัย","ä¹İä¹İ çļĦ","öffent licht","ç»´ä¹Ł 纳","A chie","Ġo ple","ĠT z","ĠE HR","Ġse cluded","åĴĮ 弦","Ġsp i","_{ +","管çIJĨ è´¹ç͍","Ġpot rav","以ä¸Ĭ ãģ®","ellig ent","ĠOff ered","Ġsto pp","ç£ģ 带","หม ูà¹Ī","ãģªãģı ãģª","Ġসà¦Ĥ à¦ĸ","çļĦåı£ æĦŁ","Ġenseñ anza","? âĢľ","d ery","ĠH wang","ĠW F","ĠL INK","Ġr ê","èĢĮ åĬ¨","äºĶ æĮĩ","ST ATE","Ġdev rait","çĥŃ èº«","rop hes","èĮ¶ åĩł","æľ« ä¸ĸ","亦 çĦ¶","ัà¸Ļ à¸Ĺ","ĠìĥĿ íĻľ","æ®· åĭ¤","Ġtransc endent","Ġдиа па","èı© èĸ©","饱åĴĮ 度","Israel i","ĠDEP ARTMENT","Ġتبد ÛĮÙĦ","æĥĨ æĢħ","\\ subset","Ġar dent","çŃī æ¯Ķ","Ġpres ided","rag ue","çĨ ¨","ĠGu illaume","éħ¸ åĮĸ","ко ÑĢа","çĨŁ äºĨ","Ġпод ви","驾 çħ§","ĠMiss ions","Ġö k","ĠMichel angelo","èĵ¬ èݱ","cze ÅĦ","ĠпÑĢоводи ÑĤÑģÑı","ĠGalile e","ĠRefuge es","ĠرÙģØª ار","ĠRei he","ĠH eng","Ġcom and","åĴĮ æľīåħ³","æĿ¥ è§£éĩĬ","Ġme ek","å¿ĥ çľ¼","Ġro ver","天 主","æĽ´ 没æľī","ä½İ è°·","çŁ¥éģĵ ä½ł","广 为","Ġпо Ñģад","(n ames","Ġmother board","â̲ =","åİļ éĩįçļĦ","_F ORM","Ġдли нÑĭ","Tu ple","ĉ go","Ġd ine","est ablished","éģĵ éķ¿","che ts","ĠUn ions","ĠWe ed","å¸ĥ ä»Ģ","éĩįè¦ģ ãģª","IJ× Ŀ","ĠMac donald","sen en","Che ers","ĠTerrit orial",". State","/ ajax","s ud","z nego","re levant","ä¸Ĭ æĿ¥çľĭ","ç͍ ä¾Ĩ","Ġë ¥","arm en","ĠIs ot","Ġdirect eur","åħ·æľī è¾ĥ强çļĦ","à¹Ĥ à¸ģ","ç¢İ è£Ĥ","inh os","ĠDar ling",".find One","ĠTob ago","Ľ ×ķף","ä¸į éĢĢ","Ġal iquot","åĩº æģ¯","é«ĺ ç²±","æľº åύçļĦ","Ġpat ië","æīĵ æĸŃäºĨ","åħī åįİ","vis ibility","çĶľ çļĦ","à¥ģ ष","Ġalert ed","******************************** ****************","ĠMans field","Ġfulfil ment","èĩªçĦ¶èĢĮ çĦ¶",". options","P ent","Ġg ara","æĹ¶ åı¯ä»¥","å°ı èĢĮ","Ġro tting","Ġthere from","å¾Ī æ£Ĵ","ä½Ĩ å®ĥ们","Ġна ÑħодÑı","åĩĨ 许","è¾¹ åĮº","åı¯èĥ½ å°±æĺ¯","-p atient","åģı è¿ľ","Ġrect ify","ĠNumer ators","Az alera","Ġjeopard y","Ġardu ous",". where","S ustainability","s orry","Ġc fg","ĠA ver","ĠP ud","âĢĻ ),","ĠThe odor","ĠL ás","Ġun ison","æĢ» æľīä¸Ģ天","Ġrep atri","ä½Ĩæĺ¯ 没æľī","è¿Ļä¸Ģ ç±»","ä¹° 个","æĶ¿çŃĸ æİªæĸ½","å¥ĸ åĵģ","丰 åİļ","åıĸå¾Ĺ æĪIJåĬŁ","Ġ{} \",","Ġà®ļ à¯Ĩ","è¿Ļ段 è¯Ŀ","Foreign Key","xj zy","ĠмÑı г","å»ļ æĪ¿","ãĤ°ãĥ« ãĥ¼ãĥĹ","à¹ģà¸Ļะ à¸Ļำ","g na","çļĦ åΤæĸŃ","ĠS SP","ur ative","os m","åľ¨ èĢģ","ĠF ilos","ĠW ieder","ib ir","Ġп ожа","ó x","Ġbl inds","à¸Ļ าม","Ġer ection","-p at","åģ¥ ç¾İ","æŀ¶ 空","Ġorganiz es","Ġcontroll o","othe k","ĠDoctor al","å°ıå¿ĥ翼翼 çļĦ","大çIJĨ çŁ³","æŃ¼ çģŃ","ĠMonter ey",". ''","c alf","¾ ¸","ĠC ullen","Ġpro getti","为 æĪijåĽ½","Ġall usion","ĠCh ic","æľ¬ æĸ¹","Ġocc idental","Ñĩи ÑĤелÑĮно","Ġма га","Det ection","رÛĮ اÙĨ","ï n","ĠImpro vements","Ġrum ours","ĠEsp ÃŃ","éķ¿æ²Ļ å¸Ĥ","åĩ³ åŃIJ","åıĹæ¬¢è¿İ çļĦ","ĠQuin cy","- und","Q W","ĉ id","ĠB atu","art hed","çŁ¥ ä¸įçŁ¥éģĵ","Ġadd er","Ġfind All","ÑģÑĤа нÑĤи","æŁIJ äºĭ","оÑģ новним","å¦Ĥä½ķ è¿Ľè¡Į","Sh ar","èįī åľ°ä¸Ĭ","触 ç͵","Ġimag inations","ĠHol m","inst ructions","Conf irmed","å°ģ建 社ä¼ļ","Ġstro de","Lu cy","ĠвÑĭÑĢаж ениÑı","Ġperts ona","itets data","( ~","O lymp","[ h","Ġ ÅĽci","ĠT SH","æĪij çľĭçĿĢ","åIJİ æīįèĥ½","天 åºŃ","Ġfl ares","ĠÎ ĸ","ĠZ ed","λ ει","ĠÑį пи","åĸ· åļı","èIJ¥åħ» ç´ł","æ¶Į åĬ¨","ĠPi aget","ध à¥įय","Ġendeav ours","ÑĨÑĸ ÑĹ","éħĮ æĥħ","ĠÙĪÛĮ ÚĺÙĩ","Opp slagsverk","ĠSiber ian",".std out","L is","M unic","_ role","Ġm TOR","il ization","ä¸Ģ éĴ±","ĠF IND","åIJĮ åľ¨","æľĪ èī²","ç² ³","ç»Ħ åĽ¢","让 å°ı","ç¥ŀ åĮ»","æµ· å²Ľ","arn s","For um","Ġpopular ly","åľ£ 女","æķ°åŃĹ è´§å¸ģ","å°±åľ¨ è¿ĻéĩĮ","ĠìĪĺ ëıĦ","ä½ĵèĤ² æ´»åĬ¨","/pro blem","Ġbull ied","ĠLenn on","Ġaccol ades","_ TRUE","为 çİĭ","du al","Ġad ored","Ġdes embre","å¹³ ä»·","St uff","æ±Ĥ ä½ł","ла Ñħ","åĮ» åѦä¼ļ","è¿Ļæł· å°±åı¯ä»¥","ĠÙħÙĨ ÙĩÙħ","-G u","×ŀ× ¨","ĠJoseph ine","ëĵľ ëĬĶ","çµ² 毫","Charl otte","Ġtheolog ian","ĠпоÑģÑĤ оÑģновним","æķĻåĬ¡ å¤Ħ","- Val","N ancy","k au","Ġa pe","Ġc ero","Ġf idd","ĠW ii","Ġch á»ī","Ġus ado","ĠK ami","åIJĪ éģ©","æŃ¤ è¯Ŀ","å±± ä¹ĭ","ĠSim ulations","æİĪ æĿĥçļĦ","алÑĮ ное","Ġë³ Ħ","è°ĭ æĿĢ","ä»¿ä½Ľ æĺ¯","ĠHun ts","ç¼ł 绵","ĠRA W","f ür","Ġd ips","Ġbe vor","ĠG overning","ä»ĸ æľĢ","Ġpre amble","å¤ĸ è¾¹","带 è´§","Ġб ок","-c le","ĠÚ© سب","IJ× Ł","è« ·","ç»ĵåIJĪ åľ¨ä¸Ģèµ·","ĠÅ ĺ","ĠAL SO","Check ing","æľŁå¾ħ çĿĢ","éĻķ åĮĹ","Ġà®ĩ à®°","ĠCharter ed","å¿ĥåĬ¨ è¿ĩ","ĠÑģеÑĢе ди","T IM","l ifting","ĠP ots","ĠG ord","ĠO X","ater ra","ĠRe ign","з мÑĭ","Ġmod ulates","èĥ ¯","èĩªå·± 没æľī","Ġmed iter","yl ate","Ġpa ediatric","Ġni pple","under stand","ĠGar rison","Ġzm ian","Ġhil ab","اÙ쨏 Ø©","ĠÙĤسÙħ ت","ĠÃŃnd ice","Scot land","æIJĸäºĨ æIJĸéłŃ","/autor itetsdata","A mer","ĠB ly","ĠH OT","æĹł æŃ¢",".s ite","æ¹ ®","æĬĵ èIJ½å®ŀ","ĠMal m","]) )ĊĊ","æ¯ı天 æĻļä¸Ĭ","à¤Ĥ त","Ġnan omaterials","���� ���","Fl oor","究竣 æĺ¯ä»Ģä¹Ī","Ġlocom ot","èĢĮå¾Ĺ åIJį","à¹Ģสà¸Ļ à¸Ń","( %)","_ contents","j w","im ely","Ġcon cerne","ong i","con es","åĪĨ å¤ĸ","-------- ---","红 è¡£","}} -\\","ä¸įè¿ĩ è¿Ļ","Ġing rained","×ľ× Ŀ","ÑģÑģ Ñĭ","Ġlif etimes","éĹª çĿĢ","Mat ches","âģ »","ạ t","ű s","elect ronics","-work er","w if","çļĦ 西","ĠR ies","ĠN CC","大 å®Ŀ","cl ip","çĤ¹ æĹ¶","åħ³ ä¸Ń","Ġjust ifying","ÑĤе ÑģÑĮ","Ġleg ion","主è¦ģ è´Łè´£","Ġport anto","Ġcat ap","ĠMac ron","Ġkne eling","lag t","Ġkle iner","Ġbou quet","ä¸Ģèµ·æĿ¥ çľĭçľĭ","Ġhandic apped","ì¼ Ģ",": X","d ialog","t imer","iv ian","ä¸Ģ 女","ä¸į å®ļæľŁ","åĴĮ 缮æłĩ","ĠPro state","èĢģ æľĭåıĭ","cent age","ĠAs king","Com plement","åĪĿ å¤ı","ĠPol ymers","оÑģ лав","æĥ¯ äºĨ","Ġкон ÑĨеп","Ġìĸ ¼","ĠLy rics","Ġتج اÙĩ","Ġdun que","Ġfonction nement","ĠPul itzer","Ġs ash","Ġw ort","Ġse jam","Ùĩ ÙĢ","æ³ķ åύ","Ġpre gunta","ĠCh ili","çľĭ å¾Ĺåĩº","天 éŃĶ","Ġz orgen","æľĪ åĴĮ","Ġdi aries","æĪĸ ç§°","ä»Ģä¹Ī éĥ½æ²¡","ç ons","ĠBl och","åįĥ ä¼ı","åĪĺ æĻĵ","ĠÑħ оÑĤи","Sc enario","ĠBr ut","è´´ åIJĪ","å·¥åħ· çļĦ","Ġtherm oplastic","PL ES","ãĥª ãĥ³ãĤ°","æİ¨èįIJ æĸĩ竳","èĦı åύ","欣 欣","ÙĪØ³ ÛĮ","èįĴ åľ°","Ġsole il","Ġpes erta","Ġapt ly","ĠVac ation","Ġگرد د","åľ¨ ä¼ļä¸Ĭ","åĩº åħ¥åı£","åİ» è§ģ","转 ä¼ļ","ĠÑĥ к",".S ecurity",")) (","çĥŃ èĥ½","Ġclaim ants","æĬĺ æĸŃ","èģĮä¸ļ åŃ¦æł¡","ĠNaz areth","éļ¶ å±ŀäºİ","Ġвек ÑĤоÑĢ","ĠØ´ÙĨ اس","Ġmisc ellaneous","Ġzeg gen","Ġst unt","pp ard","ri qu","æĪij å¹¶ä¸į","ĠJ ard","ä¸Ĭ èĤ¢","éĻ Ĥ","ĠK ew","å°± ä¸Ģå®ļ","å®¶ äºĨ","æĸ° æ°ij","ä¿¡ ä¸Ń","è¿Ļ个 è¯Ŀé¢ĺ","èijĹ è¿°","back end","ĠØ® ÙĪØ§Ø¨","ĠÑģо оÑĢÑĥж","æĬ± æľī","åħ¨çIJĥ ç»ıæµİ","èĽĭçϽ éħ¶","Ġrid icule","Ġgeb oren","ipt ic","ĠΣ Ïħ","æ¹ĺ æ½Ń","Ġné p","ĠCE LL","Ġequival ente","çļĦä¸Ģ项 æĺ¯","ĠObl ast","ĠاÙĦعÙĦ ÙĪÙħ","agnet ism","Ġangg ota","R é","Ġc ade","ist ribution","åı¯ ä¸įèĥ½","å°½ åħ¶","åĪĩå®ŀ åĬłå¼º","NC BI","æĥ© æ²»","ÑĢован ного","stell t","( theta","m oving","r ism","t ap","an je","st arter","ä¸į åīį","åħ³ æľº","ç¿ ±","-g iving","Ġcapac itive","çĬ¯ è§Ħ","Ġled ge","à¹ĥà¸Ļ à¸Ĭà¹Īวà¸ĩ","ĠGreen house","Ġalign Items","éĿĴæĺ¥ çļĦ","Ġstriking ly","笨 èĽĭ","Ġhomeschool ing","ಿà²Ĥ ದ","( Model","d ell","l ide","ol ingu","ĠC ERT","н ное","ost ridium","ep ad","åIJĪ ç͍","çĹ Ĥ","å®ī åĮº","Ġsw apped","Ġge he","ĠDis pose","å®ŀéªĮ ç»ĵæŀľ","溫 æļĸ","åįij éĦĻ","ä¸Ģç³»åĪĹ çļĦ","ĠÃģ frica","Ġoverflow ing","Ġcation ic","ĠjÄĻzy ka","ë ij","Ġd ú","ir me","ere g","èĩª åĬĽ","ors ki","åįĹ åĮº","Ġsal ut","ĠGo ff","æĥĬ åIJĵ","ĠEmp ress","æµij æµĬ","æ¿ĢåĬ± æľºåζ","/ bi","æĹ¶ æĹ¶åĪ»åĪ»","ĠK ear","Ġpr atica","ann ung","åĨľ åķĨ","λ Ïī","éķĩ æĶ¿åºľ","Ġин декÑģ","Ġnu ovi","Ġcasual ty","ĠëͰ 른","ĠиÑģкÑĥÑģ ÑģÑĤва","ĠмеÑĢопÑĢиÑı ÑĤиÑı","Ġbakter i","ĠSev illa","ĠÐŁÑĢоÑģе Ñĩан","ĠT l","æİ ²","á na",".get Class","è̳ åħī","åͱ çĿĢ","Ġadj acency","ĠCarl ton","ìĨĮ ëħĦ","Ġফ লà§ĩ","Ġkw args","Ġminist re","à¦¿à¦Ł ার","ĠMathemat ik","Balt imore","- Qu","ĠS cheduled","ĠI IS","ÑĤ ков","ĠL inking","èĩªå·± æĥ³è¦ģ","æĬĬ 头","åı£ 红","åħļ 纪","Ġsal ted","çĶ· æĢ§çļĦ","ä¼łç»Ł æĸĩåĮĸçļĦ","èĩªçͱ 度","ĠاÙĦب تÙĩ","à³įಠµ","assert Equals","åļ´ æł¼","ÃŃd os","åı¸é©¬ è¿ģ","Ġescrit a","Ġlocom otion","Ġperox idase",", .ĊĊ","Ġb oc","åĴĮ èĩªæĪij","ost ante","ĠY ar","ç»ı æĸĩ","å½ĵ åħ¶","Ġbl uff","ãĢĭ ),","ç«ĭ åł´","空 缺","ãģ¯ ãģļ","ĠSu pper","æ¯Ķè¾ĥ å¤įæĿĤ","à¦ķ র","hand s","rec ipes","Ġoxygen ation","ĠتØŃ ÙĤÙĬÙĤ","×ķ׳ ×Ļ×ij","Mar x","Ġvoy ages","Ġfist ula","ĠLiz zie","Ġiod ide","Ġzast os","à¹ĥà¸ģล à¹ī",". But","N aj","Ġm RNAs","æĺ¯ éĿł","æķ Ŀ","Ġ' );Ċ","ä¸ī æľŁ","被 å°ģ","Ġprocess os","éĢļè¿ĩ è¿ĻäºĽ","Ġgre enery","Ġaccess es","RO UGH","åĪ©ç͍ äºĨ","mon ths","ĠмÑĭ Ñģли","ä¸ĵé¢ĺ 讲座","éłIJ éĺ²","Thom pson","- outs","/ book","w ari","ä¸Ģ 念","ver m","ri u","ill ä","çŃ µ","å·² å°Ĩ","Ġб ÑĢ","Ġpot ere","æĪ¿ åŃIJéĩĮ","Ġart ikk","鼻 æ°Ĺ","ç͵影 èĬĤ","åIJĮå¿Ĺ çļĦ","è¿Ļ两 ä½į","Ġкомп леÑĤним","ĠÙĦØ£ÙĨ Ùĩ","ĠнаÑĩина еÑĤ","åĤµ åĭĻ","- ol","V m","Ġb asta","ĠL oved","ĠN IV","Ġun ab","ç͵ 车","åŁº 线","ĠAnd al","Ġsom s","åIJĥ èĤī","Ġmi asta","Ġmit ad","à¸ģาร à¹Ģรียà¸Ļรูà¹ī","æ¯Ľ ç¬Ķ","ĠReview er","åĩĿ å¿ĥ","Ġcold est","丹 åıĤ","ĠIndust ri","Equal To","çķľ çĶŁ","Ġpharmac ists","ĠRow an","omyel itis","N em","T IME","人 寿","éĩ £","çĤ¹ æĭ¨","Ġmem bre","é»ij è¢į","ĠÑģÑĤ ек","éĨĴ æĤŁ","ĠMil o","END IX","äll ä","æ°ijäºĭ 责任","åĴ¸ éĺ³","ĠÑĤоÑĩ ка","ĠFant astic",".url s","ĠHUM AN","Ġundist urbed","h asa","½ ĥ","re ferences","Ġm ÃŃt","ĠP PG","if teen","Ġab duction","Ġв Ñĥ","ä¸Ģ个 ä¸Ģ个","å·²ç»ı ä¸į","éħį ä¼į","bo emb","_t ags","éĿŀ常 ç®Ģåįķ","à¸ŀ ฤ","å·¨ é¾Ļ","å®ŀæĸ½ åĬŀæ³ķ","Ġwer de","Ġstream lining","ĠCatal yst","éĽ£ ãģĹãģĦ","ĠRh in","даÑĢ Ð½Ð¾Ð¼","ç«ĸ åIJij","ĠLis boa","ĠBurk ina","Ġl ucha","Ġy ol","æĪij æľīä¸Ģ个","Ġsh rou","к г","åIJİ åıijçݰ","éĤ£ æĬĬ","æķ° 个","Ġi ra","èī ®","Ġcor rosive","ä½ķ æĸ¹","κ ι","Ġbul ld","Ġbroad caster","Ġplas mon","á»ij ng","ĠConsult ants","à¸Ľà¸£à¸°à¸Ĭ าà¸Ĭà¸Ļ","Ġshuff led","Ġsû r","N h","S ustainable","b ib","Ġt ujuh","Ġb unga","ĠT aurus","Ġbe per","ĠB urt","äºĨ 大éĩıçļĦ","çĶŁ è¾°","int he","æ³ķ ä¸Ń","ĠCh ill","Ġinter stellar","Ġz abaw","两 æĶ¯","Ġд воÑĢ","带 çĬ¶","resent ation","Ġhead quartered","Ġé lément","ç»´ æĸ°","课 ä½Ļ","å·ŀ åİ¿","ĠpÅĻ ek","åºŁ æŃ¢","akt oren","ĠÏĢÏģ ÏĮ","çİĭå®ī çŁ³","v ajÃŃ","Ġd aw","çļĦ éĩįéĩı","ĠP ach","we zig","Ġexp iry","æĹł æģĻ","æ¯Ķ åĪ«äºº","sp re","导 çĥŃ","ç½ij è´Ń","ä¸ĩ ä¸Ī","èĬ± çĵ¶","çģ« èį¯","åħį å¾ģ","HE MAT","Ġinform ally","éĤĦ åı¯ä»¥","Ġrev ocation","Ġta per","详 å°½","ä¸Ģå¹´ å¤ļ","Ġcrow ding","å®ı 大","åµ IJ","ĠPriv ile","-j ob","沫 èĭ¥","ின à¯įà®±","ÑĦика ÑĨиÑı","ĠÑĢези денÑĤ","ĠпоÑĤе ÑĢÑı","ĠпÑĢоÑĤÑı жении","Ġl umps","ĠT ats","ĠC UR","ĠR ae","Ġse ines","è¦ģ å¡ŀ","æĹ¶ éľĢè¦ģ","åı¯ åĸľ","åĪĨ æŀĿ","ä¹ĭ åѦ","ew ód","Ġи мен","åĨħ ç»ı","éĴ ¨","на ÑĪ","ĠCl osure","声 说éģĵ","OR ES","åħ·æľī å¾Ī强çļĦ","ĠLa wn","OS P","æ»ij è½®","ĠBay lor","è¿IJç͍ åΰ","à§§ ২","prot ection","Ġgastro enter","Ġгод ов","ĠÐĺн ÑĤеÑĢ","Ġíİ ĺìĿ´ì§Ģ","严åİī æīĵåĩ»","ĠBlan co","Ġd ossier","iv ité","Ġon site","è® ¹","ĠN arrow","Ġcont ral","æīĢ å¹¸","Ġam azon","è¯Ŀ äºĨ","çα å¾·åįİ","åįĥ å®¶","éķĩ åħļå§Ķ","ĠFound ing","ĠÑĤак ом","ĠMont es","ĠâĢĻ âĢĻ","Ġס ×ij","ĠczÄĻ ÅĽÄĩ","ĠAlger ian","ĠInject able","HasColumn Type","çļĦ 太","çļĦ åį±å®³","ĠS EA","Ġv ám","Ġha ute","ber gen","ä¸ī åŃ£åº¦","失 çģµ","bl ind","ĠпÑĢо ÑħодиÑĤ","ĠاÙĦت رÙĥ","èľ ĵ","prime able","èĤĿ èĥĨ","ĠTur in","Ġ×Ķ×ŀ× ©","Ø·ÙĦ اÙĤ","åĪĩå®ŀ æıIJé«ĺ","END ING","নà§įঠ¥","ĠÏĢε Ïģιο","ĠÑĥм нож","Ġzaw od","åļ· åļ·","éĹªè¿ĩ ä¸Ģä¸Ŀ","á¹ĩ a","Ġìļ´ ëıĻ","( el","R an","T rees","st aking","Ġen fo","åı¯ èİ·å¾Ĺ","Ñĭ ÑĤÑĭ","ä¸ī éĴ±","ç²¾ å·§","áĢ Ľ","-m atched","Ġdig s","å¤ĦçIJĨ åĴĮ","ĠÑĦ ÑĢÑĥк","zi ÅĤ","Ġcu ya","Ġamb assadors","Ġве зе","Ġgod ine","onom ics","æģ¼ æĢĴ","νÏĦ ί","Ġзаболе вание","Ġê³Ħ ìĤ°","æ°¸æģĴ çļĦ","-oper ated","å¿ĥ缮 ä¸ŃçļĦ","érc ito","R ail","ĠP ÅĻÃŃ","æĺ¯ èĩªå·±çļĦ","Ġit chy","ĠH OL","æĹ¶ éĢŁ","Ġ' ;Ċ","ex ercise","... ĊĊ","Ġf airs","Ġk ie","ä¹ĭ æ°´","çľĭ æĩĤ","Ġна ÑĪи","ж ка","è¿Ļ个 æ¦Ĥ念","-m icro","ض اÙĨ","å°į æĸ¹çļĦ","Ġca o","ĠSur geons","огÑĢам ма","æ¼Ķ讲 稿","Ġblo que","Ġimpecc able","/ sw","O ak","l igen","ou wd","ĠS LA","oc les","lic ks","Ġdis band","ä¸İ æİ§åζ","ó ry","Ġpol ystyrene","Ġend oplasmic","æ£Ģ å®ļ","comm od","Ġneg ativo","ĠPre ference","éĸĵ ãģ«","è« §","æĿ¨ å®¶","秦 çļĩ","Ġré f","ĠSur round","ãĥª ãĤ¹","ĠMer riam","亿åħĥ 人æ°ijå¸ģ","ä¸īè§Ĵ åĩ½æķ°","åı¤ä»£ çļĦ","ÙĤØ· ع","æ²īé»ĺ äºĨ","Ġsubscrib ing","Ġ----- -","ĠдопÑĥ ÑģÑĤи","ahar oa","ĠIllegal ArgumentException","ĠChord ata","j ähr","ĉ ld","ĉ type","Ġb umi","ĠM bps","Ġan em","åĴĮ åīį","Ġen quiries","åı¯ åĪĨ","èĢĮ åĿIJ","æľ¬ æłĩåĩĨ","Ġlong s","Ġм ÑĸлÑĮ","hes ive","çļĦä¸Ģ åı¥","åİĭ åĢĴ","çĶŁäº§ æĬĢæľ¯","èĩªçĦ¶ ä¸įä¼ļ","è¿İ æĸ°","çīĻ é¾Ī","æ³Ľ æĮĩ","iform is","Ġgrass y","Ġasc ended","à¸ŀืà¹īà¸Ļ à¸IJาà¸Ļ","Ġautobi ographical","è¯ķ管 å©´åĦ¿","Õ §","ĠB ingham","ĠR och","Ġob owiÄħ","ĠSe ab","co ordinates","ger icht","è¿Ļç§į ä¸ľè¥¿","åĨ° å±±","æĩĤ å¾ĹäºĨ","Ġtro isième","éĻª 审","Ġê·¸ ìĿĺ","çݰ代åĮĸ 建设","Ġjew eils","è¿Ļä¹Īå¤ļ 人","ĠAdapt ed","éĢĤå®ľ çļĦ","Brit ain","Dam n","à¹Ģรีย à¸ģ","è¯ĿéŁ³ åĪļèIJ½","H elen","ou vert","Ġg arg","Ġis omorphic","ĠC aring","Ġsu ces","大 æ°´","æĪIJ æ´»","äºĮ èĥİ","çļ® çĤİ","ĠRep rod","å·´ èı²çī¹","EG FR","æµĻæ±Ł 大åѦ","Ġbin aries",".read lines","Ġнеп иÑģмено","ĠÒ Ĺ","ä¸į åĢĴ","ĠH arl","ä¸Ń è¾ĵåħ¥","Ġle tech","ÑĢа ÑĤ","Ġdef ends","ĠZ ad","è®° äºĭ","ö hn","ĠÙģ Ø§ÙĦ","ĠØ¢ ÙĪØ±","æ´Ĺ åıij","çĬ¯ éĶĻ","à¸Ľà¸£à¸° สà¸ļ","å·¥åķĨ è¡ĮæĶ¿","ĠJama ican","å¹´è¼ķ 人","Ġদà§įব ারা","ĠDort mund","incorpor ated","ĠпÑĢедела Ñħ","T J","_ book","Ġ ï¼ĭ","ĠM SE","ĠE id","ah ara","管 å±Ģ","Ġche g","è© IJ","çıŃ éĩĮ","æĭ¿ çĿĢä¸Ģ","ìŀ ¡","åģ¥åº· çłģ","åĬĽéĩı åĴĮ","ä½³ 绩","تÛĮ ب","ில à¯įல","Ġëĵ± ìĿĦ","Ġneurom uscular","Ġprese ason","-Mus lim","âĢ ¬","ĠT uk","ĠB ers","ä»ĸ ç»Īäºİ","对 åĨ²","ĠÑģ наÑĩала","çłĶ çϼ","Ġpress urized","åģľ ä¸ĭäºĨ","å·¥ç¨ĭ éĢłä»·","空æ°Ķ ä¸ŃçļĦ","Ġнов ой","Ġreign ing","-gl ass","ĠGeme inde","à®¿à®Ł à¯įà®Ł","Ġtolu ene","' > bool: return model in litellm.anthropic_models and "claude-3" not in model -HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "llama2", "llama3"] +HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"] + +_DEEPSEEK_TOKENIZER_MODEL: Final = re.compile( + r"deepseek[-_.](?:chat[-_.])?v[34]|(?:^|/)deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE +) + +_BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading.Lock]] = MappingProxyType( + {"anthropic": threading.Lock(), "deepseek": threading.Lock()} +) def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: @@ -2598,6 +2607,8 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return "cohere" if uses_anthropic_tokenizer(model): return "anthropic" + if _DEEPSEEK_TOKENIZER_MODEL.search(model): + return "deepseek" if "llama-2" in model.lower() or "replicate" in model.lower(): return "llama2" if "llama-3" in model.lower(): @@ -2609,10 +2620,9 @@ def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None - return { - "type": "huggingface_tokenizer", - "tokenizer": _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()), - } + with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): + tokenizer: Final = _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()) + return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) @@ -2623,6 +2633,12 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") case "anthropic": return tokenizer_dispatch.anthropic() + case "deepseek": + return tokenizer_dispatch.from_str( + resources.files("litellm.litellm_core_utils.tokenizers") + .joinpath("deepseek_tokenizer.json") + .read_text(encoding="utf-8") + ) case "llama2": return tokenizer_dispatch.from_pretrained("hf-internal-testing/llama-tokenizer") case "llama3": diff --git a/tests/integration/_support/deepseek_tokens.py b/tests/integration/_support/deepseek_tokens.py new file mode 100644 index 00000000000..fa4bdd13d48 --- /dev/null +++ b/tests/integration/_support/deepseek_tokens.py @@ -0,0 +1,35 @@ +from collections.abc import Callable +from functools import cache +from importlib import resources +from typing import Final + +from litellm.litellm_core_utils.default_encoding import encoding as cl100k_base +from tokenizers import Tokenizer + +CHINESE_TEXT: Final = "深度求索的分词器会把这段中文切成更少的词元。" * 20 +SHORT_TEXT: Final = "Find record 14." + + +@cache +def _deepseek_tokenizer() -> Tokenizer: + bundled: Final = resources.files("litellm.litellm_core_utils.tokenizers").joinpath("deepseek_tokenizer.json") + return Tokenizer.from_str(bundled.read_text(encoding="utf-8")) + + +def deepseek_tokens(text: str) -> int: + return len(_deepseek_tokenizer().encode(text).ids) + + +def cl100k_tokens(text: str) -> int: + return len(cl100k_base.encode(text)) + + +def chinese_minus_short(tokens: Callable[[str], int]) -> int: + return tokens(CHINESE_TEXT) - tokens(SHORT_TEXT) + + +def assert_counted_with_deepseek(counted: int, expected: Callable[[Callable[[str], int]], int]) -> None: + cl100k_count: Final = expected(cl100k_tokens) + assert counted != cl100k_count, f"counted {counted}, the cl100k_base count, not DeepSeek's tokenizer" + deepseek_count: Final = expected(deepseek_tokens) + assert counted == deepseek_count, f"counted {counted}, DeepSeek's tokenizer counts {deepseek_count}" diff --git a/tests/integration/providers/test_chat_template_tool_count_wire.py b/tests/integration/providers/test_chat_template_tool_count_wire.py index d2b85e7168f..a8b97fa732e 100644 --- a/tests/integration/providers/test_chat_template_tool_count_wire.py +++ b/tests/integration/providers/test_chat_template_tool_count_wire.py @@ -1,7 +1,8 @@ import json import re import uuid -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from dataclasses import dataclass from itertools import chain @@ -19,6 +20,13 @@ from integration._support.client import ( object_value, string_value, ) +from integration._support.deepseek_tokens import ( + CHINESE_TEXT, + SHORT_TEXT, + assert_counted_with_deepseek, + chinese_minus_short, + cl100k_tokens, +) from integration._support.openai_wire import chat_reply from integration._support.process import graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server @@ -29,6 +37,7 @@ pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) _API_KEY: Final = "scripted-provider-key" _DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731" +_DEEPSEEK_R1: Final = "azure_ai/DeepSeek-R1-0528" _QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b" _GROK: Final = "azure_ai/grok-4-1" _DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222 @@ -47,7 +56,7 @@ _OPENAI_TOOL: Final[dict[str, JsonValue]] = {"type": "function", "function": _FU _RESPONSES_TOOL: Final[dict[str, JsonValue]] = {"type": "function", **_FUNCTION} _NAMED_TOOL_CHOICE: Final[dict[str, JsonValue]] = {"type": "function", "function": {"name": "lookup"}} _WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {"type": "web_search_20250305", "name": "web_search", "max_uses": 3} -_PROMPT: Final = "Find record 14." +_PROMPT: Final = SHORT_TEXT _MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": _PROMPT}] _INPUT_PRICE: Final = 0.001 _OUTPUT_PRICE: Final = 0.002 @@ -59,8 +68,8 @@ def _tokens(text: str) -> int: return len(cl100k_base.encode(text)) -def _deepseek_v4_tool_tokens(function: Mapping[str, JsonValue]) -> int: - return _DEEPSEEK_V4_TOOL_PREAMBLE + _tokens(json.dumps(function, ensure_ascii=False)) +def _deepseek_v4_tool_tokens(function: Mapping[str, JsonValue], tokens: Callable[[str], int]) -> int: + return _DEEPSEEK_V4_TOOL_PREAMBLE + tokens(json.dumps(function, ensure_ascii=False)) def _qwen3_5_tool_tokens(function: Mapping[str, JsonValue], *, with_system: bool) -> int: @@ -93,9 +102,11 @@ class _CountRoute: field: str tool: dict[str, JsonValue] - def body(self, model: str, tools: Sequence[JsonValue]) -> dict[str, JsonValue]: + def body(self, model: str, tools: Sequence[JsonValue], text: str = _PROMPT) -> dict[str, JsonValue]: prompt: Final[dict[str, JsonValue]] = ( - {"input": _PROMPT} if self.path == "/v1/responses/input_tokens" else {"messages": _MESSAGES} + {"input": text} + if self.path == "/v1/responses/input_tokens" + else {"messages": [{"role": "user", "content": text}]} ) return {"model": model, **prompt, **({"tools": list(tools)} if tools else {})} @@ -116,18 +127,85 @@ def _count(gateway: Gateway, route: _CountRoute, body: Mapping[str, JsonValue]) return counted -@pytest.mark.parametrize("route", sorted(_COUNT_ROUTES)) +_TOOL_COUNTS: Final[Mapping[str, tuple[str, str]]] = { + **{f"{route}-azure_ai-v4": (route, _DEEPSEEK_V4) for route in _COUNT_ROUTES}, + "messages-count-tokens-deepseek-chat": ("messages-count-tokens", "deepseek/deepseek-chat"), + "messages-count-tokens-deepseek-reasoner": ("messages-count-tokens", "deepseek/deepseek-reasoner"), + "messages-count-tokens-openrouter-deepseek-chat": ("messages-count-tokens", "openrouter/deepseek/deepseek-chat"), +} + + +@pytest.mark.parametrize("case", sorted(_TOOL_COUNTS)) def test_count_routes_count_deepseek_v4_tools_the_way_its_chat_template_renders_them( - gateway: Gateway, route: str + gateway: Gateway, case: str ) -> None: + route, backend = _TOOL_COUNTS[case] selected: Final = _COUNT_ROUTES[route] with wire_server(_completion) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, "count", _DEEPSEEK_V4, wire.url) + model: Final = _deployment(scenario, "count", backend, wire.url) without_tools: Final = _count(gateway, selected, selected.body(model, ())) with_tool: Final = _count(gateway, selected, selected.body(model, (selected.tool,))) upstream: Final = wire.drain() assert upstream == () - assert with_tool - without_tools == _deepseek_v4_tool_tokens(_FUNCTION) + tool_tokens: Final = with_tool - without_tools + assert tool_tokens > _DEEPSEEK_V4_TOOL_PREAMBLE, f"counted {tool_tokens} tool tokens, short of V4's tool preamble" + assert_counted_with_deepseek(tool_tokens, lambda tokens: _deepseek_v4_tool_tokens(_FUNCTION, tokens)) + + +_TEXT_COUNTS: Final[Mapping[str, tuple[str, str]]] = { + "messages-count-tokens-azure_ai-v4": ("messages-count-tokens", _DEEPSEEK_V4), + "responses-input-tokens-azure_ai-v4": ("responses-input-tokens", _DEEPSEEK_V4), + "messages-count-tokens-deepseek-chat": ("messages-count-tokens", "deepseek/deepseek-chat"), +} + + +def _text_count_delta(gateway: Gateway, route: _CountRoute, model: str) -> int: + chinese: Final = _count(gateway, route, route.body(model, (), CHINESE_TEXT)) + short: Final = _count(gateway, route, route.body(model, (), SHORT_TEXT)) + return chinese - short + + +@pytest.mark.parametrize("case", sorted(_TEXT_COUNTS)) +def test_count_routes_count_deepseek_v3_and_v4_text_with_deepseeks_tokenizer(gateway: Gateway, case: str) -> None: + route, backend = _TEXT_COUNTS[case] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + delta: Final = _text_count_delta( + gateway, _COUNT_ROUTES[route], _deployment(scenario, "count", backend, wire.url) + ) + upstream: Final = wire.drain() + assert upstream == () + assert_counted_with_deepseek(delta, chinese_minus_short) + + +def _counted_with_deepseek(delta: int) -> None: + assert_counted_with_deepseek(delta, chinese_minus_short) + + +def _counted_with_cl100k(delta: int) -> None: + assert delta == chinese_minus_short(cl100k_tokens) + + +@pytest.mark.parametrize( + ("backend", "tokenizer_type", "counted_with"), + ( + (_DEEPSEEK_V4, "huggingface_tokenizer", _counted_with_deepseek), + (_DEEPSEEK_R1, "openai_tokenizer", _counted_with_cl100k), + ), + ids=("deepseek-v4", "deepseek-r1"), +) +def test_utils_token_counter_names_the_tokenizer_that_counted_deepseek_text( + gateway: Gateway, backend: str, tokenizer_type: str, counted_with: Callable[[int], None] +) -> None: + route: Final = _COUNT_ROUTES["utils-token-counter"] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", backend, wire.url) + delta: Final = _text_count_delta(gateway, route, model) + response: Final = gateway.request("POST", route.path, route.body(model, (), CHINESE_TEXT)) + upstream: Final = wire.drain() + assert upstream == () + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content)["tokenizer_type"] == tokenizer_type, response.text + counted_with(delta) @pytest.mark.parametrize("with_system", (False, True), ids=("no-system", "system")) @@ -162,7 +240,12 @@ def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_fo upstream: Final = wire.drain() assert upstream == () assert with_web_search == plain - assert deferred - plain == _deepseek_v4_tool_tokens(deferred_function) - _deepseek_v4_tool_tokens(_FUNCTION) + assert_counted_with_deepseek( + deferred - plain, + lambda tokens: ( + _deepseek_v4_tool_tokens(deferred_function, tokens) - _deepseek_v4_tool_tokens(_FUNCTION, tokens) + ), + ) def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: @@ -215,17 +298,36 @@ def _send(gateway: Gateway, endpoint: _Endpoint, group: str, key: str, *, tools: @dataclass(frozen=True, slots=True) -class _BudgetRig: +class _OwnedRig: gateway: Gateway wire: Wire group: str + itpm_wire: Wire + itpm_group: str -def _strict_budget_config(directory: Path, group: str, api_base: str) -> Path: +def _owned_config( + directory: Path, + model_list: Sequence[JsonValue], + general_settings: Mapping[str, JsonValue], + router_settings: Mapping[str, JsonValue], +) -> Path: base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) config: Final = { **base, - "model_list": [ + "model_list": list(model_list), + "general_settings": {**object_value(base["general_settings"]), **general_settings}, + "router_settings": {**object_value(base["router_settings"]), **router_settings}, + } + path: Final = directory / f"chat-template-tool-count-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _owned_rig_config(directory: Path, group: str, api_base: str, itpm_group: str, itpm_api_base: str) -> Path: + return _owned_config( + directory, + [ { "model_name": group, "litellm_params": { @@ -235,26 +337,35 @@ def _strict_budget_config(directory: Path, group: str, api_base: str) -> Path: "input_cost_per_token": _INPUT_PRICE, "output_cost_per_token": _OUTPUT_PRICE, }, - } + }, + { + "model_name": itpm_group, + "litellm_params": { + "model": _DEEPSEEK_V4, + "api_base": itpm_api_base, + "api_key": _API_KEY, + "itpm": cl100k_tokens(CHINESE_TEXT), + }, + "model_info": {"id": uuid.uuid4().hex}, + }, ], - "general_settings": {**object_value(base["general_settings"]), "fail_closed_budget_enforcement": True}, - } - path: Final = directory / f"chat-template-tool-budget-{uuid.uuid4().hex[:8]}.yaml" - path.write_text(yaml.safe_dump(config)) - return path + {"fail_closed_budget_enforcement": True}, + {"optional_pre_call_checks": ["enforce_model_rate_limits"]}, + ) @pytest.fixture(scope="module") -def strict_budget(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_BudgetRig]: - directory: Final = tmp_path_factory.mktemp("chat-template-tool-budget") +def owned_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_OwnedRig]: + directory: Final = tmp_path_factory.mktemp("chat-template-tool-count") group: Final = f"deepseek-v4-flash-{uuid.uuid4().hex}" + itpm_group: Final = f"itpm-{uuid.uuid4().hex}" with ExitStack() as stack: rig_gateway: Final = stack.enter_context(gateway_from_environment()) wire: Final = stack.enter_context(wire_server(_completion)) - owned: Final = stack.enter_context( - owned_proxy_process(rig_gateway, directory, {}, config=_strict_budget_config(directory, group, wire.url)) - ) - yield _BudgetRig(owned.gateway, wire, group) + itpm_wire: Final = stack.enter_context(wire_server(_completion)) + config: Final = _owned_rig_config(directory, group, wire.url, itpm_group, itpm_wire.url) + owned: Final = stack.enter_context(owned_proxy_process(rig_gateway, directory, {}, config=config)) + yield _OwnedRig(owned.gateway, wire, group, itpm_wire, itpm_group) def _estimated_cost(response: httpx.Response) -> float: @@ -267,30 +378,47 @@ def _estimated_cost(response: httpx.Response) -> float: @pytest.mark.parametrize("endpoint", ("chat-completions", "messages")) def test_budget_reservation_estimates_deepseek_v4_tools_the_way_its_chat_template_renders_them( - strict_budget: _BudgetRig, endpoint: str + owned_rig: _OwnedRig, endpoint: str ) -> None: selected: Final = _ENDPOINTS[endpoint] - with strict_budget.gateway.scenario() as scenario: - key: Final = scenario.key(models=[strict_budget.group], max_budget=_TINY_BUDGET) - without_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=False)) - with_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=True)) - assert strict_budget.wire.drain() == () - assert with_tools - without_tools == pytest.approx(_deepseek_v4_tool_tokens(_FUNCTION) * _INPUT_PRICE) + with owned_rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[owned_rig.group], max_budget=_TINY_BUDGET) + without_tools: Final = _estimated_cost(_send(owned_rig.gateway, selected, owned_rig.group, key, tools=False)) + with_tools: Final = _estimated_cost(_send(owned_rig.gateway, selected, owned_rig.group, key, tools=True)) + assert owned_rig.wire.drain() == () + assert_counted_with_deepseek( + round((with_tools - without_tools) / _INPUT_PRICE), lambda tokens: _deepseek_v4_tool_tokens(_FUNCTION, tokens) + ) -def _project_key(scenario: Scenario, group: str) -> str: +def _chinese_chat(group: str) -> dict[str, JsonValue]: + return {"model": group, "messages": [{"role": "user", "content": CHINESE_TEXT}], "max_tokens": 1} + + +def test_router_itpm_reserves_deepseek_v4_text_with_deepseeks_tokenizer(owned_rig: _OwnedRig) -> None: + response: Final = owned_rig.gateway.request("POST", "/v1/chat/completions", _chinese_chat(owned_rig.itpm_group)) + upstream: Final = owned_rig.itpm_wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + + +def _project_key(scenario: Scenario, group: str, itpm_limit: int) -> str: team: Final = scenario.team() - project: Final = scenario.project(team, model_itpm_limit={group: _deepseek_v4_tool_tokens(_FUNCTION)}) + project: Final = scenario.project(team, model_itpm_limit={group: itpm_limit}) return scenario.key(team_id=team, project_id=project) +_TOOL_FRAMING_ITPM_LIMIT: Final = _deepseek_v4_tool_tokens(_FUNCTION, _tokens) + + @pytest.mark.parametrize("endpoint", sorted(_ENDPOINTS)) def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_renders_them( gateway: Gateway, endpoint: str ) -> None: with wire_server(_completion) as wire, gateway.scenario() as scenario: group: Final = _deployment(scenario, "deepseek-v4-flash", _DEEPSEEK_V4, wire.url) - response: Final = _send(gateway, _ENDPOINTS[endpoint], group, _project_key(scenario, group), tools=True) + key: Final = _project_key(scenario, group, _TOOL_FRAMING_ITPM_LIMIT) + response: Final = _send(gateway, _ENDPOINTS[endpoint], group, key, tools=True) upstream: Final = wire.drain() assert response.status_code == 429, response.text assert "model_per_project_itpm" in response.text, response.text @@ -305,13 +433,23 @@ def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gatewa "POST", chat.path, {"model": group, **chat.body, "tools": [_OPENAI_TOOL]}, - key=_project_key(scenario, group), + key=_project_key(scenario, group, _TOOL_FRAMING_ITPM_LIMIT), ) upstream: Final = wire.drain() assert response.status_code == 200, response.text assert len(upstream) == 1, upstream +def test_project_itpm_reserves_deepseek_v4_text_with_deepseeks_tokenizer(gateway: Gateway) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "deepseek-v4-flash", _DEEPSEEK_V4, wire.url) + key: Final = _project_key(scenario, group, cl100k_tokens(CHINESE_TEXT)) + response: Final = gateway.request("POST", "/v1/chat/completions", _chinese_chat(group), key=key) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + + _FILES: Final = 3 _READ_FILE_FUNCTION: Final[dict[str, JsonValue]] = { "name": "read_file", @@ -323,7 +461,7 @@ _READ_FILE_TOOL: Final[dict[str, JsonValue]] = { } -def _read_file_round(marker: str, index: int) -> tuple[JsonValue, ...]: +def _read_file_round(marker: str, index: int, file_body: str) -> tuple[JsonValue, ...]: return ( {"role": "user", "content": f"Read file {index}."}, { @@ -338,15 +476,15 @@ def _read_file_round(marker: str, index: int) -> tuple[JsonValue, ...]: { "type": "tool_result", "tool_use_id": f"toolu_{index}", - "content": f"short file body.", + "content": f"{file_body}", } ], }, ) -def _agent_body(group: str, marker: str) -> dict[str, JsonValue]: - rounds: Final = tuple(chain.from_iterable(_read_file_round(marker, index) for index in range(_FILES))) +def _agent_body(group: str, marker: str, file_body: str, trigger: int) -> dict[str, JsonValue]: + rounds: Final = tuple(chain.from_iterable(_read_file_round(marker, index, file_body) for index in range(_FILES))) return { "model": group, "max_tokens": 16, @@ -356,7 +494,7 @@ def _agent_body(group: str, marker: str) -> dict[str, JsonValue]: "edits": [ { "type": "clear_tool_uses_20250919", - "trigger": {"type": "input_tokens", "value": _deepseek_v4_tool_tokens(_READ_FILE_FUNCTION)}, + "trigger": {"type": "input_tokens", "value": trigger}, "keep": {"type": "tool_uses", "value": 1}, } ] @@ -375,9 +513,60 @@ def test_messages_context_management_trigger_counts_tools_the_way_the_models_cha marker: Final = uuid.uuid4().hex with wire_server(_completion) as wire, gateway.scenario() as scenario: group: Final = _deployment(scenario, "context", model, wire.url) - response: Final = gateway.request("POST", "/v1/messages", _agent_body(group, marker)) + trigger: Final = _deepseek_v4_tool_tokens(_READ_FILE_FUNCTION, _tokens) + response: Final = gateway.request( + "POST", "/v1/messages", _agent_body(group, marker, "short file body.", trigger) + ) upstream: Final = wire.drain() assert response.status_code == 200, response.text assert len(upstream) == 1, upstream sent: Final = upstream[0].body.decode() assert tuple(f"" in sent for index in range(_FILES)) == files_kept + + +def test_messages_context_management_trigger_counts_deepseek_v4_text_with_deepseeks_tokenizer(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "context", _DEEPSEEK_V4, wire.url) + trigger: Final = _FILES * cl100k_tokens(CHINESE_TEXT) + response: Final = gateway.request("POST", "/v1/messages", _agent_body(group, marker, CHINESE_TEXT, trigger)) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + sent: Final = upstream[0].body.decode() + kept: Final = tuple(f"" in sent for index in range(_FILES)) + assert kept == (True,) * _FILES, f"cleared tool results under a trigger only the cl100k_base count reaches: {kept}" + + +_BURST: Final = (CHINESE_TEXT, SHORT_TEXT) * 8 + + +def test_concurrent_first_deepseek_v4_counts_in_a_fresh_proxy_agree(tmp_path: Path) -> None: + group: Final = f"count-{uuid.uuid4().hex}" + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + wire: Final = stack.enter_context(wire_server(_completion)) + config: Final = _owned_config( + tmp_path, + [ + { + "model_name": group, + "litellm_params": {"model": _DEEPSEEK_V4, "api_base": wire.url, "api_key": _API_KEY}, + } + ], + {}, + {}, + ) + owned: Final = stack.enter_context(owned_proxy_process(rig_gateway, tmp_path, {}, config=config)) + with ThreadPoolExecutor(max_workers=len(_BURST)) as pool: + counts: Final = tuple( + pool.map( + lambda text: _count(owned.gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(group, (), text)), _BURST + ) + ) + upstream: Final = wire.drain() + assert upstream == () + chinese: Final = frozenset(counts[0::2]) + short: Final = frozenset(counts[1::2]) + assert len(chinese) == 1 and len(short) == 1, counts + assert_counted_with_deepseek(min(chinese) - min(short), chinese_minus_short) diff --git a/tests/integration/sdk/test_huggingface_tokenizer_downloads_sdk.py b/tests/integration/sdk/test_huggingface_tokenizer_downloads_sdk.py new file mode 100644 index 00000000000..1ec554de099 --- /dev/null +++ b/tests/integration/sdk/test_huggingface_tokenizer_downloads_sdk.py @@ -0,0 +1,54 @@ +import os +import subprocess +import sys +import threading +from pathlib import Path +from queue import SimpleQueue +from typing import Final + +import litellm +from integration._support.wire import Reply, Request, wire_server + +_SCRIPT_SECONDS: Final = 60 +_CONCURRENT_LLAMA_COUNTS: Final = """ +import sys +from concurrent.futures import ThreadPoolExecutor +sys.path.insert(0, sys.argv[1]) +from litellm.utils import _select_tokenizer_helper +with ThreadPoolExecutor(max_workers=2) as pool: + tuple(pool.map(lambda _: _select_tokenizer_helper("meta-llama/llama-3-8b"), range(2))) +""" + + +def test_concurrent_first_llama_counts_download_without_waiting_on_each_other(tmp_path: Path) -> None: + both_downloading: Final = threading.Barrier(2, timeout=5) + downloads: Final = SimpleQueue[str]() + + def stalled_hub(request: Request) -> Reply: + if request.target.endswith("/tokenizer.json"): + try: + both_downloading.wait() + downloads.put("concurrent") + except threading.BrokenBarrierError: + downloads.put("serialized") + return Reply(status=404) + + with wire_server(stalled_hub) as hub: + result: Final = subprocess.run( + [sys.executable, "-I", "-c", _CONCURRENT_LLAMA_COUNTS, str(Path(litellm.__file__).parent.parent)], + capture_output=True, + text=True, + timeout=_SCRIPT_SECONDS, + check=False, + env={ + **os.environ, + "HF_ENDPOINT": hub.url, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_HUB_OFFLINE": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert downloads.get_nowait() == "concurrent" diff --git a/tests/integration/spend/test_failed_dispatch_tokens.py b/tests/integration/spend/test_failed_dispatch_tokens.py index db5c49623eb..b92606ebc88 100644 --- a/tests/integration/spend/test_failed_dispatch_tokens.py +++ b/tests/integration/spend/test_failed_dispatch_tokens.py @@ -6,6 +6,12 @@ from typing import Final import pytest from integration._support.client import Gateway, eventually, list_value from integration._support.database import read_rows +from integration._support.deepseek_tokens import ( + CHINESE_TEXT, + SHORT_TEXT, + assert_counted_with_deepseek, + chinese_minus_short, +) from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue @@ -126,3 +132,38 @@ def test_a_provider_500_failure_row_keeps_the_tags_the_caller_sent( _, row = _provider_500_failure_row(gateway, route, body, {"x-litellm-tags": tag}) assert tag in list_value(row["request_tags"]), row + + +def _provider_500(request: Request) -> Reply: + assert request.method == "POST" + return Reply( + status=500, + body=b'{"error":{"message":"synthetic provider outage","type":"server_error","code":"500"}}', + ) + + +def _failure_row_prompt_tokens(gateway: Gateway, model: str, key: str, text: str) -> int: + failed: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, text), key=key) + assert failed.status_code == 500, failed.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (failed.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "failure", rows + prompt_tokens: Final = rows[0]["prompt_tokens"] + assert isinstance(prompt_tokens, int), rows + return prompt_tokens + + +def test_provider_500_failure_rows_count_deepseek_v4_prompt_tokens_with_deepseeks_tokenizer(gateway: Gateway) -> None: + with wire_server(_provider_500) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="azure_ai/DeepSeek-V4-Flash-0731", api_base=wire.url, num_retries=0) + key: Final = scenario.key(models=[model]) + chinese: Final = _failure_row_prompt_tokens(gateway, model, key, CHINESE_TEXT) + short: Final = _failure_row_prompt_tokens(gateway, model, key, SHORT_TEXT) + assert wire.drain() + assert_counted_with_deepseek(chinese - short, chinese_minus_short) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 6126d919180..309cbd33131 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -10,7 +10,7 @@ import sys import threading import time from collections.abc import Callable, Mapping -from concurrent.futures import Future, wait +from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain from pathlib import Path from typing import Final @@ -586,10 +586,14 @@ def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates( [ ("DeepSeek-V4-Pro", "DeepSeek-V4-Flash-0731"), ("accounts/fireworks/models/deepseek-v4-flash", "DeepSeek-V4-Flash-0731"), + ("deepseek/deepseek-chat", "DeepSeek-V4-Flash-0731"), + ("deepseek/deepseek-chat:free", "DeepSeek-V4-Flash-0731"), + ("deepseek-reasoner", "DeepSeek-V4-Flash-0731"), + ("deepseek/deepseek-flash", "DeepSeek-V4-Flash-0731"), ("DeepSeek-V4.1-Flash", "deepseek-v3.2"), ("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"), ("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"), - ("qwen3-235b-a22b", "deepseek-v3.2"), + ("qwen3-235b-a22b", "mistral-large-latest"), ], ) def test_token_counter_counts_tools_by_the_models_chat_template_family(model: str, counted_like: str): @@ -1035,6 +1039,83 @@ class TestTokenizerSelection(unittest.TestCase): monkeypatch.undo() +_CODE_SAMPLE: Final = " for item in items:\n total += item.price * item.quantity\n" * 20 + + +def _deepseek_token_ids(text: str) -> list[int]: + deepseek_tokenizer: Final = Tokenizer.from_str( + importlib.resources.files("litellm.litellm_core_utils.tokenizers") + .joinpath("deepseek_tokenizer.json") + .read_text(encoding="utf-8") + ) + return deepseek_tokenizer.encode(text).ids + + +def _deepseek_token_count(text: str) -> int: + return len(_deepseek_token_ids(text)) + + +@pytest.mark.parametrize( + "model", + [ + "azure_ai/DeepSeek-V4-Flash-0731", + "deepseek/deepseek-v4-pro", + "fireworks_ai/deepseek-v4p1-flash", + "bedrock/us-east-1/deepseek.v3.2", + "together_ai/deepseek-ai/DeepSeek-V3.1", + "databricks/databricks-deepseek-v4-flash-0731", + "replicate/deepseek-ai/deepseek-v3", + "deepseek-chat", + "deepseek/deepseek-reasoner", + "openrouter/deepseek/deepseek-chat-v3-0324", + "openrouter/deepseek/deepseek-chat:free", + ], +) +def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_tokenizer(model: str): + expected: Final = _deepseek_token_count(_CODE_SAMPLE) + + assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_CODE_SAMPLE)) + assert litellm.token_counter(model=model, text=_CODE_SAMPLE) == expected + + +@pytest.mark.parametrize( + "model", + ["together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "fireworks_ai/accounts/fireworks/models/deepseek-v2p5"], +) +def test_token_counter_skips_the_deepseek_tokenizer_for_deepseek_models_built_on_other_vocabularies(model: str): + assert litellm.token_counter(model=model, text=_CODE_SAMPLE) != _deepseek_token_count(_CODE_SAMPLE) + + +def test_encode_and_decode_use_deepseeks_token_ids_for_deepseek_v4(): + token_ids: Final = encode(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) + + assert token_ids == _deepseek_token_ids(_CODE_SAMPLE) + assert decode(model="deepseek/deepseek-v4-flash", tokens=token_ids) == _CODE_SAMPLE + + +def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + + counted: Final = litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) + + assert counted != _deepseek_token_count(_CODE_SAMPLE) + assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE) + + +def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer(): + _load_huggingface_tokenizer.cache_clear() + start: Final = threading.Barrier(8) + + def first_tokenizer(_: int) -> object: + start.wait() + return _select_tokenizer_helper("deepseek/deepseek-v4-flash")["tokenizer"] + + with ThreadPoolExecutor(max_workers=8) as pool: + tokenizers: Final = tuple(pool.map(first_tokenizer, range(8))) + + assert len({id(tokenizer) for tokenizer in tokenizers}) == 1 + + def test_token_counter_with_anthropic_tool_use(): """ Test that _count_anthropic_content() correctly handles tool_use blocks. From b690225956eb7d91c02b308e690943dd440effd6 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:21:53 -0500 Subject: [PATCH 057/124] fix(proxy): resync configuration after Redis reconnect (#43313) * fix(proxy): resync configuration after Redis reconnect * chore(ci): rerun checks after upstream documentation fix --- .../proxy/common_utils/config_sync_pubsub.py | 19 +++++++++---- .../common_utils/test_config_sync_pubsub.py | 28 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index af5441c2e94..da74851b2d8 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -185,6 +185,7 @@ class ConfigSyncSubscriber: "_monotonic", "_redis_cache", "_resync_callbacks", + "_resync_required", "_rng", "_sleep", "_task", @@ -215,6 +216,7 @@ class ConfigSyncSubscriber: self._monotonic = monotonic self._task: asyncio.Task[None] | None = None self._last_resync_at: float | None = None + self._resync_required = False def start(self) -> None: if self._task is not None: @@ -240,6 +242,9 @@ class ConfigSyncSubscriber: pubsub = client.pubsub() try: await pubsub.subscribe(config_sync_channel(self._redis_cache)) + if self._resync_required: + await self._debounce_and_resync(pubsub) + self._resync_required = False backoff_seconds = self._backoff_initial_seconds await self._consume(pubsub) finally: @@ -247,6 +252,7 @@ class ConfigSyncSubscriber: except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 # any redis failure falls through to backoff and reconnect + self._resync_required = True verbose_proxy_logger.warning( "config sync subscriber redis error: %s; reconnecting in %.0fs", e, @@ -260,11 +266,14 @@ class ConfigSyncSubscriber: message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS) if message is None: continue - await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) - await self._wait_for_min_resync_interval() - await self._drain_pending(pubsub) - await self._run_resync_callbacks() - self._last_resync_at = self._monotonic() + await self._debounce_and_resync(pubsub) + + async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None: + await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) + await self._wait_for_min_resync_interval() + await self._drain_pending(pubsub) + await self._run_resync_callbacks() + self._last_resync_at = self._monotonic() async def _wait_for_min_resync_interval(self) -> None: if self._last_resync_at is None: diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index 83ed3afa293..92cfb6a71e5 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -1,7 +1,7 @@ import asyncio import json import random -from typing import Callable, Coroutine, Iterable, List, Optional, Tuple +from typing import Callable, Coroutine, Final, Iterable, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -521,6 +521,32 @@ async def test_redis_error_leads_to_backoff_and_resubscribe() -> None: assert resyncs == ["resync"] +@pytest.mark.parametrize("broken_type", (_BrokenPubSub, _CloseFailingBrokenPubSub)) +async def test_reconnect_recovers_missed_config_without_another_message(broken_type: type[_BrokenPubSub]) -> None: + broken: Final = broken_type() + healthy: Final = _QueuePubSub() + cache: Final = _FakeRedisCache(_ScriptedPubSubRedisClient((broken, healthy))) + recovered: Final = asyncio.Event() + reload_config: Final = AsyncMock(side_effect=recovered.set) + subscriber: Final = ConfigSyncSubscriber( + redis_cache=cache, + resync_callbacks=(reload_config,), + debounce_seconds=0.0, + jitter_max_seconds=0.0, + backoff_initial_seconds=0.0, + backoff_max_seconds=0.0, + ) + + subscriber.start() + try: + await asyncio.wait_for(recovered.wait(), timeout=1) + finally: + await subscriber.stop() + + assert healthy.subscribed_channels == [CONFIG_SYNC_CHANNEL] + reload_config.assert_awaited_once_with() + + async def test_failing_resync_callback_does_not_kill_subscriber() -> None: pubsub = _QueuePubSub() cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) From 6edbc4bc5e126fe23e33c5658c26490a6cdbc41b Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:26:02 -0700 Subject: [PATCH 058/124] chore(pricing): add github_copilot deprecation dates for models retired on 2026-10-19 (#47487) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 7 +++++++ model_prices_and_context_window.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e923369d8fe..95285d50f26 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32674,6 +32674,7 @@ }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 7.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 200000, @@ -32834,6 +32835,7 @@ }, "github_copilot/gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 128000, @@ -32841,6 +32843,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -32867,6 +32870,7 @@ "github_copilot/gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "litellm_provider": "github_copilot", @@ -32889,6 +32893,7 @@ }, "github_copilot/gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 7.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 272000, @@ -32909,6 +32914,7 @@ "github_copilot/gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "deprecation_date": "2026-10-19", "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "litellm_provider": "github_copilot", @@ -76034,6 +76040,7 @@ "github_copilot/grok-4.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "github_copilot", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e923369d8fe..95285d50f26 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32674,6 +32674,7 @@ }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 7.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 200000, @@ -32834,6 +32835,7 @@ }, "github_copilot/gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 128000, @@ -32841,6 +32843,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, + "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, @@ -32867,6 +32870,7 @@ "github_copilot/gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "litellm_provider": "github_copilot", @@ -32889,6 +32893,7 @@ }, "github_copilot/gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "deprecation_date": "2026-10-19", "input_cost_per_token": 7.5e-07, "litellm_provider": "github_copilot", "max_input_tokens": 272000, @@ -32909,6 +32914,7 @@ "github_copilot/gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "deprecation_date": "2026-10-19", "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "litellm_provider": "github_copilot", @@ -76034,6 +76040,7 @@ "github_copilot/grok-4.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "deprecation_date": "2026-10-19", "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "github_copilot", From 8884f461af3c46d632c535db94f5f0c00b717ea6 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 19 Oct 2026 21:29:17 -0700 Subject: [PATCH 059/124] feat(proxy): spend log storage probe and the reclaimable-bytes count (#47449) * feat(proxy): spend log size settings, storage probe and the reclaimable-bytes count Adds maximum_spend_logs_size, maximum_spend_logs_cleanup_target_size and maximum_spend_logs_size_check_interval, size_in_bytes, the pg_stat_user_tables storage probe, and SizeCleanupState / advance_size_state, the running count of reclaimable bytes kept in LiteLLM_Config that the size pass (next PR) compares against the limit and the target. Nothing is scheduled or deleted yet. * refactor(proxy): keep the spend log size settings out of the state layer The three general_settings keys, their schema.d.ts rows, the SizeRule parsing and the scheduler constants move up to the PR that acts on them, so merging this layer alone changes nothing an operator can set * test(proxy): cover the storage probe and the size state round trip The probe, the LiteLLM_Config load and save and SpendLogStorage.after_deleting had no caller in this layer, so codecov/patch sat at 67 percent * fix(proxy): a stale storage probe carries the size state forward instead of reseeding Bugbot on #47449: with zero live and dead rows the reseed made bytes_per_row the whole file. advance_size_state now keeps the prior count on a stale probe and, with no prior count, treats the files as fully reclaimable so nothing is deleted on its say-so --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../db_transaction_queue/spend_log_cleanup.py | 266 +++++++++++++++- tests/unit/proxy/test_spend_log_cleanup.py | 286 ++++++++++++------ 2 files changed, 459 insertions(+), 93 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index c82b34dfcf1..b57952b4992 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -1,11 +1,12 @@ import asyncio +import re import time from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Final, Literal, TypeAlias -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.caching import RedisCache @@ -36,6 +37,166 @@ from litellm.types.llms.base import LiteLLMBaseModel StopReason: TypeAlias = Literal["exhausted", "budget_exhausted", "batch_cap_reached", "aborted"] +_SIZE_RE: Final = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?)I?B?\s*$", re.IGNORECASE) +_SIZE_UNITS: Final = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4} + + +def size_in_bytes(value: str | int) -> int: + """ + Parse a storage size such as '180GB', '512 MiB' or '1.5TB' into bytes; a bare int is bytes already. + + Units are powers of 1024, the same convention pg_size_pretty uses, so the number an operator reads + off `SELECT pg_size_pretty(pg_total_relation_size(...))` is the number they can put in the config. + """ + if isinstance(value, int): + if value <= 0: + raise ValueError(f"size must be positive, got {value}") + return value + match: Final = _SIZE_RE.match(value) + if match is None: + raise ValueError(f"invalid size {value!r}, expected a number with a unit such as '180GB'") + size: Final = int(float(match.group(1)) * _SIZE_UNITS[match.group(2).upper()]) + if size <= 0: + raise ValueError(f"size must be positive, got {value!r}") + return size + + +@dataclass(frozen=True, slots=True) +class StorageProbe: + """ + What the catalog says about the spend-log tables, read without scanning them. + + `allocated_bytes` is what the files take on disk (heap, indexes and TOAST of LiteLLM_SpendLogs and + LiteLLM_SpendLogToolIndex, partitions included). Deleting rows does not shrink it: deleted rows become dead + tuples, autovacuum later turns them into free space that new rows reuse, and only VACUUM FULL or pg_repack + hands the bytes back to the filesystem. Neither dead tuples nor free space are live data, and Postgres + tracks only the former (`n_dead_tup`, zeroed by vacuum), so the size rule keeps its own count of the + reclaimable bytes, see `SizeCleanupState`. `inserted_total` and `deleted_total` are the cumulative + `n_tup_ins` and `n_tup_del` counters that count is driven by. + """ + + allocated_bytes: int + live_rows: int + dead_rows: int + inserted_total: int + deleted_total: int + + @property + def stats_are_stale(self) -> bool: + """No row counters at all while the files hold data: the statistics were reset, e.g. by a crash.""" + return self.allocated_bytes > 0 and self.live_rows + self.dead_rows == 0 + + +class SizeCleanupState(LiteLLMBaseModel): + """ + The size rule's running count of reclaimable bytes, kept in LiteLLM_Config between checks. + + `reclaimable_bytes` is the space inside the files that holds no live row: the bytes of every row deleted + since the count started (by this job or anyone else, from `n_tup_del`), less the free space new rows have + since been written into (bytes inserted beyond what the files grew by), less what a VACUUM FULL handed back. + Live bytes are `allocated_bytes - reclaimable_bytes`, whether or not autovacuum has run yet. `bytes_per_row` + converts rows to bytes and is refreshed from live bytes over live rows on every check. + """ + + model_config = ConfigDict(frozen=True) + allocated_bytes: int + reclaimable_bytes: int + bytes_per_row: float + inserted_total: int + deleted_total: int + + +def advance_size_state(state: SizeCleanupState | None, probe: StorageProbe) -> SizeCleanupState: + """ + Roll the reclaimable count forward to what the catalog shows now. + + Without a prior state, or after the cumulative counters went backwards (a statistics reset), the count + starts from the dead rows Postgres still knows about, at the files' average bytes per row. A probe with no + row counters at all (`stats_are_stale`) cannot seed anything: the prior count is carried forward, and with + no prior count the files are taken as fully reclaimable so nothing is deleted on its say-so. + """ + if probe.stats_are_stale: + return SizeCleanupState( + allocated_bytes=probe.allocated_bytes, + reclaimable_bytes=probe.allocated_bytes + if state is None + else min(probe.allocated_bytes, state.reclaimable_bytes), + bytes_per_row=1.0 if state is None else state.bytes_per_row, + inserted_total=probe.inserted_total, + deleted_total=probe.deleted_total, + ) + if state is None or probe.inserted_total < state.inserted_total or probe.deleted_total < state.deleted_total: + seed_bytes_per_row: Final = probe.allocated_bytes / max(probe.live_rows + probe.dead_rows, 1) + return SizeCleanupState( + allocated_bytes=probe.allocated_bytes, + reclaimable_bytes=min(probe.allocated_bytes, round(probe.dead_rows * seed_bytes_per_row)), + bytes_per_row=max(seed_bytes_per_row, 1.0), + inserted_total=probe.inserted_total, + deleted_total=probe.deleted_total, + ) + growth: Final = probe.allocated_bytes - state.allocated_bytes + bytes_deleted: Final = (probe.deleted_total - state.deleted_total) * state.bytes_per_row + free_space_refilled: Final = max( + 0.0, (probe.inserted_total - state.inserted_total) * state.bytes_per_row - max(0, growth) + ) + returned_to_filesystem: Final = max(0, -growth) + reclaimable: Final = min( + probe.allocated_bytes, + max(0, round(state.reclaimable_bytes + bytes_deleted - free_space_refilled - returned_to_filesystem)), + ) + live_bytes: Final = probe.allocated_bytes - reclaimable + return SizeCleanupState( + allocated_bytes=probe.allocated_bytes, + reclaimable_bytes=reclaimable, + bytes_per_row=max(live_bytes / probe.live_rows, 1.0) if probe.live_rows > 0 else state.bytes_per_row, + inserted_total=probe.inserted_total, + deleted_total=probe.deleted_total, + ) + + +@dataclass(frozen=True, slots=True) +class SpendLogStorage: + """What the spend-log tables weigh at one point: the files on disk and the live bytes inside them.""" + + allocated_bytes: int + reclaimable_bytes: int + bytes_per_row: float + live_rows: int + + @property + def estimated_live_bytes(self) -> int: + return max(0, self.allocated_bytes - self.reclaimable_bytes) + + def after_deleting(self, rows: int, allocated_bytes: int) -> "SpendLogStorage": + """ + The same storage once `rows` more rows are gone and the files measure `allocated_bytes`. + + Files that grew while the pass only deleted (a visibility map or free space map fork a DELETE creates, + an index page split) hold no live row, so the growth counts as reclaimable; rows inserted meanwhile are + taken back out at the next check from n_tup_ins. + """ + return SpendLogStorage( + allocated_bytes=allocated_bytes, + reclaimable_bytes=min( + allocated_bytes, + round( + self.reclaimable_bytes + rows * self.bytes_per_row + max(0, allocated_bytes - self.allocated_bytes) + ), + ), + bytes_per_row=self.bytes_per_row, + live_rows=max(0, self.live_rows - rows), + ) + + def to_state(self, inserted_total: int, deleted_total: int) -> SizeCleanupState: + return SizeCleanupState( + allocated_bytes=self.allocated_bytes, + reclaimable_bytes=self.reclaimable_bytes, + bytes_per_row=self.bytes_per_row, + inserted_total=inserted_total, + deleted_total=deleted_total, + ) + + Cutoff: TypeAlias = datetime | str """Rows strictly older than this are expired: a timestamp, or an ISO calendar day for tables keyed by day""" @@ -87,6 +248,57 @@ class _RemainingRow(LiteLLMBaseModel): _REMAINING_ROWS: Final = TypeAdapter(list[_RemainingRow]) + +class _StorageRow(LiteLLMBaseModel): + """One row of the storage probe, validated out of prisma's untyped result.""" + + model_config = ConfigDict(frozen=True) + allocated_bytes: int + live_rows: int + dead_rows: int + inserted_total: int + deleted_total: int + + +_STORAGE_ROWS: Final = TypeAdapter(list[_StorageRow]) + + +class _SizeStateRow(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + state_json: str + + +_SIZE_STATE_ROWS: Final = TypeAdapter(list[_SizeStateRow]) +SPEND_LOG_SIZE_CLEANUP_STATE_PARAM: Final = "spend_logs_size_cleanup_state" +_LOAD_SIZE_STATE_SQL: Final = 'SELECT param_value::text AS state_json FROM "LiteLLM_Config" WHERE param_name = $1' +_SAVE_SIZE_STATE_SQL: Final = """ + INSERT INTO "LiteLLM_Config" (param_name, param_value) VALUES ($1, $2::jsonb) + ON CONFLICT (param_name) DO UPDATE SET param_value = EXCLUDED.param_value + """ + + +_SPEND_LOG_TABLES: Final = ("LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex") + +# Partition children are counted with their parent: a partitioned parent has no storage of its own. +_STORAGE_SQL: Final = """ + SELECT coalesce(sum(pg_total_relation_size(s.relid)), 0)::bigint AS allocated_bytes, + coalesce(sum(s.n_live_tup), 0)::bigint AS live_rows, + coalesce(sum(s.n_dead_tup), 0)::bigint AS dead_rows, + coalesce(sum(s.n_tup_ins), 0)::bigint AS inserted_total, + coalesce(sum(s.n_tup_del), 0)::bigint AS deleted_total + FROM pg_stat_user_tables s + WHERE s.schemaname = current_schema() + AND ( + s.relname = ANY($1::text[]) + OR s.relid IN ( + SELECT i.inhrelid FROM pg_inherits i + JOIN pg_class p ON p.oid = i.inhparent + JOIN pg_namespace n ON n.oid = p.relnamespace + WHERE p.relname = ANY($1::text[]) AND n.nspname = current_schema() + ) + ) + """ + SPEND_LOG_CLEANUP_BOUND_SETTINGS: Final = ( "maximum_spend_logs_cleanup_batch_size", "maximum_spend_logs_cleanup_max_batches", @@ -244,6 +456,58 @@ class SpendLogCleanup: self.retention_seconds = self._retention_seconds_for("maximum_spend_logs_retention_period") return self.retention_seconds is not None + async def _probe_storage(self, prisma_client: PrismaClient, deadline: float) -> StorageProbe | None: + """Read what the spend-log tables weigh from the catalog, or None when the probe fails or has no budget.""" + if time.monotonic() >= deadline: + return None + timeout_ms: Final = self._timeout_ms(deadline) + try: + async with db_span("measure_storage", "LiteLLM_SpendLogs"), bounded_tx(prisma_client, timeout_ms) as tx: + await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}")) + rows: Final = _STORAGE_ROWS.validate_python( + await tx.query_raw(trusted_sql(_STORAGE_SQL), list(_SPEND_LOG_TABLES)) + ) + except Exception as e: # noqa: BLE001 - a failed measurement skips the size pass, it must not fail the run + verbose_proxy_logger.warning("Could not measure spend-log storage: %s: %s", type(e).__name__, e) + return None + if not rows: + return None + row: Final = rows[0] + return StorageProbe( + allocated_bytes=row.allocated_bytes, + live_rows=row.live_rows, + dead_rows=row.dead_rows, + inserted_total=row.inserted_total, + deleted_total=row.deleted_total, + ) + + async def _load_size_state(self, prisma_client: PrismaClient, deadline: float) -> SizeCleanupState | None: + """The reclaimable count saved by the last check, or None on the first check or when it cannot be read.""" + if time.monotonic() >= deadline: + return None + timeout_ms: Final = self._timeout_ms(deadline) + try: + async with db_span("load_size_state", "LiteLLM_Config"), bounded_tx(prisma_client, timeout_ms) as tx: + await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}")) + rows: Final = _SIZE_STATE_ROWS.validate_python( + await tx.query_raw(trusted_sql(_LOAD_SIZE_STATE_SQL), SPEND_LOG_SIZE_CLEANUP_STATE_PARAM) + ) + return SizeCleanupState.model_validate_json(rows[0].state_json) if rows else None + except Exception as e: # noqa: BLE001 - an unreadable count restarts from the catalog, it must not fail the run + verbose_proxy_logger.warning("Could not read the spend-log size cleanup state: %s: %s", type(e).__name__, e) + return None + + async def _save_size_state(self, prisma_client: PrismaClient, deadline: float, state: SizeCleanupState) -> None: + timeout_ms: Final = max(self._timeout_ms(deadline), 1000) + try: + async with db_span("save_size_state", "LiteLLM_Config"), bounded_tx(prisma_client, timeout_ms) as tx: + await tx.execute_raw(trusted_sql(f"SET LOCAL statement_timeout = {timeout_ms}")) + await tx.execute_raw( + trusted_sql(_SAVE_SIZE_STATE_SQL), SPEND_LOG_SIZE_CLEANUP_STATE_PARAM, state.model_dump_json() + ) + except Exception as e: # noqa: BLE001 - a lost save costs one re-seed from the catalog, it must not fail the run + verbose_proxy_logger.warning("Could not save the spend-log size cleanup state: %s: %s", type(e).__name__, e) + def _timeout_ms(self, deadline: float) -> int: """ The per-statement bound in milliseconds: the batch timeout, or whatever diff --git a/tests/unit/proxy/test_spend_log_cleanup.py b/tests/unit/proxy/test_spend_log_cleanup.py index de4e5c90fe0..a9bc82d141d 100644 --- a/tests/unit/proxy/test_spend_log_cleanup.py +++ b/tests/unit/proxy/test_spend_log_cleanup.py @@ -20,6 +20,10 @@ from litellm.constants import ( SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS, ) from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( + SizeCleanupState, + SpendLogStorage, + StorageProbe, + advance_size_state, SPEND_LOG_CLEANUP_BOUND_SETTINGS, SpendLogCleanup, TableCleanupResult, @@ -88,10 +92,10 @@ def test_spend_log_cleanup_cron_scheduling(): assert trigger_weekly is not None # Invalid cron expression should raise ValueError - with pytest.raises(ValueError, match='Wrong number of fields; got'): + with pytest.raises(ValueError, match="Wrong number of fields; got"): CronTrigger.from_crontab("invalid cron") - with pytest.raises(ValueError, match='is higher than the maximum value'): + with pytest.raises(ValueError, match="is higher than the maximum value"): CronTrigger.from_crontab("60 25 * * *") # Invalid minute and hour @@ -151,15 +155,11 @@ def test_spend_log_cleanup_cron_scheduler_integration(): # No cron, so it should fall back to interval } - cleanup_cron_fallback = general_settings_interval.get( - "maximum_spend_logs_cleanup_cron" - ) + cleanup_cron_fallback = general_settings_interval.get("maximum_spend_logs_cleanup_cron") assert cleanup_cron_fallback is None # No cron configured # Simulate interval-based scheduling fallback - retention_interval = general_settings_interval.get( - "maximum_spend_logs_retention_interval", "1d" - ) + retention_interval = general_settings_interval.get("maximum_spend_logs_retention_interval", "1d") from litellm.litellm_core_utils.duration_parser import duration_in_seconds interval_seconds = duration_in_seconds(retention_interval) @@ -187,27 +187,19 @@ async def test_should_delete_spend_logs(): assert cleaner._should_delete_spend_logs() is False # Test case 2: Valid seconds string - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "3600s"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "3600s"}) assert cleaner._should_delete_spend_logs() is True # Test case 3: Valid days string - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "30d"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "30d"}) assert cleaner._should_delete_spend_logs() is True # Test case 4: Valid hours string - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "24h"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "24h"}) assert cleaner._should_delete_spend_logs() is True # Test case 5: Invalid format - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "invalid"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "invalid"}) assert cleaner._should_delete_spend_logs() is False @@ -338,9 +330,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): # Verify the cutoff date is correct cutoff_date = mock_db.execute_raw.call_args[0][1] expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400) - assert ( - abs((cutoff_date - expected_cutoff).total_seconds()) < 1 - ) # Allow 1 second difference for test execution time + assert abs((cutoff_date - expected_cutoff).total_seconds()) < 1 # Allow 1 second difference for test execution time @pytest.mark.asyncio @@ -360,9 +350,7 @@ async def test_cleanup_drops_partitions_when_enabled_and_partitioned(): partition_manager = MagicMock() partition_manager.is_partitioned = AsyncMock(return_value=True) partition_manager.ensure_partitions = AsyncMock(return_value=["p1"]) - partition_manager.drop_partitions_older_than = AsyncMock( - return_value=["LiteLLM_SpendLogs_p20260601"] - ) + partition_manager.drop_partitions_older_than = AsyncMock(return_value=["LiteLLM_SpendLogs_p20260601"]) cleaner = SpendLogCleanup( general_settings={ @@ -500,9 +488,7 @@ async def test_integer_retention_treated_as_days(): An integer value for maximum_spend_logs_retention_period should be treated as days (e.g., 3 → '3d' → 259200 seconds). """ - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": 3} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": 3}) result = cleaner._should_delete_spend_logs() assert result is True assert cleaner.retention_seconds == 3 * 86400 # 3 days in seconds @@ -519,13 +505,11 @@ def test_string_retention_still_works(): ("2w", 2 * 604800), ] for setting, expected_seconds in cases: - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": setting} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": setting}) assert cleaner._should_delete_spend_logs() is True, f"Failed for {setting}" - assert ( - cleaner.retention_seconds == expected_seconds - ), f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}" + assert cleaner.retention_seconds == expected_seconds, ( + f"Expected {expected_seconds} for {setting}, got {cleaner.retention_seconds}" + ) @pytest.mark.asyncio @@ -539,9 +523,7 @@ async def test_delete_old_logs_aborts_on_non_int_execute_raw_return(): mock_db.execute_raw = AsyncMock(return_value=None) mock_prisma_client.db = mock_db - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline()) @@ -560,9 +542,7 @@ async def test_delete_old_logs_continues_on_valid_int_return(): mock_db.execute_raw = AsyncMock(side_effect=[500, 300, 0]) mock_prisma_client.db = mock_db - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline()) @@ -609,9 +589,7 @@ async def test_delete_old_tool_index_rows_deletes_on_composite_key(): mock_db.execute_raw = AsyncMock(side_effect=[5, 0]) mock_prisma_client.db = mock_db - cleaner = SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_tool_index_rows(mock_prisma_client, cutoff_date, _far_deadline()) @@ -631,9 +609,7 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch) import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module # Zero out the failure backoff so the test doesn't take ~0.5s of real sleep. - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 - ) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0) mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) @@ -641,14 +617,10 @@ async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch) _wire_tx(mock_db) # batch 1 succeeds, batch 2 raises (one-off DB timeout), batches 3-4 succeed, # batch 5 returns 0 → loop exits naturally. - mock_db.execute_raw = AsyncMock( - side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0] - ) + mock_db.execute_raw = AsyncMock(side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0]) mock_prisma_client.db = mock_db - cleaner = cleanup_module.SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline()) @@ -665,26 +637,18 @@ async def test_delete_old_logs_aborts_after_consecutive_failures(monkeypatch): import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module # Lower the threshold so the test is fast and deterministic. - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3 - ) - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 - ) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0) mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) mock_db = MagicMock() _wire_tx(mock_db) # Every batch raises — must abort after exactly 3 attempts, not loop forever. - mock_db.execute_raw = AsyncMock( - side_effect=ConnectionError("simulated persistent DB outage") - ) + mock_db.execute_raw = AsyncMock(side_effect=ConnectionError("simulated persistent DB outage")) mock_prisma_client.db = mock_db - cleaner = cleanup_module.SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline()) @@ -699,12 +663,8 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc intermittent timeouts don't trip the abort threshold.""" import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3 - ) - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 - ) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0) mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) @@ -725,9 +685,7 @@ async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatc ) mock_prisma_client.db = mock_db - cleaner = cleanup_module.SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) result = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date, _far_deadline()) @@ -748,9 +706,7 @@ async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch): mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) # Force the outer try/except to fire by making _should_delete_spend_logs raise. - cleaner = cleanup_module.SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cleaner.pod_lock_manager = None def boom(): @@ -775,12 +731,8 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch must still be released so the next scheduled run isn't permanently blocked.""" import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2 - ) - monkeypatch.setattr( - cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 - ) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2) + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0) mock_prisma_client = MagicMock() _wire_tx(mock_prisma_client.db) @@ -794,9 +746,7 @@ async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) mock_pod_lock_manager.release_lock = AsyncMock() - cleaner = cleanup_module.SpendLogCleanup( - general_settings={"maximum_spend_logs_retention_period": "7d"} - ) + cleaner = cleanup_module.SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) cleaner.pod_lock_manager = mock_pod_lock_manager await cleaner.cleanup_old_spend_logs(mock_prisma_client) @@ -1081,9 +1031,7 @@ async def test_each_batch_carries_a_statement_and_lock_timeout(): } ) - await cleaner._delete_old_logs( - mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline() - ) + await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()) assert "SET LOCAL statement_timeout = 12000" in recorded assert "SET LOCAL lock_timeout = 12000" in recorded @@ -1219,9 +1167,7 @@ async def test_remaining_rows_probe_is_capped_so_it_cannot_scan_the_table(): cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) - await cleaner._delete_old_logs( - mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline() - ) + await cleaner._delete_old_logs(mock_prisma_client, datetime.now(timezone.utc) - timedelta(days=7), _far_deadline()) count_sql = mock_db.query_raw.call_args[0][0] assert "count(*)" in count_sql @@ -1733,3 +1679,159 @@ async def test_a_cleanup_delete_batch_renders_a_postgres_delete_span_for_its_tab ) assert await postgres_span_names() == ("postgres.delete LiteLLM_SpendLogs",) + + +# --- spend log storage measurement and the reclaimable-bytes state --------------------------------------- + + +def _storage(allocated: int, live: int, dead: int, inserted: int | None = None, deleted: int | None = None) -> dict: + return { + "allocated_bytes": allocated, + "live_rows": live, + "dead_rows": dead, + "inserted_total": live + dead if inserted is None else inserted, + "deleted_total": dead if deleted is None else deleted, + } + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("180GB", 180 * 1024**3), + ("20 GiB", 20 * 1024**3), + ("512MB", 512 * 1024**2), + ("1.5TB", int(1.5 * 1024**4)), + ("4096", 4096), + (4096, 4096), + ], +) +def test_size_in_bytes_reads_the_units_pg_size_pretty_prints(value, expected): + from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import size_in_bytes + + assert size_in_bytes(value) == expected + + +@pytest.mark.parametrize("value", ["", "big", "-1GB", "0", 0, "1PB"]) +def test_size_in_bytes_rejects_what_cannot_be_a_size(value): + from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import size_in_bytes + + with pytest.raises(ValueError, match="size"): + size_in_bytes(value) + + +_RUN_STATE = SizeCleanupState( + allocated_bytes=32_030_720, + reclaimable_bytes=24_015_000, + bytes_per_row=1601.0, + inserted_total=20_000, + deleted_total=15_000, +) + + +@pytest.mark.parametrize( + ("name", "probe", "reclaimable_expected"), + [ + ( + "vacuum zeroed the dead rows, nothing else changed", + _storage(32_030_720, 5000, 0, 20_000, 15_000), + 24_015_000, + ), + ("7000 new rows written into the free space", _storage(32_030_720, 12_000, 0, 27_000, 15_000), 12_808_000), + ( + "7000 new rows that all grew the files", + _storage(32_030_720 + 7000 * 1601, 12_000, 0, 27_000, 15_000), + 24_015_000, + ), + ("VACUUM FULL handed the free space back", _storage(8_015_720, 5000, 0, 20_000, 15_000), 0), + ( + "statistics wiped while the files still hold data: the count is carried forward, not reseeded", + _storage(32_030_720, 0, 0, 0, 0), + 24_015_000, + ), + ("the retention job deleted 2000 more rows", _storage(32_030_720, 3000, 2000, 20_000, 17_000), 27_217_000), + ( + "statistics reset: counters restart from the dead rows", + _storage(32_030_720, 5000, 1000, 5000, 1000), + 5_338_453, + ), + ], +) +def test_advance_size_state_tracks_free_space_through_vacuum_inserts_and_other_deletes( + name, probe, reclaimable_expected +): + advanced = advance_size_state(_RUN_STATE, StorageProbe(**probe)) + + assert advanced.reclaimable_bytes == reclaimable_expected, name + assert 0 <= probe["allocated_bytes"] - advanced.reclaimable_bytes <= probe["allocated_bytes"], name + + +def test_after_deleting_counts_the_rows_and_any_file_growth_as_reclaimable_and_to_state_keeps_the_counters(): + storage = SpendLogStorage(allocated_bytes=32_030_720, reclaimable_bytes=0, bytes_per_row=1601.0, live_rows=20_000) + + after = storage.after_deleting(15_000, 32_030_720 + 8192) + + assert (after.reclaimable_bytes, after.live_rows, after.estimated_live_bytes) == (24_023_192, 5000, 8_015_720) + assert after.to_state(inserted_total=20_000, deleted_total=15_000) == SizeCleanupState( + allocated_bytes=32_038_912, + reclaimable_bytes=24_023_192, + bytes_per_row=1601.0, + inserted_total=20_000, + deleted_total=15_000, + ) + assert StorageProbe( + allocated_bytes=8192, live_rows=0, dead_rows=0, inserted_total=0, deleted_total=0 + ).stats_are_stale + + +@pytest.mark.asyncio +async def test_probe_storage_reads_the_catalog_row_for_both_tables_and_a_failed_probe_is_none(caplog): + prisma_client = MagicMock() + _wire_tx(prisma_client.db) + prisma_client.db.query_raw = AsyncMock(return_value=[_storage(32_030_720, 20_000, 0)]) + cleaner = SpendLogCleanup(general_settings={}) + + probe = await cleaner._probe_storage(prisma_client, _far_deadline()) + + assert probe is not None + assert (probe.allocated_bytes, probe.live_rows, probe.dead_rows, probe.inserted_total, probe.deleted_total) == ( + 32_030_720, + 20_000, + 0, + 20_000, + 0, + ) + assert prisma_client.db.query_raw.call_args.args[1] == ["LiteLLM_SpendLogs", "LiteLLM_SpendLogToolIndex"] + + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("pg_stat_user_tables unavailable")) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + assert await cleaner._probe_storage(prisma_client, _far_deadline()) is None + assert any("Could not measure spend-log storage" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_size_state_round_trips_through_its_litellm_config_row_and_a_lost_save_is_only_a_warning(caplog): + saved = {} + prisma_client = MagicMock() + _wire_tx(prisma_client.db) + + async def execute_raw(sql, *args): + saved[args[0]] = args[1] + return 1 + + prisma_client.db.execute_raw = execute_raw + prisma_client.db.query_raw = AsyncMock( + side_effect=lambda sql, param: [{"state_json": saved[param]}] if param in saved else [] + ) + cleaner = SpendLogCleanup(general_settings={}) + + assert await cleaner._load_size_state(prisma_client, _far_deadline()) is None + await cleaner._save_size_state(prisma_client, _far_deadline(), _RUN_STATE) + assert list(saved) == ["spend_logs_size_cleanup_state"] + loaded = await cleaner._load_size_state(prisma_client, _far_deadline()) + assert loaded is not None + assert loaded.model_dump() == _RUN_STATE.model_dump() + + prisma_client.db.execute_raw = AsyncMock(side_effect=RuntimeError("LiteLLM_Config is read only")) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await cleaner._save_size_state(prisma_client, _far_deadline(), _RUN_STATE) + assert any("Could not save the spend-log size cleanup state" in record.getMessage() for record in caplog.records) From 424687ce09b6907070ed71e9a1768345ad36de2a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:31:37 +0000 Subject: [PATCH 060/124] fix(proxy): build the OpenAPI schema off the event loop so a cold /openapi.json no longer stalls the worker (#47352) * test(proxy): keep answering requests while a cold /openapi.json builds * fix(proxy): build the OpenAPI schema off the event loop so a cold /openapi.json no longer stalls the worker * perf(proxy): reuse the rendered /openapi.json body until the schema changes * fix(proxy): scope the openapi build's pydantic patch to the build The schema build swapped a process-wide pydantic hook, which live requests saw once the build moved to a thread, and overlapping builds could leave it installed. The hook now checks a context variable set only inside the build. The proxy test counts liveliness answers instead of timing them. * test(proxy): hold the cold openapi build on an event instead of pacing on a clock * fix(proxy): serve a replaced app.openapi like FastAPI's own docs route * test(proxy): build the cold-docs test on the proxy module's current app * test(proxy): freeze the model that holds the cold docs build --- litellm/proxy/common_utils/openapi_route.py | 82 ++++++++++ .../common_utils/openapi_schema_compat.py | 61 +++---- litellm/proxy/proxy_server.py | 67 +++++--- .../proxy/common_utils/test_openapi_route.py | 154 ++++++++++++++++++ .../test_openapi_schema_compat.py | 59 +++++++ .../test_openapi_customization.py | 41 +++++ 6 files changed, 407 insertions(+), 57 deletions(-) create mode 100644 litellm/proxy/common_utils/openapi_route.py create mode 100644 tests/unit/proxy/common_utils/test_openapi_route.py create mode 100644 tests/unit/proxy/common_utils/test_openapi_schema_compat.py diff --git a/litellm/proxy/common_utils/openapi_route.py b/litellm/proxy/common_utils/openapi_route.py new file mode 100644 index 00000000000..69fbeb67277 --- /dev/null +++ b/litellm/proxy/common_utils/openapi_route.py @@ -0,0 +1,82 @@ +import asyncio +import functools +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Final, TypeAlias, TypeVar + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from pydantic import TypeAdapter +from starlette.routing import BaseRoute, Route + +from litellm.proxy.common_utils.custom_openapi_spec import JsonObject + +OpenAPISchema: TypeAlias = JsonObject +Inputs: Final = TypeVar("Inputs") +_SERVERS: Final = TypeAdapter(list[dict[str, object]]) + + +@dataclass(frozen=True, slots=True, eq=False) +class _SameSchema: + schema: Mapping[str, object] + + def __eq__(self, other: object) -> bool: + return isinstance(other, _SameSchema) and other.schema is self.schema + + def __hash__(self) -> int: + return id(self.schema) + + +def serve_openapi_off_the_event_loop( + app: FastAPI, + snapshot: Callable[[], Inputs], + build: Callable[[Inputs], OpenAPISchema], +) -> None: + openapi_url: Final = app.openapi_url + if openapi_url is None: + return + build_lock: Final = asyncio.Lock() + installed_openapi: Final = app.openapi + + async def built_schema() -> OpenAPISchema: + async with build_lock: + if app.openapi_schema: + return app.openapi_schema + inputs: Final = snapshot() + schema: Final = await asyncio.to_thread(build, inputs) + if app.openapi_schema is None and snapshot() == inputs: + app.openapi_schema = schema + return schema + + async def served_schema() -> OpenAPISchema: + if app.openapi != installed_openapi: + return app.openapi() + return app.openapi_schema or await built_schema() + + async def openapi(request: Request) -> Response: + schema: Final = await served_schema() + scope_root_path: Final[object] = request.scope.get("root_path") + root_path: Final = scope_root_path.rstrip("/") if isinstance(scope_root_path, str) else "" + servers_root_path: Final = root_path if app.root_path_in_servers else "" + return Response(_rendered(_SameSchema(schema), servers_root_path), media_type="application/json") + + app.router.routes[:] = [ # rebind-ok: the app owns its route table + Route(openapi_url, openapi, include_in_schema=False) if _serves(route, openapi_url) else route + for route in app.router.routes + ] + + +@functools.lru_cache(maxsize=1) +def _rendered(served: _SameSchema, root_path: str) -> bytes | memoryview: + return JSONResponse(_with_root_path_server(served.schema, root_path) if root_path else served.schema).body + + +def _with_root_path_server(schema: Mapping[str, object], root_path: str) -> Mapping[str, object]: + servers: Final = _SERVERS.validate_python(schema.get("servers", [])) + if any(server.get("url") == root_path for server in servers): + return schema + return {**schema, "servers": [{"url": root_path}, *servers]} + + +def _serves(route: BaseRoute, path: str) -> bool: + return isinstance(route, Route) and route.path == path diff --git a/litellm/proxy/common_utils/openapi_schema_compat.py b/litellm/proxy/common_utils/openapi_schema_compat.py index 1826daa2762..5bc9121858c 100644 --- a/litellm/proxy/common_utils/openapi_schema_compat.py +++ b/litellm/proxy/common_utils/openapi_schema_compat.py @@ -5,10 +5,33 @@ FastAPI 0.120+ has stricter schema generation that fails on certain types like o This module provides a compatibility layer to handle these cases gracefully. """ +import contextvars +import functools from typing import Final from litellm._logging import verbose_proxy_logger +_BUILDING_OPENAPI_SCHEMA: Final = contextvars.ContextVar("building_openapi_schema", default=False) + + +@functools.cache +def _accept_unknown_types_while_building() -> None: + from pydantic._internal._generate_schema import GenerateSchema + from pydantic_core import core_schema + + reject_unknown_type: Final = ( + GenerateSchema._unknown_type_schema # pyright: ignore[reportPrivateUsage] # Pydantic schema internals + ) + + def unknown_type_schema(self: GenerateSchema, obj: object) -> core_schema.CoreSchema: + if not _BUILDING_OPENAPI_SCHEMA.get(): + return reject_unknown_type(self, obj) + if getattr(obj, "__module__", "") == "openai" and "Timeout" in str(obj): + return core_schema.str_schema() + return core_schema.any_schema() + + setattr(GenerateSchema, "_unknown_type_schema", unknown_type_schema) + def get_openapi_schema_with_compat( get_openapi_func, @@ -33,40 +56,9 @@ def get_openapi_schema_with_compat( Returns: OpenAPI schema dictionary """ - # FastAPI 0.120+ may fail schema generation for certain types (e.g., openai.Timeout) - # Patch Pydantic's schema generation to handle unknown types gracefully try: - from pydantic._internal._generate_schema import GenerateSchema - from pydantic_core import core_schema - - # Store original method - original_unknown_type_schema: Final = ( - GenerateSchema._unknown_type_schema # pyright: ignore[reportPrivateUsage] # Pydantic schema internals - ) - - def patched_unknown_type_schema(self, obj): - """Patch to handle openai.Timeout and other non-serializable types""" - # Check if it's openai.Timeout or similar types - obj_str: Final = str(obj) - obj_module: Final = getattr(obj, "__module__", "") - - if (obj_module == "openai" and "Timeout" in obj_str) or ( - hasattr(obj, "__name__") and obj.__name__ == "Timeout" and obj_module == "openai" - ): - # Return a simple string schema for Timeout types - return core_schema.str_schema() - - # For other unknown types, try to return a default schema - # This prevents the error from propagating - try: - return core_schema.any_schema() - except Exception: - # Last resort: return string schema - return core_schema.str_schema() - - # Apply patch - setattr(GenerateSchema, "_unknown_type_schema", patched_unknown_type_schema) - + _accept_unknown_types_while_building() + building: Final = _BUILDING_OPENAPI_SCHEMA.set(True) try: openapi_schema: Final = get_openapi_func( title=title, @@ -75,8 +67,7 @@ def get_openapi_schema_with_compat( routes=routes, ) finally: - # Restore original method - setattr(GenerateSchema, "_unknown_type_schema", original_unknown_type_schema) + _BUILDING_OPENAPI_SCHEMA.reset(building) return openapi_schema diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bbdc996a027..581698626d5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1995,6 +1995,9 @@ vertex_live_passthrough_vertex_base: Final = VertexBase() ### CUSTOM API DOCS [ENTERPRISE FEATURE] ### # Custom OpenAPI schema generator to include only selected routes from fastapi.routing import APIWebSocketRoute +from starlette.routing import BaseRoute + +from litellm.proxy.common_utils.openapi_route import OpenAPISchema, serve_openapi_off_the_event_loop def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_routes: list) -> dict: @@ -2041,10 +2044,19 @@ def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_ return openapi_schema -def get_openapi_schema(): - if app.openapi_schema: - return app.openapi_schema +@dataclass(frozen=True, slots=True) +class _OpenAPIInputs: + routes: tuple[BaseRoute, ...] + loaded_lazy_modules: frozenset[str] + +def _openapi_inputs() -> _OpenAPIInputs: + from litellm.proxy._lazy_features import loaded_lazy_modules + + return _OpenAPIInputs(routes=tuple(app.routes), loaded_lazy_modules=loaded_lazy_modules(app)) + + +def _build_openapi_schema(inputs: _OpenAPIInputs) -> OpenAPISchema: # Use compatibility wrapper for FastAPI 0.120+ schema generation from litellm.proxy.common_utils.openapi_schema_compat import ( get_openapi_schema_with_compat, @@ -2055,11 +2067,11 @@ def get_openapi_schema(): title=app.title, version=app.version, description=app.description, - routes=app.routes, + routes=list(inputs.routes), ) # Find all WebSocket routes - websocket_routes: Final = [route for route in app.routes if isinstance(route, APIWebSocketRoute)] + websocket_routes: Final = [route for route in inputs.routes if isinstance(route, APIWebSocketRoute)] # Add a synthetic GET stub for each so they render in Swagger UI, # without clobbering existing HTTP operations on the same path. @@ -2071,31 +2083,34 @@ def get_openapi_schema(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) # Stub unloaded lazy features so they appear as Swagger sections. - from litellm.proxy._lazy_features import inject_lazy_stubs, loaded_lazy_modules + from litellm.proxy._lazy_features import inject_lazy_stubs - openapi_schema = inject_lazy_stubs(openapi_schema, loaded_lazy_modules(app)) + openapi_schema = inject_lazy_stubs(openapi_schema, inputs.loaded_lazy_modules) openapi_schema = ensure_unique_openapi_operation_ids(openapi_schema) # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] - app.openapi_schema = openapi_schema + return openapi_schema + + +def get_openapi_schema(): + if app.openapi_schema: + return app.openapi_schema + app.openapi_schema = _build_openapi_schema(_openapi_inputs()) return app.openapi_schema -def custom_openapi(): - if app.openapi_schema: - return app.openapi_schema - openapi_schema = get_openapi_schema() +def _build_docs_filtered_openapi_schema(inputs: _OpenAPIInputs) -> OpenAPISchema: + openapi_schema = _build_openapi_schema(inputs) - # Filter routes to include only specific ones - openai_routes: Final = LiteLLMRoutes.openai_routes.value - paths_to_include: Final[dict] = {} - for route in openai_routes: - if route in openapi_schema["paths"]: - paths_to_include[route] = openapi_schema["paths"][route] - openapi_schema["paths"] = paths_to_include + all_paths: Final = openapi_schema.get("paths") + openapi_schema["paths"] = ( + {route: all_paths[route] for route in LiteLLMRoutes.openai_routes.value if route in all_paths} + if isinstance(all_paths, dict) + else {} + ) # Add LLM API request schema bodies for documentation from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec @@ -2103,24 +2118,32 @@ def custom_openapi(): openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema) # Stub unloaded lazy features so they appear as Swagger sections. - from litellm.proxy._lazy_features import inject_lazy_stubs, loaded_lazy_modules + from litellm.proxy._lazy_features import inject_lazy_stubs - openapi_schema = inject_lazy_stubs(openapi_schema, loaded_lazy_modules(app)) + openapi_schema = inject_lazy_stubs(openapi_schema, inputs.loaded_lazy_modules) openapi_schema = ensure_unique_openapi_operation_ids(openapi_schema) # Fix Swagger UI execute path error when server_root_path is set if server_root_path: openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}] - app.openapi_schema = openapi_schema + return openapi_schema + + +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + app.openapi_schema = _build_docs_filtered_openapi_schema(_openapi_inputs()) return app.openapi_schema if os.getenv("DOCS_FILTERED", "False") == "True" and premium_user: app.openapi = custom_openapi + serve_openapi_off_the_event_loop(app, _openapi_inputs, _build_docs_filtered_openapi_schema) else: # For regular users, use get_openapi_schema to include LLM API schemas app.openapi = get_openapi_schema + serve_openapi_off_the_event_loop(app, _openapi_inputs, _build_openapi_schema) class UserAPIKeyCacheTTLEnum(enum.Enum): diff --git a/tests/unit/proxy/common_utils/test_openapi_route.py b/tests/unit/proxy/common_utils/test_openapi_route.py new file mode 100644 index 00000000000..1fe4aaf9e67 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_openapi_route.py @@ -0,0 +1,154 @@ +import asyncio +import itertools +import threading +from _collections_abc import dict_items +from collections.abc import Callable +from typing import Final + +import httpx +import pytest +from fastapi import FastAPI + +from litellm.proxy.common_utils.custom_openapi_spec import JsonValue +from litellm.proxy.common_utils.openapi_route import OpenAPISchema, serve_openapi_off_the_event_loop + +UNBLOCK_TIMEOUT_SECONDS: Final = 5 + + +def _route_paths(app: FastAPI) -> Callable[[], tuple[str, ...]]: + return lambda: tuple(str(getattr(route, "path", "")) for route in app.routes) + + +def _client(app: FastAPI, root_path: str = "") -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app, root_path=root_path), base_url="http://proxy") + + +async def _let_other_tasks_run() -> None: + for _ in range(50): + await asyncio.sleep(0) + + +async def test_other_requests_are_answered_while_the_schema_builds(): + app: Final = FastAPI() + app.add_api_route("/ping", lambda: "pong") + build_started: Final = threading.Event() + ping_answered: Final = threading.Event() + + def build(paths: tuple[str, ...]) -> OpenAPISchema: + build_started.set() + return {"ping_answered_during_build": ping_answered.wait(UNBLOCK_TIMEOUT_SECONDS)} + + serve_openapi_off_the_event_loop(app, _route_paths(app), build) + async with _client(app) as client: + schema_request: Final = asyncio.ensure_future(client.get("/openapi.json")) + await asyncio.to_thread(build_started.wait, UNBLOCK_TIMEOUT_SECONDS) + ping: Final = await client.get("/ping") + ping_answered.set() + schema: Final = await schema_request + + assert (ping.json(), schema.json()) == ("pong", {"ping_answered_during_build": True}) + + +async def test_a_route_added_during_the_build_is_in_the_next_schema(): + app: Final = FastAPI() + build_started: Final = threading.Event() + may_finish: Final = threading.Event() + + def build(paths: tuple[str, ...]) -> OpenAPISchema: + build_started.set() + may_finish.wait(UNBLOCK_TIMEOUT_SECONDS) + return {"paths": [*paths]} + + serve_openapi_off_the_event_loop(app, _route_paths(app), build) + async with _client(app) as client: + first_request: Final = asyncio.ensure_future(client.get("/openapi.json")) + await asyncio.to_thread(build_started.wait, UNBLOCK_TIMEOUT_SECONDS) + app.add_api_route("/registered-mid-build", lambda: "late") + may_finish.set() + first: Final = await first_request + second: Final = await client.get("/openapi.json") + + assert ("/registered-mid-build" in first.json()["paths"], "/registered-mid-build" in second.json()["paths"]) == ( + False, + True, + ) + + +async def test_concurrent_cold_requests_share_one_build(): + app: Final = FastAPI() + build_numbers: Final = itertools.count() + build_started: Final = threading.Event() + may_finish: Final = threading.Event() + + def build(paths: tuple[str, ...]) -> OpenAPISchema: + build_started.set() + may_finish.wait(UNBLOCK_TIMEOUT_SECONDS) + return {"build": next(build_numbers)} + + serve_openapi_off_the_event_loop(app, _route_paths(app), build) + async with _client(app) as client: + first_request: Final = asyncio.ensure_future(client.get("/openapi.json")) + await asyncio.to_thread(build_started.wait, UNBLOCK_TIMEOUT_SECONDS) + second_request: Final = asyncio.ensure_future(client.get("/openapi.json")) + await _let_other_tasks_run() + may_finish.set() + responses: Final = await asyncio.gather(first_request, second_request) + + assert [response.json() for response in responses] == [{"build": 0}, {"build": 0}] + + +async def test_warm_requests_reuse_the_rendered_schema(): + app: Final = FastAPI() + renders: Final = itertools.count() + + class RenderCountingSchema(dict[str, JsonValue]): + def items(self) -> "dict_items[str, JsonValue]": + _ = next(renders) + return super().items() + + serve_openapi_off_the_event_loop(app, _route_paths(app), lambda paths: RenderCountingSchema(paths=[*paths])) + async with _client(app) as client: + bodies: Final = [(await client.get("/openapi.json")).content for _ in range(3)] + + assert (len(set(bodies)), next(renders)) == (1, 1) + + +async def test_a_rebuilt_schema_is_served_instead_of_the_previous_one(): + app: Final = FastAPI() + builds: Final = itertools.count() + serve_openapi_off_the_event_loop(app, _route_paths(app), lambda paths: {"build": next(builds)}) + + async with _client(app) as client: + first: Final = await client.get("/openapi.json") + app.openapi_schema = None + second: Final = await client.get("/openapi.json") + + assert (first.json(), second.json()) == ({"build": 0}, {"build": 1}) + + + +async def test_an_app_that_replaces_its_openapi_function_serves_that_schema(): + app: Final = FastAPI() + serve_openapi_off_the_event_loop(app, _route_paths(app), lambda paths: {"built_by": "the proxy"}) + app.openapi = lambda: {"built_by": "the operator"} + + async with _client(app) as client: + response: Final = await client.get("/openapi.json") + + assert response.json() == {"built_by": "the operator"} + +@pytest.mark.parametrize( + ("servers", "expected"), + [ + ([{"url": "https://docs.example"}], [{"url": "/api"}, {"url": "https://docs.example"}]), + ([{"url": "/api"}], [{"url": "/api"}]), + ], +) +async def test_the_root_path_is_listed_first_among_the_servers(servers: JsonValue, expected: JsonValue): + app: Final = FastAPI() + serve_openapi_off_the_event_loop(app, _route_paths(app), lambda paths: {"servers": servers}) + + async with _client(app, root_path="/api") as client: + response: Final = await client.get("/openapi.json") + + assert response.json()["servers"] == expected diff --git a/tests/unit/proxy/common_utils/test_openapi_schema_compat.py b/tests/unit/proxy/common_utils/test_openapi_schema_compat.py new file mode 100644 index 00000000000..39c508319f0 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_openapi_schema_compat.py @@ -0,0 +1,59 @@ +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Final + +import pytest +from pydantic import PydanticSchemaGenerationError, TypeAdapter + +from litellm.proxy.common_utils.openapi_schema_compat import get_openapi_schema_with_compat + +UNBLOCK_TIMEOUT_SECONDS: Final = 5 + + +@dataclass(frozen=True, slots=True) +class _BlockingBuild: + started: threading.Event = field(default_factory=threading.Event) + may_finish: threading.Event = field(default_factory=threading.Event) + + def __call__(self, **_: object) -> dict[str, object]: + self.started.set() + _ = self.may_finish.wait(UNBLOCK_TIMEOUT_SECONDS) + return {} + + +def test_only_the_build_documents_a_type_pydantic_cannot_describe(): + class Opaque: + pass + + def build(**_: object) -> dict[str, object]: + return {"opaque": TypeAdapter(Opaque).json_schema()} + + schema: Final = get_openapi_schema_with_compat(build, "title", "1.0.0", "", []) + + with pytest.raises(PydanticSchemaGenerationError): + _ = TypeAdapter(Opaque) + assert schema == {"opaque": {}} + + +def test_code_outside_overlapping_builds_still_rejects_unknown_types(): + class Opaque: + pass + + first: Final = _BlockingBuild() + second: Final = _BlockingBuild() + + with ThreadPoolExecutor(max_workers=2) as pool: + first_result: Final = pool.submit(get_openapi_schema_with_compat, first, "title", "1.0.0", "", []) + _ = first.started.wait(UNBLOCK_TIMEOUT_SECONDS) + second_result: Final = pool.submit(get_openapi_schema_with_compat, second, "title", "1.0.0", "", []) + _ = second.started.wait(UNBLOCK_TIMEOUT_SECONDS) + with pytest.raises(PydanticSchemaGenerationError): + _ = TypeAdapter(Opaque) + first.may_finish.set() + _ = first_result.result() + second.may_finish.set() + _ = second_result.result() + + with pytest.raises(PydanticSchemaGenerationError): + _ = TypeAdapter(Opaque) diff --git a/tests/unit/proxy/proxy_server/test_openapi_customization.py b/tests/unit/proxy/proxy_server/test_openapi_customization.py index 141b9f2a98a..6c588976efc 100644 --- a/tests/unit/proxy/proxy_server/test_openapi_customization.py +++ b/tests/unit/proxy/proxy_server/test_openapi_customization.py @@ -13,16 +13,24 @@ Pins covered: from __future__ import annotations +import asyncio +import threading from types import SimpleNamespace +from typing import Final +import httpx import pytest from fastapi import FastAPI +from pydantic import BaseModel, ConfigDict, GetJsonSchemaHandler +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema from litellm.proxy.proxy_server import ( _generate_stable_operation_id, _get_cors_config, _inject_websocket_stubs_into_openapi_schema, _strip_operation_id_method_suffix, + app, custom_openapi, ensure_unique_openapi_operation_ids, get_openapi_schema, @@ -31,6 +39,8 @@ from litellm.proxy.proxy_server import ( from .conftest import normalize +UNBLOCK_TIMEOUT_SECONDS: Final = 5 + # --------------------------------------------------------------------------- # _generate_stable_operation_id # --------------------------------------------------------------------------- @@ -281,6 +291,37 @@ def test_get_openapi_schema_missing_app_attribute_raises_error(monkeypatch): get_openapi_schema() +async def test_a_request_is_answered_while_a_cold_openapi_json_builds(monkeypatch: pytest.MonkeyPatch): + build_started: Final = threading.Event() + liveliness_answered: Final = threading.Event() + build_saw_the_answer: Final = threading.Event() + + class HoldsTheDocsBuild(BaseModel): + model_config = ConfigDict(frozen=True) + + @classmethod + def __get_pydantic_json_schema__(cls, schema: CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: + build_started.set() + if liveliness_answered.wait(UNBLOCK_TIMEOUT_SECONDS): + build_saw_the_answer.set() + return handler(schema) + + import litellm.proxy.proxy_server as ps + + proxy_app: Final = ps.app + monkeypatch.setattr(proxy_app, "openapi_schema", None) + monkeypatch.setattr(proxy_app.router, "routes", [*proxy_app.router.routes]) + proxy_app.add_api_route("/holds-the-docs-build", lambda: None, response_model=HoldsTheDocsBuild) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=proxy_app), base_url="http://proxy") as client: + docs_request: Final = asyncio.ensure_future(client.get("/openapi.json")) + await asyncio.to_thread(build_started.wait, UNBLOCK_TIMEOUT_SECONDS) + liveliness: Final = await client.get("/health/liveliness") + liveliness_answered.set() + docs: Final = await docs_request + + assert (liveliness.status_code, docs.status_code, build_saw_the_answer.is_set()) == (200, 200, True) + + # --------------------------------------------------------------------------- # custom_openapi # --------------------------------------------------------------------------- From 96f1bc02d8c797780d90dda630caa85f90921fd1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:34:55 +0000 Subject: [PATCH 061/124] feat(router)!: apply each deployment's reasoning_effort_levels on every provider and route (#46601) * feat(router): apply each deployment's reasoning_effort_levels on every provider and route A deployment's declared reasoning_effort_levels now decide the effort sent upstream on /v1/chat/completions, /v1/messages and /v1/responses for every provider. Requests round up by default, litellm.reasoning_effort_rounding and model_info.reasoning_effort_rounding switch to down, a declaration lets providers like azure_ai accept reasoning_effort, and each remap logs one INFO line with structured fields. * fix(router): round reasoning effort only on a deployment's own declared levels Cost map levels no longer round requests or force-allow reasoning_effort, so undeclared sibling deployments and catalog-only models behave as on main. A declaring deployment lets azure_ai accept any level the resolver sends, and a hosted_vllm chat thinking budget is rounded too. * fix(router): drop undeclared reasoning effort under drop_params and round native /v1/messages effort Under drop_params a level outside the deployment's reasoning_effort_levels is dropped instead of sent. Providers that take /v1/messages natively now round output_config.effort onto the deployment's declaration. The rounding setting ignores case and surrounding spaces. * fix(router): strip line breaks from every field in the reasoning effort remap log * fix(router): round thinking-budget efforts onto a declaring deployment on every provider * fix(router): keep the caller's thinking display when rounding a thinking budget on adaptive Claude Warn on a reasoning_effort_rounding value other than up or down instead of silently rounding up * test(integration): cover deployment reasoning_effort_levels rounding on chat, messages and responses Wire cells assert the exact body each provider receives when a deployment declares reasoning_effort_levels, plus undeclared controls, malformed input, drop_params, rounding settings, fallbacks, model groups, live updates and the response cache. Chaos cells cover a killed worker, a proxy restart, an upstream outage, a slow stream and the INFO remap log. * test(integration): cut held streams when the upstream stops in the reasoning effort chaos cell The upstream-stop cell waited for the held streams before stopping the upstream, so they only ended on a client or proxy timeout and the result depended on which fired first. The upstream now stops while the streams are held, and each cut stream must carry an error and no answer * fix(router): round thinking budgets only on declaring deployments and round Converse output_config An undeclared hosted_vllm deployment serving a Claude model name now maps a thinking budget the same as any other undeclared hosted_vllm deployment. A model_info that echoes a cost map key no longer borrows that entry's levels. A non-numeric thinking.budget_tokens answers 400, or is dropped under drop_params, where main answered 500. A declaring Bedrock Converse Claude deployment now rounds output_config.effort, which /v1/messages sends for Claude Code * test(integration): cover output_config rounding and non-numeric thinking budgets in the reasoning effort cells * test(integration): keep an explicit reasoning_effort winning over a malformed thinking budget on hosted_vllm * test(integration): cover output_config rounding on openai, azure and azure_ai chat * fix(hosted_vllm): let an explicit effort win over a malformed budget and reject one on /v1/messages An explicit reasoning_effort beside a non-numeric thinking.budget_tokens keeps main's behavior again. A declared hosted_vllm deployment with a Claude model name maps a budget through the generic table instead of the adaptive Claude one. /v1/messages to a non-Claude target hands a malformed budget to completion(), which answers 400 or drops it under drop_params, where it answered 500 * fix(router): drop thinking beside an explicit effort when a declaring deployment's provider cannot read it * test(integration): cover malformed thinking budgets on /v1/messages to non-Claude targets * fix(anthropic): answer 400 for a malformed thinking budget on /v1/messages to any non-Claude model, or drop it under drop_params * test(integration): cover an explicit effort beside thinking on azure_ai and drop_params on the /v1/messages bridges * fix(anthropic): answer 400 for a malformed thinking budget through adapter_completion without drop_params * fix(anthropic): type the adapter_completion request so the drop_params read adds no unknown argument * fix(router): a default effort beside a thinking budget sends no effort on chat Chat now leaves the budget-derived effort out when the caller's effort resolves to none or asks output_config for one, matching /v1/messages. The chat thinking decision moves into resolve_chat_reasoning so completion() stays under basedpyright's complexity limit. * test: type respx_mock and mark the adapter result Final in the reasoning effort tests * fix(router): keep the caller's thinking beside a kept effort and the budget effort beside a dropped one A non-default output_config.effort beside a thinking budget now leaves the thinking as sent, so Bedrock Invoke keeps display updates. A level drop_params drops counts as never sent, so the budget still sets the effort. output_config.effort replaces a thinking budget the provider cannot read, and a non-numeric string budget on adaptive Claude answers 400 in place of 500, or drops under drop_params. * fix(router): send a kept output_config effort as reasoning_effort where the provider cannot read thinking On a declaring hosted_vllm or azure_ai deployment, a thinking budget goes out as reasoning_effort. An output_config.effort beside it now takes that slot, rounded onto the declared levels, instead of leaving only output_config, which those backends ignore. * fix(reasoning-effort): honor global drop_params for a non-numeric budget and keep the budget effort beside a dropped /v1/messages level * fix(reasoning-effort): send a level a gpt-5 deployment declares even where the cost map does not list it * fix(reasoning): apply declared levels on azure_ai gpt-5 and reject a bool thinking budget on adaptive Claude * fix(router): drop a thinking budget's effort that a deployment declaring no ranked levels excludes A deployment declaring [] or ["none"] skipped the budget check, so adaptive Claude still sent the budget-derived output_config.effort under drop_params on chat, Bedrock Converse, and native /v1/messages. The adaptive thinking now goes out without that effort * fix(anthropic): reject a false, null or empty thinking budget on adaptive Claude --- litellm/__init__.py | 1 + litellm/constants.py | 2 +- .../reasoning_effort_utils.py | 28 +- litellm/llms/anthropic/common_utils.py | 36 +- .../pass_through/adapters/handler.py | 15 +- .../pass_through/adapters/transformation.py | 22 +- .../pass_through/messages/handler.py | 61 +- .../responses_adapters/handler.py | 2 + .../responses_adapters/transformation.py | 14 +- litellm/llms/anthropic/pass_through/utils.py | 12 +- .../llms/azure/chat/gpt_5_transformation.py | 4 + litellm/llms/azure_ai/chat/transformation.py | 5 +- .../llms/hosted_vllm/chat/transformation.py | 15 +- .../llms/openai/chat/gpt_5_transformation.py | 9 +- .../llms/openai/responses/transformation.py | 17 +- litellm/main.py | 63 +- litellm/responses/main.py | 23 +- litellm/responses/utils.py | 33 +- .../reasoning_effort_capability.py | 365 ++- litellm/utils.py | 47 +- .../_reasoning_effort_levels_support.py | 296 +++ .../test_reasoning_effort_levels_chaos.py | 364 +++ .../test_reasoning_effort_levels_wire.py | 2176 +++++++++++++++++ ...al_pass_through_adapters_transformation.py | 31 + ..._handler_reasoning_effort_normalization.py | 215 ++ .../test_anthropic_messages_router.py | 161 ++ .../anthropic/test_anthropic_common_utils.py | 31 + .../test_responses_api_request_body.py | 99 + .../test_reasoning_effort_capability.py | 227 ++ tests/unit/test_main.py | 672 ++++- 30 files changed, 4971 insertions(+), 75 deletions(-) create mode 100644 tests/integration/routing/_reasoning_effort_levels_support.py create mode 100644 tests/integration/routing/test_reasoning_effort_levels_chaos.py create mode 100644 tests/integration/routing/test_reasoning_effort_levels_wire.py diff --git a/litellm/__init__.py b/litellm/__init__.py index b3e80bfece5..1f9b6c5264c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -354,6 +354,7 @@ llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False +reasoning_effort_rounding: Literal["up", "down"] = "up" ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec diff --git a/litellm/constants.py b/litellm/constants.py index 2d54a873b89..7d8311eb915 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1049,7 +1049,7 @@ openai_compatible_endpoints: Final[list[str]] = [ ] -openai_compatible_providers: Final[list] = [ +openai_compatible_providers: Final[list[str]] = [ "anyscale", "groq", "nvidia_nim", diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index 5987392d070..acd20aeb37d 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,16 +1,19 @@ -from typing import Literal +from typing import Final, Literal +import litellm from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, ) +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] def reasoning_effort_from_thinking_budget( - budget_tokens: int, + budget_tokens: float, ) -> OpenAIStyleReasoningEffort: """Bucket an Anthropic ``thinking.budget_tokens`` into an OpenAI-style ``reasoning_effort`` using the shared ``DEFAULT_REASONING_EFFORT_*_THINKING_BUDGET`` @@ -24,3 +27,24 @@ def reasoning_effort_from_thinking_budget( if budget_tokens >= DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET: return "low" return "minimal" + + +def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None: + fields: Final = as_str_mapping(thinking) + if fields is None or fields.get("type") != "enabled": + return None + budget: Final = fields.get("budget_tokens", 0) + if isinstance(budget, (int, float)) and not isinstance(budget, bool): + return reasoning_effort_from_thinking_budget(budget) + if drop_params or litellm.drop_params is True: + return None + raise non_numeric_thinking_budget_error(budget) + + +def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=( + f"thinking.budget_tokens must be a number, got {type(budget).__name__}. To drop it, set " + "`litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" + ) + ) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 578329c4c11..0db8f5c33ed 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error from litellm.llms.anthropic.wif import ( aget_anthropic_wif_token, anthropic_base_without_chat_suffix, @@ -497,6 +498,27 @@ def _fetch_anthropic_model_ids( raise Exception(f"Anthropic /v1/models did not terminate within {_MODEL_LIST_PAGE_CAP} pages.") +def _numeric_budget_tokens(budget_tokens: object) -> int | None: + if isinstance(budget_tokens, bool): + return None + if isinstance(budget_tokens, (int, float)): + return int(budget_tokens) + if not isinstance(budget_tokens, str): + return None + try: + return int(budget_tokens) + except ValueError: + return None + + +def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None: + if (numeric := _numeric_budget_tokens(budget_tokens)) is not None: + return numeric + if drop_params or litellm.drop_params is True: + return None + raise non_numeric_thinking_budget_error(budget_tokens) + + class AnthropicModelInfo(BaseLLMModelInfo): _workload_identity_eligible: ClassVar[bool] = True @@ -977,19 +999,19 @@ class AnthropicModelInfo(BaseLLMModelInfo): if thinking is None or thinking.get("type") != "enabled": return - budget_tokens: Final = thinking.get("budget_tokens") or 0 - effort: Final = AnthropicModelInfo._legacy_budget_to_effort( - model=model, - budget_tokens=int(budget_tokens) if isinstance(budget_tokens, (int, float, str)) else 0, - custom_llm_provider=custom_llm_provider, - ) - existing_output_config: Final = optional_params.get("output_config") + budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {} carried_keys: Final = carried_thinking_keys( thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display} + if budget_tokens is None: + return + effort: Final = AnthropicModelInfo._legacy_budget_to_effort( + model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider + ) + existing_output_config: Final = optional_params.get("output_config") optional_params["output_config"] = { "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 666823d6ec3..cca00a8383b 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -13,6 +13,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, @@ -549,16 +550,25 @@ class LiteLLMMessagesToCompletionTransformationHandler: model: Final = completion_kwargs.get("model", "") custom_llm_provider: Final = completion_kwargs.get("custom_llm_provider") + deployment_model_info: Final = completion_kwargs.get("model_info") if isinstance(reasoning_effort, str): normalized = normalize_reasoning_effort_value( - reasoning_effort, model=model, custom_llm_provider=custom_llm_provider + reasoning_effort, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, ) if normalized != reasoning_effort: completion_kwargs["reasoning_effort"] = normalized elif isinstance(reasoning_effort, dict) and "effort" in reasoning_effort: effort: Final = reasoning_effort["effort"] - normalized = normalize_reasoning_effort_value(effort, model=model, custom_llm_provider=custom_llm_provider) + normalized = normalize_reasoning_effort_value( + effort, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + ) if normalized != effort: completion_kwargs["reasoning_effort"] = { **reasoning_effort, @@ -639,6 +649,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + drop_params=litellm.drop_params is True or normalize_drop_params(extra_kwargs.get("drop_params")) is True, ) if openai_request is None: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index de929acbfb2..9b1297dbd75 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.pass_through.utils import ( @@ -130,7 +131,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( - reasoning_effort_from_thinking_budget, + reasoning_effort_from_thinking, ) from litellm.llms.anthropic.common_utils import ( eager_input_streaming_flag, @@ -234,7 +235,7 @@ class AnthropicAdapter: def __init__(self) -> None: pass - def translate_completion_input_params(self, kwargs) -> ChatCompletionRequest | None: + def translate_completion_input_params(self, kwargs: Mapping[str, object]) -> ChatCompletionRequest | None: """ Translate Anthropic request params to OpenAI format. @@ -244,11 +245,14 @@ class AnthropicAdapter: Note: Use translate_completion_input_params_with_tool_mapping() if you need the tool name mapping for restoring original names in responses. """ - result, _ = self.translate_completion_input_params_with_tool_mapping(kwargs) + result, _ = self.translate_completion_input_params_with_tool_mapping( + kwargs, + drop_params=litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True, + ) return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None + self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -282,6 +286,7 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + drop_params=drop_params, ) return translated_body, tool_name_mapping @@ -648,6 +653,8 @@ class LiteLLMAnthropicMessagesAdapter: @staticmethod def translate_anthropic_thinking_to_reasoning_effort( thinking: AnthropicThinkingParam, + *, + drop_params: bool = True, ) -> str | None: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -666,7 +673,7 @@ class LiteLLMAnthropicMessagesAdapter: if thinking_type == "disabled": return "none" elif thinking_type == "enabled": - return reasoning_effort_from_thinking_budget(thinking.get("budget_tokens", 0)) + return reasoning_effort_from_thinking(thinking, drop_params=drop_params) elif thinking_type == "adaptive": # Adaptive thinking: effort is controlled by output_config.effort, # not budget_tokens. Return a default; caller should override with @@ -1113,6 +1120,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs: ChatCompletionRequest, *, custom_llm_provider: str | None = None, + drop_params: bool = True, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1170,7 +1178,7 @@ class LiteLLMAnthropicMessagesAdapter: return reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking) + cast(AnthropicThinkingParam, thinking), drop_params=drop_params ) if not reasoning_effort: return @@ -1228,6 +1236,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, preserve_midturn_system: bool = False, + drop_params: bool = True, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1280,6 +1289,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, + drop_params=drop_params, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index e0d10cd37f4..a93e9106fb4 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -16,6 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm.anthropic_beta_headers_manager import aload_beta_headers_config from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( @@ -29,6 +30,14 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway +from litellm.router_utils.reasoning_effort_capability import ( + RequestedReasoningEffort, + deployment_reasoning_effort_levels, + keeps_effort, + model_group_from_metadata, + resolve_deployment_reasoning_effort, + with_adaptive_thinking_as_effort, +) from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -147,6 +156,48 @@ def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints +def _with_output_config(params: Mapping[str, object], output_config: RequestedReasoningEffort) -> Mapping[str, object]: + rest: Final = {key: value for key, value in params.items() if key != "output_config"} + return rest if output_config is None else {**rest, "output_config": output_config} + + +def _with_deployment_output_effort( + params: Mapping[str, object], + *, + model: str, + custom_llm_provider: str, + kwargs: Mapping[str, object], + drop_params: bool, +) -> Mapping[str, object]: + deployment_model_info: Final = kwargs.get("model_info") + + def resolve(requested: RequestedReasoningEffort) -> RequestedReasoningEffort: + return resolve_deployment_reasoning_effort( + requested, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + model_group=model_group_from_metadata(kwargs.get("litellm_metadata"), kwargs.get("metadata")), + drop_params=drop_params, + ).value + + def translate(source: Mapping[str, object]) -> Mapping[str, object]: + return with_adaptive_thinking_as_effort( + source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + + requested_output: Final = as_str_mapping(params.get("output_config")) + caller_output: Final = None if requested_output is None else resolve(requested_output) + caller_params: Final = params if requested_output is None else _with_output_config(params, caller_output) + if deployment_reasoning_effort_levels(deployment_model_info) is None: + return caller_params + if requested_output is not None and keeps_effort(requested_output, caller_output): + return _with_output_config(translate(params), caller_output) + translated: Final = translate(caller_params) + budget_output: Final = as_str_mapping(translated.get("output_config")) + return translated if budget_output is None else _with_output_config(translated, resolve(budget_output)) + + def _deployment_supports_cache_control_ttl(model_info: object) -> bool: return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True @@ -732,7 +783,15 @@ def anthropic_messages_handler( ), ), anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=dict(anthropic_messages_optional_request_params), + anthropic_messages_optional_request_params=dict( + _with_deployment_output_effort( + anthropic_messages_optional_request_params, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=cast(Mapping[str, object], kwargs), # cast-ok: router kwargs arrive untyped + drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True, + ) + ), _is_async=is_async, client=client, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 2cbba168ef4..b1347b0c79e 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -192,6 +192,7 @@ def _build_responses_kwargs( model, forwarded_kwargs.get("custom_llm_provider") ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), + drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), ) # Normalize reasoning effort based on model capabilities @@ -209,6 +210,7 @@ def _build_responses_kwargs( effort, model=model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, + deployment_model_info=forwarded_kwargs.get("model_info"), ) if normalized != effort: responses_kwargs["reasoning"] = {**reasoning, "effort": normalized} diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index b14a0dc0756..0fafd4abd62 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -18,7 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( - reasoning_effort_from_thinking_budget, + reasoning_effort_from_thinking, ) from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts from litellm.llms.anthropic.pass_through.messages.utils import ( @@ -542,6 +542,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: thinking: dict[str, object], output_config: dict[str, object] | None = None, auto_summary_level: str = "detailed", + *, + drop_params: bool = True, ) -> dict[str, object] | None: """ Convert Anthropic thinking param to Responses API reasoning param. @@ -562,9 +564,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if isinstance(output_config, dict) and output_config.get("effort"): effort = output_config["effort"] elif thinking_type == "enabled": - raw_budget: Final = thinking.get("budget_tokens", 0) - budget_tokens: Final = int(raw_budget) if isinstance(raw_budget, (int, float)) else 0 - effort = reasoning_effort_from_thinking_budget(budget_tokens) + budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params) + if budget_effort is None: + return None + effort = budget_effort else: return None @@ -590,6 +593,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: anthropic_request: AnthropicMessagesRequest, include_encrypted_reasoning: bool = True, auto_summary_level: str = "detailed", + *, + drop_params: bool = True, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to @@ -671,6 +676,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: thinking, output_config=cast(dict[str, Any] | None, output_config), auto_summary_level=auto_summary_level, + drop_params=drop_params, ) if reasoning: responses_kwargs["reasoning"] = reasoning diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/pass_through/utils.py index f4b587842bf..31619a04b74 100644 --- a/litellm/llms/anthropic/pass_through/utils.py +++ b/litellm/llms/anthropic/pass_through/utils.py @@ -193,9 +193,13 @@ def normalize_reasoning_effort_value( effort: str, model: str, custom_llm_provider: str | None = None, + deployment_model_info: object = None, ) -> str: """Lower a tier the deployment does not accept to the nearest one it does, leaving others alone. + A deployment whose own reasoning_effort_levels name a strength tier keeps the tier as sent, since + completion() and responses() round it onto those levels. + The accepted set is resolved by the same owner that answers ``/model_group/info``, so a level the proxy advertises is a level this path forwards. @@ -210,7 +214,13 @@ def normalize_reasoning_effort_value( if chain is None: return effort - from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + from litellm.router_utils.reasoning_effort_capability import ( + deployment_rounds_reasoning_effort, + resolve_supported_reasoning_efforts, + ) + + if deployment_rounds_reasoning_effort(deployment_model_info): + return effort from litellm.utils import get_model_info try: diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index a9b6c27a0df..4bec335064e 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -1,5 +1,6 @@ """Support for Azure OpenAI gpt-5 model family.""" +from collections.abc import Collection from typing import ClassVar, Final import litellm @@ -86,6 +87,8 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): model: str, drop_params: bool, api_version: str = "", + *, + declared_reasoning_efforts: Collection[str] = (), ) -> dict: return OpenAIGPT5Config.map_openai_params( self, @@ -93,6 +96,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): optional_params=optional_params, model=model, drop_params=drop_params, + declared_reasoning_efforts=declared_reasoning_efforts, ) def transform_request( diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 7aae2cb949d..0fa54cccbc9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,7 +1,7 @@ import copy import enum import re -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Collection, Iterator from typing import TYPE_CHECKING, Final, cast import httpx @@ -104,6 +104,8 @@ class AzureAIStudioConfig(OpenAIConfig): optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature model: str, drop_params: bool, + *, + declared_reasoning_efforts: Collection[str] = (), ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( @@ -117,6 +119,7 @@ class AzureAIStudioConfig(OpenAIConfig): optional_params=optional_params, model=model, drop_params=drop_params, + declared_reasoning_efforts=declared_reasoning_efforts, ) def _supports_stop_reason(self, model: str) -> bool: diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index f3f6bc9cbf1..63e6cb2b5da 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -11,7 +11,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type from litellm.litellm_core_utils.reasoning_effort_utils import ( - reasoning_effort_from_thinking_budget, + reasoning_effort_from_thinking, ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -96,12 +96,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): non_default_params["tools"] = _tools thinking: Final = non_default_params.pop("thinking", None) - if thinking is not None and isinstance(thinking, dict): - if thinking.get("type") == "enabled": - if "reasoning_effort" not in non_default_params: - non_default_params["reasoning_effort"] = reasoning_effort_from_thinking_budget( - thinking.get("budget_tokens", 0) - ) + thinking_effort: Final = ( + None + if "reasoning_effort" in non_default_params + else reasoning_effort_from_thinking(thinking, drop_params=drop_params) + ) + if thinking_effort is not None: + non_default_params["reasoning_effort"] = thinking_effort return super().map_openai_params(non_default_params, optional_params, model, drop_params) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 8d68e7bbad5..0311f6c6590 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -1,6 +1,7 @@ """Support for OpenAI gpt-5 model family.""" import re +from collections.abc import Collection from typing import ClassVar, Final import litellm @@ -305,6 +306,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig): optional_params: dict, model: str, drop_params: bool, + *, + declared_reasoning_efforts: Collection[str] = (), ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -333,7 +336,11 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if requested_effort is not None and self.refuses_reasoning_effort_level(model, requested_effort): + if ( + requested_effort is not None + and requested_effort not in declared_reasoning_efforts + and self.refuses_reasoning_effort_level(model, requested_effort) + ): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 464dc799c16..8f29a71d63b 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -369,6 +369,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model: str, drop_params: bool, ) -> dict: + return dict( + self.map_openai_params_for_deployment( + response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + ) + ) + + def map_openai_params_for_deployment( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + *, + declared_reasoning_efforts: Collection[str], + ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. GPT-5 models have restrictions on temperature and top_p (only temperature=1 @@ -405,6 +419,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) + and requested_effort not in declared_reasoning_efforts and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): diff --git a/litellm/main.py b/litellm/main.py index 3ecb325c5e6..8e9ff3cf910 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -136,6 +136,11 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.realtime_api.main import realtime_health_check from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway +from litellm.router_utils.reasoning_effort_capability import ( + model_group_from_metadata, + reasoning_effort_needs_allowing, + resolve_chat_reasoning, +) from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.completion import ( CompletionDispatchContext, @@ -198,7 +203,7 @@ from litellm.utils import ( from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache -from .litellm_core_utils.core_helpers import normalize_drop_params, safe_deep_copy +from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, @@ -5667,6 +5672,34 @@ def completion( cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg ), ) + routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + drop_unsupported_reasoning: Final = ( + litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True + ) + reasoning_model_group: Final = model_group_from_metadata( + routing_kwargs.get("metadata"), routing_kwargs.get("litellm_metadata") + ) + requested_output_config: Final = as_str_mapping(non_default_params.get("output_config")) + chat_reasoning: Final = resolve_chat_reasoning( + reasoning_effort, + thinking, + requested_output_config, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=routing_kwargs.get("model_info"), + model_group=reasoning_model_group, + drop_params=drop_unsupported_reasoning, + ) + resolved_reasoning_effort: Final = chat_reasoning.effort.value + resolved_output_config: Final = chat_reasoning.output_config + provider_params: Final = ( + non_default_params + if resolved_output_config is requested_output_config + else { + **{key: value for key, value in non_default_params.items() if key != "output_config"}, + **({"output_config": resolved_output_config} if resolved_output_config is not None else {}), + } + ) if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) @@ -5758,6 +5791,12 @@ def completion( allowed_openai_params: Final[list[str] | None] = ( [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] if bridges_to_responses_api + or reasoning_effort_needs_allowing( + chat_reasoning.effort, + model=model, + custom_llm_provider=custom_llm_provider, + base_model=base_model if isinstance(base_model, str) else None, + ) else kwargs.get("allowed_openai_params") ) optional_param_args: Final = { @@ -5791,9 +5830,9 @@ def completion( "api_version": api_version, "parallel_tool_calls": parallel_tool_calls, "messages": messages, - "reasoning_effort": reasoning_effort, + "reasoning_effort": resolved_reasoning_effort, "verbosity": verbosity, - "thinking": thinking, + "thinking": chat_reasoning.thinking, "web_search_options": web_search_options, "include_server_side_tool_invocations": ( include_server_side_tool_invocations @@ -5808,14 +5847,14 @@ def completion( "base_model": base_model, } provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( - reasoning_effort, - non_default_params, + resolved_reasoning_effort, + provider_params, accepts_summary=provider_config is not None and provider_config.accepts_reasoning_summary_in_reasoning_effort, ) mapping_args: Final = MappingProxyType({**optional_param_args, "reasoning_effort": provider_reasoning_effort}) - mapping_special_params: Final = MappingProxyType(dict(non_default_params)) + mapping_special_params: Final = MappingProxyType(dict(provider_params)) def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]: return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params) @@ -5852,7 +5891,7 @@ def completion( processed_non_default_params: Final = pre_process_non_default_params( model=model, passed_params=optional_param_args, - special_params=non_default_params, + special_params=provider_params, custom_llm_provider=custom_llm_provider, additional_drop_params=kwargs.get("additional_drop_params"), remove_sensitive_keys=True, @@ -6007,7 +6046,7 @@ def completion( custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, tools=tools, - reasoning_effort=bridge_reasoning_effort(reasoning_effort, optional_params.get("extra_body")), + reasoning_effort=bridge_reasoning_effort(resolved_reasoning_effort, optional_params.get("extra_body")), reasoning_summary=_reasoning_summary_for_bridge, api_base=api_base, deployment_model_info=str_keyed_mapping( @@ -6024,10 +6063,10 @@ def completion( optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) - if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: - optional_params["reasoning_effort"] = reasoning_effort + if isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort: + optional_params["reasoning_effort"] = resolved_reasoning_effort elif rs_val is not None: - eff: Final = optional_params.get("reasoning_effort", reasoning_effort) + eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort) if isinstance(eff, dict): optional_params["reasoning_effort"] = {**eff, "summary": rs_val} elif eff is not None: @@ -6089,7 +6128,7 @@ def completion( organization=organization, provider_config=provider_config, request_params=MappingProxyType( - _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **non_default_params}) + _OPTIONAL_PARAMS_ADAPTER.validate_python({**optional_param_args, **provider_params}) ), shared_session=shared_session, stream=stream, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 98475e42b99..51cd3be1b63 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -51,6 +51,11 @@ from litellm.responses.mcp.round_billing import ( ) from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.reasoning_effort_capability import ( + deployment_reasoning_effort_levels, + model_group_from_metadata, + resolve_deployment_reasoning_effort, +) from litellm.types.llms.openai import ( PromptObject, Reasoning, @@ -1557,9 +1562,22 @@ def responses( if current_reasoning is None and reasoning_effort is not None else current_reasoning ) + routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + deployment_reasoning: Final = cast( # cast-ok: the resolver hands a Reasoning back in the shape it was given + Reasoning | None, + resolve_deployment_reasoning_effort( + request_reasoning, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + model_group=model_group_from_metadata(routing_kwargs.get("litellm_metadata")), + drop_params=litellm.drop_params is True + or normalize_drop_params(routing_kwargs.get("drop_params")) is True, + ).value, + ) response_api_optional_params: Final[ResponsesAPIOptionalRequestParams] = ( ResponsesAPIRequestUtils.get_requested_response_api_optional_param( - {k: v for k, v in {**local_vars, "reasoning": request_reasoning}.items() if k != "reasoning_effort"} + {k: v for k, v in {**local_vars, "reasoning": deployment_reasoning}.items() if k != "reasoning_effort"} ) ) @@ -1576,7 +1594,7 @@ def responses( metadata=metadata, parallel_tool_calls=parallel_tool_calls, previous_response_id=previous_response_id, - reasoning=request_reasoning, + reasoning=deployment_reasoning, store=store, background=background, stream=stream, @@ -1626,6 +1644,7 @@ def responses( response_api_optional_params=response_api_optional_params, allowed_openai_params=allowed_openai_params, drop_params=normalize_drop_params(request_drop_params), + declared_reasoning_efforts=deployment_reasoning_effort_levels(deployment_model_info) or (), ) litellm_logging_obj.update_from_kwargs( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 8e5e1c1cfee..be8397c9270 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1,6 +1,6 @@ import base64 import re -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload @@ -75,6 +75,26 @@ def _as_input_text_part(part: object) -> object: _RequestInputT: Final = TypeVar("_RequestInputT") +def _provider_mapped_params( + config: BaseResponsesAPIConfig, + params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + declared_reasoning_efforts: Collection[str], +) -> Mapping[str, object]: + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + + if ( + declared_reasoning_efforts + and isinstance(config, OpenAIResponsesAPIConfig) + and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + ): + return config.map_openai_params_for_deployment( + params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts + ) + return config.map_openai_params(response_api_optional_params=params, model=model, drop_params=drop_params) + + class ResponsesAPIRequestUtils: """Helper utils for constructing ResponseAPI requests""" @@ -202,6 +222,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, + declared_reasoning_efforts: Collection[str] = (), ) -> dict: """ Get optional parameters for the responses API. @@ -234,10 +255,12 @@ class ResponsesAPIRequestUtils: # Map parameters to provider-specific format mapped_params: Final[dict[str, object]] = TypeAdapter(dict[str, object]).validate_python( - responses_api_provider_config.map_openai_params( - response_api_optional_params=response_api_optional_params, - model=model, - drop_params=should_drop_params, + _provider_mapped_params( + responses_api_provider_config, + response_api_optional_params, + model, + should_drop_params, + declared_reasoning_efforts, ), strict=True, ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index d660adf05a6..4357722970f 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -25,12 +25,24 @@ it above. """ from collections.abc import Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType -from typing import Final, get_args +from typing import Final, Literal, TypeAlias, get_args + +from typing_extensions import assert_never import litellm +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params +from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping +from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT +ReasoningEffortRounding: TypeAlias = Literal["up", "down"] +RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None + REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT) _EMPTY_ENTRY: Final[Mapping[str, object]] = MappingProxyType({}) @@ -77,7 +89,10 @@ def declared_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, . validate-model-prices-json, but an operator can put this key on a config.yaml model_info block where that schema never runs, and one mistyped level must not fail every sibling on the proxy.""" own: Final = model_info.get(_DECLARED_EFFORTS_KEY) - raw: Final = own if own is not None else _bare_model_entry(model_info).get(_DECLARED_EFFORTS_KEY) + return _declared_levels(own if own is not None else _bare_model_entry(model_info).get(_DECLARED_EFFORTS_KEY)) + + +def _declared_levels(raw: object) -> tuple[str, ...] | None: if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): return None declared: Final = frozenset(effort for effort in raw if isinstance(effort, str)) @@ -104,19 +119,347 @@ REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "x _STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}) -def nearest_declared_reasoning_effort(requested: str, declared: Sequence[str]) -> str: - """Rounds a request up to the weakest declared level at least as strong as it, and down to the - strongest declared level when it asks for more than the model has, so the caller gets no less - reasoning than it asked for instead of a rejected call. none is the off switch rather than a - strength, so it is never rounded onto the ladder and no level is rounded down to it: a caller - who turned reasoning off must not be billed for it, and a model that cannot turn it off says so - itself. A level outside the strength order is likewise returned as is for upstream to judge.""" +def nearest_declared_reasoning_effort( + requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" +) -> str: + """Rounding up takes the weakest declared level at least as strong as the request, else the + strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected + call. Rounding down mirrors it: the strongest declared level at most as strong, else the + weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder + and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and + a model that cannot turn it off says so itself. A level outside the strength order is likewise + returned as is for upstream to judge.""" ranked: Final = sorted( (effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort] ) if requested in ranked or requested not in _STRENGTH_RANK or not ranked: return requested - return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= _STRENGTH_RANK[requested]), ranked[-1]) + strength: Final = _STRENGTH_RANK[requested] + match rounding: + case "up": + return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1]) + case "down": + return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0]) + case _: + assert_never(rounding) + + +_ROUNDING_KEY: Final = "reasoning_effort_rounding" +_DEFAULT_EFFORT: Final = "default" +_EFFORT_KEY: Final = "effort" +_LOGGABLE_REQUESTED_LEVELS: Final = frozenset((*REASONING_EFFORT_ADVERTISEMENT_ORDER, _DEFAULT_EFFORT)) +_UNRECOGNIZED_LEVEL: Final = "unrecognized" +_PROVIDERS_SENDING_THINKING_AS_EFFORT: Final = frozenset(("hosted_vllm",)) + + +def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[str, ...] | None: + """Only the deployment's own declaration counts. The router copies it onto the shared + provider/model map key, and map entries describe a model rather than what a deployment's + endpoint accepts, so reading the map here would round requests for siblings and for models + nobody declared levels on.""" + own_info: Final = as_str_mapping(deployment_model_info) + return _declared_levels(own_info.get(_DECLARED_EFFORTS_KEY)) if own_info is not None else None + + +def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool: + declared: Final = deployment_reasoning_effort_levels(deployment_model_info) + return declared is not None and any(level in _STRENGTH_RANK for level in declared) + + +def with_adaptive_thinking_as_effort( + params: Mapping[str, object], *, model: str, custom_llm_provider: str, drop_params: bool +) -> Mapping[str, object]: + translated: Final = dict(params) + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + return translated + + +def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> str | None: + if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT: + return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + adaptive_config: Final = as_str_mapping( + with_adaptive_thinking_as_effort( + {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ).get("output_config") + ) + if adaptive_config is not None: + adaptive_effort: Final = adaptive_config.get(_EFFORT_KEY) + return adaptive_effort if isinstance(adaptive_effort, str) else None + if "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()): + return None + return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + + +def provider_reads_thinking(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> bool: + if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT: + return False + if "output_config" in with_adaptive_thinking_as_effort( + {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ): + return True + return "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()) + + +def adaptive_thinking_beside_effort( + thinking: AnthropicThinkingParam | None, + effort: object, + *, + model: str, + custom_llm_provider: str, + drop_params: bool, +) -> Mapping[object, object] | AnthropicThinkingParam | None: + if effort == "none": + return None + translated: Final = with_adaptive_thinking_as_effort( + {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + if "output_config" not in translated: + return None + return thinking if effort is not None else validated_object_mapping(translated.get("thinking")) + + +def adaptive_thinking_without_effort( + thinking: AnthropicThinkingParam | None, *, model: str, custom_llm_provider: str, drop_params: bool +) -> Mapping[object, object] | AnthropicThinkingParam | None: + translated: Final = with_adaptive_thinking_as_effort( + {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + adaptive: Final = validated_object_mapping(translated.get("thinking")) if "output_config" in translated else None + return thinking if adaptive is None else adaptive + + +def keeps_effort(requested: RequestedReasoningEffort, resolved: RequestedReasoningEffort) -> bool: + """A level dropped under drop_params counts as never sent, like any dropped param, so a thinking + budget beside it still decides the effort. default is kept even though it resolves to no level, + since the caller asked for the provider's own default in place of the budget's.""" + requested_level: Final = _requested_level(requested) + return requested_level is not None and ( + requested_level == _DEFAULT_EFFORT or _requested_level(resolved) is not None + ) + + +def _rounding(value: object) -> ReasoningEffortRounding | None: + match value.strip().lower() if isinstance(value, str) else value: + case "up": + return "up" + case "down": + return "down" + case None: + return None + case _: + verbose_logger.warning( + "Ignoring reasoning_effort_rounding=%s, expected 'up' or 'down'", _single_line(value) + ) + return None + + +def _deployment_rounding(deployment_model_info: object) -> ReasoningEffortRounding: + own_info: Final = as_str_mapping(deployment_model_info) + own: Final = _rounding(own_info.get(_ROUNDING_KEY)) if own_info is not None else None + return own or _rounding(litellm.reasoning_effort_rounding) or "up" + + +def _requested_level(requested: RequestedReasoningEffort) -> str | None: + if isinstance(requested, str): + return requested + fields: Final = as_str_mapping(requested) + if fields is None: + return None + level: Final = fields.get(_EFFORT_KEY) + return level if isinstance(level, str) else None + + +def _with_level(requested: RequestedReasoningEffort, level: str | None) -> RequestedReasoningEffort: + fields: Final = as_str_mapping(requested) + if fields is None: + return level + if level is not None: + return {**fields, _EFFORT_KEY: level} + rest: Final = {key: value for key, value in fields.items() if key != _EFFORT_KEY} + return rest or None + + +def model_group_from_metadata(*metadata: object) -> str | None: + mappings: Final = (as_str_mapping(entry) for entry in metadata) + groups: Final = (mapping.get("model_group") for mapping in mappings if mapping is not None) + return next((group for group in groups if isinstance(group, str)), None) + + +@dataclass(frozen=True, slots=True) +class DeploymentReasoningEffort: + value: RequestedReasoningEffort + deployment_accepts_effort: bool + + +def resolve_deployment_reasoning_effort( + requested: RequestedReasoningEffort, + *, + model: str, + custom_llm_provider: str | None, + deployment_model_info: object, + model_group: str | None, + drop_params: bool, +) -> DeploymentReasoningEffort: + requested_level: Final = _requested_level(requested) + unchanged: Final = DeploymentReasoningEffort(value=requested, deployment_accepts_effort=False) + if requested_level is None: + return unchanged + declared: Final = deployment_reasoning_effort_levels(deployment_model_info) + if declared is None: + return unchanged + rounding: Final = _deployment_rounding(deployment_model_info) + rounded_level: Final = ( + None + if requested_level == _DEFAULT_EFFORT + else nearest_declared_reasoning_effort(requested_level, declared, rounding) + ) + sent_level: Final = None if drop_params and rounded_level not in declared else rounded_level + deployment_accepts_effort: Final = sent_level is not None and len(declared) > 0 + if sent_level == requested_level: + return DeploymentReasoningEffort(value=requested, deployment_accepts_effort=deployment_accepts_effort) + _log_remap( + requested_level=requested_level, + sent_level=sent_level, + rounding=rounding, + deployment_model=_provider_model(model, custom_llm_provider), + deployment_model_info=deployment_model_info, + model_group=model_group, + ) + return DeploymentReasoningEffort( + value=_with_level(requested, sent_level), deployment_accepts_effort=deployment_accepts_effort + ) + + +@dataclass(frozen=True, slots=True) +class ChatReasoning: + effort: DeploymentReasoningEffort + thinking: Mapping[object, object] | AnthropicThinkingParam | None + output_config: RequestedReasoningEffort + + +def resolve_chat_reasoning( + reasoning_effort: RequestedReasoningEffort, + thinking: AnthropicThinkingParam | None, + output_config: Mapping[str, object] | None, + *, + model: str, + custom_llm_provider: str, + deployment_model_info: object, + model_group: str | None, + drop_params: bool, +) -> ChatReasoning: + def resolve(requested: RequestedReasoningEffort) -> DeploymentReasoningEffort: + return resolve_deployment_reasoning_effort( + requested, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + model_group=model_group, + drop_params=drop_params, + ) + + explicit: Final = resolve(reasoning_effort) + output: Final = resolve(output_config).value + if thinking is None or deployment_reasoning_effort_levels(deployment_model_info) is None: + return ChatReasoning(effort=explicit, thinking=thinking, output_config=output) + explicit_kept: Final = keeps_effort(reasoning_effort, explicit.value) + if not explicit_kept and not keeps_effort(output_config, output): + budget_level: Final = thinking_budget_effort( + thinking, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) + budget_effort: Final = resolve(budget_level) + return ChatReasoning( + effort=budget_effort, + thinking=thinking + if budget_level is None + else adaptive_thinking_beside_effort( + thinking, + budget_effort.value, + model=model, + custom_llm_provider=custom_llm_provider, + drop_params=drop_params, + ), + output_config=output, + ) + if not provider_reads_thinking( + thinking, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ): + return ChatReasoning( + effort=explicit if explicit_kept else resolve(_requested_level(output)), + thinking=None, + output_config=output, + ) + if _requested_level(reasoning_effort if explicit_kept else output_config) != _DEFAULT_EFFORT: + return ChatReasoning(effort=explicit, thinking=thinking, output_config=output) + return ChatReasoning( + effort=explicit, + thinking=adaptive_thinking_without_effort( + thinking, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ), + output_config=output, + ) + + +def _provider_model(model: str, custom_llm_provider: str | None) -> str: + if custom_llm_provider is None or model.startswith(f"{custom_llm_provider}/"): + return model + return f"{custom_llm_provider}/{model}" + + +def _log_remap( + *, + requested_level: str, + sent_level: str | None, + rounding: ReasoningEffortRounding, + deployment_model: str, + deployment_model_info: object, + model_group: str | None, +) -> None: + own_info: Final = as_str_mapping(deployment_model_info) + deployment_id: Final = own_info.get("id") if own_info is not None else None + logged_requested_level: Final = _single_line( + requested_level if requested_level in _LOGGABLE_REQUESTED_LEVELS else _UNRECOGNIZED_LEVEL + ) + logged_sent_level: Final = _single_line(sent_level) + logged_deployment_id: Final = _single_line(deployment_id) + logged_model_group: Final = _single_line(model_group) + logged_deployment_model: Final = _single_line(deployment_model) + verbose_logger.info( + "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", + logged_requested_level, + logged_sent_level, + rounding, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra={ + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + }, + ) + + +def _single_line(value: object) -> str | None: + return None if value is None else str(value).replace("\r", "").replace("\n", "") + + +def reasoning_effort_needs_allowing( + resolved: DeploymentReasoningEffort, *, model: str, custom_llm_provider: str, base_model: str | None +) -> bool: + """A provider that lists reasoning_effort maps it itself, so allowing the raw key as well would + send it twice (anthropic turns it into thinking and rejects the raw key). A level outside the + declaration is still sent once the deployment declares any, for its backend to judge.""" + if not resolved.deployment_accepts_effort: + return False + return "reasoning_effort" not in ( + get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider, base_model=base_model) or () + ) def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool: @@ -188,7 +531,7 @@ def resolve_supported_reasoning_efforts( opt_out: Final = frozenset(effort for effort in _OPT_OUT_EFFORTS if flags[effort] is not False) opt_in: Final = frozenset(effort for effort in _OPT_IN_EFFORTS if flags[effort] is True) - none_level: Final = ( + none_level: Final[frozenset[str]] = ( frozenset(("none",)) if _supports_none_reasoning_effort(model_info, flags["none"]) else frozenset() ) allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in | none_level diff --git a/litellm/utils.py b/litellm/utils.py index 3e5eed2ba9e..3990a870484 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4808,6 +4808,12 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c return optional_params +def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tuple[str, ...]: + from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels + + return deployment_reasoning_effort_levels(model_info) or () + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5336,11 +5342,21 @@ def get_optional_params( drop_params=bool(drop_params), ) elif custom_llm_provider == "openai": - optional_params = litellm.OpenAIConfig().map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=bool(drop_params), + optional_params = ( + litellm.OpenAIGPT5Config().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + declared_reasoning_efforts=_declared_reasoning_efforts(model_info), + ) + if litellm.OpenAIGPT5Config.is_model_gpt_5_model(model=model) + else litellm.OpenAIConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + ) ) elif custom_llm_provider == "nebius": optional_params = litellm.NebiusConfig().map_openai_params( @@ -5364,6 +5380,7 @@ def get_optional_params( optional_params=optional_params, model=_azure_detection_model, drop_params=bool(drop_params), + declared_reasoning_efforts=_declared_reasoning_efforts(model_info), ) else: verbose_logger.debug( @@ -5385,11 +5402,21 @@ def get_optional_params( drop_params=bool(drop_params), ) elif provider_config is not None: - optional_params = provider_config.map_openai_params( - non_default_params=non_default_params, - optional_params=optional_params, - model=model, - drop_params=bool(drop_params), + optional_params = ( + provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + declared_reasoning_efforts=_declared_reasoning_efforts(model_info), + ) + if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig) + else provider_config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=bool(drop_params), + ) ) else: # assume passing in params for openai-like api optional_params = litellm.OpenAILikeChatConfig().map_openai_params( diff --git a/tests/integration/routing/_reasoning_effort_levels_support.py b/tests/integration/routing/_reasoning_effort_levels_support.py new file mode 100644 index 00000000000..8d43695e419 --- /dev/null +++ b/tests/integration/routing/_reasoning_effort_levels_support.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import json +import re +import uuid +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit + +import yaml +from integration._support.anthropic_sse import message_json, message_stream, stream_reply +from integration._support.client import Gateway, Scenario, object_value +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.upstream import aws_event_stream_frame +from integration._support.wire import Reply, Request, Wire +from pydantic import JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +KIMI_LEVELS: Final = ("low", "high", "max", "none") +CLAUDE_LEVELS: Final = ("low", "high") +GPT_LEVELS: Final = ("low", "medium", "high", "xhigh") +KIMI: Final = "kimi-k3" +CLAUDE: Final = "claude-opus-5-5" +GPT: Final = "gpt-6.1-sol" +GLM: Final = "zai-glm-5-3" +FIREWORKS_KIMI: Final = f"accounts/fireworks/models/{KIMI}" +OPENROUTER_KIMI: Final = f"moonshotai/{KIMI}" +GEMINI: Final = "gemini-3-pro-preview" +ZAI_GLM: Final = "glm-5" +ECHOED_KEY_MODEL: Final = f"{KIMI}-echoed-key" +BEDROCK_CLAUDE: Final = f"us.anthropic.{CLAUDE}" +AZURE_API_VERSION: Final = "2025-04-01-preview" +SYNTHETIC_KEY: Final = "synthetic-provider-key" +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_SCRIPTED_STATUS: Final = re.compile(r"status=(\d{3})") +_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" + + +def new_marker() -> str: + return uuid.uuid4().hex + + +def prompt(marker: str) -> str: + return f"marker-{marker}" + + +def answer(marker: str) -> str: + return f"answer marker-{marker}" + + +def marker_of(request: Request) -> str: + found: Final = _MARKER.search(request.body.decode(errors="replace")) + assert found is not None, request.body[:400] + return found.group(1) + + +def body_of(request: Request) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(request.body) + + +def path_of(request: Request) -> str: + return urlsplit(request.target).path + + +def _converse_reply(marker: str) -> Reply: + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": answer(marker)}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14}, + "metrics": {"latencyMs": 1}, + } + ).encode() + ) + + +def _converse_frame(event_type: str, payload: Mapping[str, JsonValue]) -> bytes: + return aws_event_stream_frame( + {":event-type": event_type, ":content-type": "application/json", ":message-type": "event"}, + json.dumps(payload).encode(), + ) + + +def _converse_stream_reply(marker: str) -> Reply: + frames: Final = ( + _converse_frame("messageStart", {"role": "assistant"}), + _converse_frame("contentBlockDelta", {"delta": {"text": answer(marker)}, "contentBlockIndex": 0}), + _converse_frame("contentBlockStop", {"contentBlockIndex": 0}), + _converse_frame("messageStop", {"stopReason": "end_turn"}), + _converse_frame( + "metadata", + {"usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14}, "metrics": {"latencyMs": 1}}, + ), + ) + return Reply(body=b"".join(frames), content_type=_EVENT_STREAM) + + +def _gemini_reply(marker: str) -> Reply: + return Reply( + body=json.dumps( + { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": answer(marker)}]}, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 5, "totalTokenCount": 14}, + } + ).encode() + ) + + +def provider_reply(request: Request) -> Reply: + marker: Final = marker_of(request) + scripted: Final = _SCRIPTED_STATUS.search(request.body.decode(errors="replace")) + if scripted is not None: + return openai_error(int(scripted.group(1))) + body: Final = body_of(request) + path: Final = path_of(request) + model: Final = str(body.get("model", "")) + stream: Final = body.get("stream") is True + if path.endswith(":generateContent"): + return _gemini_reply(marker) + if path.endswith("/converse"): + return _converse_reply(marker) + if path.endswith("/converse-stream"): + return _converse_stream_reply(marker) + if path.endswith("/invoke"): + return Reply(body=message_json(f"msg_{marker}", model, answer(marker))) + if path.endswith("/messages"): + if stream: + return stream_reply(message_stream(f"msg_{marker}", model, answer(marker))) + return Reply(body=message_json(f"msg_{marker}", model, answer(marker))) + if path.endswith("/responses"): + return responses_reply(f"resp_{marker}", model, answer(marker), stream=stream) + if path.endswith("/chat/completions"): + return chat_reply(f"chatcmpl-{marker}", model, answer(marker), stream=stream) + return Reply(status=404, body=json.dumps({"error": {"message": f"no scripted route {path}"}}).encode()) + + +@dataclass(frozen=True, slots=True) +class Backend: + provider_model: str + params: Mapping[str, JsonValue] + + +def backend(kind: str, wire_url: str) -> Backend: + match kind: + case "azure_ai": + return Backend(f"azure_ai/{KIMI}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY})) + case "azure_ai_drop_params": + return Backend( + f"azure_ai/{KIMI}", + MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY, "drop_params": True}), + ) + case "fireworks_ai": + return Backend( + f"fireworks_ai/{FIREWORKS_KIMI}", + MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY}), + ) + case "openrouter": + return Backend( + f"openrouter/{OPENROUTER_KIMI}", + MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY}), + ) + case "gemini": + return Backend(f"gemini/{GEMINI}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY})) + case "zai": + return Backend(f"zai/{ZAI_GLM}", MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY})) + case "azure_ai_claude": + return Backend(f"azure_ai/{CLAUDE}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY})) + case "hosted_vllm": + return Backend( + f"hosted_vllm/{KIMI}", MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY}) + ) + case "hosted_vllm_drop_params": + return Backend( + f"hosted_vllm/{KIMI}", + MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY, "drop_params": True}), + ) + case "hosted_vllm_echoed_key": + return Backend( + f"hosted_vllm/{ECHOED_KEY_MODEL}", + MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY}), + ) + case "hosted_vllm_claude": + return Backend( + f"hosted_vllm/{CLAUDE}", MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY}) + ) + case "mistral": + return Backend(f"mistral/{GLM}", MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY})) + case "openai": + return Backend(f"openai/{GPT}", MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY})) + case "openai_drop_params": + return Backend( + f"openai/{GPT}", + MappingProxyType({"api_base": f"{wire_url}/v1", "api_key": SYNTHETIC_KEY, "drop_params": True}), + ) + case "azure": + return Backend( + f"azure/{GPT}", + MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY, "api_version": AZURE_API_VERSION}), + ) + case "anthropic": + return Backend(f"anthropic/{CLAUDE}", MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY})) + case "bedrock_converse": + return Backend( + f"bedrock/converse/{BEDROCK_CLAUDE}", + MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY, "aws_region_name": "us-east-1"}), + ) + case "bedrock_invoke": + return Backend( + f"bedrock/invoke/{BEDROCK_CLAUDE}", + MappingProxyType({"api_base": wire_url, "api_key": SYNTHETIC_KEY, "aws_region_name": "us-east-1"}), + ) + case _: + raise AssertionError(f"unknown backend kind {kind}") + + +def deployment(scenario: Scenario, kind: str, wire: Wire, model_info: Mapping[str, JsonValue]) -> str: + chosen: Final = backend(kind, wire.url) + return scenario.model(model=chosen.provider_model, model_info=model_info, **chosen.params) + + +def yaml_deployment( + model_name: str, kind: str, wire_url: str, model_info: Mapping[str, JsonValue] +) -> dict[str, JsonValue]: + chosen: Final = backend(kind, wire_url) + return { + "model_name": model_name, + "litellm_params": {"model": chosen.provider_model, **chosen.params}, + "model_info": dict(model_info), + } + + +def owned_config( + directory: Path, + model_list: Sequence[JsonValue], + *, + litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}), +) -> Path: + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + merged: Final = { + **config, + "model_list": list(model_list), + "litellm_settings": {**object_value(config["litellm_settings"]), "cache": False, **litellm_settings}, + "router_settings": {**object_value(config["router_settings"]), "num_retries": 0}, + } + path: Final = directory / f"reasoning-effort-levels-{uuid.uuid4().hex}.yaml" + path.write_text(yaml.safe_dump(merged)) + return path + + +def chat_messages(marker: str) -> list[JsonValue]: + return [{"role": "user", "content": prompt(marker)}] + + +def chat_request(model: str, marker: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": chat_messages(marker), **fields} + + +def messages_request(model: str, marker: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 4096, "messages": chat_messages(marker), **fields} + + +def responses_request(model: str, marker: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "input": prompt(marker), **fields} + + +def only_request(wire: Wire, marker: str) -> Request: + received: Final = tuple(request for request in wire.drain() if marker_of(request) == marker) + assert len(received) == 1, [(request.target, request.body[:300]) for request in received] + return received[0] + + +def sent_effort(request: Request) -> JsonValue: + body: Final = body_of(request) + path: Final = path_of(request) + if path.endswith(("/converse", "/converse-stream")): + fields: Final = body.get("additionalModelRequestFields") + config: Final = fields.get("output_config") if isinstance(fields, dict) else None + return config.get("effort") if isinstance(config, dict) else None + if path.endswith(("/messages", "/invoke")): + output_config: Final = body.get("output_config") + return output_config.get("effort") if isinstance(output_config, dict) else None + if path.endswith("/responses"): + reasoning: Final = body.get("reasoning") + return reasoning.get("effort") if isinstance(reasoning, dict) else None + return body.get("reasoning_effort") + + +def gateway_post(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple[int, str]: + response: Final = gateway.request("POST", path, body) + return response.status_code, response.text diff --git a/tests/integration/routing/test_reasoning_effort_levels_chaos.py b/tests/integration/routing/test_reasoning_effort_levels_chaos.py new file mode 100644 index 00000000000..d2c104b9ec9 --- /dev/null +++ b/tests/integration/routing/test_reasoning_effort_levels_chaos.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import dataclasses +import threading +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit + +import httpx +import pytest +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.routing._reasoning_effort_levels_support import ( + CLAUDE_LEVELS, + KIMI_LEVELS, + answer, + body_of, + chat_request, + deployment, + marker_of, + messages_request, + new_marker, + owned_config, + provider_reply, + responses_request, + sent_effort, + yaml_deployment, +) +from pydantic import JsonValue + +pytestmark = pytest.mark.timeout(3 * graceful_stop_seconds() + 180) + +_KIMI_ROUNDED: Final = MappingProxyType({"minimal": "low", "medium": "high", "xhigh": "max"}) +_CLAUDE_ROUNDED: Final = MappingProxyType({"minimal": "low", "medium": "high", "xhigh": "high"}) +_KINDS: Final = ("chat", "chat-stream", "responses", "responses-stream", "messages", "messages-stream") +_REQUESTED: Final = ("minimal", "medium", "xhigh") +_BURST: Final = 30 + + +@dataclass(frozen=True, slots=True) +class Models: + kimi: str + claude: str + + +@dataclass(frozen=True, slots=True) +class Shot: + marker: str + kind: str + requested: str + rounded: str + + +@dataclass(frozen=True, slots=True) +class Outcome: + shot: Shot + status: int | None + text: str + + +def _shot(index: int) -> Shot: + kind: Final = _KINDS[index % len(_KINDS)] + requested: Final = _REQUESTED[index // len(_KINDS) % len(_REQUESTED)] + rounded: Final = _CLAUDE_ROUNDED if kind.startswith("messages") else _KIMI_ROUNDED + return Shot(new_marker(), kind, requested, rounded[requested]) + + +def _route_and_body(shot: Shot, models: Models) -> tuple[str, dict[str, JsonValue]]: + stream: Final = shot.kind.endswith("-stream") + level: Final = shot.requested + match shot.kind.removesuffix("-stream"): + case "chat": + return "/v1/chat/completions", chat_request(models.kimi, shot.marker, reasoning_effort=level, stream=stream) + case "responses": + return "/v1/responses", responses_request( + models.kimi, shot.marker, reasoning={"effort": level}, stream=stream + ) + case "messages": + return "/v1/messages", messages_request( + models.claude, + shot.marker, + thinking={"type": "adaptive"}, + output_config={"effort": level}, + stream=stream, + ) + case other: + raise AssertionError(other) + + +def _fire(gateway: Gateway, models: Models, shot: Shot) -> Outcome: + route, body = _route_and_body(shot, models) + try: + response: Final = gateway.request("POST", route, body) + except httpx.HTTPError as error: + return Outcome(shot, None, repr(error)) + return Outcome(shot, response.status_code, response.text) + + +def _burst( + gateway: Gateway, models: Models, shots: tuple[Shot, ...], during: Callable[[], None] +) -> tuple[Outcome, ...]: + with ThreadPoolExecutor(max_workers=len(shots)) as pool: + futures: Final[tuple[Future[Outcome], ...]] = tuple(pool.submit(_fire, gateway, models, shot) for shot in shots) + during() + return tuple(future.result() for future in futures) + + +def _assert_every_upstream_effort_rounded(shots: tuple[Shot, ...], received: tuple[Request, ...]) -> None: + by_marker: Final = MappingProxyType({shot.marker: shot for shot in shots}) + observed: Final = tuple((marker_of(request), sent_effort(request)) for request in received) + assert all(marker in by_marker for marker, _ in observed), observed + wrong: Final = tuple((marker, effort) for marker, effort in observed if effort != by_marker[marker].rounded) + assert wrong == (), wrong + + +def _assert_answered_once(outcomes: tuple[Outcome, ...], received: tuple[Request, ...]) -> None: + upstream: Final = tuple(marker_of(request) for request in received) + answered: Final = tuple(outcome for outcome in outcomes if outcome.status == 200) + assert all(answer(outcome.shot.marker) in outcome.text for outcome in answered), [ + (outcome.shot, outcome.text[:300]) for outcome in answered if answer(outcome.shot.marker) not in outcome.text + ] + assert all(upstream.count(outcome.shot.marker) == 1 for outcome in answered), upstream + + +def _assert_all_answered(outcomes: tuple[Outcome, ...]) -> None: + failed: Final = tuple( + (outcome.shot, outcome.status, outcome.text[:300]) for outcome in outcomes if outcome.status != 200 + ) + assert failed == (), failed + + +def _shots(count: int) -> tuple[Shot, ...]: + return tuple(_shot(index) for index in range(count)) + + +def _yaml_models(wire_url: str) -> tuple[JsonValue, ...]: + return ( + yaml_deployment("kimi-levels", "hosted_vllm", wire_url, {"reasoning_effort_levels": list(KIMI_LEVELS)}), + yaml_deployment("claude-levels", "anthropic", wire_url, {"reasoning_effort_levels": list(CLAUDE_LEVELS)}), + ) + + +_OWNED_MODELS: Final = Models("kimi-levels", "claude-levels") + + +@contextmanager +def _owned_proxy( + directory: Path, model_list: tuple[JsonValue, ...], settings: Mapping[str, JsonValue] +) -> Iterator[OwnedProxy]: + directory.mkdir(parents=True, exist_ok=True) + with gateway_from_environment() as shared: + config: Final = owned_config(directory, model_list, litellm_settings=settings) + with owned_proxy_process(shared, directory, {"LITELLM_LOG": "INFO"}, config=config, workers=2) as owned: + yield owned + + +def _port(wire: Wire) -> int: + port: Final = urlsplit(wire.url).port + assert port is not None, wire.url + return port + + +def test_every_request_in_a_burst_is_rounded_while_a_worker_is_killed(tmp_path: Path) -> None: + shots: Final = _shots(_BURST) + with wire_server(provider_reply) as wire: + with _owned_proxy(tmp_path, _yaml_models(wire.url), {}) as owned: + workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30) + with owned.replacing_worker(min(workers)) as kill: + outcomes: Final = _burst(owned.gateway, _OWNED_MODELS, shots, kill) + probes: Final = _shots(len(_KINDS)) + probe_outcomes: Final = tuple(_fire(owned.gateway, _OWNED_MODELS, shot) for shot in probes) + received: Final = wire.drain() + assert any(outcome.status == 200 for outcome in outcomes), outcomes + _assert_every_upstream_effort_rounded((*shots, *probes), received) + _assert_answered_once((*outcomes, *probe_outcomes), received) + _assert_all_answered(probe_outcomes) + + +def test_every_request_in_a_burst_is_rounded_across_a_proxy_restart(tmp_path: Path) -> None: + shots: Final = _shots(_BURST) + probes: Final = _shots(len(_KINDS)) + with wire_server(provider_reply) as wire: + with _owned_proxy(tmp_path / "first", _yaml_models(wire.url), {}) as first: + outcomes: Final = _burst(first.gateway, _OWNED_MODELS, shots, first.begin_shutdown) + with _owned_proxy(tmp_path / "second", _yaml_models(wire.url), {}) as second: + probe_outcomes: Final = tuple(_fire(second.gateway, _OWNED_MODELS, shot) for shot in probes) + received: Final = wire.drain() + _assert_every_upstream_effort_rounded((*shots, *probes), received) + _assert_answered_once((*outcomes, *probe_outcomes), received) + _assert_all_answered(probe_outcomes) + + +def _held_streams() -> Callable[[Request], Reply]: + never_released: Final = threading.Event() + + def respond(request: Request) -> Reply: + reply: Final = provider_reply(request) + return reply if reply.chunks is None else dataclasses.replace(reply, gate_after_first=never_released) + + return respond + + +def _assert_cut_short_with_an_error(outcomes: tuple[Outcome, ...], received: tuple[Request, ...]) -> None: + upstream: Final = tuple(marker_of(request) for request in received) + wrong: Final = tuple( + (outcome.shot.kind, outcome.status, outcome.text[-300:]) + for outcome in outcomes + if answer(outcome.shot.marker) in outcome.text + or "error" not in outcome.text + or upstream.count(outcome.shot.marker) != 1 + ) + assert wrong == (), wrong + + +def _slow_streams(request: Request) -> Reply: + reply: Final = provider_reply(request) + return reply if reply.chunks is None else dataclasses.replace(reply, pause_between_chunks=0.2) + + +def test_requests_stay_rounded_when_the_upstream_stops_mid_burst_and_comes_back(gateway: Gateway) -> None: + before: Final = _shots(_BURST) + during: Final = tuple(shot for shot in _shots(12) if not shot.kind.endswith("-stream")) + after: Final = _shots(_BURST) + with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=_BURST) as pool: + with wire_server(_held_streams()) as first: + models: Final = Models( + deployment(scenario, "hosted_vllm", first, {"reasoning_effort_levels": list(KIMI_LEVELS)}), + deployment(scenario, "anthropic", first, {"reasoning_effort_levels": list(CLAUDE_LEVELS)}), + ) + port: Final = _port(first) + pending: Final = tuple(pool.submit(_fire, gateway, models, shot) for shot in before) + eventually(first.received.qsize, lambda count: count == _BURST, seconds=30) + first_received: Final = first.drain() + held_outcomes: Final = tuple(future.result() for future in pending) + stopped_outcomes: Final = tuple(_fire(gateway, models, shot) for shot in during) + with wire_server(provider_reply, port=port) as second: + after_outcomes: Final = _burst(gateway, models, after, lambda: None) + second_received: Final = second.drain() + completed: Final = tuple(outcome for outcome in held_outcomes if not outcome.shot.kind.endswith("-stream")) + cut_short: Final = tuple(outcome for outcome in held_outcomes if outcome.shot.kind.endswith("-stream")) + _assert_every_upstream_effort_rounded(before, first_received) + _assert_answered_once(completed, first_received) + _assert_all_answered(completed) + _assert_cut_short_with_an_error(cut_short, first_received) + assert all(outcome.status is not None and outcome.status >= 500 for outcome in stopped_outcomes), [ + (outcome.status, outcome.text[:200]) for outcome in stopped_outcomes + ] + assert all("error" in outcome.text for outcome in stopped_outcomes), [ + outcome.text[:200] for outcome in stopped_outcomes + ] + _assert_every_upstream_effort_rounded(after, second_received) + _assert_answered_once(after_outcomes, second_received) + _assert_all_answered(after_outcomes) + + +def test_a_slow_streaming_upstream_neither_deadlocks_nor_duplicates_rounded_requests(gateway: Gateway) -> None: + shots: Final = _shots(_BURST) + with wire_server(_slow_streams) as wire, gateway.scenario() as scenario: + models: Final = Models( + deployment(scenario, "hosted_vllm", wire, {"reasoning_effort_levels": list(KIMI_LEVELS)}), + deployment(scenario, "anthropic", wire, {"reasoning_effort_levels": list(CLAUDE_LEVELS)}), + ) + outcomes: Final = _burst(gateway, models, shots, lambda: None) + received: Final = wire.drain() + _assert_all_answered(outcomes) + _assert_every_upstream_effort_rounded(shots, received) + _assert_answered_once(outcomes, received) + assert len(received) == len(shots) + + +def _remap_lines(log: Path, needle: str) -> tuple[str, ...]: + return tuple(line for line in log.read_text(errors="replace").splitlines() if needle in line) + + +def test_yaml_global_rounding_yields_to_a_deployment_override_and_logs_each_remap(tmp_path: Path) -> None: + markers: Final = MappingProxyType({name: new_marker() for name in ("global", "override", "forged")}) + with wire_server(provider_reply) as wire: + model_list: Final = ( + yaml_deployment("kimi-global", "hosted_vllm", wire.url, {"reasoning_effort_levels": list(KIMI_LEVELS)}), + yaml_deployment( + "kimi-override", + "hosted_vllm", + wire.url, + {"reasoning_effort_levels": list(KIMI_LEVELS), "reasoning_effort_rounding": "up"}, + ), + ) + with _owned_proxy(tmp_path, model_list, {"reasoning_effort_rounding": "down"}) as owned: + global_response: Final = owned.gateway.request( + "POST", + "/v1/chat/completions", + chat_request("kimi-global", markers["global"], reasoning_effort="medium"), + ) + override_response: Final = owned.gateway.request( + "POST", + "/v1/chat/completions", + chat_request("kimi-override", markers["override"], reasoning_effort="medium"), + ) + forged_response: Final = owned.gateway.request( + "POST", + "/v1/chat/completions", + chat_request( + "kimi-global", + markers["forged"], + reasoning_effort="turbo\nreasoning_effort high sent as forged", + drop_params=True, + ), + ) + remaps: Final = eventually( + lambda: _remap_lines(owned.log, "reasoning_effort "), + lambda lines: sum(" sent as " in line for line in lines) >= 2, + seconds=30, + ) + received: Final = MappingProxyType({marker_of(request): request for request in wire.drain()}) + assert [global_response.status_code, override_response.status_code, forged_response.status_code] == [200] * 3, ( + global_response.text, + override_response.text, + forged_response.text, + ) + assert sent_effort(received[markers["global"]]) == "low" + assert sent_effort(received[markers["override"]]) == "high" + assert sent_effort(received[markers["forged"]]) is None + assert any( + "reasoning_effort medium sent as low (rounding down)" in line and "kimi-global" in line for line in remaps + ), remaps + assert any( + "reasoning_effort medium sent as high (rounding up)" in line and "kimi-override" in line for line in remaps + ), remaps + assert not any(line.lstrip().startswith("reasoning_effort high sent as forged") for line in remaps), remaps + + +def test_yaml_global_drop_params_drops_a_malformed_budget_on_messages_to_every_non_claude_bridge( + tmp_path: Path, +) -> None: + targets: Final = MappingProxyType( + {"fireworks_ai": "kimi-fireworks", "openai": "gpt-responses", "azure_ai": "kimi-azure"} + ) + markers: Final = MappingProxyType({name: new_marker() for name in targets.values()}) + with wire_server(provider_reply) as wire: + model_list: Final = tuple(yaml_deployment(name, kind, wire.url, {}) for kind, name in targets.items()) + with _owned_proxy(tmp_path, model_list, {"drop_params": True}) as owned: + responses: Final = MappingProxyType( + { + name: owned.gateway.request( + "POST", + "/v1/messages", + messages_request(name, markers[name], thinking={"type": "enabled", "budget_tokens": "2048"}), + ) + for name in targets.values() + } + ) + received: Final = MappingProxyType({marker_of(request): body_of(request) for request in wire.drain()}) + assert {name: response.status_code for name, response in responses.items()} == dict.fromkeys( + targets.values(), 200 + ), {name: response.text for name, response in responses.items()} + assert sorted(received) == sorted(markers.values()) + assert [ + sorted(set(received[marker]) & {"thinking", "reasoning_effort", "reasoning"}) for marker in markers.values() + ] == [[], [], []], received diff --git a/tests/integration/routing/test_reasoning_effort_levels_wire.py b/tests/integration/routing/test_reasoning_effort_levels_wire.py new file mode 100644 index 00000000000..3d88aad547c --- /dev/null +++ b/tests/integration/routing/test_reasoning_effort_levels_wire.py @@ -0,0 +1,2176 @@ +import asyncio +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import openai +import pytest +from integration._support.client import Gateway, eventually, list_value, object_value, string_value +from integration._support.openai_wire import openai_error +from integration._support.wire import Request, wire_server +from integration.routing._reasoning_effort_levels_support import ( + AZURE_API_VERSION, + CLAUDE, + CLAUDE_LEVELS, + ECHOED_KEY_MODEL, + FIREWORKS_KIMI, + GEMINI, + GLM, + GPT, + GPT_LEVELS, + JSON_OBJECT, + KIMI, + KIMI_LEVELS, + OPENROUTER_KIMI, + ZAI_GLM, + answer, + body_of, + chat_messages, + chat_request, + deployment, + marker_of, + messages_request, + new_marker, + prompt, + provider_reply, + responses_request, + sent_effort, +) +from pydantic import JsonValue + +CHAT: Final = "/v1/chat/completions" +MESSAGES: Final = "/v1/messages" +RESPONSES: Final = "/v1/responses" +_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 2048} +_ADAPTIVE: Final[dict[str, JsonValue]] = {"type": "adaptive"} +_STRING_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": "2048"} +_XHIGH_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": 9000} +_BOOL_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": True} +_NULL_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": None} +_LIST_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": [2048]} +_EMPTY_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": ""} +_LONG_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": "9" * 5000} +_MISSING_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled"} +_DISABLED: Final[dict[str, JsonValue]] = {"type": "disabled"} +_BUDGET_WITH_UNKNOWN_KEY: Final[dict[str, JsonValue]] = {**_BUDGET, "not_a_thinking_key": True} +_ADAPTIVE_WITH_UNKNOWN_KEY: Final[dict[str, JsonValue]] = {**_ADAPTIVE, "not_a_thinking_key": True} +_WORD_BUDGET: Final[dict[str, JsonValue]] = {"type": "enabled", "budget_tokens": "lots"} +_BUDGET_WITH_DISPLAY_UPDATES: Final[dict[str, JsonValue]] = {**_BUDGET, "display": "updates"} +_ADAPTIVE_WITH_DISPLAY_UPDATES: Final[dict[str, JsonValue]] = {**_ADAPTIVE, "display": "updates"} +_DISPLAY_UPDATES_BETA: Final = "thinking-display-updates-2026-08-18" +_NON_NUMERIC_BUDGET: Final = "thinking.budget_tokens must be a number" +_GEMINI_HIGH_ONLY: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": ["high"]}) +_KIMI_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": list(KIMI_LEVELS)}) +_CLAUDE_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": list(CLAUDE_LEVELS)}) +_GPT_INFO: Final[Mapping[str, JsonValue]] = MappingProxyType({"reasoning_effort_levels": list(GPT_LEVELS)}) +_UNDECLARED: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +_ECHOED_MAP_KEY: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"key": "azure/chat-latest", "litellm_provider": "azure"} +) +_LONG_LEVEL: Final = "x" * 5000 +_JSON_SCHEMA_FORMAT: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}, +} +_NATIVE_OUTPUT_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, +} +_CLAUDE_INFO_WITH_NATIVE_OUTPUT: Final[Mapping[str, JsonValue]] = MappingProxyType( + {**_CLAUDE_INFO, "supports_bedrock_native_structured_output": True} +) + +Expected = Callable[[str], dict[str, JsonValue]] + + +def _openai_compatible(effort: JsonValue, *, model: str = KIMI, stream: bool = False) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + usage: Final[dict[str, JsonValue]] = {"stream_options": {"include_usage": True}} if stream else {} + sent: Final[dict[str, JsonValue]] = { + "model": model, + "messages": chat_messages(marker), + "stream": stream, + **usage, + } + return sent if effort is None else {**sent, "reasoning_effort": effort} + + return expected + + +def _openai_chat(effort: str) -> Expected: + return lambda marker: {"model": GPT, "messages": chat_messages(marker), "reasoning_effort": effort} + + +def _azure_chat(effort: str) -> Expected: + return lambda marker: {"model": GPT, "messages": chat_messages(marker), "reasoning_effort": effort, "stream": False} + + +def _anthropic_chat(effort: str, thinking: Mapping[str, JsonValue]) -> Expected: + return lambda marker: { + "model": CLAUDE, + "messages": [{"role": "user", "content": [{"type": "text", "text": prompt(marker)}]}], + "thinking": dict(thinking), + "output_config": {"effort": effort}, + "max_tokens": 128000, + } + + +def _anthropic_chat_without_effort(thinking: Mapping[str, JsonValue]) -> Expected: + return lambda marker: { + "model": CLAUDE, + "messages": [{"role": "user", "content": [{"type": "text", "text": prompt(marker)}]}], + "thinking": dict(thinking), + "max_tokens": 128000, + } + + +def _invoke_chat(effort: str, thinking: Mapping[str, JsonValue], betas: tuple[str, ...]) -> Expected: + return lambda marker: { + "messages": [{"role": "user", "content": [{"type": "text", "text": prompt(marker)}]}], + "thinking": dict(thinking), + "output_config": {"effort": effort}, + "max_tokens": 4096, + "anthropic_version": "bedrock-2023-05-31", + "anthropic_beta": list(betas), + } + + +def _with_output_config(expected: Expected, output_config: JsonValue) -> Expected: + return lambda marker: {**expected(marker), "output_config": output_config} + + +def _converse_chat(effort: str, thinking: Mapping[str, JsonValue]) -> Expected: + return lambda marker: { + "messages": [{"role": "user", "content": [{"text": prompt(marker)}]}], + "inferenceConfig": {}, + "additionalModelRequestFields": {"thinking": dict(thinking), "output_config": {"effort": effort}}, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], + } + + +def _converse_chat_without_effort(thinking: Mapping[str, JsonValue]) -> Expected: + return lambda marker: { + "messages": [{"role": "user", "content": [{"text": prompt(marker)}]}], + "inferenceConfig": {}, + "additionalModelRequestFields": {"thinking": dict(thinking)}, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], + } + + +def _native_messages( + output_config: Mapping[str, JsonValue] | None, thinking: Mapping[str, JsonValue] | None +) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + sent: Final[dict[str, JsonValue]] = { + "messages": chat_messages(marker), + "max_tokens": 4096, + "model": CLAUDE, + "stream": False, + } + with_thinking: Final = sent if thinking is None else {**sent, "thinking": dict(thinking)} + return with_thinking if output_config is None else {**with_thinking, "output_config": dict(output_config)} + + return expected + + +def _invoke_messages(effort: str) -> Expected: + return lambda marker: { + "messages": chat_messages(marker), + "max_tokens": 4096, + "thinking": dict(_ADAPTIVE), + "output_config": {"effort": effort}, + "anthropic_version": "bedrock-2023-05-31", + } + + +def _bridged_messages(effort: str | None, *, model: str = KIMI) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + sent: Final[dict[str, JsonValue]] = {"model": model, "messages": chat_messages(marker), "max_tokens": 4096} + return sent if effort is None else {**sent, "reasoning_effort": effort} + + return expected + + +def _openrouter_messages(effort: str | None) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + return {**_bridged_messages(effort, model=OPENROUTER_KIMI)(marker), "usage": {"include": True}} + + return expected + + +def _bridged_stream_messages(effort: str | None) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + sent: Final[dict[str, JsonValue]] = { + "model": KIMI, + "messages": chat_messages(marker), + "stream": True, + "stream_options": {"include_usage": True}, + "max_tokens": 4096, + } + return sent if effort is None else {**sent, "reasoning_effort": effort} + + return expected + + +def _azure_messages(effort: str | None) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + sent: Final[dict[str, JsonValue]] = { + "model": GPT, + "messages": chat_messages(marker), + "max_completion_tokens": 4096, + } + return sent if effort is None else {**sent, "reasoning_effort": effort} + + return expected + + +def _gemini_messages(level: str | None) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + thinking: Final[dict[str, JsonValue]] = ( + {} if level is None else {"thinkingConfig": {"thinkingLevel": level, "includeThoughts": True}} + ) + return { + "contents": [{"role": "user", "parts": [{"text": prompt(marker)}]}], + "generationConfig": {"max_output_tokens": 4096, **thinking}, + } + + return expected + + +def _gemini_chat(level: str) -> Expected: + return lambda marker: { + "contents": [{"role": "user", "parts": [{"text": prompt(marker)}]}], + "generationConfig": {"thinkingConfig": {"thinkingLevel": level, "includeThoughts": True}}, + } + + +def _responses_bridged_messages_without_effort() -> Expected: + return lambda marker: { + "model": GPT, + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt(marker)}]}], + "include": ["reasoning.encrypted_content"], + "max_output_tokens": 4096, + } + + +def _responses_bridged_messages(effort: str) -> Expected: + return lambda marker: { + "model": GPT, + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt(marker)}]}], + "include": ["reasoning.encrypted_content"], + "max_output_tokens": 4096, + "reasoning": {"effort": effort}, + } + + +def _converse_messages(effort: str | None) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + effort_fields: Final[dict[str, JsonValue]] = {} if effort is None else {"output_config": {"effort": effort}} + return { + "messages": [{"role": "user", "content": [{"text": prompt(marker)}]}], + "inferenceConfig": {"maxTokens": 4096}, + "additionalModelRequestFields": {"thinking": dict(_ADAPTIVE), **effort_fields}, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], + } + + return expected + + +def _native_responses(reasoning: Mapping[str, JsonValue] | None, *, model: str = KIMI) -> Expected: + def expected(marker: str) -> dict[str, JsonValue]: + sent: Final[dict[str, JsonValue]] = {"model": model, "input": prompt(marker)} + return sent if reasoning is None else {**sent, "reasoning": dict(reasoning)} + + return expected + + +def _bridged_responses(effort: str) -> Expected: + return lambda marker: {"model": KIMI, "messages": chat_messages(marker), "reasoning_effort": effort} + + +@dataclass(frozen=True, slots=True) +class Cell: + route: str + kind: str + model_info: Mapping[str, JsonValue] + fields: Mapping[str, JsonValue] + expected: Expected | None + target: str + status: int = 200 + error: str = "" + stream: bool = False + + +_VLLM_CHAT: Final = "/v1/chat/completions" +_AZURE_AI_CHAT: Final = "/chat/completions" +_OPENAI_CHAT: Final = "/v1/chat/completions" +_AZURE_CHAT: Final = f"/openai/deployments/{GPT}/chat/completions?api-version={AZURE_API_VERSION}" +_ANTHROPIC: Final = "/v1/messages" +_CONVERSE: Final = f"/model/us.anthropic.{CLAUDE}/converse" +_CONVERSE_STREAM: Final = f"/model/us.anthropic.{CLAUDE}/converse-stream" +_INVOKE: Final = f"/model/us.anthropic.{CLAUDE}/invoke" +_AZURE_AI_ANTHROPIC: Final = "/anthropic/v1/messages" +_VLLM_RESPONSES: Final = "/v1/responses" +_OPENAI_RESPONSES: Final = "/v1/responses" +_AZURE_RESPONSES: Final = f"/openai/responses?api-version={AZURE_API_VERSION}" +_UNSUPPORTED: Final = "does not support parameters" +_GEMINI: Final = f"/models/{GEMINI}:generateContent" + +CELLS: Final[Mapping[str, Cell]] = MappingProxyType( + { + "chat-azure_ai-medium-to-high": Cell( + CHAT, "azure_ai", _KIMI_INFO, {"reasoning_effort": "medium"}, _openai_compatible("high"), _AZURE_AI_CHAT + ), + "chat-azure_ai-xhigh-to-max": Cell( + CHAT, "azure_ai", _KIMI_INFO, {"reasoning_effort": "xhigh"}, _openai_compatible("max"), _AZURE_AI_CHAT + ), + "chat-azure_ai-stream-medium-to-high": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"reasoning_effort": "medium", "stream": True}, + _openai_compatible("high", stream=True), + _AZURE_AI_CHAT, + stream=True, + ), + "chat-azure_ai-budget-to-high": Cell( + CHAT, "azure_ai", _KIMI_INFO, {"thinking": _BUDGET}, _openai_compatible("high"), _AZURE_AI_CHAT + ), + "chat-azure_ai-none-undeclared-passes": Cell( + CHAT, + "azure_ai", + {"reasoning_effort_levels": ["low", "high"]}, + {"reasoning_effort": "none"}, + _openai_compatible("none"), + _AZURE_AI_CHAT, + ), + "chat-azure_ai-undeclared-control-400": Cell( + CHAT, + "azure_ai", + _UNDECLARED, + {"reasoning_effort": "medium"}, + None, + _AZURE_AI_CHAT, + status=400, + error=_UNSUPPORTED, + ), + "chat-azure_ai-empty-declaration-400": Cell( + CHAT, + "azure_ai", + {"reasoning_effort_levels": []}, + {"reasoning_effort": "medium"}, + None, + _AZURE_AI_CHAT, + status=400, + error=_UNSUPPORTED, + ), + "chat-azure_ai-int-effort-400": Cell( + CHAT, "azure_ai", _KIMI_INFO, {"reasoning_effort": 5}, None, _AZURE_AI_CHAT, status=400, error=_UNSUPPORTED + ), + "chat-azure_ai-explicit-effort-wins-over-budget-thinking": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"reasoning_effort": "xhigh", "thinking": _BUDGET}, + _openai_compatible("max"), + _AZURE_AI_CHAT, + ), + "chat-azure_ai-output-config-beside-budget-sends-its-effort": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"output_config": {"effort": "minimal"}, "thinking": _BUDGET}, + _with_output_config(_openai_compatible("low"), {"effort": "low"}), + _AZURE_AI_CHAT, + ), + "chat-azure_ai-explicit-effort-wins-over-string-budget-thinking": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"reasoning_effort": "xhigh", "thinking": _STRING_BUDGET}, + _openai_compatible("max"), + _AZURE_AI_CHAT, + ), + "chat-azure_ai-explicit-effort-wins-over-adaptive-thinking": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"reasoning_effort": "xhigh", "thinking": _ADAPTIVE}, + _openai_compatible("max"), + _AZURE_AI_CHAT, + ), + "chat-azure_ai-stream-explicit-effort-wins-over-budget-thinking": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"reasoning_effort": "xhigh", "thinking": _BUDGET, "stream": True}, + _openai_compatible("max", stream=True), + _AZURE_AI_CHAT, + stream=True, + ), + "chat-azure_ai-undeclared-explicit-effort-beside-budget-400": Cell( + CHAT, + "azure_ai", + _UNDECLARED, + {"reasoning_effort": "xhigh", "thinking": _BUDGET}, + None, + _AZURE_AI_CHAT, + status=400, + error=_UNSUPPORTED, + ), + "chat-vllm-medium-to-high": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": "medium"}, _openai_compatible("high"), _VLLM_CHAT + ), + "chat-vllm-minimal-to-low": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": "minimal"}, _openai_compatible("low"), _VLLM_CHAT + ), + "chat-vllm-stream-xhigh-to-max": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "xhigh", "stream": True}, + _openai_compatible("max", stream=True), + _VLLM_CHAT, + stream=True, + ), + "chat-vllm-none-kept": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": "none"}, _openai_compatible("none"), _VLLM_CHAT + ), + "chat-vllm-default-sends-nothing": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": "default"}, _openai_compatible(None), _VLLM_CHAT + ), + "chat-vllm-unknown-level-passes": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": "turbo"}, _openai_compatible("turbo"), _VLLM_CHAT + ), + "chat-vllm-unknown-level-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "turbo", "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-undeclared-none-drop-params": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["low", "high", "max"]}, + {"reasoning_effort": "none", "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-declared-none-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "none", "drop_params": True}, + _openai_compatible("none"), + _VLLM_CHAT, + ), + "chat-vllm-budget-to-high": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"thinking": _BUDGET}, _openai_compatible("high"), _VLLM_CHAT + ), + "chat-vllm-budget-undeclared-control": Cell( + CHAT, "hosted_vllm", _UNDECLARED, {"thinking": _BUDGET}, _openai_compatible("medium"), _VLLM_CHAT + ), + "chat-vllm-budget-only-none-drop-params": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["none"]}, + {"thinking": _BUDGET, "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-float-budget-to-high": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": 2048.5}}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-string-budget-400": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": "2048"}}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-vllm-string-budget-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": "2048"}, "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-disabled-thinking": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"thinking": {"type": "disabled"}}, _openai_compatible(None), _VLLM_CHAT + ), + "chat-vllm-explicit-effort-beats-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "minimal", "thinking": _BUDGET}, + _openai_compatible("low"), + _VLLM_CHAT, + ), + "chat-vllm-output-config-beside-budget-sends-its-effort": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": "minimal"}, "thinking": _BUDGET}, + _with_output_config(_openai_compatible("low"), {"effort": "low"}), + _VLLM_CHAT, + ), + "chat-vllm-deployment-rounds-down": Cell( + CHAT, + "hosted_vllm", + {**_KIMI_INFO, "reasoning_effort_rounding": "down"}, + {"reasoning_effort": "medium"}, + _openai_compatible("low"), + _VLLM_CHAT, + ), + "chat-vllm-rounding-down-floor": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["high", "max"], "reasoning_effort_rounding": "down"}, + {"reasoning_effort": "minimal"}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-rounding-up-ceiling": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["minimal", "low"]}, + {"reasoning_effort": "max"}, + _openai_compatible("low"), + _VLLM_CHAT, + ), + "chat-vllm-rounding-mixed-case": Cell( + CHAT, + "hosted_vllm", + {**_KIMI_INFO, "reasoning_effort_rounding": " DOWN "}, + {"reasoning_effort": "medium"}, + _openai_compatible("low"), + _VLLM_CHAT, + ), + "chat-vllm-rounding-unknown-falls-back-up": Cell( + CHAT, + "hosted_vllm", + {**_KIMI_INFO, "reasoning_effort_rounding": "sideways"}, + {"reasoning_effort": "medium"}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-rounding-int-falls-back-up": Cell( + CHAT, + "hosted_vllm", + {**_KIMI_INFO, "reasoning_effort_rounding": 5}, + {"reasoning_effort": "medium"}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-uppercase-levels-ignored": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["LOW", "HIGH"]}, + {"reasoning_effort": "medium"}, + _openai_compatible("medium"), + _VLLM_CHAT, + ), + "chat-vllm-string-levels-ignored": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": "high"}, + {"reasoning_effort": "medium"}, + _openai_compatible("medium"), + _VLLM_CHAT, + ), + "chat-vllm-null-levels-ignored": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": None}, + {"reasoning_effort": "medium"}, + _openai_compatible("medium"), + _VLLM_CHAT, + ), + "chat-vllm-mixed-junk-levels": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": ["high", 7, None, "turbo"]}, + {"reasoning_effort": "low"}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-undeclared-control": Cell( + CHAT, "hosted_vllm", _UNDECLARED, {"reasoning_effort": "medium"}, _openai_compatible("medium"), _VLLM_CHAT + ), + "chat-vllm-int-effort-passes": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": 5}, _openai_compatible(5), _VLLM_CHAT + ), + "chat-vllm-list-effort-passes": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": ["high"]}, _openai_compatible(["high"]), _VLLM_CHAT + ), + "chat-vllm-empty-effort-passes": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"reasoning_effort": ""}, _openai_compatible(""), _VLLM_CHAT + ), + "chat-vllm-empty-effort-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "", "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-5kb-effort-passes": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": _LONG_LEVEL}, + _openai_compatible(_LONG_LEVEL), + _VLLM_CHAT, + ), + "chat-vllm-5kb-effort-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": _LONG_LEVEL, "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-claude-name-undeclared-budget-unchanged": Cell( + CHAT, + "hosted_vllm_claude", + _UNDECLARED, + {"thinking": {"type": "enabled", "budget_tokens": 9000}}, + _openai_compatible("high", model=CLAUDE), + _VLLM_CHAT, + ), + "messages-vllm-claude-name-undeclared-budget-unchanged": Cell( + MESSAGES, + "hosted_vllm_claude", + _UNDECLARED, + {"thinking": {"type": "enabled", "budget_tokens": 9000}}, + _bridged_messages("high", model=CLAUDE), + _VLLM_CHAT, + ), + "chat-vllm-echoed-map-key-undeclared-unchanged": Cell( + CHAT, + "hosted_vllm_echoed_key", + _ECHOED_MAP_KEY, + {"reasoning_effort": "high"}, + _openai_compatible("high", model=ECHOED_KEY_MODEL), + _VLLM_CHAT, + ), + "chat-vllm-undeclared-string-budget-400": Cell( + CHAT, + "hosted_vllm", + _UNDECLARED, + {"thinking": {"type": "enabled", "budget_tokens": "2048"}}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-mistral-declared-level-is-sent": Cell( + CHAT, + "mistral", + {"reasoning_effort_levels": ["low", "medium"]}, + {"reasoning_effort": "medium"}, + lambda marker: {"model": GLM, "messages": chat_messages(marker), "reasoning_effort": "medium"}, + _OPENAI_CHAT, + ), + "chat-mistral-deployment-rounds-down": Cell( + CHAT, + "mistral", + {"reasoning_effort_levels": ["low", "high"], "reasoning_effort_rounding": "down"}, + {"reasoning_effort": "medium"}, + lambda marker: {"model": GLM, "messages": chat_messages(marker), "reasoning_effort": "low"}, + _OPENAI_CHAT, + ), + "chat-openai-minimal-to-low": Cell( + CHAT, "openai", _GPT_INFO, {"reasoning_effort": "minimal"}, _openai_chat("low"), _OPENAI_CHAT + ), + "chat-openai-max-to-xhigh": Cell( + CHAT, "openai", _GPT_INFO, {"reasoning_effort": "max"}, _openai_chat("xhigh"), _OPENAI_CHAT + ), + "chat-openai-undeclared-minimal-control-400": Cell( + CHAT, + "openai", + _UNDECLARED, + {"reasoning_effort": "minimal"}, + None, + _OPENAI_CHAT, + status=400, + error="reasoning_effort=minimal is not supported", + ), + "chat-azure-medium-to-high": Cell( + CHAT, + "azure", + {"reasoning_effort_levels": ["low", "high"]}, + {"reasoning_effort": "medium"}, + _azure_chat("high"), + _AZURE_CHAT, + ), + "chat-anthropic-medium-to-high": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"reasoning_effort": "medium"}, + _anthropic_chat("high", {"type": "adaptive", "display": "summarized"}), + _ANTHROPIC, + ), + "chat-anthropic-budget-to-high": Cell( + CHAT, "anthropic", _CLAUDE_INFO, {"thinking": _BUDGET}, _anthropic_chat("high", _ADAPTIVE), _ANTHROPIC + ), + "chat-anthropic-budget-keeps-display": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": {**_BUDGET, "display": "omitted"}}, + _anthropic_chat("high", {"type": "adaptive", "display": "omitted"}), + _ANTHROPIC, + ), + "chat-anthropic-undeclared-budget-control": Cell( + CHAT, "anthropic", _UNDECLARED, {"thinking": _BUDGET}, _anthropic_chat("medium", _ADAPTIVE), _ANTHROPIC + ), + "chat-converse-medium-to-high": Cell( + CHAT, + "bedrock_converse", + _CLAUDE_INFO, + {"reasoning_effort": "medium"}, + _converse_chat("high", {"type": "adaptive", "display": "summarized"}), + _CONVERSE, + ), + "chat-converse-budget-to-high": Cell( + CHAT, "bedrock_converse", _CLAUDE_INFO, {"thinking": _BUDGET}, _converse_chat("high", _ADAPTIVE), _CONVERSE + ), + "chat-converse-undeclared-control": Cell( + CHAT, + "bedrock_converse", + _UNDECLARED, + {"reasoning_effort": "medium"}, + _converse_chat("medium", {"type": "adaptive", "display": "summarized"}), + _CONVERSE, + ), + "messages-anthropic-claude-code-medium-to-high": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _native_messages({"effort": "high"}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-budget-to-high": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _BUDGET}, + _native_messages({"effort": "high"}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-budget-keeps-display": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": {**_BUDGET, "display": "omitted"}}, + _native_messages({"effort": "high"}, {"type": "adaptive", "display": "omitted"}), + _ANTHROPIC, + ), + "messages-anthropic-output-effort-only": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"output_config": {"effort": "medium"}}, + _native_messages({"effort": "high"}, None), + _ANTHROPIC, + ), + "messages-anthropic-drop-params-keeps-format": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"output_config": {"effort": "turbo", "format": _JSON_SCHEMA_FORMAT}, "drop_params": True}, + _native_messages({"format": _JSON_SCHEMA_FORMAT}, None), + _ANTHROPIC, + ), + "messages-anthropic-drop-params-dropped-effort-beside-budget-keeps-the-budget-effort": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + { + "output_config": {"effort": "turbo", "format": _JSON_SCHEMA_FORMAT}, + "thinking": _BUDGET, + "drop_params": True, + }, + _native_messages({"effort": "high", "format": _JSON_SCHEMA_FORMAT}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-default-sends-no-effort": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "default"}}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-undeclared-control": Cell( + MESSAGES, + "anthropic", + _UNDECLARED, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _native_messages({"effort": "medium"}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-azure_ai-claude-code-medium-to-high": Cell( + MESSAGES, + "azure_ai", + _KIMI_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _bridged_messages("high"), + _AZURE_AI_CHAT, + ), + "messages-azure_ai-budget-to-high": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _BUDGET}, _bridged_messages("high"), _AZURE_AI_CHAT + ), + "messages-vllm-claude-code-xhigh-to-max": Cell( + MESSAGES, + "hosted_vllm", + _KIMI_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "xhigh"}}, + _bridged_messages("max"), + _VLLM_CHAT, + ), + "messages-vllm-budget-to-high": Cell( + MESSAGES, "hosted_vllm", _KIMI_INFO, {"thinking": _BUDGET}, _bridged_messages("high"), _VLLM_CHAT + ), + "messages-vllm-undeclared-budget-control": Cell( + MESSAGES, "hosted_vllm", _UNDECLARED, {"thinking": _BUDGET}, _bridged_messages("medium"), _VLLM_CHAT + ), + "messages-openai-claude-code-medium-to-high": Cell( + MESSAGES, + "openai", + {"reasoning_effort_levels": ["low", "high"]}, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _responses_bridged_messages("high"), + _OPENAI_RESPONSES, + ), + "messages-converse-budget-to-high": Cell( + MESSAGES, "bedrock_converse", _CLAUDE_INFO, {"thinking": _BUDGET}, _converse_messages("high"), _CONVERSE + ), + "messages-converse-claude-code-medium-to-high": Cell( + MESSAGES, + "bedrock_converse", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _converse_messages("high"), + _CONVERSE, + ), + "messages-invoke-claude-code-medium-to-high": Cell( + MESSAGES, + "bedrock_invoke", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _invoke_messages("high"), + _INVOKE, + ), + "messages-azure_ai-claude-claude-code-medium-to-high": Cell( + MESSAGES, + "azure_ai_claude", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _native_messages({"effort": "high"}, _ADAPTIVE), + _AZURE_AI_ANTHROPIC, + ), + "responses-vllm-minimal-to-low": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "minimal"}}, + _native_responses({"effort": "low"}), + _VLLM_RESPONSES, + ), + "responses-vllm-keeps-summary": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "medium", "summary": "auto"}}, + _native_responses({"effort": "high", "summary": "auto"}), + _VLLM_RESPONSES, + ), + "responses-vllm-default-keeps-summary": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "default", "summary": "auto"}}, + _native_responses({"summary": "auto"}), + _VLLM_RESPONSES, + ), + "responses-vllm-default-alone-sends-nothing": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "default"}}, + _native_responses(None), + _VLLM_RESPONSES, + ), + "responses-vllm-unknown-drop-params": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "turbo"}, "drop_params": True}, + _native_responses(None), + _VLLM_RESPONSES, + ), + "responses-vllm-stream-medium-to-high": Cell( + RESPONSES, + "hosted_vllm", + _KIMI_INFO, + {"reasoning": {"effort": "medium"}, "stream": True}, + lambda marker: {"model": KIMI, "input": prompt(marker), "reasoning": {"effort": "high"}, "stream": True}, + _VLLM_RESPONSES, + stream=True, + ), + "responses-vllm-deployment-rounds-down": Cell( + RESPONSES, + "hosted_vllm", + {**_KIMI_INFO, "reasoning_effort_rounding": "down"}, + {"reasoning": {"effort": "medium"}}, + _native_responses({"effort": "low"}), + _VLLM_RESPONSES, + ), + "responses-vllm-undeclared-control": Cell( + RESPONSES, + "hosted_vllm", + _UNDECLARED, + {"reasoning": {"effort": "medium"}}, + _native_responses({"effort": "medium"}), + _VLLM_RESPONSES, + ), + "responses-azure_ai-minimal-to-low": Cell( + RESPONSES, + "azure_ai", + _KIMI_INFO, + {"reasoning": {"effort": "minimal"}}, + _bridged_responses("low"), + _AZURE_AI_CHAT, + ), + "responses-openai-minimal-to-low": Cell( + RESPONSES, + "openai", + {"reasoning_effort_levels": ["low", "high"]}, + {"reasoning": {"effort": "minimal", "summary": "auto"}}, + _native_responses({"effort": "low", "summary": "auto"}, model=GPT), + _OPENAI_RESPONSES, + ), + "responses-azure-medium-to-high": Cell( + RESPONSES, + "azure", + {"reasoning_effort_levels": ["low", "high"]}, + {"reasoning": {"effort": "medium"}}, + _native_responses({"effort": "high"}, model=GPT), + _AZURE_RESPONSES, + ), + "chat-anthropic-output-config-medium-to-high": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _anthropic_chat("high", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-output-config-undeclared-control": Cell( + CHAT, + "anthropic", + _UNDECLARED, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _anthropic_chat("medium", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-output-config-default-sends-no-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "default"}}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-output-config-unknown-drop-params": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "turbo"}, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-converse-output-config-medium-to-high": Cell( + CHAT, + "bedrock_converse", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _converse_chat("high", _ADAPTIVE), + _CONVERSE, + ), + "chat-converse-output-config-undeclared-control": Cell( + CHAT, + "bedrock_converse", + _UNDECLARED, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _converse_chat("medium", _ADAPTIVE), + _CONVERSE, + ), + "chat-vllm-output-config-medium-to-high": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": "medium"}}, + _with_output_config(_openai_compatible(None), {"effort": "high"}), + _VLLM_CHAT, + ), + "chat-vllm-output-config-keeps-other-keys": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": "minimal", "format": _JSON_SCHEMA_FORMAT}}, + _with_output_config(_openai_compatible(None), {"effort": "low", "format": _JSON_SCHEMA_FORMAT}), + _VLLM_CHAT, + ), + "chat-vllm-output-config-string-passes": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": "medium"}, + _with_output_config(_openai_compatible(None), "medium"), + _VLLM_CHAT, + ), + "chat-vllm-output-config-int-effort-passes": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": 5}}, + _with_output_config(_openai_compatible(None), {"effort": 5}), + _VLLM_CHAT, + ), + "chat-vllm-output-config-5kb-effort-drop-params": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": _LONG_LEVEL}, "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-output-config-undeclared-control": Cell( + CHAT, + "hosted_vllm", + _UNDECLARED, + {"output_config": {"effort": "medium"}}, + _with_output_config(_openai_compatible(None), {"effort": "medium"}), + _VLLM_CHAT, + ), + "chat-openai-output-config-minimal-to-low": Cell( + CHAT, + "openai", + _GPT_INFO, + {"output_config": {"effort": "minimal"}}, + lambda marker: {"model": GPT, "messages": chat_messages(marker), "output_config": {"effort": "low"}}, + _OPENAI_CHAT, + ), + "chat-openai-output-config-undeclared-control": Cell( + CHAT, + "openai", + _UNDECLARED, + {"output_config": {"effort": "minimal"}}, + lambda marker: {"model": GPT, "messages": chat_messages(marker), "output_config": {"effort": "minimal"}}, + _OPENAI_CHAT, + ), + "chat-azure-output-config-medium-to-high": Cell( + CHAT, + "azure", + {"reasoning_effort_levels": ["low", "high"]}, + {"output_config": {"effort": "medium"}}, + lambda marker: { + "model": GPT, + "messages": chat_messages(marker), + "stream": False, + "output_config": {"effort": "high"}, + }, + _AZURE_CHAT, + ), + "chat-azure_ai-output-config-medium-to-high": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"output_config": {"effort": "medium"}}, + _with_output_config(_openai_compatible(None), {"effort": "high"}), + _AZURE_AI_CHAT, + ), + "messages-converse-claude-code-undeclared-control": Cell( + MESSAGES, + "bedrock_converse", + _UNDECLARED, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _converse_messages("medium"), + _CONVERSE, + ), + "messages-converse-claude-code-unknown-drop-params": Cell( + MESSAGES, + "bedrock_converse", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "turbo"}, "drop_params": True}, + _converse_messages(None), + _CONVERSE, + ), + "messages-converse-claude-code-default-sends-no-effort": Cell( + MESSAGES, + "bedrock_converse", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "default"}}, + _converse_messages(None), + _CONVERSE, + ), + "messages-converse-claude-code-rounds-down": Cell( + MESSAGES, + "bedrock_converse", + {**_CLAUDE_INFO, "reasoning_effort_rounding": "down"}, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}}, + _converse_messages("low"), + _CONVERSE, + ), + "messages-converse-claude-code-stream-medium-to-high": Cell( + MESSAGES, + "bedrock_converse", + _CLAUDE_INFO, + {"thinking": _ADAPTIVE, "output_config": {"effort": "medium"}, "stream": True}, + _converse_messages("high"), + _CONVERSE_STREAM, + stream=True, + ), + "chat-vllm-bool-budget-400": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": True}}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-vllm-null-budget-400": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": None}}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-vllm-list-budget-400": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"thinking": {"type": "enabled", "budget_tokens": [2048]}}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-vllm-missing-budget-to-low": Cell( + CHAT, "hosted_vllm", _KIMI_INFO, {"thinking": {"type": "enabled"}}, _openai_compatible("low"), _VLLM_CHAT + ), + "chat-vllm-explicit-effort-wins-over-string-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": _STRING_BUDGET}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-explicit-effort-wins-over-bool-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": {"type": "enabled", "budget_tokens": True}}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-explicit-effort-beside-string-budget-drop-params": Cell( + CHAT, + "hosted_vllm_drop_params", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": _STRING_BUDGET}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-undeclared-explicit-effort-wins-over-string-budget": Cell( + CHAT, + "hosted_vllm", + _UNDECLARED, + {"reasoning_effort": "medium", "thinking": _STRING_BUDGET}, + _openai_compatible("medium"), + _VLLM_CHAT, + ), + "chat-vllm-string-budget-deployment-drop-params": Cell( + CHAT, + "hosted_vllm_drop_params", + _KIMI_INFO, + {"thinking": _STRING_BUDGET}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-undeclared-string-budget-drop-params": Cell( + CHAT, + "hosted_vllm", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-vllm-empty-levels-budget-unchanged": Cell( + CHAT, + "hosted_vllm", + {"reasoning_effort_levels": []}, + {"thinking": _BUDGET}, + _openai_compatible("medium"), + _VLLM_CHAT, + ), + "chat-azure_ai-string-budget-400": Cell( + CHAT, + "azure_ai", + _KIMI_INFO, + {"thinking": _STRING_BUDGET}, + None, + _AZURE_AI_CHAT, + status=400, + error="budget_tokens", + ), + "chat-anthropic-numeric-string-budget-to-high": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": _STRING_BUDGET}, + _anthropic_chat("high", _ADAPTIVE), + _ANTHROPIC, + ), + "messages-vllm-string-budget-400": Cell( + MESSAGES, + "hosted_vllm", + _KIMI_INFO, + {"thinking": _STRING_BUDGET}, + None, + _VLLM_CHAT, + status=400, + error="budget_tokens", + ), + "chat-vllm-echoed-map-key-own-levels-win": Cell( + CHAT, + "hosted_vllm_echoed_key", + {**_ECHOED_MAP_KEY, "reasoning_effort_levels": ["low", "high"]}, + {"reasoning_effort": "medium"}, + _openai_compatible("high", model=ECHOED_KEY_MODEL), + _VLLM_CHAT, + ), + "responses-vllm-echoed-map-key-undeclared-unchanged": Cell( + RESPONSES, + "hosted_vllm_echoed_key", + _ECHOED_MAP_KEY, + {"reasoning": {"effort": "high"}}, + _native_responses({"effort": "high"}, model=ECHOED_KEY_MODEL), + _VLLM_RESPONSES, + ), + "chat-vllm-claude-name-declared-budget-generic-table": Cell( + CHAT, + "hosted_vllm_claude", + {"reasoning_effort_levels": ["low", "high", "max"]}, + {"thinking": _XHIGH_BUDGET}, + _openai_compatible("high", model=CLAUDE), + _VLLM_CHAT, + ), + "messages-vllm-claude-name-declared-budget-generic-table": Cell( + MESSAGES, + "hosted_vllm_claude", + {"reasoning_effort_levels": ["low", "high", "max"]}, + {"thinking": _XHIGH_BUDGET}, + _bridged_messages("high", model=CLAUDE), + _VLLM_CHAT, + ), + "messages-azure_ai-string-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _STRING_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-undeclared-string-budget-400": Cell( + MESSAGES, "azure_ai", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-string-budget-drop-params": Cell( + MESSAGES, + "azure_ai", + _KIMI_INFO, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _bridged_messages(None), + _AZURE_AI_CHAT, + ), + "messages-azure_ai-undeclared-string-budget-drop-params": Cell( + MESSAGES, + "azure_ai", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _bridged_messages(None), + _AZURE_AI_CHAT, + ), + "messages-azure_ai-string-budget-deployment-drop-params": Cell( + MESSAGES, + "azure_ai_drop_params", + _KIMI_INFO, + {"thinking": _STRING_BUDGET}, + _bridged_messages(None), + _AZURE_AI_CHAT, + ), + "messages-azure_ai-undeclared-string-budget-deployment-drop-params": Cell( + MESSAGES, + "azure_ai_drop_params", + _UNDECLARED, + {"thinking": _STRING_BUDGET}, + _bridged_messages(None), + _AZURE_AI_CHAT, + ), + "messages-azure_ai-bool-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _BOOL_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-null-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _NULL_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-list-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _LIST_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-empty-string-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _EMPTY_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-5kb-budget-400": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _LONG_BUDGET}, None, _AZURE_AI_CHAT, 400, "thinking" + ), + "messages-azure_ai-stream-string-budget-400": Cell( + MESSAGES, + "azure_ai", + _KIMI_INFO, + {"thinking": _STRING_BUDGET, "stream": True}, + None, + _AZURE_AI_CHAT, + 400, + "thinking", + stream=True, + ), + "messages-azure_ai-stream-string-budget-drop-params": Cell( + MESSAGES, + "azure_ai", + _KIMI_INFO, + {"thinking": _STRING_BUDGET, "stream": True, "drop_params": True}, + _bridged_stream_messages(None), + _AZURE_AI_CHAT, + stream=True, + ), + "messages-azure_ai-missing-budget-to-low": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _MISSING_BUDGET}, _bridged_messages("low"), _AZURE_AI_CHAT + ), + "messages-azure_ai-disabled-thinking-sends-none": Cell( + MESSAGES, "azure_ai", _KIMI_INFO, {"thinking": _DISABLED}, _bridged_messages("none"), _AZURE_AI_CHAT + ), + "messages-azure-budget-control": Cell( + MESSAGES, "azure", _UNDECLARED, {"thinking": _BUDGET}, _azure_messages("medium"), _AZURE_CHAT + ), + "messages-azure-string-budget-400": Cell( + MESSAGES, "azure", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _AZURE_CHAT, 400, "thinking" + ), + "messages-azure-string-budget-drop-params": Cell( + MESSAGES, + "azure", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _azure_messages(None), + _AZURE_CHAT, + ), + "messages-azure-bool-budget-400": Cell( + MESSAGES, "azure", _UNDECLARED, {"thinking": _BOOL_BUDGET}, None, _AZURE_CHAT, 400, "thinking" + ), + "messages-azure-declared-string-budget-400": Cell( + MESSAGES, "azure", _GPT_INFO, {"thinking": _STRING_BUDGET}, None, _AZURE_CHAT, 400, "budget_tokens" + ), + "messages-azure-declared-string-budget-drop-params": Cell( + MESSAGES, + "azure", + _GPT_INFO, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _azure_messages(None), + _AZURE_CHAT, + ), + "messages-mistral-string-budget-400": Cell( + MESSAGES, "mistral", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _VLLM_CHAT, 400, "thinking" + ), + "messages-openai-budget-control": Cell( + MESSAGES, + "openai", + _UNDECLARED, + {"thinking": _BUDGET}, + _responses_bridged_messages("medium"), + _OPENAI_RESPONSES, + ), + "messages-openai-string-budget-400": Cell( + MESSAGES, "openai", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _OPENAI_RESPONSES, 400, "thinking" + ), + "messages-openai-string-budget-drop-params": Cell( + MESSAGES, + "openai", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _responses_bridged_messages_without_effort(), + _OPENAI_RESPONSES, + ), + "messages-openai-bool-budget-400": Cell( + MESSAGES, "openai", _UNDECLARED, {"thinking": _BOOL_BUDGET}, None, _OPENAI_RESPONSES, 400, "thinking" + ), + "messages-openai-string-budget-deployment-drop-params": Cell( + MESSAGES, + "openai_drop_params", + _UNDECLARED, + {"thinking": _STRING_BUDGET}, + _responses_bridged_messages_without_effort(), + _OPENAI_RESPONSES, + ), + "messages-fireworks-budget-control": Cell( + MESSAGES, + "fireworks_ai", + _UNDECLARED, + {"thinking": _BUDGET}, + _bridged_messages("medium", model=FIREWORKS_KIMI), + _VLLM_CHAT, + ), + "messages-fireworks-string-budget-400": Cell( + MESSAGES, "fireworks_ai", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _VLLM_CHAT, 400, "thinking" + ), + "messages-fireworks-string-budget-drop-params": Cell( + MESSAGES, + "fireworks_ai", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _bridged_messages(None, model=FIREWORKS_KIMI), + _VLLM_CHAT, + ), + "messages-fireworks-declared-string-budget-400": Cell( + MESSAGES, + "fireworks_ai", + {"reasoning_effort_levels": ["low", "high"]}, + {"thinking": _STRING_BUDGET}, + None, + _VLLM_CHAT, + 400, + "thinking", + ), + "messages-fireworks-bool-budget-400": Cell( + MESSAGES, "fireworks_ai", _UNDECLARED, {"thinking": _BOOL_BUDGET}, None, _VLLM_CHAT, 400, "thinking" + ), + "messages-openrouter-budget-control": Cell( + MESSAGES, + "openrouter", + _UNDECLARED, + {"thinking": _BUDGET}, + _openrouter_messages("medium"), + _VLLM_CHAT, + ), + "messages-openrouter-string-budget-drop-params": Cell( + MESSAGES, + "openrouter", + _UNDECLARED, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _openrouter_messages(None), + _VLLM_CHAT, + ), + "messages-gemini-budget-control": Cell( + MESSAGES, "gemini", _UNDECLARED, {"thinking": _BUDGET}, _gemini_messages("medium"), _GEMINI + ), + "messages-gemini-string-budget-400": Cell( + MESSAGES, "gemini", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _GEMINI, 400, "thinking" + ), + "messages-gemini-bool-budget-400": Cell( + MESSAGES, "gemini", _UNDECLARED, {"thinking": _BOOL_BUDGET}, None, _GEMINI, 400, "thinking" + ), + "messages-zai-bool-budget-400": Cell( + MESSAGES, "zai", _UNDECLARED, {"thinking": _BOOL_BUDGET}, None, _VLLM_CHAT, 400 + ), + "messages-zai-bool-budget-drop-params": Cell( + MESSAGES, + "zai", + _UNDECLARED, + {"thinking": _BOOL_BUDGET, "drop_params": True}, + _bridged_messages(None, model=ZAI_GLM), + _VLLM_CHAT, + ), + "messages-vllm-undeclared-string-budget-400": Cell( + MESSAGES, "hosted_vllm", _UNDECLARED, {"thinking": _STRING_BUDGET}, None, _VLLM_CHAT, 400, "budget_tokens" + ), + "messages-vllm-string-budget-drop-params": Cell( + MESSAGES, + "hosted_vllm", + _KIMI_INFO, + {"thinking": _STRING_BUDGET, "drop_params": True}, + _bridged_messages(None), + _VLLM_CHAT, + ), + "messages-vllm-string-budget-deployment-drop-params": Cell( + MESSAGES, + "hosted_vllm_drop_params", + _KIMI_INFO, + {"thinking": _STRING_BUDGET}, + _bridged_messages(None), + _VLLM_CHAT, + ), + "messages-vllm-list-budget-400": Cell( + MESSAGES, "hosted_vllm", _KIMI_INFO, {"thinking": _LIST_BUDGET}, None, _VLLM_CHAT, 400, "budget_tokens" + ), + "messages-vllm-bool-budget-400": Cell( + MESSAGES, "hosted_vllm", _KIMI_INFO, {"thinking": _BOOL_BUDGET}, None, _VLLM_CHAT, 400, "budget_tokens" + ), + "messages-vllm-claude-name-declared-string-budget-400": Cell( + MESSAGES, + "hosted_vllm_claude", + {"reasoning_effort_levels": ["low", "high", "max"]}, + {"thinking": _STRING_BUDGET}, + None, + _VLLM_CHAT, + 400, + "budget_tokens", + ), + "chat-vllm-claude-name-declared-adaptive-thinking-sends-no-effort": Cell( + CHAT, + "hosted_vllm_claude", + {"reasoning_effort_levels": ["low", "high", "max"]}, + {"thinking": _ADAPTIVE}, + _openai_compatible(None, model=CLAUDE), + _VLLM_CHAT, + ), + "chat-vllm-explicit-effort-wins-over-null-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": _NULL_BUDGET}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-explicit-effort-wins-over-list-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": _LIST_BUDGET}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-vllm-explicit-effort-wins-over-5kb-budget": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"reasoning_effort": "medium", "thinking": _LONG_BUDGET}, + _openai_compatible("high"), + _VLLM_CHAT, + ), + "chat-anthropic-default-effort-beside-budget-sends-no-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"reasoning_effort": "default", "thinking": _BUDGET}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-output-config-default-beside-budget-sends-no-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"output_config": {"effort": "default"}, "thinking": _BUDGET}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-output-config-low-beside-xhigh-budget-keeps-low": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"output_config": {"effort": "low"}, "thinking": _XHIGH_BUDGET}, + _anthropic_chat("low", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-unknown-effort-drop-params-beside-budget-keeps-the-budget-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"reasoning_effort": "turbo", "thinking": _BUDGET, "drop_params": True}, + _anthropic_chat("high", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-default-beside-budget-drops-unknown-thinking-key-under-drop-params": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"reasoning_effort": "default", "thinking": _BUDGET_WITH_UNKNOWN_KEY, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "chat-anthropic-default-beside-budget-keeps-unknown-thinking-key": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"reasoning_effort": "default", "thinking": _BUDGET_WITH_UNKNOWN_KEY}, + _anthropic_chat_without_effort(_ADAPTIVE_WITH_UNKNOWN_KEY), + _ANTHROPIC, + ), + "chat-converse-default-effort-beside-budget-sends-no-effort": Cell( + CHAT, + "bedrock_converse", + _CLAUDE_INFO, + {"reasoning_effort": "default", "thinking": _BUDGET}, + _converse_chat_without_effort(_ADAPTIVE), + _CONVERSE, + ), + "messages-anthropic-budget-drops-unknown-thinking-key-under-drop-params": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _BUDGET_WITH_UNKNOWN_KEY, "drop_params": True}, + _native_messages({"effort": "high"}, _ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-budget-keeps-unknown-thinking-key": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _BUDGET_WITH_UNKNOWN_KEY}, + _native_messages({"effort": "high"}, _ADAPTIVE_WITH_UNKNOWN_KEY), + _ANTHROPIC, + ), + "messages-anthropic-output-config-default-beside-budget-sends-no-effort": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _BUDGET, "output_config": {"effort": "default"}}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "chat-gemini-declared-low-with-summary-rounds-up": Cell( + CHAT, + "gemini", + _GEMINI_HIGH_ONLY, + {"reasoning_effort": "low", "reasoning_summary": "auto"}, + _gemini_chat("high"), + _GEMINI, + ), + "responses-gemini-declared-low-with-summary-rounds-up": Cell( + RESPONSES, + "gemini", + _GEMINI_HIGH_ONLY, + {"reasoning": {"effort": "low", "summary": "auto"}}, + _gemini_chat("high"), + _GEMINI, + ), + "chat-vllm-output-config-default-beside-budget-sends-no-effort": Cell( + CHAT, + "hosted_vllm", + _KIMI_INFO, + {"output_config": {"effort": "default"}, "thinking": _BUDGET}, + _openai_compatible(None), + _VLLM_CHAT, + ), + "chat-anthropic-output-config-unknown-drop-params-beside-budget-keeps-the-budget-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"output_config": {"effort": "turbo"}, "thinking": _BUDGET, "drop_params": True}, + _anthropic_chat("high", _ADAPTIVE), + _ANTHROPIC, + ), + "chat-invoke-output-config-beside-budget-keeps-display-updates": Cell( + CHAT, + "bedrock_invoke", + _CLAUDE_INFO, + {"output_config": {"effort": "medium"}, "thinking": _BUDGET_WITH_DISPLAY_UPDATES}, + _invoke_chat("high", _ADAPTIVE_WITH_DISPLAY_UPDATES, (_DISPLAY_UPDATES_BETA,)), + _INVOKE, + ), + "chat-anthropic-undeclared-non-numeric-budget-400": Cell( + CHAT, + "anthropic", + _UNDECLARED, + {"thinking": _WORD_BUDGET}, + None, + _ANTHROPIC, + status=400, + error=_NON_NUMERIC_BUDGET, + ), + "chat-anthropic-non-numeric-budget-drop-params-sends-adaptive-without-effort": Cell( + CHAT, + "anthropic", + _CLAUDE_INFO, + {"thinking": _WORD_BUDGET, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + "messages-anthropic-undeclared-non-numeric-budget-400": Cell( + MESSAGES, + "anthropic", + _UNDECLARED, + {"thinking": _WORD_BUDGET}, + None, + _ANTHROPIC, + status=400, + error=_NON_NUMERIC_BUDGET, + ), + "messages-anthropic-non-numeric-budget-drop-params-sends-adaptive-without-effort": Cell( + MESSAGES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _WORD_BUDGET, "drop_params": True}, + _native_messages(None, _ADAPTIVE), + _ANTHROPIC, + ), + "responses-anthropic-non-numeric-budget-400": Cell( + RESPONSES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _WORD_BUDGET}, + None, + _ANTHROPIC, + status=400, + error=_NON_NUMERIC_BUDGET, + ), + "responses-anthropic-non-numeric-budget-drop-params-sends-adaptive-without-effort": Cell( + RESPONSES, + "anthropic", + _CLAUDE_INFO, + {"thinking": _WORD_BUDGET, "drop_params": True}, + _anthropic_chat_without_effort(_ADAPTIVE), + _ANTHROPIC, + ), + } +) + + +def _request_body(cell: Cell, model: str, marker: str) -> dict[str, JsonValue]: + match cell.route: + case "/v1/chat/completions": + return chat_request(model, marker, **cell.fields) + case "/v1/messages": + return messages_request(model, marker, **cell.fields) + case "/v1/responses": + return responses_request(model, marker, **cell.fields) + case _: + raise AssertionError(cell.route) + + +def _assert_answered(cell: Cell, response: httpx.Response, marker: str) -> None: + assert response.status_code == cell.status, response.text[:2000] + if cell.status != 200: + assert cell.error in response.text, response.text[:2000] + return + assert answer(marker) in response.text, response.text[:2000] + if cell.stream: + return + payload: Final = object_value(response.json()) + if cell.route == CHAT and cell.kind in ( + "hosted_vllm", + "hosted_vllm_claude", + "hosted_vllm_echoed_key", + "hosted_vllm_drop_params", + "azure_ai", + "openai", + "azure", + ): + assert payload["id"] == f"chatcmpl-{marker}", payload + if cell.route == MESSAGES and cell.kind in ("anthropic", "azure_ai_claude", "bedrock_invoke"): + assert payload["id"] == f"msg_{marker}", payload + + +@pytest.mark.parametrize("cell", [pytest.param(cell, id=name) for name, cell in CELLS.items()]) +def test_reasoning_effort_reaches_a_declaring_deployment_as_one_of_its_levels(gateway: Gateway, cell: Cell) -> None: + marker: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, cell.kind, wire, cell.model_info) + response: Final = gateway.request("POST", cell.route, _request_body(cell, model, marker)) + received: Final = wire.drain() + _assert_answered(cell, response, marker) + if cell.expected is None: + assert received == (), [request.body[:300] for request in received] + return + assert [(request.method, request.target) for request in received] == [("POST", cell.target)] + assert marker_of(received[0]) == marker + assert body_of(received[0]) == cell.expected(marker) + + +def _bedrock_native_schema(body: Mapping[str, JsonValue], *, invoke: bool) -> JsonValue: + if invoke: + return object_value(object_value(body.get("output_config") or {}).get("format") or {}).get("schema") + text_format: Final = object_value(body.get("outputConfig") or {}).get("textFormat") + if text_format is None: + return None + json_schema: Final = object_value(object_value(object_value(text_format)["structure"])["jsonSchema"]) + return JSON_OBJECT.validate_json(string_value(json_schema["schema"])) + + +@pytest.mark.parametrize( + ("kind", "target", "fields"), + [ + pytest.param("bedrock_converse", _CONVERSE, {"thinking": _BUDGET}, id="converse-budget"), + pytest.param( + "bedrock_invoke", + _INVOKE, + {"thinking": _BUDGET, "output_config": {"effort": "medium"}}, + id="invoke-output-config-beside-budget", + ), + ], +) +def test_a_deployment_opted_into_bedrock_native_output_keeps_it_beside_its_declared_level( + gateway: Gateway, kind: str, target: str, fields: Mapping[str, JsonValue] +) -> None: + marker: Final = new_marker() + response_format: Final[dict[str, JsonValue]] = { + "type": "json_schema", + "json_schema": {"name": "answer", "schema": _NATIVE_OUTPUT_SCHEMA}, + } + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, kind, wire, _CLAUDE_INFO_WITH_NATIVE_OUTPUT) + response: Final = gateway.request( + "POST", CHAT, chat_request(model, marker, response_format=response_format, **fields) + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text[:2000] + assert answer(marker) in response.text, response.text[:2000] + assert [(request.method, request.target) for request in received] == [("POST", target)] + body: Final = body_of(received[0]) + assert _bedrock_native_schema(body, invoke=target == _INVOKE) == _NATIVE_OUTPUT_SCHEMA, body + assert {"toolConfig", "tools"}.isdisjoint(body), body + assert sent_effort(received[0]) == "high", body + + +def _single(received: tuple[Request, ...], marker: str) -> Request: + matching: Final = tuple(request for request in received if marker_of(request) == marker) + assert len(matching) == 1, [request.body[:300] for request in received] + return matching[0] + + +def _proxy_url(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +async def _streamed_openai_text(base_url: str, key: str, model: str, marker: str) -> str: + client: Final = openai.AsyncOpenAI(base_url=base_url, api_key=key, max_retries=0) + stream: Final = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt(marker)}], + reasoning_effort="minimal", + stream=True, + ) + parts: Final = [chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices] + return "".join(parts) + + +def test_openai_sdk_sync_and_async_clients_reach_azure_ai_with_the_declared_level(gateway: Gateway) -> None: + sync_marker: Final = new_marker() + stream_marker: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "azure_ai", wire, _KIMI_INFO) + base_url: Final = f"{_proxy_url(gateway)}/v1" + completion: Final = openai.OpenAI( + base_url=base_url, api_key=gateway.key, max_retries=0 + ).chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt(sync_marker)}], + reasoning_effort="medium", + ) + streamed_text: Final = asyncio.run(_streamed_openai_text(base_url, gateway.key, model, stream_marker)) + received: Final = wire.drain() + assert completion.id == f"chatcmpl-{sync_marker}" + assert completion.choices[0].message.content == answer(sync_marker) + assert streamed_text == answer(stream_marker) + assert len(received) == 2 + assert sent_effort(_single(received, sync_marker)) == "high" + assert sent_effort(_single(received, stream_marker)) == "low" + + +def test_anthropic_sdk_sync_and_streaming_clients_reach_claude_with_the_declared_level(gateway: Gateway) -> None: + sync_marker: Final = new_marker() + stream_marker: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "anthropic", wire, _CLAUDE_INFO) + client: Final = anthropic.Anthropic(base_url=_proxy_url(gateway), api_key=gateway.key, max_retries=0) + message: Final = client.messages.create( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": prompt(sync_marker)}], + extra_body={"thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}, + ) + with client.messages.stream( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": prompt(stream_marker)}], + thinking={"type": "enabled", "budget_tokens": 2048}, + ) as stream: + streamed_text: Final = "".join(stream.text_stream) + received: Final = wire.drain() + assert message.id == f"msg_{sync_marker}" + assert [block.text for block in message.content if block.type == "text"] == [answer(sync_marker)] + assert streamed_text == answer(stream_marker) + assert len(received) == 2 + assert sent_effort(_single(received, sync_marker)) == "high" + assert sent_effort(_single(received, stream_marker)) == "high" + + +def test_a_duplicated_effort_key_rounds_the_value_the_proxy_parsed(gateway: Gateway) -> None: + marker: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "hosted_vllm", wire, _KIMI_INFO) + raw: Final = ( + f'{{"model": "{model}", "messages": [{{"role": "user", "content": "{prompt(marker)}"}}],' + ' "reasoning_effort": "max", "reasoning_effort": "minimal"}' + ) + response: Final = gateway.client.post( + CHAT, + content=raw.encode(), + headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}, + ) + received: Final = wire.drain() + assert response.status_code == 200, response.text + assert object_value(response.json())["id"] == f"chatcmpl-{marker}" + assert sent_effort(_single(received, marker)) == "low" + + +def _raw_messages_post(gateway: Gateway, model: str, marker: str, budgets: str) -> httpx.Response: + raw: Final = ( + f'{{"model": "{model}", "max_tokens": 4096, "messages": [{{"role": "user", "content": "{prompt(marker)}"}}],' + f' "thinking": {{"type": "enabled", {budgets}}}}}' + ) + return gateway.client.post( + MESSAGES, + content=raw.encode(), + headers={"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}, + ) + + +def test_a_duplicated_budget_key_on_messages_is_judged_by_the_value_the_proxy_parsed(gateway: Gateway) -> None: + malformed_last: Final = new_marker() + numeric_last: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "azure_ai", wire, _KIMI_INFO) + rejected: Final = _raw_messages_post( + gateway, model, malformed_last, '"budget_tokens": 9000, "budget_tokens": "2048"' + ) + answered: Final = _raw_messages_post( + gateway, model, numeric_last, '"budget_tokens": "2048", "budget_tokens": 9000' + ) + received: Final = wire.drain() + assert rejected.status_code == 400, rejected.text + assert "thinking" in rejected.text + assert answered.status_code == 200, answered.text + assert answer(numeric_last) in answered.text + assert [marker_of(request) for request in received] == [numeric_last] + assert sent_effort(received[0]) == "high" + + +def test_anthropic_sdk_gets_a_400_for_a_malformed_budget_to_a_non_claude_deployment_and_an_answer_under_drop_params( + gateway: Gateway, +) -> None: + rejected_marker: Final = new_marker() + sync_marker: Final = new_marker() + stream_marker: Final = new_marker() + malformed: Final = {"thinking": {"type": "enabled", "budget_tokens": "2048"}} + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "azure_ai", wire, _KIMI_INFO) + client: Final = anthropic.Anthropic(base_url=_proxy_url(gateway), api_key=gateway.key, max_retries=0) + with pytest.raises(anthropic.BadRequestError) as rejected: + client.messages.create( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": prompt(rejected_marker)}], + extra_body=malformed, + ) + message: Final = client.messages.create( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": prompt(sync_marker)}], + extra_body={**malformed, "drop_params": True}, + ) + with client.messages.stream( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": prompt(stream_marker)}], + extra_body={**malformed, "drop_params": True}, + ) as stream: + streamed_text: Final = "".join(stream.text_stream) + received: Final = wire.drain() + assert rejected.value.status_code == 400 + assert "thinking" in str(rejected.value) + assert [block.text for block in message.content if block.type == "text"] == [answer(sync_marker)] + assert streamed_text == answer(stream_marker) + assert sorted(marker_of(request) for request in received) == sorted((sync_marker, stream_marker)) + assert [sorted(set(body_of(request)) & {"thinking", "reasoning_effort"}) for request in received] == [[], []] + + +def _served_by(response: httpx.Response) -> str: + assert response.status_code == 200, response.text + return response.headers["x-litellm-model-id"] + + +def test_each_deployment_of_a_model_group_gets_its_own_declared_level(gateway: Gateway) -> None: + markers: Final = tuple(new_marker() for _ in range(10)) + with ( + wire_server(provider_reply) as low_wire, + wire_server(provider_reply) as high_wire, + gateway.scenario() as scenario, + ): + group: Final = deployment(scenario, "hosted_vllm", low_wire, {"reasoning_effort_levels": ["minimal", "low"]}) + sibling: Final = scenario.gateway.post( + "/model/new", + { + "model_name": group, + "litellm_params": {"model": f"hosted_vllm/{KIMI}", "api_base": f"{high_wire.url}/v1", "api_key": "k"}, + "model_info": {"reasoning_effort_levels": ["high", "max"]}, + }, + ) + sibling_id: Final = string_value(object_value(sibling["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, sibling_id) + served: Final = tuple( + _served_by(gateway.request("POST", CHAT, chat_request(group, marker, reasoning_effort="medium"))) + for marker in markers + ) + low_received: Final = low_wire.drain() + high_received: Final = high_wire.drain() + expected_high: Final = frozenset(marker for marker, model_id in zip(markers, served) if model_id == sibling_id) + assert {marker_of(request) for request in high_received} == expected_high + assert {marker_of(request) for request in low_received} == frozenset(markers) - expected_high + assert len(low_received) + len(high_received) == len(markers) + assert {sent_effort(request) for request in low_received} <= {"low"} + assert {sent_effort(request) for request in high_received} <= {"high"} + + +def test_a_fallback_deployment_rounds_the_callers_level_onto_its_own_declaration(gateway: Gateway) -> None: + marker: Final = new_marker() + with ( + wire_server(lambda request: openai_error(500)) as failing, + wire_server(provider_reply) as serving, + gateway.scenario() as scenario, + ): + primary: Final = deployment(scenario, "hosted_vllm", failing, {"reasoning_effort_levels": ["minimal", "high"]}) + fallback: Final = deployment(scenario, "hosted_vllm", serving, {"reasoning_effort_levels": ["low", "max"]}) + response: Final = gateway.request( + "POST", + CHAT, + chat_request(primary, marker, reasoning_effort="medium", fallbacks=[fallback], num_retries=0), + ) + failed: Final = failing.drain() + served: Final = serving.drain() + assert response.status_code == 200, response.text + assert object_value(response.json())["id"] == f"chatcmpl-{marker}" + assert failed and {sent_effort(request) for request in failed} == {"high"} + assert [sent_effort(request) for request in served] == ["max"] + + +def _deployment_id(gateway: Gateway, model: str) -> str: + entries: Final = tuple(object_value(entry) for entry in list_value(gateway.get("/model/info", {})["data"])) + matching: Final = tuple(entry for entry in entries if entry["model_name"] == model) + assert len(matching) == 1, [entry["model_name"] for entry in entries] + return string_value(object_value(matching[0]["model_info"])["id"]) + + +def _declared_levels(gateway: Gateway, identity: str) -> JsonValue: + entries: Final = list_value(gateway.get("/model/info", {"litellm_model_id": identity})["data"]) + return object_value(object_value(entries[0])["model_info"]).get("reasoning_effort_levels") + + +def test_a_declaration_updated_under_traffic_applies_to_the_next_request(gateway: Gateway) -> None: + from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + + first: Final = new_marker() + second: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "hosted_vllm", wire, {"reasoning_effort_levels": ["low", "high"]}) + identity: Final = _deployment_id(gateway, model) + before: Final = gateway.request("POST", CHAT, chat_request(model, first, reasoning_effort="medium")) + updated: Final = gateway.request( + "PATCH", f"/model/{identity}/update", {"model_info": {"reasoning_effort_levels": ["minimal", "max"]}} + ) + eventually( + lambda: _declared_levels(gateway, identity), + lambda levels: levels == ["minimal", "max"], + seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ) + after: Final = gateway.request("POST", CHAT, chat_request(model, second, reasoning_effort="medium")) + received: Final = wire.drain() + assert before.status_code == 200, before.text + assert updated.status_code == 200, updated.text + assert after.status_code == 200, after.text + assert sent_effort(_single(received, first)) == "high" + assert sent_effort(_single(received, second)) == "max" + + +def test_a_repeated_identical_request_is_served_from_cache_after_one_rounded_upstream_call(gateway: Gateway) -> None: + marker: Final = new_marker() + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "hosted_vllm", wire, _KIMI_INFO) + body: Final = chat_request(model, marker, reasoning_effort="medium") + first: Final = gateway.request("POST", CHAT, body) + second: Final = gateway.request("POST", CHAT, body) + received: Final = wire.drain() + assert first.status_code == 200 and second.status_code == 200, (first.text, second.text) + assert object_value(first.json())["id"] == object_value(second.json())["id"] == f"chatcmpl-{marker}" + assert [sent_effort(request) for request in received] == ["high"] + + +def test_model_group_info_advertises_the_declared_levels(gateway: Gateway) -> None: + with wire_server(provider_reply) as wire, gateway.scenario() as scenario: + model: Final = deployment(scenario, "hosted_vllm", wire, {**_KIMI_INFO, "supports_reasoning": True}) + info: Final = gateway.get("/model_group/info", {"model_group": model}) + entries: Final = list_value(info["data"]) + assert len(entries) == 1, info + assert object_value(entries[0])["supported_reasoning_efforts"] == ["none", "low", "high", "max"] + + +def _create_swapping_prompt(gateway: Gateway, prompt_id: str, model: str, marker: str) -> None: + created: Final = gateway.request( + "POST", + "/prompts", + { + "prompt_id": prompt_id, + "litellm_params": { + "prompt_id": prompt_id, + "prompt_integration": "dotprompt", + "dotprompt_content": f"---\nmodel: {model}\n---\nUser: {prompt(marker)}", + }, + "prompt_info": {"prompt_type": "db"}, + }, + ) + assert created.status_code == 200, created.text + + +def test_a_prompt_that_swaps_the_model_is_rounded_onto_the_serving_deployments_levels(gateway: Gateway) -> None: + marker: Final = new_marker() + prompt_id: Final = f"prompt-{new_marker()}" + with ( + wire_server(provider_reply) as routed_wire, + wire_server(provider_reply) as swapped_wire, + gateway.scenario() as scenario, + ): + routed: Final = deployment( + scenario, "hosted_vllm", routed_wire, {"reasoning_effort_levels": ["minimal", "max"]} + ) + swapped: Final = deployment(scenario, "hosted_vllm", swapped_wire, {"reasoning_effort_levels": ["low", "high"]}) + _create_swapping_prompt(gateway, prompt_id, swapped, marker) + scenario.cleanups.callback(gateway.request, "DELETE", f"/prompts/{prompt_id}") + response: Final = gateway.request( + "POST", CHAT, chat_request(routed, marker, reasoning_effort="medium", prompt_id=prompt_id) + ) + routed_received: Final = routed_wire.drain() + swapped_received: Final = swapped_wire.drain() + assert response.status_code == 200, response.text + assert object_value(response.json())["id"] == f"chatcmpl-{marker}" + efforts: Final = [("routed", sent_effort(request)) for request in routed_received] + [ + ("swapped", sent_effort(request)) for request in swapped_received + ] + assert efforts in ([("routed", "max")], [("swapped", "high")]), efforts diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 0e918cd49c5..658aa009683 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -2850,6 +2850,37 @@ def test_translate_completion_input_params_keeps_provider_native_tools(): assert translated["tools"] == [{"googleMaps": {}}] +def _gemini_request_with_budget(budget: object, **extra: object) -> dict[str, object]: + return { + "model": "gemini/gemini-2.5-flash", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Say ok"}], + "thinking": {"type": "enabled", "budget_tokens": budget}, + **extra, + } + + +@pytest.mark.parametrize("budget", ["2048", True]) +def test_translate_completion_input_params_rejects_a_non_numeric_budget_without_drop_params( + budget: object, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match="budget_tokens must be a number"): + AnthropicAdapter().translate_completion_input_params(_gemini_request_with_budget(budget)) + + +@pytest.mark.parametrize("budget", ["2048", True]) +def test_translate_completion_input_params_drops_a_non_numeric_budget_under_drop_params( + budget: object, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "drop_params", False) + translated: Final = AnthropicAdapter().translate_completion_input_params( + _gemini_request_with_budget(budget, drop_params=True) + ) + assert translated is not None + assert not {"reasoning_effort", "thinking"} & translated.keys() + + def test_translate_openai_content_to_anthropic_reasoning_content_without_thinking_blocks(): """ Test that reasoning_content is converted to thinking block when thinking_blocks is not present. diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py index 7a4b5badb3a..5a82b9af468 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -6,16 +6,21 @@ regression they guard is the one a caller sees: a tier the proxy advertises has leaves the adapter, in the shape the target expects. """ +import functools +import json from collections.abc import Mapping from typing import Final +import httpx import pytest +import respx import litellm from litellm.llms.anthropic.pass_through.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig +from litellm.router import Router from litellm.utils import reasoning_effort_with_summary_alias MESSAGES = [{"role": "user", "content": "hello"}] @@ -359,3 +364,213 @@ class TestTheNormalizedTierIsTheTierSent: def test_the_openai_hosted_twin_still_sends_max(self, local_model_cost_map): assert _reasoning_effort_sent("gpt-6-astra", "openai", "max") == "max" + + +_CHAT_COMPLETION: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "my-kimi", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_RESPONSE: Final = { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, +} +_KIMI_LEVELS: Final = {"reasoning_effort_levels": ["low", "high", "max", "none"]} +_SELF_HOSTED: Final = { + "model": "hosted_vllm/my-kimi", + "api_base": "https://vllm.example.test/v1", + "api_key": "fake-key", +} +_VLLM_CHAT_URL: Final = r"https://vllm\.example\.test/v1/chat/completions" +_MALFORMED_BUDGET_TARGETS: Final = [ + pytest.param(_SELF_HOSTED, _KIMI_LEVELS, _VLLM_CHAT_URL, _CHAT_COMPLETION, id="declaring-hosted_vllm"), + pytest.param(_SELF_HOSTED, {}, _VLLM_CHAT_URL, _CHAT_COMPLETION, id="hosted_vllm"), + pytest.param( + {"model": "fireworks_ai/accounts/fireworks/models/kimi-k3", "api_key": "fake-key"}, + {}, + r"https://api\.fireworks\.ai/inference/v1/chat/completions", + _CHAT_COMPLETION, + id="fireworks_ai", + ), + pytest.param( + {"model": "openai/gpt-5.5-pro", "api_key": "fake-key"}, + {}, + r"https://api\.openai\.com/v1/responses", + _RESPONSE, + id="openai-responses-bridge", + ), +] + + +async def _body_sent_upstream( + respx_mock: respx.MockRouter, + litellm_params: Mapping[str, object], + model_info: Mapping[str, object], + upstream_url: str, + upstream_body: Mapping[str, object], + **messages_params: object, +) -> Mapping[str, object]: + route: Final = respx_mock.post(url__regex=upstream_url).mock(return_value=httpx.Response(200, json=upstream_body)) + router: Final = Router( + model_list=[{"model_name": "group", "litellm_params": {**litellm_params}, "model_info": {**model_info}}] + ) + await router.aanthropic_messages(model="group", messages=MESSAGES, max_tokens=4096, **messages_params) + return json.loads(route.calls.last.request.read()) + + +class TestADeclaredDeploymentRoundsTheMessagesTierOntoItsLevels: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "messages_params, expected", + [ + ({"thinking": {"type": "enabled", "budget_tokens": 2048}}, "high"), + ({"thinking": {"type": "adaptive"}, "output_config": {"effort": "xhigh"}}, "max"), + ], + ) + async def test_a_self_hosted_deployment_receives_its_declared_level( + self, respx_mock: respx.MockRouter, messages_params: Mapping[str, object], expected: str + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + {"model": "hosted_vllm/my-kimi", "api_base": "https://vllm.example.test/v1", "api_key": "fake-key"}, + _KIMI_LEVELS, + r"https://vllm\.example\.test/v1/chat/completions", + _CHAT_COMPLETION, + **messages_params, + ) + + assert sent["reasoning_effort"] == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize("model_info", [_KIMI_LEVELS, {}]) + async def test_a_claude_named_self_hosted_deployment_maps_a_budget_like_any_other_model_name( + self, respx_mock: respx.MockRouter, model_info: Mapping[str, object] + ) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": 9000} + + claude_named: Final = await _body_sent_upstream( + respx_mock, + {**_SELF_HOSTED, "model": "hosted_vllm/claude-opus-5-5"}, + model_info, + _VLLM_CHAT_URL, + _CHAT_COMPLETION, + thinking=thinking, + ) + kimi_named: Final = await _body_sent_upstream( + respx_mock, _SELF_HOSTED, model_info, _VLLM_CHAT_URL, _CHAT_COMPLETION, thinking=thinking + ) + + assert claude_named["reasoning_effort"] == kimi_named["reasoning_effort"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("budget", ["2048", True]) + @pytest.mark.parametrize("litellm_params, model_info, upstream_url, upstream_body", _MALFORMED_BUDGET_TARGETS) + async def test_a_non_numeric_budget_is_rejected_with_a_400( + self, + local_model_cost_map: None, + respx_mock: respx.MockRouter, + litellm_params: Mapping[str, object], + model_info: Mapping[str, object], + upstream_url: str, + upstream_body: Mapping[str, object], + budget: object, + ) -> None: + with pytest.raises(litellm.BadRequestError, match=r"thinking\.budget_tokens must be a number") as rejected: + await _body_sent_upstream( + respx_mock, + litellm_params, + model_info, + upstream_url, + upstream_body, + thinking={"type": "enabled", "budget_tokens": budget}, + ) + + assert rejected.value.status_code == 400 + + @pytest.mark.asyncio + @pytest.mark.parametrize("budget", ["2048", True]) + @pytest.mark.parametrize("litellm_params, model_info, upstream_url, upstream_body", _MALFORMED_BUDGET_TARGETS) + async def test_drop_params_drops_a_non_numeric_budget( + self, + local_model_cost_map: None, + respx_mock: respx.MockRouter, + litellm_params: Mapping[str, object], + model_info: Mapping[str, object], + upstream_url: str, + upstream_body: Mapping[str, object], + budget: object, + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + {**litellm_params, "drop_params": True}, + model_info, + upstream_url, + upstream_body, + thinking={"type": "enabled", "budget_tokens": budget}, + ) + + assert not {"reasoning_effort", "reasoning", "thinking"} & sent.keys() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, api_base, upstream_url, upstream_body, effort_path", + [ + ("openai/gpt-5.5-pro", None, r"https://api\.openai\.com/v1/responses", _RESPONSE, ("reasoning", "effort")), + ( + "hosted_vllm/my-kimi", + "https://vllm.example.test/v1", + r"https://vllm\.example\.test/v1/chat/completions", + _CHAT_COMPLETION, + ("reasoning_effort",), + ), + ], + ) + async def test_the_deployment_declaration_beats_the_flag_based_chain( + self, + local_model_cost_map: None, + respx_mock: respx.MockRouter, + model: str, + api_base: str | None, + upstream_url: str, + upstream_body: Mapping[str, object], + effort_path: tuple[str, ...], + ) -> None: + """The map says gpt-5.5-pro refuses minimal and knows nothing of my-kimi, so the chain alone + would send medium and low.""" + route: Final = respx_mock.post(url__regex=upstream_url).mock( + return_value=httpx.Response(200, json=upstream_body) + ) + + await litellm.anthropic.messages.acreate( + model=model, + api_base=api_base, + api_key="fake-key", + messages=MESSAGES, + max_tokens=4096, + thinking={"type": "adaptive"}, + output_config={"effort": "minimal"}, + model_info={"reasoning_effort_levels": ["minimal", "medium", "high"]}, + ) + + sent: Final = json.loads(route.calls.last.request.read()) + assert functools.reduce(lambda body, key: body[key], effort_path, sent) == "minimal" diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py index 409e64635f7..19e03d8ce46 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py @@ -11,6 +11,7 @@ from zlib import crc32 import httpx import pytest +import respx import litellm from litellm.integrations.custom_logger import CustomLogger @@ -619,6 +620,166 @@ async def test_tool_search_forwards_deferred_tools_and_beta_header( ] +async def _native_output_config_sent( + respx_mock: respx.MockRouter, effort: str, model_info: Mapping[str, object] | None, **params: object +) -> object: + route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body("claude-sonnet-4-6")) + ) + deployment: Final = { + "model_name": _ALIAS, + "litellm_params": {"model": "anthropic/claude-sonnet-4-6", "api_key": "fake-key"}, + **({"model_info": model_info} if model_info is not None else {}), + } + await Router(model_list=[deployment]).aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], + model=_ALIAS, + max_tokens=100, + thinking={"type": "adaptive"}, + output_config={"effort": effort}, + **params, + ) + return json.loads(route.calls.last.request.read()).get("output_config") + + +@pytest.mark.asyncio +async def test_native_messages_round_output_config_effort_onto_the_deployment_declaration( + respx_mock: respx.MockRouter, +) -> None: + declared: Final = {"reasoning_effort_levels": ["low", "high"]} + + assert await _native_output_config_sent(respx_mock, "medium", declared) == {"effort": "high"} + assert await _native_output_config_sent(respx_mock, "medium", None) == {"effort": "medium"} + + +@pytest.mark.asyncio +async def test_native_messages_drop_an_undeclared_effort_under_drop_params(respx_mock: respx.MockRouter) -> None: + declared: Final = {"reasoning_effort_levels": ["low", "high"]} + + assert await _native_output_config_sent(respx_mock, "turbo", declared, drop_params=True) is None + assert await _native_output_config_sent(respx_mock, "turbo", declared) == {"effort": "turbo"} + + +_DECLARING_LOW_AND_HIGH: Final = {"reasoning_effort_levels": ["low", "high"]} + + +async def _native_opus_body_sent( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, **params: object +) -> Mapping[str, object]: + route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body("claude-opus-5-5")) + ) + deployment: Final = { + "model_name": _ALIAS, + "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "fake-key"}, + **({"model_info": model_info} if model_info is not None else {}), + } + await Router(model_list=[deployment]).aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], model=_ALIAS, max_tokens=4096, **params + ) + return json.loads(route.calls.last.request.read()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_info, expected_effort", [(_DECLARING_LOW_AND_HIGH, "high"), (None, "medium")]) +async def test_native_messages_round_a_thinking_budget_on_an_adaptive_claude_deployment( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, expected_effort: str +) -> None: + sent: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"} + ) + + assert sent["output_config"] == {"effort": expected_effort} + assert sent["thinking"] == {"type": "adaptive", "display": "summarized"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dropped_effort", ["turbo", "none"]) +async def test_native_messages_drop_params_leave_the_thinking_budget_to_set_the_effort_beside_a_dropped_level( + respx_mock: respx.MockRouter, dropped_effort: str +) -> None: + budget: Final = {"type": "enabled", "budget_tokens": 2048} + + dropped: Final = await _native_opus_body_sent( + respx_mock, + _DECLARING_LOW_AND_HIGH, + thinking=budget, + output_config={"effort": dropped_effort, "format": {"type": "json_schema", "schema": {"type": "object"}}}, + drop_params=True, + ) + budget_alone: Final = await _native_opus_body_sent( + respx_mock, + _DECLARING_LOW_AND_HIGH, + thinking=budget, + output_config={"format": {"type": "json_schema", "schema": {"type": "object"}}}, + drop_params=True, + ) + + assert budget_alone["output_config"] == { + "effort": "high", + "format": {"type": "json_schema", "schema": {"type": "object"}}, + } + assert dropped == budget_alone + + +@pytest.mark.asyncio +@pytest.mark.parametrize("declared", [["none"], []]) +async def test_native_messages_drop_params_drop_a_budget_effort_the_deployment_does_not_declare( + respx_mock: respx.MockRouter, declared: list[str] +) -> None: + model_info: Final = {"reasoning_effort_levels": declared} + + sent: Final = await _native_opus_body_sent( + respx_mock, + model_info, + thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, + drop_params=True, + ) + + assert "output_config" not in sent + assert sent["thinking"] == {"type": "adaptive", "display": "summarized"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", [False, True]) +async def test_native_messages_send_no_effort_for_a_default_effort_beside_a_thinking_budget( + respx_mock: respx.MockRouter, drop_params: bool +) -> None: + sent: Final = await _native_opus_body_sent( + respx_mock, + _DECLARING_LOW_AND_HIGH, + thinking={"type": "enabled", "budget_tokens": 2048}, + output_config={"effort": "default"}, + drop_params=drop_params, + ) + + assert "output_config" not in sent + assert sent["thinking"] == {"type": "adaptive"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_info", [None, _DECLARING_LOW_AND_HIGH]) +@pytest.mark.parametrize("drop_globally", [False, True]) +async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model_info: Mapping[str, object] | None, + drop_globally: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + request_drop: Final = {} if drop_globally else {"drop_params": True} + + dropped: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + adaptive_alone: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop + ) + + assert "output_config" not in adaptive_alone + assert dropped == adaptive_alone + + _MIXED_GROUP: Final = "gpt-5.6-mixed" _MIXED_AZURE_API_BASE: Final = "https://mixed-group.openai.azure.com" _MIXED_AZURE_CHAT_URL: Final = r".*mixed-group\.openai\.azure\.com/openai/deployments/.*/chat/completions.*" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index f1d8d46216b..093b4d6cce8 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -4491,3 +4491,34 @@ def test_anthropic_error_frame_exception_classes_an_overloaded_frame_as_internal error: Final = anthropic_error_frame_exception("overloaded_error", "Overloaded", 503, "claude-sonnet-4-5") assert type(error) is litellm.InternalServerError + + +_NON_NUMERIC_BUDGETS: Final = (True, False, None, "", "lots", [4096]) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("budget_tokens", _NON_NUMERIC_BUDGETS) +def test_shared_legacy_thinking_translation_rejects_a_non_numeric_budget(budget_tokens: object) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + with pytest.raises(litellm.UnsupportedParamsError, match=r"thinking\.budget_tokens must be a number"): + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", + optional_params={"thinking": {"type": "enabled", "budget_tokens": budget_tokens}}, + custom_llm_provider="azure_ai", + drop_params=False, + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("budget_tokens", _NON_NUMERIC_BUDGETS) +def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params(budget_tokens: object) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + optional_params: Final = {"thinking": {"type": "enabled", "budget_tokens": budget_tokens}} + + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True + ) + + assert optional_params == {"thinking": {"type": "adaptive"}} diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 4a2fc0627ab..ea3eab7e002 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -6,6 +6,7 @@ in expected_responses_api_request/. import copy import json +import logging from pathlib import Path from importlib import import_module from typing import Final @@ -16,6 +17,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, @@ -279,6 +281,103 @@ async def test_acompletion_with_tools_forwards_non_enum_reasoning_effort_over_th assert response.id == "resp_bridge_int" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, api_base", + [ + ("hosted_vllm/my-kimi", "https://vllm.example.test/v1"), + ("azure_ai/my-kimi", "https://kimi.services.ai.azure.com/openai/v1"), + ], +) +async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, + model: str, + api_base: str, +): + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(url__regex=rf"{api_base}/responses.*").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_declared_effort", "my-kimi")) + ) + + response: Final = await litellm.aresponses( + model=model, + api_base=api_base, + api_key="fake-api-key", + input="hi", + reasoning={"effort": "minimal", "summary": "auto"}, + model_info={"id": "kimi-1", "reasoning_effort_levels": ["low", "high", "max", "none"]}, + litellm_metadata={"model_group": "kimi-k3"}, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["reasoning"] == {"effort": "low", "summary": "auto"} + assert response.output[0].content[0].text == "Done." + assert [ + ( + getattr(record, "reasoning_effort_deployment_id"), + getattr(record, "reasoning_effort_model_group"), + getattr(record, "reasoning_effort_sent"), + ) + for record in caplog.records + if hasattr(record, "reasoning_effort_sent") + ] == [("kimi-1", "kimi-k3", "low")] + + +def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]: + return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False) + + +def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]: + return next( + (key.removeprefix(f"{provider}/"), _effort_levels_disabled_by(entry)[0]) + for key, entry in litellm.model_cost.items() + if entry.get("litellm_provider") == provider and "gpt-5" in key and _effort_levels_disabled_by(entry) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider, api_base", + [("openai", "https://gpt5-gateway.example.test/v1"), ("azure", "https://gpt5.openai.azure.com")], +) +async def test_aresponses_sends_a_declared_level_the_model_map_disables( + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + provider: str, + api_base: str, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "drop_params", False) + litellm.in_memory_llm_clients_cache.flush_cache() + model, level = _gpt_5_row_disabling_a_level(provider) + upstream: Final = respx_mock.post(url__regex=rf"{api_base}/.*responses.*").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_declared_disabled", model)) + ) + request: Final = { + "model": f"{provider}/{model}", + "api_base": api_base, + "api_version": "2025-04-01-preview", + "api_key": "fake-api-key", + "input": "hi", + "reasoning": {"effort": level}, + } + + with pytest.raises(litellm.UnsupportedParamsError): + await litellm.aresponses(**request, model_info={"id": "undeclared"}) + await litellm.aresponses(**request, model_info={"id": "declared", "reasoning_effort_levels": [level, "high"]}) + + assert upstream.call_count == 1 + assert json.loads(upstream.calls[0].request.read())["reasoning"] == {"effort": level} + + @pytest.mark.asyncio async def test_aresponses_forwards_prompt_managed_reasoning_effort( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 2226b1f39df..d634e239ea9 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -1,11 +1,21 @@ +import json +import logging +from collections.abc import Mapping +from typing import Final + import pytest import litellm +from litellm._logging import JsonFormatter, verbose_logger from litellm.router_utils.reasoning_effort_capability import ( + REASONING_EFFORT_STRENGTH_ORDER, + RequestedReasoningEffort, deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, nearest_declared_reasoning_effort, + resolve_deployment_reasoning_effort, resolve_supported_reasoning_efforts, + thinking_budget_effort, ) @@ -462,6 +472,203 @@ class TestNearestDeclaredReasoningEffort: assert nearest_declared_reasoning_effort("turbo", ("none", "high")) == "turbo" assert nearest_declared_reasoning_effort("medium", ()) == "medium" + def test_rounding_down_takes_the_strongest_declared_level_at_most_as_strong(self) -> None: + assert nearest_declared_reasoning_effort("medium", ("low", "high", "max"), "down") == "low" + assert nearest_declared_reasoning_effort("xhigh", ("low", "high", "max"), "down") == "high" + assert nearest_declared_reasoning_effort("high", ("low", "high", "max"), "down") == "high" + + def test_rounding_down_below_the_floor_takes_the_weakest_declared_one(self) -> None: + assert nearest_declared_reasoning_effort("minimal", ("none", "low", "high"), "down") == "low" + + def test_rounding_down_leaves_none_and_unknown_levels_alone(self) -> None: + assert nearest_declared_reasoning_effort("none", ("low", "high"), "down") == "none" + assert nearest_declared_reasoning_effort("turbo", ("low", "high"), "down") == "turbo" + + +KIMI_DEPLOYMENT: Final = {"id": "deployment-1", "reasoning_effort_levels": ["low", "high", "max", "none"]} + + +def _sent( + requested: RequestedReasoningEffort, + deployment_model_info: Mapping[str, object] | None = KIMI_DEPLOYMENT, + *, + model: str = "my-kimi", + provider: str = "hosted_vllm", + drop_params: bool = False, + model_group: str = "kimi-k3", +) -> RequestedReasoningEffort: + return resolve_deployment_reasoning_effort( + requested, + model=model, + custom_llm_provider=provider, + deployment_model_info=deployment_model_info, + model_group=model_group, + drop_params=drop_params, + ).value + + +def _bare_map_model_declaring_fewer_levels() -> tuple[str, str]: + def declares_fewer(key: str, entry: Mapping[str, object]) -> bool: + levels: Final = entry.get("reasoning_effort_levels") + return "/" not in key and isinstance(levels, list) and not set(REASONING_EFFORT_STRENGTH_ORDER) <= set(levels) + + model, entry = next((key, entry) for key, entry in litellm.model_cost.items() if declares_fewer(key, entry)) + return str(entry["litellm_provider"]), model + + +class TestResolveDeploymentReasoningEffort: + @pytest.mark.parametrize( + "requested, expected", + [ + ("minimal", "low"), + ("low", "low"), + ("medium", "high"), + ("xhigh", "max"), + ("max", "max"), + ("none", "none"), + ("turbo", "turbo"), + ("default", None), + ({"effort": "medium", "summary": "auto"}, {"effort": "high", "summary": "auto"}), + ({"effort": "default", "summary": "auto"}, {"summary": "auto"}), + ({"effort": "default"}, None), + ], + ) + def test_a_declaring_deployment_receives_a_level_it_declared( + self, requested: RequestedReasoningEffort, expected: RequestedReasoningEffort + ) -> None: + assert _sent(requested) == expected + + def test_a_deployment_that_declares_nothing_gets_the_request_untouched(self, local_model_cost_map: None) -> None: + requested: Final = {"effort": "minimal", "summary": "auto"} + + assert _sent(requested, {"id": "deployment-1"}) == requested + assert _sent("default", None) == "default" + + def test_the_deployment_rounding_wins_over_the_global(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "reasoning_effort_rounding", "up") + assert _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": "down"}) == "low" + + monkeypatch.setattr(litellm, "reasoning_effort_rounding", "down") + assert _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": "up"}) == "high" + assert _sent("medium") == "low" + + @pytest.mark.parametrize("spelling", ["DOWN", " Down "]) + def test_the_rounding_setting_ignores_case_and_surrounding_spaces( + self, monkeypatch: pytest.MonkeyPatch, spelling: str + ) -> None: + monkeypatch.setattr(litellm, "reasoning_effort_rounding", "up") + assert _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": spelling}) == "low" + + @pytest.mark.parametrize("invalid", ["sideways", 1, None]) + def test_an_invalid_rounding_falls_back_to_the_next_setting( + self, monkeypatch: pytest.MonkeyPatch, invalid: str | int | None + ) -> None: + monkeypatch.setattr(litellm, "reasoning_effort_rounding", "down") + assert _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": invalid}) == "low" + + monkeypatch.setattr(litellm, "reasoning_effort_rounding", invalid) + assert _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": invalid}) == "high" + + def test_a_misspelled_rounding_is_reported_rather_than_silently_rounding_up( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + monkeypatch.setattr(litellm, "reasoning_effort_rounding", None) + + _sent("medium", {**KIMI_DEPLOYMENT, "reasoning_effort_rounding": "donw\nforged"}) + + assert [record.getMessage() for record in caplog.records if record.levelno == logging.WARNING] == [ + "Ignoring reasoning_effort_rounding=donwforged, expected 'up' or 'down'" + ] + + def test_only_the_deployment_declaration_rounds_never_the_model_map(self, local_model_cost_map: None) -> None: + assert _sent("xhigh", None, model="kimi-k3", provider="fireworks_ai") == "xhigh" + assert _sent("xhigh", {"id": "kimi-1"}, model="kimi-k3", provider="fireworks_ai") == "xhigh" + assert _sent("xhigh", {"reasoning_effort_levels": ["medium"]}, model="kimi-k3", provider="fireworks_ai") == ( + "medium" + ) + + def test_a_model_info_echoing_a_map_key_rounds_nothing_the_deployment_did_not_declare( + self, local_model_cost_map: None + ) -> None: + provider, model = _bare_map_model_declaring_fewer_levels() + echoed: Final = {"key": f"{provider}/{model}", "litellm_provider": provider} + + sent: Final = [_sent(level, echoed, model=model, provider=provider) for level in REASONING_EFFORT_STRENGTH_ORDER] + + assert sent == list(REASONING_EFFORT_STRENGTH_ORDER) + + @pytest.mark.parametrize( + "requested, expected", + [ + ("turbo", None), + ({"effort": "turbo", "summary": "auto"}, {"summary": "auto"}), + ("none", "none"), + ("medium", "high"), + ], + ) + def test_drop_params_drops_a_level_the_deployment_does_not_declare( + self, requested: RequestedReasoningEffort, expected: RequestedReasoningEffort + ) -> None: + assert _sent(requested, drop_params=True) == expected + + def test_drop_params_drops_none_only_where_the_deployment_leaves_it_out(self) -> None: + assert _sent("none", {"reasoning_effort_levels": ["low", "high"]}, drop_params=True) is None + assert _sent("none", {"reasoning_effort_levels": ["low", "high"]}) == "none" + + def test_a_dropped_unrecognized_level_is_not_written_to_the_log(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + + _sent("turbo\nforged line", drop_params=True) + + assert [record.getMessage() for record in caplog.records] == [ + "reasoning_effort unrecognized sent as None (rounding up) to deployment deployment-1, " + "model group kimi-k3, model hosted_vllm/my-kimi" + ] + + def test_line_breaks_in_logged_names_cannot_forge_a_log_line(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + + _sent( + "medium", + {"id": "deployment-1\r\nforged id", "reasoning_effort_levels": ["low", "high"]}, + model="my-kimi\nforged model", + model_group="kimi-k3\nforged group", + ) + + assert [record.getMessage() for record in caplog.records] == [ + "reasoning_effort medium sent as high (rounding up) to deployment deployment-1forged id, " + "model group kimi-k3forged group, model hosted_vllm/my-kimiforged model" + ] + + def test_a_remap_logs_one_record_naming_the_deployment_and_both_levels( + self, caplog: pytest.LogCaptureFixture + ) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + + _sent("high") + _sent("medium") + + assert len(caplog.records) == 1 + logged: Final = json.loads(JsonFormatter().format(caplog.records[0])) + assert logged["level"] == "INFO" + assert {key: value for key, value in logged.items() if key.startswith("reasoning_effort_")} == { + "reasoning_effort_deployment_id": "deployment-1", + "reasoning_effort_model_group": "kimi-k3", + "reasoning_effort_deployment_model": "hosted_vllm/my-kimi", + "reasoning_effort_requested": "medium", + "reasoning_effort_sent": "high", + "reasoning_effort_rounding": "up", + } + class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: @pytest.mark.parametrize( @@ -488,3 +695,23 @@ class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: assert resolve_supported_reasoning_efforts( azure_info, deployment_is_mapped=True ) == resolve_supported_reasoning_efforts(openai_info, deployment_is_mapped=True) + + +class TestThinkingBudgetEffort: + @pytest.mark.parametrize( + "model, custom_llm_provider, expected", + [ + ("my-kimi", "hosted_vllm", "medium"), + ("FW-Kimi-K3", "azure_ai", "medium"), + ("claude-opus-5-5", "anthropic", "medium"), + ("claude-sonnet-4-6", "anthropic", None), + ], + ) + def test_only_a_provider_that_would_send_an_effort_gets_one( + self, model: str, custom_llm_provider: str, expected: str | None + ) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": 2048} + + assert thinking_budget_effort(thinking, model=model, custom_llm_provider=custom_llm_provider, drop_params=False) == ( + expected + ) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index aa15893e395..c3794ea7544 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -8,7 +8,7 @@ import logging import os import threading import urllib.parse -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime @@ -8798,6 +8798,676 @@ def test_model_alias_map_resolves_the_outbound_model( assert json.loads(route.calls[0].request.content)["model"] == "resolved-model" +_KIMI_DEPLOYMENT: Final = {"id": "kimi-vllm-1", "reasoning_effort_levels": ["low", "high", "max", "none"]} +_VLLM_API_BASE: Final = "https://vllm.example.test/v1" +_HELLO: Final = [{"role": "user", "content": "hello"}] + + +def _chat_body_sent( + respx_mock: respx.MockRouter, + model: str, + reasoning_effort: str | None, + model_info: Mapping[str, object] | None, + **params: object, +) -> Mapping[str, object]: + route: Final = respx_mock.post(f"{_VLLM_API_BASE}/chat/completions").mock( + return_value=_mocked_openai_chat_response(model) + ) + completion( + model=model, + api_base=_VLLM_API_BASE, + api_key="fake-key", + messages=_HELLO, + reasoning_effort=reasoning_effort, + model_info=model_info, + **params, + ) + return json.loads(route.calls.last.request.content) + + +def _reasoning_effort_remaps(caplog: pytest.LogCaptureFixture) -> list[logging.LogRecord]: + return [record for record in caplog.records if hasattr(record, "reasoning_effort_sent")] + + +@pytest.fixture +def remap_log(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture: + from litellm._logging import verbose_logger + + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + return caplog + + +@pytest.mark.parametrize( + "model, requested, expected", + [ + ("hosted_vllm/my-kimi", "minimal", "low"), + ("hosted_vllm/my-kimi", "medium", "high"), + ("hosted_vllm/my-kimi", "xhigh", "max"), + ("hosted_vllm/my-kimi", "none", "none"), + ("azure_ai/my-kimi", "medium", "high"), + ], +) +def test_a_deployment_declaring_its_levels_receives_the_level_rounded_up( + respx_mock: respx.MockRouter, model: str, requested: str, expected: str +) -> None: + assert _chat_body_sent(respx_mock, model, requested, _KIMI_DEPLOYMENT)["reasoning_effort"] == expected + + +def test_a_deployment_rounding_down_receives_the_declared_level_below(respx_mock: respx.MockRouter) -> None: + rounding_down: Final = {**_KIMI_DEPLOYMENT, "reasoning_effort_rounding": "down"} + + assert _chat_body_sent(respx_mock, "hosted_vllm/my-kimi", "medium", rounding_down)["reasoning_effort"] == "low" + + +def test_default_on_a_declaring_deployment_leaves_the_effort_to_the_provider(respx_mock: respx.MockRouter) -> None: + assert "reasoning_effort" not in _chat_body_sent(respx_mock, "hosted_vllm/my-kimi", "default", _KIMI_DEPLOYMENT) + + +def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + _chat_body_sent(respx_mock, "azure_ai/my-kimi", "medium", {"id": "kimi-azure-1"}) + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + _chat_body_sent(respx_mock, "azure_ai/my-kimi", "medium", {"reasoning_effort_levels": []}) + + assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "medium", _KIMI_DEPLOYMENT)["reasoning_effort"] == "high" + assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == ( + "adaptive" + ) + assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ + "reasoning_effort" + ] == "none" + + +def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params( + local_model_cost_map: None, respx_mock: respx.MockRouter +) -> None: + assert "reasoning_effort" not in _chat_body_sent( + respx_mock, "azure_ai/FW-Kimi-K3", "medium", {"id": "kimi-azure-1"}, drop_params=True + ) + + +@pytest.mark.parametrize("model_info, expected", [(_KIMI_DEPLOYMENT, "high"), (None, "medium")]) +def test_a_hosted_vllm_thinking_budget_is_rounded_onto_the_declared_levels( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, expected: str +) -> None: + sent: Final = _chat_body_sent( + respx_mock, "hosted_vllm/my-kimi", None, model_info, thinking={"type": "enabled", "budget_tokens": 2048} + ) + + assert sent["reasoning_effort"] == expected + + +def test_an_undeclared_sibling_of_a_declaring_deployment_gets_the_request_untouched( + respx_mock: respx.MockRouter, +) -> None: + from litellm.router import Router + + route: Final = respx_mock.post(f"{_VLLM_API_BASE}/chat/completions").mock( + return_value=_mocked_openai_chat_response("my-kimi") + ) + shared_params: Final = {"model": "hosted_vllm/my-kimi", "api_base": _VLLM_API_BASE, "api_key": "fake-key"} + router: Final = Router( + model_list=[ + {"model_name": "kimi-declared", "litellm_params": {**shared_params}, "model_info": _KIMI_DEPLOYMENT}, + {"model_name": "kimi-plain", "litellm_params": {**shared_params}, "model_info": {"id": "kimi-plain-1"}}, + ] + ) + + router.completion(model="kimi-plain", messages=_HELLO, reasoning_effort="medium") + + assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "medium" + + +def _anthropic_chat_body_sent( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, **params: object +) -> Mapping[str, object]: + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + ) + completion(model="anthropic/claude-opus-5-5", messages=_HELLO, model_info=model_info, max_tokens=4096, **params) + return json.loads(route.calls.last.request.content) + + +_OPUS_DECLARING_LOW_AND_HIGH: Final = {"id": "opus-1", "reasoning_effort_levels": ["low", "high"]} + + +def test_an_anthropic_deployment_declaring_its_levels_sends_the_rounded_tier_in_its_own_shape( + respx_mock: respx.MockRouter, +) -> None: + rounded: Final = _anthropic_chat_body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, reasoning_effort="medium") + asked_for_high: Final = _anthropic_chat_body_sent(respx_mock, None, reasoning_effort="high") + + assert "reasoning_effort" not in rounded + assert rounded == asked_for_high + + +def _bedrock_converse_request_fields_sent( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, **params: object +) -> Mapping[str, object]: + route: Final = respx_mock.post( + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-opus-5-5/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + ) + completion( + model="bedrock/converse/us.anthropic.claude-opus-5-5", + messages=_HELLO, + model_info=model_info, + max_tokens=4096, + api_key="test-bedrock-token", + aws_region_name="us-west-2", + **params, + ) + return json.loads(route.calls.last.request.content)["additionalModelRequestFields"] + + +@pytest.mark.parametrize("display", [{}, {"display": "omitted"}, {"display": "summarized"}]) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_a_thinking_budget_on_a_declaring_adaptive_claude_deployment_sends_the_rounded_tier( + respx_mock: respx.MockRouter, + body_sent: Callable[..., Mapping[str, object]], + display: Mapping[str, str], +) -> None: + budget: Final = {"type": "enabled", "budget_tokens": 2048, **display} + + rounded: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking=budget) + undeclared: Final = body_sent(respx_mock, None, thinking=budget) + + assert undeclared["output_config"] == {"effort": "medium"} + assert rounded == {**undeclared, "output_config": {"effort": "high"}} + + +@pytest.mark.parametrize("display", [{}, {"display": "omitted"}]) +@pytest.mark.parametrize("default_effort", [{"reasoning_effort": "default"}, {"output_config": {"effort": "default"}}]) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declaring_adaptive_claude_deployment( + respx_mock: respx.MockRouter, + body_sent: Callable[..., Mapping[str, object]], + default_effort: Mapping[str, object], + display: Mapping[str, str], +) -> None: + defaulted: Final = body_sent( + respx_mock, + _OPUS_DECLARING_LOW_AND_HIGH, + thinking={"type": "enabled", "budget_tokens": 2048, **display}, + **default_effort, + ) + adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}) + + assert "output_config" not in defaulted + assert defaulted == adaptive_alone + + +def _bedrock_invoke_body_sent( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, **params: object +) -> Mapping[str, object]: + route: Final = respx_mock.post( + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-opus-5-5/invoke" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + ) + completion( + model="bedrock/invoke/us.anthropic.claude-opus-5-5", + messages=_HELLO, + model_info=model_info, + max_tokens=4096, + api_key="test-bedrock-token", + aws_region_name="us-west-2", + **params, + ) + return json.loads(route.calls.last.request.content) + + +@pytest.mark.parametrize("display", ["updates", "summarized"]) +@pytest.mark.parametrize( + "body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent, _bedrock_invoke_body_sent] +) +def test_an_output_config_effort_beside_a_thinking_budget_reaches_a_declaring_claude_deployment_unchanged( + respx_mock: respx.MockRouter, body_sent: Callable[..., Mapping[str, object]], display: str +) -> None: + params: Final = { + "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}, + "output_config": {"effort": "high"}, + } + + declared: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, **params) + undeclared: Final = body_sent(respx_mock, None, **params) + + assert undeclared["output_config"] == {"effort": "high"} + assert declared == undeclared + + +@pytest.mark.parametrize("dropped_effort", ["none", "HIGH"]) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_drop_params_leaves_the_thinking_budget_to_set_the_effort_beside_a_dropped_level( + respx_mock: respx.MockRouter, body_sent: Callable[..., Mapping[str, object]], dropped_effort: str +) -> None: + budget: Final = {"type": "enabled", "budget_tokens": 2048} + + dropped: Final = body_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking=budget, reasoning_effort=dropped_effort, drop_params=True + ) + budget_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking=budget, drop_params=True) + + assert budget_alone["output_config"] == {"effort": "high"} + assert dropped == budget_alone + + +@pytest.mark.parametrize( + "model_info, effort", + [(None, {}), (_OPUS_DECLARING_LOW_AND_HIGH, {}), (_OPUS_DECLARING_LOW_AND_HIGH, {"reasoning_effort": "default"})], +) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_a_non_numeric_thinking_budget_on_adaptive_claude_is_rejected_with_a_400( + respx_mock: respx.MockRouter, + body_sent: Callable[..., Mapping[str, object]], + model_info: Mapping[str, object] | None, + effort: Mapping[str, str], +) -> None: + with pytest.raises(litellm.BadRequestError, match=r"thinking\.budget_tokens must be a number") as rejected: + body_sent(respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **effort) + + assert rejected.value.status_code == 400 + + +@pytest.mark.parametrize("drop_globally", [False, True]) +@pytest.mark.parametrize("model_info", [None, _OPUS_DECLARING_LOW_AND_HIGH]) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + body_sent: Callable[..., Mapping[str, object]], + model_info: Mapping[str, object] | None, + drop_globally: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + request_drop: Final = {} if drop_globally else {"drop_params": True} + + dropped: Final = body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + adaptive_alone: Final = body_sent(respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop) + + assert "output_config" not in adaptive_alone + assert dropped == adaptive_alone + + +@pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) +@pytest.mark.parametrize("drop_params", [False, True]) +def test_an_output_config_effort_replaces_a_thinking_budget_the_provider_cannot_read( + respx_mock: respx.MockRouter, model: str, drop_params: bool +) -> None: + sent: Final = _chat_body_sent( + respx_mock, + model, + None, + _KIMI_DEPLOYMENT, + thinking={"type": "enabled", "budget_tokens": 2048}, + output_config={"effort": "minimal"}, + drop_params=drop_params, + ) + + assert sent["reasoning_effort"] == "low" + assert sent["output_config"] == {"effort": "low"} + assert "thinking" not in sent + + +@pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) +def test_a_default_output_config_effort_beside_a_thinking_budget_sends_no_effort_where_thinking_is_unread( + respx_mock: respx.MockRouter, model: str +) -> None: + sent: Final = _chat_body_sent( + respx_mock, + model, + None, + _KIMI_DEPLOYMENT, + thinking={"type": "enabled", "budget_tokens": 2048}, + output_config={"effort": "default"}, + ) + + assert "reasoning_effort" not in sent + assert "thinking" not in sent + + +@pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) +def test_a_thinking_budget_reaches_a_declaring_deployment_as_its_rounded_effort( + respx_mock: respx.MockRouter, model: str +) -> None: + sent: Final = _chat_body_sent( + respx_mock, model, None, _KIMI_DEPLOYMENT, thinking={"type": "enabled", "budget_tokens": 2048} + ) + + assert sent["reasoning_effort"] == "high" + assert "thinking" not in sent + + +@pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) +def test_an_explicit_reasoning_effort_replaces_a_thinking_budget_the_provider_cannot_read( + respx_mock: respx.MockRouter, model: str +) -> None: + sent: Final = _chat_body_sent( + respx_mock, model, "medium", _KIMI_DEPLOYMENT, thinking={"type": "enabled", "budget_tokens": 2048} + ) + + assert sent["reasoning_effort"] == "high" + assert "thinking" not in sent + + +def test_an_explicit_reasoning_effort_keeps_the_thinking_a_claude_deployment_reads( + respx_mock: respx.MockRouter, +) -> None: + budget: Final = {"type": "enabled", "budget_tokens": 2048} + + declared: Final = _anthropic_chat_body_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, reasoning_effort="medium", thinking=budget + ) + undeclared: Final = _anthropic_chat_body_sent(respx_mock, None, reasoning_effort="medium", thinking=budget) + + assert "thinking" in undeclared + assert declared["thinking"] == undeclared["thinking"] + + +@pytest.mark.parametrize("declared", [["none"], []]) +def test_drop_params_drops_a_thinking_budget_whose_effort_the_deployment_does_not_declare( + respx_mock: respx.MockRouter, declared: list[str] +) -> None: + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + None, + {"reasoning_effort_levels": declared}, + thinking={"type": "enabled", "budget_tokens": 2048}, + drop_params=True, + ) + + assert "reasoning_effort" not in sent + + +@pytest.mark.parametrize("declared", [["none"], []]) +@pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) +def test_drop_params_keeps_adaptive_thinking_but_drops_a_budget_effort_the_claude_deployment_does_not_declare( + respx_mock: respx.MockRouter, body_sent: Callable[..., Mapping[str, object]], declared: list[str] +) -> None: + model_info: Final = {"reasoning_effort_levels": declared} + + dropped: Final = body_sent( + respx_mock, + model_info, + thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, + drop_params=True, + ) + adaptive_alone: Final = body_sent( + respx_mock, model_info, thinking={"type": "adaptive", "display": "summarized"}, drop_params=True + ) + + assert "output_config" not in dropped + assert dropped == adaptive_alone + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +@pytest.mark.parametrize("budget_tokens", [500, 9000]) +def test_a_hosted_vllm_deployment_maps_a_thinking_budget_the_same_for_a_claude_model_name( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, budget_tokens: int +) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": budget_tokens} + + claude_named: Final = _chat_body_sent( + respx_mock, "hosted_vllm/claude-opus-5-5", None, model_info, thinking=thinking + ) + kimi_named: Final = _chat_body_sent(respx_mock, "hosted_vllm/my-kimi", None, model_info, thinking=thinking) + + assert claude_named["reasoning_effort"] == kimi_named["reasoning_effort"] + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +@pytest.mark.parametrize("budget_tokens", ["2048", None, True]) +def test_a_non_numeric_thinking_budget_is_rejected_with_a_400( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, budget_tokens: str | bool | None +) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": budget_tokens} + + with pytest.raises(litellm.BadRequestError, match=r"thinking\.budget_tokens must be a number") as rejected: + _chat_body_sent(respx_mock, "hosted_vllm/my-kimi", None, model_info, thinking=thinking) + + assert rejected.value.status_code == 400 + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +@pytest.mark.parametrize("budget_tokens", ["2048", None, True]) +def test_an_explicit_reasoning_effort_wins_over_a_non_numeric_thinking_budget( + respx_mock: respx.MockRouter, model_info: Mapping[str, object] | None, budget_tokens: str | bool | None +) -> None: + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + "low", + model_info, + thinking={"type": "enabled", "budget_tokens": budget_tokens}, + ) + + assert sent["reasoning_effort"] == "low" + assert "thinking" not in sent + + +@pytest.mark.parametrize("drop_globally", [False, True]) +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +def test_drop_params_drops_a_non_numeric_thinking_budget( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model_info: Mapping[str, object] | None, + drop_globally: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + request_drop: Final = {} if drop_globally else {"drop_params": True} + + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + None, + model_info, + thinking={"type": "enabled", "budget_tokens": "2048"}, + **request_drop, + ) + + assert "reasoning_effort" not in sent + assert "thinking" not in sent + + +def test_an_adaptive_effort_on_a_declaring_bedrock_converse_claude_deployment_is_rounded( + respx_mock: respx.MockRouter, +) -> None: + adaptive: Final = {"type": "adaptive"} + + rounded: Final = _bedrock_converse_request_fields_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking=adaptive, output_config={"effort": "medium"} + ) + asked_for_high: Final = _bedrock_converse_request_fields_sent( + respx_mock, None, thinking=adaptive, output_config={"effort": "high"} + ) + + assert rounded == asked_for_high + + +def test_a_routed_remap_is_logged_with_the_deployment_id_and_model_group( + respx_mock: respx.MockRouter, remap_log: pytest.LogCaptureFixture +) -> None: + from litellm.router import Router + + route: Final = respx_mock.post(f"{_VLLM_API_BASE}/chat/completions").mock( + return_value=_mocked_openai_chat_response("my-kimi") + ) + router: Final = Router( + model_list=[ + { + "model_name": "kimi-k3", + "litellm_params": {"model": "hosted_vllm/my-kimi", "api_base": _VLLM_API_BASE, "api_key": "fake-key"}, + "model_info": _KIMI_DEPLOYMENT, + } + ] + ) + + router.completion(model="kimi-k3", messages=_HELLO, reasoning_effort="minimal") + + assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "low" + assert [ + ( + getattr(record, "reasoning_effort_deployment_id"), + getattr(record, "reasoning_effort_model_group"), + getattr(record, "reasoning_effort_requested"), + getattr(record, "reasoning_effort_sent"), + ) + for record in _reasoning_effort_remaps(remap_log) + ] == [("kimi-vllm-1", "kimi-k3", "minimal", "low")] + + +def test_a_chat_request_bridged_to_responses_is_rounded_and_logged_once( + respx_mock: respx.MockRouter, remap_log: pytest.LogCaptureFixture +) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_FOUNDRY_RESPONSES_FUNCTION_CALL_BODY) + ) + + completion( + model="openai/gpt-5.5", + messages=_HELLO, + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + reasoning_effort="medium", + model_info={"id": "gpt-5.5-1", "reasoning_effort_levels": ["low", "high"]}, + ) + + assert json.loads(route.calls.last.request.content)["reasoning"]["effort"] == "high" + assert [getattr(record, "reasoning_effort_sent") for record in _reasoning_effort_remaps(remap_log)] == ["high"] + + +def test_default_on_a_declaring_deployment_keeps_a_tool_call_on_chat_completions(respx_mock: respx.MockRouter) -> None: + gateway: Final = "https://openai-gateway.example.test/v1" + chat: Final = respx_mock.post(f"{gateway}/chat/completions").mock( + return_value=_mocked_openai_chat_response("gpt-5.5") + ) + responses: Final = respx_mock.post(f"{gateway}/responses") + + completion( + model="openai/gpt-5.5", + api_base=gateway, + api_key="fake-key", + messages=_HELLO, + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + reasoning_effort="default", + model_info={"id": "gpt-5.5-1", "reasoning_effort_levels": ["low", "high"]}, + ) + + assert not responses.called + assert "reasoning_effort" not in json.loads(chat.calls.last.request.content) + + +@pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) +def test_drop_params_drops_a_level_the_deployment_does_not_declare(respx_mock: respx.MockRouter, model: str) -> None: + without_none: Final = {"reasoning_effort_levels": ["low", "high", "max"]} + + assert "reasoning_effort" not in _chat_body_sent(respx_mock, model, "turbo", without_none, drop_params=True) + assert "reasoning_effort" not in _chat_body_sent(respx_mock, model, "none", without_none, drop_params=True) + assert _chat_body_sent(respx_mock, model, "none", _KIMI_DEPLOYMENT, drop_params=True)["reasoning_effort"] == "none" + assert _chat_body_sent(respx_mock, model, "turbo", without_none)["reasoning_effort"] == "turbo" + + +_GPT_5_GATEWAY: Final = "https://gpt5-gateway.example.test/v1" +_AZURE_GPT_5_BASE: Final = "https://gpt5.openai.azure.com" + + +def _gpt_5_chat_body_sent( + respx_mock: respx.MockRouter, model: str, reasoning_effort: str, model_info: Mapping[str, object] +) -> Mapping[str, object]: + route: Final = respx_mock.post(url__regex=r"https://gpt5[^/]*/.*chat/completions.*").mock( + return_value=_mocked_openai_chat_response(model) + ) + completion( + model=model, + api_base=_AZURE_GPT_5_BASE if model.startswith("azure/") else _GPT_5_GATEWAY, + api_version="2025-04-01-preview", + api_key="fake-key", + messages=_HELLO, + reasoning_effort=reasoning_effort, + model_info=model_info, + ) + return json.loads(route.calls.last.request.content) if route.called else {} + + +def _openai_gpt_5_entry_disabling(level: str) -> str: + return next( + model + for model, entry in litellm.model_cost.items() + if entry.get("litellm_provider") == "openai" and entry.get(f"supports_{level}_reasoning_effort") is False + ) + + +@pytest.mark.parametrize( + "model, requested, declared, expected", + [ + ("openai/gpt-5-in-house", "high", ["low", "xhigh"], "xhigh"), + ("openai/gpt-5-in-house", "xhigh", ["xhigh"], "xhigh"), + ("azure/gpt-5-in-house", "none", ["none", "high"], "none"), + ("azure_ai/gpt-5-in-house", "xhigh", ["xhigh"], "xhigh"), + ], +) +def test_a_gpt_5_deployment_receives_a_level_it_declares_that_the_model_map_does_not_list( + respx_mock: respx.MockRouter, model: str, requested: str, declared: list[str], expected: str +) -> None: + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + _gpt_5_chat_body_sent(respx_mock, model, expected, {"id": "gpt-5-in-house-1"}) + + sent: Final = _gpt_5_chat_body_sent(respx_mock, model, requested, {"reasoning_effort_levels": declared}) + + assert sent["reasoning_effort"] == expected + + +def test_a_gpt_5_deployment_receives_a_declared_level_the_model_map_disables( + local_model_cost_map: None, respx_mock: respx.MockRouter +) -> None: + model: Final = f"openai/{_openai_gpt_5_entry_disabling('low')}" + responses: Final = respx_mock.post(f"{_GPT_5_GATEWAY}/responses").mock( + return_value=httpx.Response(200, json=_FOUNDRY_RESPONSES_FUNCTION_CALL_BODY) + ) + + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + _gpt_5_chat_body_sent(respx_mock, model, "low", {"id": "gpt-5-pro-1"}) + _gpt_5_chat_body_sent(respx_mock, model, "low", {"reasoning_effort_levels": ["low", "high"]}) + + assert json.loads(responses.calls.last.request.content)["reasoning"]["effort"] == "low" + + @pytest.mark.asyncio async def test_aspeech_routes_a_bare_model_name_to_its_explicit_provider(respx_mock: respx.MockRouter) -> None: api_base: Final = "http://localhost:12351/v1" From 7b82b2a3c38cd3b641b581b08970aeaed3f03f0c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:36:13 +0000 Subject: [PATCH 062/124] feat(bedrock): route token counts from model metadata (#47441) * feat(bedrock): route token counts from model metadata * fix(bedrock): keep counting metadata compatible with older proxies --- .../count_tokens/bedrock_token_counter.py | 18 +- ...odel_prices_and_context_window_backup.json | 603 +++++++++++++++--- .../common_utils/token_count_settings.py | 28 +- litellm/proxy/proxy_server.py | 9 +- litellm/types/token_count.py | 2 + model_prices_and_context_window.json | 603 +++++++++++++++--- model_prices_and_context_window.schema.json | 9 +- .../test_bedrock_token_counter.py | 47 ++ .../common_utils/test_token_count_settings.py | 35 +- tests/unit/proxy/test_proxy_token_counter.py | 69 +- 10 files changed, 1213 insertions(+), 210 deletions(-) diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 97f28b9fbc5..068cf4f2e67 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -59,8 +59,7 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: class BedrockTokenCounter(BaseTokenCounter): - """Token counter for AWS Bedrock: bedrock-runtime CountTokens first, and Anthropic's count_tokens - on bedrock-mantle for Claude models bedrock-runtime answers 400 or 404 for""" + """Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model""" def __init__( self, @@ -127,17 +126,22 @@ class BedrockTokenCounter(BaseTokenCounter): **({"tools": tools} if tools else {}), **({"system": system} if system else {}), } - resolved_model: Final = get_bedrock_base_model(model_to_use) + use_mantle: Final = model_to_use.startswith("bedrock_mantle/") + resolved_model: Final = get_bedrock_base_model(model_to_use.removeprefix("bedrock_mantle/")) - runtime: Final = await self._count_with( - self._runtime_handler, RUNTIME_TOKENIZER_TYPE, request_data, litellm_params, resolved_model + primary: Final = await self._count_with( + self._mantle_handler if use_mantle else self._runtime_handler, + MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE, + request_data, + litellm_params, + resolved_model, ) outcome: Final = ( await self._count_with( self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model ) - if _runtime_rejected_claude_model(runtime, resolved_model) - else runtime + if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) + else primary ) match outcome: case CountedTokens(): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 95285d50f26..7f8a2c2ae93 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -922,7 +922,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5e-07, - "output_cost_per_token_batches": 2.5e-06 + "output_cost_per_token_batches": 2.5e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -1076,7 +1080,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -1135,7 +1143,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1171,7 +1183,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1207,7 +1223,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1243,7 +1263,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1279,7 +1303,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1315,7 +1343,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1353,7 +1385,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-mythos-preview": { "cache_creation_input_token_cost": 3.4375e-05, @@ -1411,7 +1448,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1449,7 +1491,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1487,7 +1534,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1525,7 +1577,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1563,7 +1620,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1640,7 +1702,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1717,7 +1784,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1870,7 +1942,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -1910,7 +1987,12 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1948,7 +2030,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -1988,7 +2075,12 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2026,7 +2118,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2066,7 +2163,12 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2104,7 +2206,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2182,7 +2289,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2222,7 +2334,12 @@ "prompt_cache_min_tokens": 512, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2340,7 +2457,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2380,7 +2502,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2420,7 +2547,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2460,7 +2592,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2500,7 +2637,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2540,7 +2682,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -2577,7 +2724,12 @@ "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2617,7 +2769,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2657,7 +2814,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2697,7 +2859,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2737,7 +2904,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2777,7 +2949,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2852,7 +3029,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2887,7 +3068,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2922,7 +3107,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2957,7 +3146,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2992,7 +3185,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -3027,7 +3224,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3060,7 +3261,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3099,7 +3304,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -3312,7 +3521,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3345,7 +3558,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -3396,7 +3613,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "azure/ada": { "deprecation_date": "2028-02-09", @@ -14651,7 +14872,11 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -14828,7 +15053,11 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -24589,7 +24818,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -24686,7 +24919,11 @@ "supports_tool_choice": true, "supports_vision": true, "prompt_cache_min_tokens": 1024, - "deprecation_date": "2027-01-08" + "deprecation_date": "2027-01-08", + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -24745,7 +24982,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -24784,7 +25025,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "eu.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -33929,7 +34174,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -33962,7 +34211,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -33990,7 +34243,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5e-07, - "output_cost_per_token_batches": 2.5e-06 + "output_cost_per_token_batches": 2.5e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -39238,7 +39495,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -39266,7 +39527,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -48338,7 +48603,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -48436,7 +48705,11 @@ "supports_tool_choice": true, "supports_vision": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -48475,7 +48748,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -48506,7 +48783,11 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "us-gov.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -48541,7 +48822,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -48576,7 +48862,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -48609,7 +48900,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -48644,7 +48940,12 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, @@ -48678,7 +48979,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-fable-5-1", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -48831,7 +49137,11 @@ "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -48890,7 +49200,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -48923,7 +49237,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -48956,7 +49274,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -48989,7 +49311,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "us.deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", @@ -61055,7 +61381,11 @@ "supports_native_structured_output": true, "supports_pdf_input": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -61081,7 +61411,11 @@ "supports_native_structured_output": true, "supports_pdf_input": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, @@ -67226,7 +67560,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -67260,7 +67599,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -67292,7 +67636,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -67327,7 +67676,12 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, @@ -67360,7 +67714,12 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "token_count_model": { + "model": "anthropic.claude-fable-5-1", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -72763,7 +73122,12 @@ "cache_read_input_token_cost": 1.1e-06, "output_cost_per_token": 5.5e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-mythos-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "apac.anthropic.claude-fable-5": { "litellm_provider": "bedrock_converse", @@ -72862,7 +73226,12 @@ "input_cost_per_token": 5.5e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "apac.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -72897,7 +73266,12 @@ "output_cost_per_token": 2.75e-05, "cache_read_input_token_cost": 5.5e-07, "cache_creation_input_token_cost": 6.875e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -72930,7 +73304,12 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -72965,7 +73344,12 @@ "input_cost_per_token": 4.4e-06, "output_cost_per_token": 2.2e-05, "cache_read_input_token_cost": 2.2e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "apac.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -72995,7 +73379,11 @@ "cache_read_input_token_cost": 3.3e-07, "cache_creation_input_token_cost": 4.125e-06, "input_cost_per_token": 3.3e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "apac.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -73030,7 +73418,12 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_read_input_token_cost": 2.2e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "us.anthropic.claude-mythos-preview": { "litellm_provider": "bedrock_converse", @@ -73154,6 +73547,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -73231,6 +73625,7 @@ "au.anthropic.claude-sonnet-5-5": { "token_count_model": { "model": "au.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" }, "bedrock_converse_supports_strict_tools": false, @@ -73382,7 +73777,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "eu.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -73435,6 +73835,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -73637,7 +74038,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -73648,6 +74054,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "us.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -74627,6 +75034,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -74709,6 +75117,7 @@ "au.anthropic.claude-haiku-5-5": { "token_count_model": { "model": "au.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" }, "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -74880,7 +75289,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-haiku-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "eu.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -74943,6 +75357,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -75078,7 +75493,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-haiku-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -75094,6 +75514,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "us.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 6f805616820..22b35bd1001 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -9,7 +9,7 @@ import litellm from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException -from litellm.types.token_count import TokenCountSourceFields, TokenCountTokenizer +from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer from litellm.types.utils import TokenCountResponse CANNOT_COUNT_MODEL_STATUSES: Final = frozenset({400, 404}) @@ -46,6 +46,7 @@ def _with_hint(message: str, hint: str) -> str: class TokenCountModel: model: str region: str | None + litellm_provider: TokenCountProvider | None = None @dataclass(frozen=True, slots=True) @@ -100,9 +101,27 @@ def _non_blank(value: object) -> str | None: return value if isinstance(value, str) and value.strip() else None +def _count_provider(value: object) -> TokenCountProvider | None: + match value: + case "bedrock": + return "bedrock" + case "bedrock_mantle": + return "bedrock_mantle" + case _: + return None + + def _count_model(value: Mapping[str, object]) -> TokenCountModel | None: model: Final = _non_blank(value.get("model")) - return None if model is None else TokenCountModel(model=model, region=_non_blank(value.get("region"))) + return ( + None + if model is None + else TokenCountModel( + model=model, + region=_non_blank(value.get("region")), + litellm_provider=_count_provider(value.get("litellm_provider")), + ) + ) def _geography(region: str) -> str: @@ -112,7 +131,10 @@ def _geography(region: str) -> str: def _within_deployment_geography( count_model: TokenCountModel | None, deployment_region: Callable[[], str] ) -> TokenCountModel | None: - if count_model is None or count_model.region is None or count_model.model.startswith("global."): + if count_model is None or count_model.region is None: + return count_model + model: Final = count_model.model.removeprefix("bedrock/").removeprefix("bedrock_mantle/") + if model.startswith("global."): return count_model if _geography(deployment_region()) == _geography(count_model.region): return count_model diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 581698626d5..811d4b8601e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14851,9 +14851,14 @@ async def _provider_token_count( Returns None when the count should be estimated locally, or the provider's failure when the framed local estimate stands in for it and that failure is the answer if the estimate cannot read the request.""" count_model: Final = settings.count_model - counted: Final = await ( - count_with(model_to_use, None) if count_model is None else count_with(count_model.model, count_model.region) + counting_model: Final = ( + model_to_use + if count_model is None + else count_model.model + if count_model.litellm_provider is None + else f"{count_model.litellm_provider}/{count_model.model.removeprefix('bedrock/').removeprefix('bedrock_mantle/')}" ) + counted: Final = await count_with(counting_model, None if count_model is None else count_model.region) if counted is None: return None if isinstance(counted, TokenCountResponse): diff --git a/litellm/types/token_count.py b/litellm/types/token_count.py index 3b21401066c..90ccefea177 100644 --- a/litellm/types/token_count.py +++ b/litellm/types/token_count.py @@ -4,11 +4,13 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"] TokenCountTokenizer: TypeAlias = Literal["kimi_k3"] +TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"] class TokenCountModelInfo(TypedDict): model: ReadOnly[str] region: ReadOnly[NotRequired[str]] + litellm_provider: ReadOnly[NotRequired[TokenCountProvider]] class TokenCountSourceFields(TypedDict, total=False): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 95285d50f26..7f8a2c2ae93 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -922,7 +922,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5e-07, - "output_cost_per_token_batches": 2.5e-06 + "output_cost_per_token_batches": 2.5e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -1076,7 +1080,11 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -1135,7 +1143,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1171,7 +1183,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1207,7 +1223,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1243,7 +1263,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1279,7 +1303,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1315,7 +1343,11 @@ "bedrock_output_config_effort_ceiling": "max", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-6-v1", + "litellm_provider": "bedrock" + } }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1353,7 +1385,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-mythos-preview": { "cache_creation_input_token_cost": 3.4375e-05, @@ -1411,7 +1448,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1449,7 +1491,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1487,7 +1534,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1525,7 +1577,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1563,7 +1620,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1640,7 +1702,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.25e-05, @@ -1717,7 +1784,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-fable-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.375e-05, @@ -1870,7 +1942,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -1910,7 +1987,12 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -1948,7 +2030,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -1988,7 +2075,12 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2026,7 +2118,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2066,7 +2163,12 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/", "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2104,7 +2206,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2182,7 +2289,12 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2222,7 +2334,12 @@ "prompt_cache_min_tokens": 512, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -2340,7 +2457,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2380,7 +2502,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2420,7 +2547,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2460,7 +2592,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2500,7 +2637,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -2540,7 +2682,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -2577,7 +2724,12 @@ "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 2048, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2617,7 +2769,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "global.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2657,7 +2814,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "global.anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "us.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2697,7 +2859,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "eu.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2737,7 +2904,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "eu-west-1" + } }, "au.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2777,7 +2949,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "jp.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -2852,7 +3029,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2887,7 +3068,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2922,7 +3107,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2957,7 +3146,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2992,7 +3185,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -3027,7 +3224,11 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3060,7 +3261,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3099,7 +3304,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -3312,7 +3521,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -3345,7 +3558,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -3396,7 +3613,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "azure/ada": { "deprecation_date": "2028-02-09", @@ -14651,7 +14872,11 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -14828,7 +15053,11 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -24589,7 +24818,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -24686,7 +24919,11 @@ "supports_tool_choice": true, "supports_vision": true, "prompt_cache_min_tokens": 1024, - "deprecation_date": "2027-01-08" + "deprecation_date": "2027-01-08", + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -24745,7 +24982,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -24784,7 +25025,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "eu.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -33929,7 +34174,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -33962,7 +34211,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -33990,7 +34243,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5e-07, - "output_cost_per_token_batches": 2.5e-06 + "output_cost_per_token_batches": 2.5e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -39238,7 +39495,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -39266,7 +39527,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -48338,7 +48603,11 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06 + "output_cost_per_token_batches": 2.75e-06, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -48436,7 +48705,11 @@ "supports_tool_choice": true, "supports_vision": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-1-20250805-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -48475,7 +48748,11 @@ "prompt_cache_min_tokens": 1024, "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -48506,7 +48783,11 @@ "supports_native_structured_output": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "litellm_provider": "bedrock" + } }, "us-gov.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -48541,7 +48822,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -48576,7 +48862,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -48609,7 +48900,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -48644,7 +48940,12 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, - "supports_forced_tool_use": false + "supports_forced_tool_use": false, + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, @@ -48678,7 +48979,12 @@ "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-fable-5-1", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us-gov.nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -48831,7 +49137,11 @@ "prompt_cache_min_tokens": 4096, "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -48890,7 +49200,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -48923,7 +49237,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -48956,7 +49274,11 @@ "bedrock_output_config_effort_ceiling": "high", "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 4096, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-4-5-20251101-v1:0", + "litellm_provider": "bedrock" + } }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -48989,7 +49311,11 @@ "supports_vision": true, "bedrock_converse_supports_strict_tools": false, "prompt_cache_min_tokens": 1024, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "litellm_provider": "bedrock" + } }, "us.deepseek.r1-v1:0": { "deprecation_date": "2027-03-30", @@ -61055,7 +61381,11 @@ "supports_native_structured_output": true, "supports_pdf_input": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -61081,7 +61411,11 @@ "supports_native_structured_output": true, "supports_pdf_input": true, "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096 + "prompt_cache_min_tokens": 4096, + "token_count_model": { + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_provider": "bedrock" + } }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, @@ -67226,7 +67560,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -67260,7 +67599,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -67292,7 +67636,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -67327,7 +67676,12 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true, "supports_forced_tool_use": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, @@ -67360,7 +67714,12 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 512 + "prompt_cache_min_tokens": 512, + "token_count_model": { + "model": "anthropic.claude-fable-5-1", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 7.2e-08, @@ -72763,7 +73122,12 @@ "cache_read_input_token_cost": 1.1e-06, "output_cost_per_token": 5.5e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-mythos-5", + "litellm_provider": "bedrock_mantle", + "region": "us-east-1" + } }, "apac.anthropic.claude-fable-5": { "litellm_provider": "bedrock_converse", @@ -72862,7 +73226,12 @@ "input_cost_per_token": 5.5e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-7", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "apac.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -72897,7 +73266,12 @@ "output_cost_per_token": 2.75e-05, "cache_read_input_token_cost": 5.5e-07, "cache_creation_input_token_cost": 6.875e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-4-8", + "litellm_provider": "bedrock_mantle", + "region": "ap-northeast-1" + } }, "apac.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, @@ -72930,7 +73304,12 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -72965,7 +73344,12 @@ "input_cost_per_token": 4.4e-06, "output_cost_per_token": 2.2e-05, "cache_read_input_token_cost": 2.2e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "apac.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -72995,7 +73379,11 @@ "cache_read_input_token_cost": 3.3e-07, "cache_creation_input_token_cost": 4.125e-06, "input_cost_per_token": 3.3e-06, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-4-6", + "litellm_provider": "bedrock" + } }, "apac.anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -73030,7 +73418,12 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 1.1e-05, "cache_read_input_token_cost": 2.2e-07, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "token_count_model": { + "model": "anthropic.claude-sonnet-5", + "litellm_provider": "bedrock_mantle", + "region": "ap-southeast-4" + } }, "us.anthropic.claude-mythos-preview": { "litellm_provider": "bedrock_converse", @@ -73154,6 +73547,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -73231,6 +73625,7 @@ "au.anthropic.claude-sonnet-5-5": { "token_count_model": { "model": "au.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" }, "bedrock_converse_supports_strict_tools": false, @@ -73382,7 +73777,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "eu.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -73435,6 +73835,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -73637,7 +74038,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-sonnet-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -73648,6 +74054,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "us.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -74627,6 +75034,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -74709,6 +75117,7 @@ "au.anthropic.claude-haiku-5-5": { "token_count_model": { "model": "au.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" }, "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -74880,7 +75289,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-haiku-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "eu.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -74943,6 +75357,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "global.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, @@ -75078,7 +75493,12 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "model": "anthropic.claude-haiku-5-5", + "litellm_provider": "bedrock_mantle", + "region": "us-gov-west-1" + } }, "us.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, @@ -75094,6 +75514,7 @@ "litellm_provider": "bedrock_converse", "token_count_model": { "model": "us.anthropic.claude-opus-5-5", + "litellm_provider": "bedrock_mantle", "region": "us-east-1" }, "supports_tool_search": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index de57fc0616c..ce9af67eb94 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1229,9 +1229,16 @@ "region": { "type": "string", "pattern": "\\S" + }, + "litellm_provider": { + "type": "string", + "enum": [ + "bedrock", + "bedrock_mantle" + ] } }, - "description": "Model that counts tokens for this one, and the AWS region its count API runs in." + "description": "Model that counts tokens for this one, its counting provider, and the AWS region its count API runs in." }, "token_count_tokenizer": { "type": "string", diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 8a21027872a..dd0d6429f8a 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -174,3 +174,50 @@ async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> N "system": "You are a terse assistant.", "tools": tools, } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "host", "tokenizer", "expected_tokens"), + ( + ("bedrock_mantle/anthropic.claude-opus-5-5", MANTLE_HOST, "bedrock_mantle_api", 271), + ("bedrock/anthropic.claude-3-haiku-20240307-v1:0", RUNTIME_HOST, "bedrock_api", 193), + ), +) +async def test_qualified_counting_model_uses_its_endpoint_without_a_failed_probe( + model: str, host: str, tokenizer: str, expected_tokens: int +) -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 193}), mantle=(200, {"input_tokens": 271})) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use=model, + messages=MESSAGES, + contents=None, + deployment=DEPLOYMENT, + request_model="served-model", + ) + + assert result is not None + assert result.total_tokens == expected_tokens + assert result.tokenizer_type == tokenizer + assert bedrock.posted_hosts() == (host,) + if host == MANTLE_HOST: + assert _JSON_BODY.validate_json(bedrock.requests[0].content)["model"] == model.split("/", 1)[1] + + +@pytest.mark.asyncio +async def test_explicit_mantle_failure_does_not_probe_runtime() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 193}), mantle=(404, UNSUPPORTED)) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="bedrock_mantle/anthropic.claude-opus-5-5", + messages=MESSAGES, + contents=None, + deployment=DEPLOYMENT, + ) + + assert result is not None + assert result.error is True + assert result.status_code == 404 + assert result.tokenizer_type == "bedrock_mantle_api" + assert bedrock.posted_hosts() == (MANTLE_HOST,) diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index 8bd5ced1b58..d7314f1a6d3 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -4,6 +4,7 @@ from typing import Final import pytest +from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.proxy.common_utils.token_count_settings import ( ProviderCountFailure, TokenCountModel, @@ -33,6 +34,13 @@ def _in_us_west_2() -> str: {"token_count_framing": 5, "token_count_model": {"model": "custom-model", "region": "ap-southeast-4"}}, TokenCountSettings(framing=5, count_model=TokenCountModel(model="custom-model", region="ap-southeast-4")), ), + ( + {"token_count_model": {"model": "custom-model", "litellm_provider": "bedrock_mantle"}}, + TokenCountSettings( + framing=97, + count_model=TokenCountModel(model="custom-model", region=None, litellm_provider="bedrock_mantle"), + ), + ), ( {"token_count_model": {"model": "custom-model"}}, TokenCountSettings(framing=97, count_model=TokenCountModel(model="custom-model", region=None)), @@ -94,13 +102,15 @@ def test_shipped_count_region_never_takes_the_prompt_out_of_the_deployment_geogr assert settings.count_model == TokenCountModel(model="anthropic.claude-opus-5-5", region=expected_region) -def test_shipped_global_count_model_keeps_its_region_for_a_deployment_in_any_geography() -> None: +@pytest.mark.parametrize("prefix", ("", "bedrock/", "bedrock_mantle/")) +def test_shipped_global_count_model_keeps_its_region_for_a_deployment_in_any_geography(prefix: str) -> None: + model: Final = f"{prefix}global.anthropic.claude-opus-5-5" settings: Final = resolve_token_count_settings( configured={}, - default={"token_count_model": {"model": "global.anthropic.claude-opus-5-5", "region": "us-east-1"}}, + default={"token_count_model": {"model": model, "region": "us-east-1"}}, deployment_region=lambda: "eu-west-1", ) - assert settings.count_model == TokenCountModel(model="global.anthropic.claude-opus-5-5", region="us-east-1") + assert settings.count_model == TokenCountModel(model=model, region="us-east-1") def test_shipped_australian_count_region_stays_with_an_australian_deployment() -> None: @@ -157,21 +167,24 @@ def test_count_source_fields_name_the_counting_model_only_when_token_count_model assert count_source_fields(_counted(source)) == expected -def test_every_shipped_token_count_model_is_another_mapped_model_of_the_same_provider_in_a_named_region() -> None: +def test_every_shipped_token_count_model_resolves_to_a_mapped_aws_model() -> None: cost_map: Final = json.loads(_COST_MAP.read_text()) + aws_models: Final = frozenset( + get_bedrock_base_model(name) + for name, entry in cost_map.items() + if isinstance(entry, dict) and entry.get("litellm_provider") in ("bedrock", "bedrock_converse") + ) count_models: Final = { name: entry["token_count_model"] for name, entry in cost_map.items() if isinstance(entry, dict) and "token_count_model" in entry } assert count_models - assert { - name: count_model - for name, count_model in count_models.items() - if cost_map.get(count_model["model"], {}).get("litellm_provider") != cost_map[name]["litellm_provider"] - or count_model["model"] == name - or not count_model.get("region") - } == {} + for name, count_model in count_models.items(): + model: Final = count_model["model"] + assert get_bedrock_base_model(model) in aws_models + assert cost_map[name]["litellm_provider"] in ("bedrock", "bedrock_converse") + assert "region" not in count_model or count_model["region"].strip() _UPSTREAM_404: Final = { diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 5cbda9d320c..f1b8d3fd591 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -67,6 +67,7 @@ from litellm.proxy.common_utils.token_count_settings import ( TokenCountModel, TokenCountSettings, cost_map_entry, + resolve_token_count_settings, ) from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request from litellm.proxy.proxy_server import ( @@ -1636,6 +1637,65 @@ async def test_model_without_token_count_model_counts_itself_in_the_deployment_r assert ([call[0] for call in counter.calls], counter.regions) == (["us.anthropic.claude-sonnet-5-5"], ["us-west-2"]) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "expected_host", "expected_tokens"), + ( + ("bedrock", "bedrock-runtime.us-east-1.amazonaws.com", 193), + ("bedrock_mantle", "bedrock-mantle.us-east-1.api.aws", 271), + ), +) +async def test_token_count_model_provider_selects_the_endpoint_and_preserves_the_public_model( + provider: str, expected_host: str, expected_tokens: int, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + model: Final = "anthropic.claude-opus-5-5" + + def peer(request: httpx.Request) -> httpx.Response: + assert request.url.host == expected_host + if provider == "bedrock_mantle": + assert json.loads(request.content)["model"] == model + return httpx.Response(200, json={"input_tokens": expected_tokens}) + assert request.url.path == f"/model/{model}/count-tokens" + return httpx.Response(200, json={"inputTokens": expected_tokens}) + + settings: Final = resolve_token_count_settings( + configured={}, + default={"token_count_model": {"model": model, "litellm_provider": provider, "region": "us-east-1"}}, + deployment_region=lambda: "us-west-2", + ) + counter: Final = BedrockTokenCounter(client=AsyncHTTPHandler(transport=httpx.MockTransport(peer))) + result: Final = await _provider_token_count( + count_with=partial( + _try_provider_token_count, + counter, + "bedrock", + messages=_COUNT_REQUEST.messages, + contents=_COUNT_REQUEST.contents, + request_model=_COUNT_REQUEST.model, + deployment={ + "litellm_params": { + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + } + }, + tools=_COUNT_REQUEST.tools, + system=_COUNT_REQUEST.system, + ), + model_to_use="us.anthropic.claude-sonnet-5-5", + settings=settings, + local_estimate_allowed=False, + ) + + assert isinstance(result, TokenCountResponse) + assert (result.total_tokens, result.model_used, result.count_source) == ( + expected_tokens, + model, + "token_count_model", + ) + + @pytest.mark.parametrize( ("count_model", "region"), [ @@ -2658,7 +2718,7 @@ async def test_uncountable_model_keeps_the_provider_error_when_local_counting_is @pytest.mark.parametrize( ("model_info", "counted_with"), [ - ({}, ("us.anthropic.claude-opus-5-5", "us-east-1", 447)), + ({}, ("bedrock_mantle/us.anthropic.claude-opus-5-5", "us-east-1", 447)), ( {"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": "ap-southeast-4"}}, ("us.anthropic.claude-opus-5-5", "ap-southeast-4", 447), @@ -2674,6 +2734,7 @@ async def test_configured_token_count_model_overrides_the_shipped_one_and_an_emp { "us.anthropic.claude-sonnet-5-5": _provider_reply("sonnet", total_tokens=300), "us.anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447), + "bedrock_mantle/us.anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447), } ) monkeypatch.setattr( @@ -2712,7 +2773,7 @@ async def test_configured_token_count_model_overrides_the_shipped_one_and_an_emp async def test_shipped_token_count_model_keeps_the_prompt_in_the_deployment_geography( monkeypatch: pytest.MonkeyPatch, aws_region_name: str, counted_in: str ): - counter: Final = _ScriptedCounter({"anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447)}) + counter: Final = _ScriptedCounter({"bedrock_mantle/anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447)}) monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", @@ -2736,7 +2797,7 @@ async def test_shipped_token_count_model_keeps_the_prompt_in_the_deployment_geog request=TokenCountRequest(model="sonnet", messages=[{"role": "user", "content": "hi"}]), call_endpoint=True ) - assert (counter.calls[0][0], counter.regions[0]) == ("anthropic.claude-opus-5-5", counted_in) + assert (counter.calls[0][0], counter.regions[0]) == ("bedrock_mantle/anthropic.claude-opus-5-5", counted_in) class _SlowVault: @@ -2756,7 +2817,7 @@ async def _longest_stall(done: asyncio.Event) -> float: async def test_region_lookup_for_a_shipped_count_model_leaves_the_event_loop_free(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server - counter: Final = _ScriptedCounter({"anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447)}) + counter: Final = _ScriptedCounter({"bedrock_mantle/anthropic.claude-opus-5-5": _provider_reply("opus", total_tokens=447)}) def bedrock_counter(deployment: object, model: str) -> tuple[_ScriptedCounter, str, str]: return counter, model, "bedrock" From daa133da5cb5092c61f5653e1e19d5002afb23ff Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:37:31 +0000 Subject: [PATCH 063/124] fix(mcp)!: bill both model calls of a chat completions MCP auto-execute on one spend row (#47422) * test(mcp): cover one spend row billing both rounds of a chat completions MCP auto-execute * fix(mcp): bill both model calls of a chat completions MCP auto-execute on one spend row * fix(mcp): keep provider usage fields when summing both chat rounds of an MCP auto-execute * fix(mcp): log each auto-executed MCP tool call under its own spend-log id A chat MCP auto-execute logged its tool calls under the request's call id, which is also the id of the request's failure row. When the follow-up call failed, the database kept the tool row and dropped the failure row that bills the paid first call * test(mcp): cover the margin, cache, failure row and parallel tool rows of MCP auto-execute billing --- .../responses/mcp/chat_completions_handler.py | 109 ++++---- .../mcp/litellm_proxy_mcp_handler.py | 4 +- litellm/responses/mcp/round_billing.py | 77 ++++-- .../integration/mcp/test_mcp_llm_endpoints.py | 237 +++++++++++++----- .../mcp/test_chat_completions_handler.py | 234 ++++++++++++++++- .../mcp/test_litellm_proxy_mcp_handler.py | 27 +- .../unit/responses/mcp/test_round_billing.py | 71 +++++- 7 files changed, 619 insertions(+), 140 deletions(-) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index a8c93eebff3..b100858b306 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -6,11 +6,18 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import TypedDict, Unpack +from litellm._internal_context import internal_sub_call from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.responses.mcp.round_billing import ( + billed_for_every_round, + first_round_billed_on_failure, + summed_cost_breakdown, +) from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper @@ -588,19 +595,21 @@ async def acompletion_with_mcp( return cast(CustomStreamWrapper, MCPStreamWrapper(initial_stream, iterator)) - # Non-streaming mode: use existing logic initial_call_args = dict(base_call_args) initial_call_args["stream"] = False if mock_tool_calls is not None: initial_call_args["mock_tool_calls"] = mock_tool_calls - # Make initial call - initial_response: Final = await litellm_acompletion(**initial_call_args) + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.carry_failed_attempt_usage() + with internal_sub_call(): + initial_response: Final = await litellm_acompletion(**initial_call_args) + first_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None if not isinstance(initial_response, ModelResponse): return initial_response - # Extract tool calls from response tool_calls: Final = LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response=initial_response) if not tool_calls: @@ -610,48 +619,62 @@ async def acompletion_with_mcp( ) return initial_response - # Execute tool calls - tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( - tool_server_map=tool_server_map, - tool_calls=tool_calls, - served_tools=deduplicated_mcp_tools, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_call_id=context.litellm_call_id, - litellm_trace_id=context.litellm_trace_id, - request_tags=request_tags, - guardrail_context=context.guardrail_context, - ) - - if not tool_results: - _add_mcp_metadata_to_response( - response=initial_response, - openai_tools=openai_tools, + with first_round_billed_on_failure(logging_obj, initial_response): + tool_results: Final = await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( + tool_server_map=tool_server_map, tool_calls=tool_calls, + served_tools=deduplicated_mcp_tools, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_call_id=context.litellm_call_id, + litellm_trace_id=context.litellm_trace_id, + request_tags=request_tags, + guardrail_context=context.guardrail_context, ) - return initial_response - # Create follow-up messages with tool results - follow_up_messages: Final = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( - original_messages=messages, - response=initial_response, - tool_results=tool_results, - ) + if not tool_results: + _add_mcp_metadata_to_response( + response=initial_response, + openai_tools=openai_tools, + tool_calls=tool_calls, + ) + return initial_response - # Make follow-up call with original stream setting - follow_up_call_args: Final = dict(base_call_args) - follow_up_call_args["messages"] = follow_up_messages - follow_up_call_args["stream"] = stream - - response = await litellm_acompletion(**follow_up_call_args) - if isinstance(response, (ModelResponse, CustomStreamWrapper)): - _add_mcp_metadata_to_response( - response=response, - openai_tools=openai_tools, - tool_calls=tool_calls, + follow_up_messages: Final = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( + original_messages=messages, + response=initial_response, tool_results=tool_results, ) - return response + + follow_up_call_args: Final = dict(base_call_args) + follow_up_call_args["messages"] = follow_up_messages + follow_up_call_args["stream"] = stream + + with internal_sub_call(): + final_response: Final = await litellm_acompletion(**follow_up_call_args) + + if not isinstance(final_response, ModelResponse): + return final_response + _add_mcp_metadata_to_response( + response=final_response, + openai_tools=openai_tools, + tool_calls=tool_calls, + tool_results=tool_results, + ) + final_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.cost_breakdown = summed_cost_breakdown( + first=initial_response, + first_breakdown=first_round_breakdown, + final=final_response, + final_breakdown=final_round_breakdown, + ) + return billed_for_every_round( + first=initial_response, + first_breakdown=first_round_breakdown, + final=final_response, + final_breakdown=final_round_breakdown, + ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index e66a7d1102b..e3c3ab671ea 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -773,9 +773,7 @@ class LiteLLM_Proxy_MCP_Handler: } ] tool_logging_call_id = ( - f"{litellm_call_id}_{tool_call_id or uuid.uuid4()}" - if request_trace_id and litellm_call_id - else litellm_call_id or str(uuid.uuid4()) + f"{litellm_call_id}_{tool_call_id or uuid.uuid4()}" if litellm_call_id else str(uuid.uuid4()) ) logging_metadata: dict[str, object] = { "tool_call_id": tool_call_id, diff --git a/litellm/responses/mcp/round_billing.py b/litellm/responses/mcp/round_billing.py index f20d01785a5..ae34ab31633 100644 --- a/litellm/responses/mcp/round_billing.py +++ b/litellm/responses/mcp/round_billing.py @@ -2,21 +2,24 @@ from asyncio import CancelledError from collections.abc import Generator, Mapping from contextlib import contextmanager from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from pydantic import TypeAdapter +from pydantic_core import to_jsonable_python -from litellm.litellm_core_utils.hidden_params import served_from_cache +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import CostBreakdown +from litellm.types.utils import CostBreakdown, ModelResponse, Usage _PER_REQUEST_KEYS: Final = frozenset({"discount_percent", "margin_percent", "margin_fixed_amount"}) _TOTALS_WITH_FIXED_MARGIN: Final = ("total_cost", "margin_total_amount") _COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown) _STR_KEYED: Final = TypeAdapter(Mapping[str, object]) +_Round = TypeVar("_Round", ResponsesAPIResponse, ModelResponse) + def _summed(first: object, final: object) -> object: if first is None: @@ -49,10 +52,43 @@ def merged_round_usage(first: ResponseAPIUsage | None, final: ResponseAPIUsage | return ResponseAPIUsage.model_validate(_summed_mapping(first.model_dump(), final.model_dump())) +def _merged_chat_usage(first: Usage, final: Usage) -> Usage: + summed: Final = _summed_mapping(first.model_dump(), final.model_dump()) + return Usage.model_validate(to_jsonable_python(summed, fallback=dict)) + + +def _chat_usage(round_response: ResponsesAPIResponse | ModelResponse) -> Usage | None: + if isinstance(round_response, ResponsesAPIResponse): + return ( + None + if round_response.usage is None + else ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(round_response.usage) + ) + usage: Final = getattr(round_response, "usage", None) + return usage if isinstance(usage, Usage) else None + + +def _response_cost(round_response: ResponsesAPIResponse | ModelResponse) -> object: + hidden_params: Final = get_hidden_params(round_response) + return None if hidden_params is None else hidden_params.get("response_cost") + + +def _usage_of_both_rounds( + first: ResponsesAPIResponse | ModelResponse, final: ResponsesAPIResponse | ModelResponse +) -> ResponseAPIUsage | Usage | None: + if isinstance(first, ResponsesAPIResponse) and isinstance(final, ResponsesAPIResponse): + return merged_round_usage(first.usage, final.usage) + first_usage: Final = _chat_usage(first) + final_usage: Final = _chat_usage(final) + if first_usage is None or final_usage is None: + return final_usage if first_usage is None else first_usage + return _merged_chat_usage(first_usage, final_usage) + + def _repeated_fixed_margin( - first: ResponsesAPIResponse, + first: ResponsesAPIResponse | ModelResponse, first_breakdown: CostBreakdown | None, - final: ResponsesAPIResponse, + final: ResponsesAPIResponse | ModelResponse, final_breakdown: CostBreakdown | None, ) -> float: if first_breakdown is None or final_breakdown is None or first_breakdown is final_breakdown: @@ -70,9 +106,9 @@ def _less_repeated_fixed_margin(summed: Mapping[str, object], repeated: float) - def summed_cost_breakdown( - first: ResponsesAPIResponse, + first: ResponsesAPIResponse | ModelResponse, first_breakdown: CostBreakdown | None, - final: ResponsesAPIResponse, + final: ResponsesAPIResponse | ModelResponse, final_breakdown: CostBreakdown | None, ) -> CostBreakdown | None: """The breakdown of the charged rounds, or None when a charged round was priced without its own breakdown, since a @@ -94,41 +130,38 @@ def summed_cost_breakdown( def billed_for_every_round( - first: ResponsesAPIResponse, + first: _Round, first_breakdown: CostBreakdown | None, - final: ResponsesAPIResponse, + final: _Round, final_breakdown: CostBreakdown | None, -) -> ResponsesAPIResponse: +) -> _Round: """The final round's response carrying the usage and cost of both rounds, so the one logged row bills the request. Costs are summed per round rather than repriced from the summed usage, because a round's price can depend on its own size (tiered rates) or on charges the usage does not show (built-in tools, provider-reported costs). The fixed margin both rounds added is charged once, since it is a per-request fee. """ - first_cost: Final = first.hidden_params.get("response_cost") - final_cost: Final = final.hidden_params.get("response_cost") + first_cost: Final = _response_cost(first) + final_cost: Final = _response_cost(final) total_cost: Final = ( first_cost + final_cost - _repeated_fixed_margin(first, first_breakdown, final, final_breakdown) if isinstance(first_cost, (int, float)) and isinstance(final_cost, (int, float)) else None ) - billed: Final = final.model_copy(update=MappingProxyType({"usage": merged_round_usage(first.usage, final.usage)})) - billed.hidden_params = { - **final.hidden_params, - "response_cost": total_cost, - } + billed: Final = final.model_copy(update=MappingProxyType({"usage": _usage_of_both_rounds(first, final)})) + set_hidden_params(billed, {**(get_hidden_params(final) or {}), "response_cost": total_cost}) return billed @contextmanager -def first_round_billed_on_failure(logging_obj: object, first: ResponsesAPIResponse) -> Generator[None]: +def first_round_billed_on_failure(logging_obj: object, first: ResponsesAPIResponse | ModelResponse) -> Generator[None]: try: yield except (Exception, CancelledError): - first_cost: Final = first.hidden_params.get("response_cost") - if isinstance(logging_obj, LiteLLMLoggingObj) and first.usage is not None and not served_from_cache(first): + first_cost: Final = _response_cost(first) + first_usage: Final = _chat_usage(first) + if isinstance(logging_obj, LiteLLMLoggingObj) and first_usage is not None and not served_from_cache(first): logging_obj.record_partial_usage_for_failure( - ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(first.usage), - float(first_cost) if isinstance(first_cost, (int, float)) else 0.0, + first_usage, float(first_cost) if isinstance(first_cost, (int, float)) else 0.0 ) raise diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index 4edf7b620f2..c5ffb88506a 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass from hashlib import sha256 +from itertools import chain from pathlib import Path from typing import Final, Literal, TypeVar @@ -54,6 +55,10 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat { **stock, "general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True}, + "litellm_settings": { + **object_value(stock["litellm_settings"]), + "cost_margin_config": {"hosted_vllm": {"fixed_amount": FIXED_MARGIN}}, + }, } ) ) @@ -81,6 +86,8 @@ ANSWER: Final = "the sum is 5" GATEWAY_REF: Final = {"type": "mcp", "server_url": "litellm_proxy", "server_label": "litellm"} AUTO: Final = {**GATEWAY_REF, "require_approval": "never"} OUTAGE: Final = "bridge-outage" +DIRECT: Final = "answer-without-tools" +FIXED_MARGIN: Final = 0.25 JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -169,17 +176,27 @@ def _tool_result_block(item: Mapping[str, JsonValue]) -> str | None: return None -def _responses_stream(response: Mapping[str, JsonValue], item: Mapping[str, JsonValue]) -> Reply: +def _item_events(index: int, item: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + return ( + {"type": "response.output_item.added", "output_index": index, "item": dict(item)}, + {"type": "response.output_item.done", "output_index": index, "item": dict(item)}, + ) + + +def _responses_stream(response: Mapping[str, JsonValue], items: Sequence[Mapping[str, JsonValue]]) -> Reply: + item_events: Final = tuple(chain.from_iterable(_item_events(index, item) for index, item in enumerate(items))) events: Final = ( - {"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}}, - {"type": "response.in_progress", "sequence_number": 1, "response": {**response, "status": "in_progress"}}, - {"type": "response.output_item.added", "sequence_number": 2, "output_index": 0, "item": item}, - {"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": item}, - {"type": "response.completed", "sequence_number": 4, "response": response}, + {"type": "response.created", "response": {**response, "status": "in_progress"}}, + {"type": "response.in_progress", "response": {**response, "status": "in_progress"}}, + *item_events, + {"type": "response.completed", "response": response}, ) return Reply( content_type="text/event-stream", - chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + chunks=tuple( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode() + for number, event in enumerate(events) + ), ) @@ -233,7 +250,7 @@ def _anthropic_tool_result_rule(body: Mapping[str, JsonValue]) -> Reply | None: def _model_double( - plan: Callable[[Mapping[str, JsonValue]], Turn], billing: Billing = EVEN_ROUNDS + plan: Callable[[Mapping[str, JsonValue]], Turn], billing: Billing = EVEN_ROUNDS, parallel: int = 1 ) -> Callable[[Request], Reply]: def respond(request: Request) -> Reply: if request.method == "GET" and request.target.endswith("/models"): @@ -243,11 +260,13 @@ def _model_double( outage: Final = {"error": {"message": "scripted provider outage", "type": "server_error", "code": None}} return Reply(status=500, body=json.dumps(outage).encode()) turn: Final = plan(body) - done: Final = _has_tool_result(body) - if done and billing.follow_up_fails: + answered: Final = _has_tool_result(body) + if answered and billing.follow_up_fails: failure: Final = {"error": {"message": FOLLOW_UP_OUTAGE, "type": "server_error", "code": None}} return Reply(status=500, body=json.dumps(failure).encode()) - prompt_tokens, completion_tokens = billing.follow_up if done else billing.first + done: Final = answered or DIRECT in _prompt(body) + calls: Final = range(1, parallel + 1) + prompt_tokens, completion_tokens = billing.follow_up if answered else billing.first identity: Final = uuid.uuid4().hex[:12] usage: Final = { "prompt_tokens": prompt_tokens, @@ -255,27 +274,21 @@ def _model_double( "total_tokens": prompt_tokens + completion_tokens, } if request.target.endswith("/chat/completions"): + requested: Final = tuple( + {"id": f"call_{call}", "type": "function", "function": {"name": turn.tool, "arguments": turn.arguments}} + for call in calls + ) message: Final = ( {"role": "assistant", "content": turn.answer} if done - else { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": turn.tool, "arguments": turn.arguments}, - } - ], - } + else {"role": "assistant", "content": None, "tool_calls": [*requested]} ) finish: Final = "stop" if done else "tool_calls" if body.get("stream") is True: delta: Final = ( - {**message, "tool_calls": [{**call, "index": 0} for call in message["tool_calls"]]} - if "tool_calls" in message - else message + message + if done + else {**message, "tool_calls": [{**call, "index": index} for index, call in enumerate(requested)]} ) chunk: Final = { "id": f"chatcmpl-{identity}", @@ -307,7 +320,10 @@ def _model_double( content: Final = ( [{"type": "text", "text": turn.answer}] if done - else [{"type": "tool_use", "id": "toolu_1", "name": turn.tool, "input": json.loads(turn.arguments)}] + else [ + {"type": "tool_use", "id": f"toolu_{call}", "name": turn.tool, "input": json.loads(turn.arguments)} + for call in calls + ] ) return _json( { @@ -322,23 +338,28 @@ def _model_double( } ) assert request.target.endswith("/responses"), request.target - item: Final[dict[str, JsonValue]] = ( - { - "type": "message", - "id": f"msg_{identity}", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": turn.answer, "annotations": []}], - } + items: Final[tuple[dict[str, JsonValue], ...]] = ( + ( + { + "type": "message", + "id": f"msg_{identity}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": turn.answer, "annotations": []}], + }, + ) if done - else { - "type": "function_call", - "id": f"fc_{identity}", - "call_id": f"call_{identity}", - "name": turn.tool, - "arguments": turn.arguments, - "status": "completed", - } + else tuple( + { + "type": "function_call", + "id": f"fc_{identity}_{call}", + "call_id": f"call_{identity}_{call}", + "name": turn.tool, + "arguments": turn.arguments, + "status": "completed", + } + for call in calls + ) ) response: Final[dict[str, JsonValue]] = { "id": f"resp_{identity}", @@ -346,14 +367,14 @@ def _model_double( "created_at": 1, "status": "completed", "model": "gpt-4o-mini", - "output": [item], + "output": [*items], "usage": { "input_tokens": prompt_tokens, "output_tokens": completion_tokens, "total_tokens": prompt_tokens + completion_tokens, }, } - return _responses_stream(response, item) if body.get("stream") is True else _json(response) + return _responses_stream(response, items) if body.get("stream") is True else _json(response) return respond @@ -373,17 +394,19 @@ class Rig: def tool(self) -> str: return f"{self.alias}-add" - def send(self, key: str, tools: Sequence[Mapping[str, object]], **extra: object) -> httpx.Response: - prompt: Final = f"add {self.alias}" + def send( + self, key: str, tools: Sequence[Mapping[str, object]], prompt: str | None = None, **extra: object + ) -> httpx.Response: + text: Final = prompt or f"add {self.alias}" headers: Final = {"Authorization": f"Bearer {key}"} if self.surface == "chat": - body: Final = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "tools": list(tools)} + body: Final = {"model": self.model, "messages": [{"role": "user", "content": text}], "tools": list(tools)} return self.gateway.client.post("/v1/chat/completions", headers=headers, json={**body, **extra}, timeout=90) if self.surface == "responses": return self.gateway.client.post( "/v1/responses", headers=headers, - json={"model": self.model, "input": prompt, "tools": list(tools), **extra}, + json={"model": self.model, "input": text, "tools": list(tools), **extra}, timeout=90, ) return self.gateway.client.post( @@ -392,7 +415,7 @@ class Rig: json={ "model": self.model, "max_tokens": 64, - "messages": [{"role": "user", "content": prompt}], + "messages": [{"role": "user", "content": text}], "tools": list(tools), **extra, }, @@ -451,11 +474,12 @@ def _rig( upstream: Surface | None = None, billing: Billing = EVEN_ROUNDS, prices: Mapping[str, JsonValue] | None = None, + parallel: int = 1, ) -> Iterator[Rig]: alias: Final = "llm" + uuid.uuid4().hex[:8] with ( mcp_peer() as peer, - wire_server(_model_double(_fixed_turn(f"{alias}-add"), billing)) as wire, + wire_server(_model_double(_fixed_turn(f"{alias}-add"), billing, parallel)) as wire, gateway.scenario() as scenario, ): server_id: Final = register_mcp(scenario, peer, alias) @@ -753,12 +777,12 @@ def _price(*rounds: tuple[int, int]) -> float: return sum(prompt * input_price + completion * output_price for prompt, completion in rounds) -def _responses_rows(call_id: str, status: str) -> list[dict[str, JsonValue]]: +def _spend_rows(call_id: str, call_type: str, status: str) -> list[dict[str, JsonValue]]: return eventually( lambda: read_rows( "SELECT request_id, spend, prompt_tokens, completion_tokens, session_id " 'FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s AND status=%s', - (call_id, "aresponses", status), + (call_id, call_type, status), ), lambda found: len(found) >= 1, seconds=70, @@ -791,7 +815,7 @@ def test_nonstream_responses_auto_execute_logs_the_answer_and_bills_both_rounds( assert response.status_code == 200, response.text body: Final = JSON_OBJECT.validate_json(response.content) assert rig.final_text(body) == ANSWER, response.text - rows: Final = _responses_rows(response.headers["x-litellm-call-id"], "success") + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "aresponses", "success") assert len(rows) == 1, rows row: Final = rows[0] assert row["request_id"] == body["id"], row @@ -810,24 +834,123 @@ def test_nonstream_responses_auto_execute_logs_the_answer_and_bills_both_rounds( ) -def test_nonstream_responses_failed_follow_up_bills_round_one_on_the_failure_row(gateway: Gateway) -> None: +def test_nonstream_chat_auto_execute_logs_the_answer_and_bills_both_rounds(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO]) + assert response.status_code == 200, response.text + body: Final = JSON_OBJECT.validate_json(response.content) + assert rig.final_text(body) == ANSWER, response.text + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1, rows + row: Final = rows[0] + assert row["request_id"] == body["id"], row + assert (row["prompt_tokens"], row["completion_tokens"]) == ( + FIRST_ROUND[0] + FOLLOW_UP_ROUND[0], + FIRST_ROUND[1] + FOLLOW_UP_ROUND[1], + ), row + assert object_value(body["usage"])["prompt_tokens"] == row["prompt_tokens"], body + assert float(str(row["spend"])) == pytest.approx(_price(FIRST_ROUND, FOLLOW_UP_ROUND)), row + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(float(str(row["spend"]))) + + +def test_nonstream_chat_auto_execute_without_a_tool_call_bills_the_one_call_it_made(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], prompt=f"{DIRECT} {rig.alias}") + assert response.status_code == 200, response.text + body: Final = JSON_OBJECT.validate_json(response.content) + assert rig.final_text(body) == ANSWER, response.text + assert len(rig.upstream_tools()) == 1 and _peer_add_calls(rig.peer) == () + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1 and rows[0]["request_id"] == body["id"], rows + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == FIRST_ROUND, rows + assert float(str(rows[0]["spend"])) == pytest.approx(_price(FIRST_ROUND)), rows + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(float(str(rows[0]["spend"]))) + + +def test_nonstream_chat_auto_execute_charges_the_fixed_margin_once(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", upstream="messages_bridge", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO]) + assert response.status_code == 200, response.text + assert len(rig.upstream_tools()) == 2 + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1, rows + assert float(str(rows[0]["spend"])) == pytest.approx(_price(FIRST_ROUND, FOLLOW_UP_ROUND) + FIXED_MARGIN), rows + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(float(str(rows[0]["spend"]))) + + +def test_nonstream_chat_auto_execute_whose_calls_both_come_from_the_response_cache_bills_nothing( + gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + key: Final = _granted_key(rig) + paid: Final = rig.send(key, [AUTO]) + assert paid.status_code == 200, paid.text + assert len(rig.upstream_tools()) == 2 and len(_peer_add_calls(rig.peer)) == 1 + cached: Final = rig.send(key, [{**AUTO, "server_label": "relabeled"}]) + assert cached.status_code == 200, cached.text + assert rig.final_text(JSON_OBJECT.validate_json(cached.content)) == ANSWER, cached.text + assert rig.upstream_tools() == () and len(_peer_add_calls(rig.peer)) == 1 + rows: Final = _spend_rows(cached.headers["x-litellm-call-id"], "acompletion", "success") + assert all(float(str(row["spend"])) == 0.0 for row in rows), rows + assert float(cached.headers["x-litellm-response-cost"]) == 0.0 + + +@pytest.mark.parametrize(("surface", "call_type"), (("responses", "aresponses"), ("chat", "acompletion"))) +def test_nonstream_failed_follow_up_bills_round_one_on_the_failure_row( + gateway: Gateway, surface: Surface, call_type: str +) -> None: billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) - with _rig(gateway, "responses", billing=billing, prices=PRICES) as rig: + with _rig(gateway, surface, billing=billing, prices=PRICES) as rig: response: Final = rig.send(_granted_key(rig), [AUTO]) assert response.status_code == 500 and FOLLOW_UP_OUTAGE in response.text, response.text call_id: Final = response.headers["x-litellm-call-id"] - rows: Final = _responses_rows(call_id, "failure") + rows: Final = _spend_rows(call_id, call_type, "failure") assert len(rows) == 1, rows assert float(str(rows[0]["spend"])) == pytest.approx(_price(FIRST_ROUND)), rows assert ( read_rows( 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s AND status=%s', - (call_id, "aresponses", "success"), + (call_id, call_type, "success"), ) == [] ) +def _tool_rows(call_id: str, expected: int) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE call_type=%s AND strpos(request_id, %s) = 1', + ("call_mcp_tool", call_id), + ), + lambda found: len(found) >= expected, + seconds=70, + ) + + +@pytest.mark.parametrize( + ("surface", "stream"), + (("chat", False), ("chat", True), ("messages", False), ("responses", True), ("responses", False)), + ids=("chat", "chat-stream", "messages", "responses-stream", "responses"), +) +def test_every_auto_executed_tool_call_lands_its_own_spend_row_under_the_request_call_id( + gateway: Gateway, surface: Surface, stream: bool +) -> None: + with _rig(gateway, surface, parallel=2) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=stream) + assert response.status_code == 200, response.text + assert [_add_call_arguments(call) for call in _peer_add_calls(rig.peer)] == [ADD, ADD] + call_id: Final = response.headers["x-litellm-call-id"] + request_ids: Final = frozenset(str(row["request_id"]) for row in _tool_rows(call_id, 2)) + assert len(request_ids) == 2 and all(request_id.startswith(f"{call_id}_") for request_id in request_ids), ( + call_id, + request_ids, + ) + + def test_bridged_responses_auto_execute_answers_every_tool_use_with_its_result(gateway: Gateway) -> None: with _rig(gateway, "responses", upstream="messages") as rig: response: Final = rig.send(_granted_key(rig), [AUTO]) diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index 27477f24d92..9064eba161d 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -1,14 +1,23 @@ +import asyncio import json import sys import types +from collections.abc import Mapping, Sequence +from datetime import datetime +from types import MappingProxyType +from typing import Final, cast import pytest import respx from httpx import Response +from mcp.types import Tool as MCPTool from unittest.mock import AsyncMock, patch import litellm -from litellm.types.utils import ModelResponse +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.utils import ModelResponse, StandardLoggingPayload from litellm.responses.mcp import chat_completions_handler from litellm.responses.mcp.chat_completions_handler import ( @@ -1479,3 +1488,226 @@ async def test_request_selected_mcp_guardrail_blocks_before_upstream(monkeypatch assert upstream.await_count == (0 if selected else 1) tool_message = model_call.await_args.kwargs["messages"][-1] assert ("request-selected MCP block" in tool_message["content"]) is selected + + +_ECHO_ARGUMENTS: Final = '{"value": "PING"}' +_CHAT_ROUND_ONE: Final = MappingProxyType( + { + "id": "chatcmpl-round-one", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": ( + MappingProxyType( + { + "index": 0, + "finish_reason": "tool_calls", + "message": MappingProxyType( + { + "role": "assistant", + "content": None, + "tool_calls": ( + MappingProxyType( + { + "id": "call_echo", + "type": "function", + "function": MappingProxyType({"name": "echo", "arguments": _ECHO_ARGUMENTS}), + } + ), + ), + } + ), + } + ), + ), + "usage": MappingProxyType( + { + "prompt_tokens": 71, + "completion_tokens": 19, + "total_tokens": 90, + "prompt_tokens_details": MappingProxyType({"cached_tokens": 32}), + } + ), + } +) +_CHAT_ROUND_TWO: Final = MappingProxyType( + { + "id": "chatcmpl-round-two", + "object": "chat.completion", + "created": 2, + "model": "gpt-5.6", + "choices": ( + MappingProxyType( + { + "index": 0, + "finish_reason": "stop", + "message": MappingProxyType({"role": "assistant", "content": "PING"}), + } + ), + ), + "usage": MappingProxyType( + { + "prompt_tokens": 149, + "completion_tokens": 5, + "total_tokens": 154, + "prompt_tokens_details": MappingProxyType({"cached_tokens": 64}), + } + ), + } +) +_CONTEXT_WINDOW_ERROR: Final = MappingProxyType( + { + "error": MappingProxyType( + { + "message": "Your input exceeds the context window of this model.", + "type": "invalid_request_error", + "param": "messages", + "code": "context_length_exceeded", + } + ) + } +) + + +class _ChatRoundRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged: tuple[StandardLoggingPayload, ...] = () + self.failed: tuple[StandardLoggingPayload, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.logged = (*self.logged, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.failed = (*self.failed, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + + +def _chat_round_cost(body: Mapping[str, object]) -> float: + return litellm.completion_cost( + completion_response=ModelResponse(**json.loads(json.dumps(body, default=dict))), + model="gpt-5.6", + custom_llm_provider="openai", + ) + + +async def _settled() -> None: + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) + + +async def _auto_execute_chat_echo( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str +) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://mcp-chat-rounds.example.com/v1/chat/completions").mock(side_effect=list(rounds)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = Logging( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + stream=False, + call_type="acompletion", + start_time=datetime(2026, 1, 1), + litellm_call_id=call_id, + function_id=call_id, + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + echo_tool: Final = MCPTool.model_validate( + {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, + by_name=False, + ) + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + AsyncMock(return_value=([echo_tool], {"echo": "demo"})), + ), + patch.object( + LiteLLM_Proxy_MCP_Handler, + "execute_tool_calls", + AsyncMock(return_value=[{"tool_call_id": "call_echo", "result": "PING", "name": "echo"}]), + ), + ): + try: + response: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + litellm_logging_obj=logging_obj, + ) + except Exception as error: + await _settled() + return recorder, error + await _settled() + assert isinstance(response, ModelResponse) + return recorder, response + + +def _ok(body: Mapping[str, object]) -> Response: + return Response(200, content=json.dumps(body, default=dict)) + + +@pytest.mark.asyncio +async def test_non_stream_auto_execute_logs_one_row_billing_both_rounds_and_matching_the_client_cost( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + recorder, response = await _auto_execute_chat_echo( + respx_mock, monkeypatch, (_ok(_CHAT_ROUND_ONE), _ok(_CHAT_ROUND_TWO)), "mcp-chat-rounds-both" + ) + + assert isinstance(response, ModelResponse) + assert response.choices[0].message.content == "PING" + both_rounds: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + assert row["id"] == response.id + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["metadata"]["usage_object"]["prompt_tokens_details"]["cached_tokens"] == 32 + 64 + assert row["response_cost"] == pytest.approx(both_rounds) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(both_rounds) + assert response.hidden_params["response_cost"] == pytest.approx(both_rounds) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (71 + 149, 19 + 5) + assert recorder.failed == () + + +@pytest.mark.asyncio +async def test_non_stream_auto_execute_without_a_tool_call_logs_the_one_round_it_made( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + recorder, response = await _auto_execute_chat_echo( + respx_mock, monkeypatch, (_ok(_CHAT_ROUND_TWO),), "mcp-chat-rounds-no-tool-call" + ) + + assert isinstance(response, ModelResponse) + assert len(recorder.logged) == 1 + assert recorder.logged[0]["id"] == response.id + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (149, 5) + assert recorder.logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_TWO)) + assert response.hidden_params["response_cost"] == pytest.approx(recorder.logged[0]["response_cost"]) + + +@pytest.mark.asyncio +async def test_non_stream_auto_execute_bills_the_paid_first_round_when_the_follow_up_fails( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + recorder, outcome = await _auto_execute_chat_echo( + respx_mock, + monkeypatch, + (_ok(_CHAT_ROUND_ONE), Response(400, content=json.dumps(_CONTEXT_WINDOW_ERROR, default=dict))), + "mcp-chat-rounds-follow-up-fails", + ) + + assert isinstance(outcome, litellm.BadRequestError) + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71, 19) + assert recorder.failed[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + assert recorder.failed[0]["response_cost"] > 0 diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index f43e15cb342..26553558298 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -420,20 +420,16 @@ async def test_execute_tool_calls_logs_failure_via_post_call_failure_hook(monkey @pytest.mark.asyncio -async def test_execute_tool_calls_passes_litellm_call_id_and_trace_id_to_function_setup( +async def test_execute_tool_calls_logs_each_tool_call_under_its_own_id_on_the_request_trace( monkeypatch, ): - """ - Regression test for ae4d92ad...: - Ensure litellm_call_id / litellm_trace_id are forwarded into function_setup kwargs. - """ _setup_proxy_logging(monkeypatch) call_tool_mock = _setup_mcp_call_environment(monkeypatch) - captured = {} + captured: list[dict[str, object]] = [] def fake_function_setup(*_args, **kwargs): - captured.update(kwargs) + captured.append(kwargs) return None, None # NOTE: Don't patch via dotted string path here because `litellm.responses` @@ -443,7 +439,10 @@ async def test_execute_tool_calls_passes_litellm_call_id_and_trace_id_to_functio monkeypatch.setattr(handler_module, "function_setup", fake_function_setup) tool_name = "deepwiki-read_wiki_structure" - tool_calls = [{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}] + tool_calls = [ + {"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}, + {"id": "call-2", "function": {"name": tool_name, "arguments": "{}"}}, + ] await LiteLLM_Proxy_MCP_Handler.execute_tool_calls( tool_server_map={tool_name: "deepwiki"}, @@ -453,11 +452,13 @@ async def test_execute_tool_calls_passes_litellm_call_id_and_trace_id_to_functio litellm_trace_id="tid", ) - # Ensure the tool call was attempted (sanity) - assert call_tool_mock.await_count == 1 - - assert captured.get("litellm_call_id") == "cid" - assert captured.get("litellm_trace_id") == "tid" + assert call_tool_mock.await_count == 2 + tool_call_ids: Final = [kwargs.get("litellm_call_id") for kwargs in captured] + assert len(tool_call_ids) == 2 + assert "cid" not in tool_call_ids + assert len(set(tool_call_ids)) == 2 + assert all(isinstance(tool_call_id, str) and tool_call_id.startswith("cid") for tool_call_id in tool_call_ids) + assert [kwargs.get("litellm_trace_id") for kwargs in captured] == ["tid", "tid"] @pytest.mark.asyncio diff --git a/tests/unit/responses/mcp/test_round_billing.py b/tests/unit/responses/mcp/test_round_billing.py index 993b38de16a..99bb931bc27 100644 --- a/tests/unit/responses/mcp/test_round_billing.py +++ b/tests/unit/responses/mcp/test_round_billing.py @@ -4,9 +4,10 @@ from typing import Final import pytest +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.responses.mcp.round_billing import billed_for_every_round, merged_round_usage, summed_cost_breakdown from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import CostBreakdown +from litellm.types.utils import CostBreakdown, ModelResponse, Usage def _round(response_id: str, response_cost: object, served_from_cache: bool = False) -> ResponsesAPIResponse: @@ -216,3 +217,71 @@ def test_a_round_served_from_the_response_cache_drops_out_of_the_breakdown( ) assert breakdown == (None if kept_breakdown is None else breakdowns[kept_breakdown]) + + +def _anthropic_chat_round(response_id: str, anthropic_usage: Mapping[str, object]) -> ModelResponse: + response: Final = ModelResponse( + id=response_id, + model="claude-haiku-5-5", + usage=AnthropicConfig().calculate_usage(anthropic_usage, reasoning_content=None), + ) + response._hidden_params["response_cost"] = 0.001 + return response + + +def _usage_of(response: ModelResponse) -> Usage: + usage: Final = response.usage + assert isinstance(usage, Usage) + return usage + + +def test_billed_chat_usage_sums_every_provider_counter_and_adds_no_field_the_rounds_lacked(): + first: Final = _anthropic_chat_round( + "msg_first", + MappingProxyType( + { + "input_tokens": 400, + "output_tokens": 10, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 30, + "cache_creation": MappingProxyType({"ephemeral_5m_input_tokens": 50, "ephemeral_1h_input_tokens": 0}), + "server_tool_use": MappingProxyType({"web_search_requests": 1}), + "service_tier": "standard", + } + ), + ) + final: Final = _anthropic_chat_round( + "msg_final", + MappingProxyType( + { + "input_tokens": 500, + "output_tokens": 20, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 80, + "cache_creation": MappingProxyType({"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}), + "server_tool_use": MappingProxyType({"web_search_requests": 2}), + "service_tier": "priority", + } + ), + ) + + billed: Final = _usage_of(billed_for_every_round(first=first, first_breakdown=None, final=final, final_breakdown=None)) + first_usage: Final = _usage_of(first) + final_usage: Final = _usage_of(final) + + assert billed.model_dump(exclude_none=True).keys() == first_usage.model_dump(exclude_none=True).keys() + assert billed.prompt_tokens == first_usage.prompt_tokens + final_usage.prompt_tokens + assert billed.completion_tokens == first_usage.completion_tokens + final_usage.completion_tokens + assert billed["cache_read_input_tokens"] == 30 + 80 + assert billed["cache_creation_input_tokens"] == 50 + 0 + assert billed.server_tool_use is not None + assert billed.server_tool_use.web_search_requests == 1 + 2 + assert billed.prompt_tokens_details is not None + assert billed.prompt_tokens_details.cached_tokens == 30 + 80 + assert billed.prompt_tokens_details.model_dump(exclude_none=True) == { + key: first_value + final_value + for key, first_value in first_usage.prompt_tokens_details.model_dump(exclude_none=True).items() + if isinstance(first_value, int) + and isinstance(final_value := final_usage.prompt_tokens_details.model_dump()[key], int) + } | {"cache_creation_token_details": {"ephemeral_5m_input_tokens": 50, "ephemeral_1h_input_tokens": 0}} + assert billed["service_tier"] == "priority" From c4d24ec2f118e2bc60588d9681fc82754fd15325 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:39:35 -0700 Subject: [PATCH 064/124] test(integration): remove BUG skips for 3 tests that pass on main (#47364) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../observability/test_arize_otel_v2_openinference_spans.py | 1 - .../observability/test_guardrail_stream_scope_chaos.py | 1 - tests/integration/providers/test_vector_store_create_wire.py | 3 --- 3 files changed, 5 deletions(-) diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py index 122ada65f0e..ab79e81289e 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_spans.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_spans.py @@ -925,7 +925,6 @@ def test_arize_otel_v2_a_cache(surface: str, gateway: Gateway, tmp_path: Path) - def test_arize_otel_v2_cache_hit_exports_llm_span(gateway: Gateway, tmp_path: Path) -> None: - pytest.skip("BUG: LIT-9127 response-cache hits emit no OTel v2 LLM span") marker: Final = "a-cache-hit-" + uuid.uuid4().hex sentinel: Final = f"{marker}-sentinel" diff --git a/tests/integration/observability/test_guardrail_stream_scope_chaos.py b/tests/integration/observability/test_guardrail_stream_scope_chaos.py index eb534161ab4..a332f951ba1 100644 --- a/tests/integration/observability/test_guardrail_stream_scope_chaos.py +++ b/tests/integration/observability/test_guardrail_stream_scope_chaos.py @@ -961,7 +961,6 @@ def test_h6_spend_rows_for_requests_served_during_postgres_restart_land_once( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - pytest.skip("BUG: LIT-9050 spend rows for requests served during a Postgres restart are dropped") postgres_directory: Final = Path(os.environ["INTEGRATION_RESULTS_DIR"]) / f"owned-postgres-{uuid.uuid4().hex}" with _owned_postgres(postgres_directory) as database, wire_server(_sink) as sink: monkeypatch.setenv("INTEGRATION_PROXY_DATABASE_URL", database.database_url) diff --git a/tests/integration/providers/test_vector_store_create_wire.py b/tests/integration/providers/test_vector_store_create_wire.py index fd166da1763..0968bf72827 100644 --- a/tests/integration/providers/test_vector_store_create_wire.py +++ b/tests/integration/providers/test_vector_store_create_wire.py @@ -419,9 +419,6 @@ def test_vector_store_create_translates_managed_file_ids(gateway: Gateway) -> No def test_vector_store_create_forwards_only_caller_metadata(gateway: Gateway) -> None: - pytest.skip( - "BUG: vector store create forwards proxy-internal metadata (user_api_key_hash, endpoint, api_base, ...) to the provider" - ) provider_key: Final = f"provider-create-metadata-{uuid.uuid4().hex}" provider_id: Final = f"vs-metadata-{uuid.uuid4().hex}" From da8055289773ef4751f45c09de117ba1ed698970 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:40:03 -0700 Subject: [PATCH 065/124] test(translation): add sonnet 5.5 converse and gemini direct cross-model mid rows (#47328) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_completions/cross_model/README.md | 4 +- .../cross_model/bedrock_converse.py | 440 ++++++++++ .../chat_completions/cross_model/case.py | 16 + .../chat_completions/cross_model/gemini.py | 769 ++++++++++++++++++ 4 files changed, 1227 insertions(+), 2 deletions(-) create mode 100644 tests/integration/translation/chat_completions/cross_model/gemini.py diff --git a/tests/integration/translation/chat_completions/cross_model/README.md b/tests/integration/translation/chat_completions/cross_model/README.md index ee7c178da81..90710d5597c 100644 --- a/tests/integration/translation/chat_completions/cross_model/README.md +++ b/tests/integration/translation/chat_completions/cross_model/README.md @@ -8,7 +8,7 @@ Every row sends `CODING_AGENT_REQUEST` to `/v1/chat/completions`, changing only | Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 Terra | | High | Fable 5 and 5.1, Gemini 4 Argon, GPT 6 Astra | -The 48 rows cover Anthropic direct, OpenAI direct, Vertex, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented +The 51 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented The request contains system instructions, conversation history, previous assistant tool calls and results, four function definitions, automatic parallel tools, low reasoning effort and an output limit. Each row exercises a tool-call response, raw streaming, SDK stream assembly and a follow-up that replays the returned assistant message and tool-call IDs @@ -33,7 +33,7 @@ Streaming cases store `expected_litellm_message` and `expected_litellm_usage`. M Usage requires nonnegative integer counts, positive prompt and completion counts, and `total_tokens == prompt_tokens + completion_tokens`. The complete normalized usage must equal the independently expected counters, including cached and reasoning tokens and derived text/cache details. Missing, changed or unexpected counters fail. Only null optional fields, the explicitly listed provider metadata/duplicate counters and computed `cost` are excluded. Thus a wrong prompt count with an adjusted total still fails -The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 48 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips +The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 51 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips ## Recorded upstreams diff --git a/tests/integration/translation/chat_completions/cross_model/bedrock_converse.py b/tests/integration/translation/chat_completions/cross_model/bedrock_converse.py index 14a2986d132..49f8edea1be 100644 --- a/tests/integration/translation/chat_completions/cross_model/bedrock_converse.py +++ b/tests/integration/translation/chat_completions/cross_model/bedrock_converse.py @@ -14,6 +14,7 @@ from integration.translation.chat_completions.bases.bedrock_converse import ( CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_OPUS_5_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, GPT_5_6_LUNA_TEST_CASE, GPT_5_6_SOL_TEST_CASE, GPT_5_6_TERRA_TEST_CASE, @@ -3192,6 +3193,445 @@ CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( ) +BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST: Final = { + **BEDROCK_CONVERSE_AGENT_REQUEST, + "additionalModelRequestFields": { + "thinking": {"type": "adaptive", "display": "summarized"}, + "tool_choice": {"disable_parallel_tool_use": False}, + "output_config": {"effort": "low"}, + }, + "additionalModelResponseFieldPaths": ["/usage/output_tokens_details"], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + } + }, + "name": "read_file", + "description": "Read a UTF-8 file in the repository.", + } + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + } + }, + "name": "search_code", + "description": "Search repository source for a literal query.", + } + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + } + }, + "name": "run_tests", + "description": "Run a specific pytest test path and return the exit code and output.", + } + }, + { + "toolSpec": { + "inputSchema": { + "json": { + "type": "object", + "properties": {"patch": {"type": "string"}}, + "required": ["patch"], + } + }, + "name": "apply_patch", + "description": "Apply a unified diff to repository files.", + } + }, + ], + "toolChoice": {"auto": {}}, + }, +} + + +CLAUDE_SONNET_5_5_AGENT_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="agent_tools", + litellm_request={**CODING_AGENT_REQUEST, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5"}, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_headers={ + "authorization": "Bearer synthetic-bedrock-key", + "content-type": "application/json", + }, + expected_provider_request=BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST, + mock_provider_response={ + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}}, + "metrics": {"latencyMs": 1294}, + "output": { + "message": { + "content": [ + { + "toolUse": { + "input": {"query": "retries_remaining"}, + "name": "search_code", + "toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G", + "type": "tool_use", + } + }, + { + "toolUse": { + "input": {"path": "tests/test_retry.py"}, + "name": "run_tests", + "toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa", + "type": "tool_use", + } + }, + ], + "role": "assistant", + } + }, + "stopReason": "tool_use", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1144, + "outputTokens": 105, + "serverToolUsage": {}, + "totalTokens": 1249, + }, + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "", + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "function": {"arguments": '{"query": "retries_remaining"}', "name": "search_code"}, + "id": "tooluse_2Z6A2nogMa1RGHgnwZXH6G", + "type": "function", + }, + { + "index": 1, + "function": {"arguments": '{"path": "tests/test_retry.py"}', "name": "run_tests"}, + "id": "tooluse_mwNqPITfkD3zc6JHkg5GJa", + "type": "function", + }, + ], + }, + } + ], + "usage": { + "completion_tokens": 105, + "prompt_tokens": 1144, + "total_tokens": 1249, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1144, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + + +CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="agent_streaming", + litellm_endpoint=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_endpoint, + litellm_request={ + **CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request, + "stream": True, + "stream_options": {"include_usage": True}, + }, + expected_provider_headers=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse-stream", + expected_provider_request=BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST, + mock_provider_frames=( + EventStreamFrame( + data={"p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX", "role": "assistant"}, + headers={":event-type": "messageStart", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={ + "contentBlockIndex": 0, + "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY", + "start": { + "toolUse": { + "name": "search_code", + "toolUseId": "tooluse_6cLTZNBlXnKXY9x3Rp4wNo", + "type": "tool_use", + } + }, + }, + headers={":event-type": "contentBlockStart", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": ""}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": '{"qu'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": 'ery": '}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": '"retries_r'}}, "p": "abcdefghijk"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "delta": {"toolUse": {"input": 'emaining"}'}}, "p": "abcdef"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 0, "p": "abcdefghijklmnopqrstu"}, + headers={":event-type": "contentBlockStop", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={ + "contentBlockIndex": 1, + "p": "abcdefghijklmnopqrstuvwxyz", + "start": { + "toolUse": { + "name": "run_tests", + "toolUseId": "tooluse_uoozKiVjrkiFbYLgq52Lh1", + "type": "tool_use", + } + }, + }, + headers={":event-type": "contentBlockStart", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": ""}}, "p": "abcdefghijklmnopqrstuvwxyzA"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": '{"p'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": 'ath": "tests'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": "/tes"}}, "p": "abcdefghijklmnopqrstuvw"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={ + "contentBlockIndex": 1, + "delta": {"toolUse": {"input": "t_retry."}}, + "p": "abcdefghijklmnopqrstuvwxyzAB", + }, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "delta": {"toolUse": {"input": 'py"}'}}, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRST"}, + headers={":event-type": "contentBlockDelta", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={"contentBlockIndex": 1, "p": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU"}, + headers={":event-type": "contentBlockStop", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={ + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}}, + "p": "abcdefghijklmnopqr", + "stopReason": "tool_use", + }, + headers={":event-type": "messageStop", ":content-type": "application/json", ":message-type": "event"}, + ), + EventStreamFrame( + data={ + "metrics": {"latencyMs": 1258}, + "p": "abcde", + "usage": {"inputTokens": 1144, "outputTokens": 105, "serverToolUsage": {}, "totalTokens": 1249}, + }, + headers={":event-type": "metadata", ":content-type": "application/json", ":message-type": "event"}, + ), + ), + expected_litellm_message=coding_agent_stream_message( + ("tooluse_6cLTZNBlXnKXY9x3Rp4wNo", "tooluse_uoozKiVjrkiFbYLgq52Lh1"), + ), + expected_litellm_usage={ + "prompt_tokens": 1144, + "completion_tokens": 105, + "total_tokens": 1249, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1144, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + }, +) + + +CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = { + **BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST, + "messages": [ + *list_value(BEDROCK_CONVERSE_AGENT_PROVIDER_REQUEST["messages"]), + { + "role": "assistant", + "content": [ + { + "toolUse": { + "input": {"query": "retries_remaining"}, + "name": "search_code", + "toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G", + } + }, + { + "toolUse": { + "input": {"path": "tests/test_retry.py"}, + "name": "run_tests", + "toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa", + } + }, + ], + }, + { + "role": "user", + "content": [ + { + "toolResult": { + "content": [ + { + "text": ( + '{"matches": [{"path": "src/worker.py", "line": 12, "text": ' + '"remaining = retries_remaining(3, attempts)"}]}' + ) + } + ], + "toolUseId": "tooluse_2Z6A2nogMa1RGHgnwZXH6G", + } + }, + { + "toolResult": { + "content": [ + { + "text": ( + '{"exit_code": 1, "output": "FAILED tests/test_retry.py::' + 'test_retries_remaining - assert 4 == 2"}' + ) + } + ], + "toolUseId": "tooluse_mwNqPITfkD3zc6JHkg5GJa", + } + }, + { + "text": ( + "Stop after these diagnostics. Do not call more tools or change files. " + "Summarize the bug, the caller found and the failed test in two short sentences." + ) + }, + ], + }, + ], +} + + +CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_AGENT_TEST_CASE, + scenario="agent_followup", + litellm_request=followup_request( + CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request, + CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_litellm_response, + ), + expected_provider_endpoint="/model/us.anthropic.claude-sonnet-5-5/converse", + expected_provider_request=CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_PROVIDER_REQUEST, + mock_provider_response={ + "additionalModelResponseFields": {"usage": {"output_tokens_details": {"thinking_tokens": 0}}}, + "metrics": {"latencyMs": 1976}, + "output": { + "message": { + "content": [ + { + "text": ( + "The bug is that `retries_remaining` in `src/retry.py` returns `limit + attempts` " + "instead of `max(0, limit - attempts)`. That gives 4 for `(3, 1)` where the test " + "expects 2, and it can't be clamped to zero. The only other caller is " + "`src/worker.py:12` (`retries_remaining(3, attempts)`), and " + "`tests/test_retry.py::test_retries_remaining` fails with `assert 4 == 2`. " + "I haven't changed any files." + ) + } + ], + "role": "assistant", + } + }, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1452, + "outputTokens": 163, + "serverToolUsage": {}, + "totalTokens": 1615, + }, + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": ( + "The bug is that `retries_remaining` in `src/retry.py` returns `limit + attempts` " + "instead of `max(0, limit - attempts)`. That gives 4 for `(3, 1)` where the test " + "expects 2, and it can't be clamped to zero. The only other caller is " + "`src/worker.py:12` (`retries_remaining(3, attempts)`), and " + "`tests/test_retry.py::test_retries_remaining` fails with `assert 4 == 2`. " + "I haven't changed any files." + ), + "role": "assistant", + }, + } + ], + "usage": { + "completion_tokens": 163, + "prompt_tokens": 1452, + "total_tokens": 1615, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 163}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1452, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, +) + + CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE: Final = replace( CLAUDE_OPUS_5_TEST_CASE, scenario="agent_followup", diff --git a/tests/integration/translation/chat_completions/cross_model/case.py b/tests/integration/translation/chat_completions/cross_model/case.py index d81e6a269f6..189592b26c2 100644 --- a/tests/integration/translation/chat_completions/cross_model/case.py +++ b/tests/integration/translation/chat_completions/cross_model/case.py @@ -6,6 +6,7 @@ from integration.translation.chat_completions.cross_model import ( azure, azure_ai, bedrock_converse, + gemini, openai, vertex_ai, ) @@ -126,6 +127,11 @@ MID: Final[tuple[AgentCases, ...]] = ( bedrock_converse.CLAUDE_OPUS_5_5_AGENT_FOLLOWUP_TEST_CASE, bedrock_converse.CLAUDE_OPUS_5_5_AGENT_STREAMING_TEST_CASE, ), + ( + bedrock_converse.CLAUDE_SONNET_5_5_AGENT_TEST_CASE, + bedrock_converse.CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE, + bedrock_converse.CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE, + ), ( bedrock_converse.CLAUDE_OPUS_5_TEST_CASE, bedrock_converse.CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE, @@ -187,11 +193,21 @@ MID: Final[tuple[AgentCases, ...]] = ( vertex_ai.GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE, vertex_ai.GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE, ), + ( + gemini.GEMINI_3_8_FLASH_AGENT_TEST_CASE, + gemini.GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE, + gemini.GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE, + ), ( vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE, vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE, vertex_ai.GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE, ), + ( + gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE, + gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE, + gemini.GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE, + ), ) HIGH: Final[tuple[AgentCases, ...]] = ( diff --git a/tests/integration/translation/chat_completions/cross_model/gemini.py b/tests/integration/translation/chat_completions/cross_model/gemini.py new file mode 100644 index 00000000000..2bac5f99947 --- /dev/null +++ b/tests/integration/translation/chat_completions/cross_model/gemini.py @@ -0,0 +1,769 @@ +from dataclasses import replace +from typing import Final + +from integration._support.client import list_value +from integration.translation.case import SSEFrame, StreamingTranslationTestCase +from integration.translation.chat_completions.bases.gemini import ( + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + GEMINI_3_8_FLASH_TEST_CASE, +) +from integration.translation.chat_completions.cross_model.contracts import followup_request +from integration.translation.chat_completions.templates import ( + CODING_AGENT_REQUEST, + CODING_AGENT_RESPONSE, + coding_agent_stream_message, +) +from integration.translation.templates.vertex_ai import VERTEX_AI_AGENT_REQUEST + +GEMINI_SKIP_VALIDATION_SIGNATURE: Final = "c2tpcF90aG91Z2h0X3NpZ25hdHVyZV92YWxpZGF0b3I=" + +GEMINI_3_1_PRO_PREVIEW_THOUGHT: Final = ( + "Okay, here's my summary, framed as the internal thought process:\n\n" + "**Concurrent Debugging of `retries_remaining` Usage**\n\n" + "Alright, the objective is clear: I need to track down the root cause. My gut tells me this " + "`retries_remaining` issue isn't isolated. My first move will be to perform a comprehensive search " + "for all instances where `retries_remaining` is referenced within the codebase. We're talking " + "about a full, concurrent search. I don't want to waste time waiting.\n\n" + "Once I have a list, the plan is to launch the relevant test suite for each of those callers, " + "*concurrently*. That's the only way to reproduce and identify the specific failure scenario. " + "Speed and thoroughness are key here. I'll need to instrument each test run to capture as much " + "diagnostic information as possible - logging, stack traces, maybe even memory dumps if the issue " + "seems complex. I need a clear picture of what's happening within those retry loops to figure out " + "what's causing the problem.\n\n\n" +) + +GEMINI_3_8_FLASH_FOLLOWUP_TEXT: Final = ( + "The function `retries_remaining` erroneously adds `limit` and `attempts` instead of subtracting " + "them with a minimum of zero, causing `test_retries_remaining` to fail when " + "`retries_remaining(3, 1)` returns `4` instead of `2`. In addition to the tests, `src/worker.py` " + "was found as a caller invoking `retries_remaining(3, attempts)`." +) + +GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT: Final = ( + "**My Analysis of the Bug Report**\n\n" + "Okay, let's break this down efficiently. The key is to avoid unnecessary verbosity and, " + "importantly, not invoke any tooling or editing operations. This is about summarizing the defect " + "itself. I see the caller is in `src/worker.py` and the failing test is " + "`tests/test_retry.py::test_retries_remaining`.\n\n" + "The core issue appears to be an incorrect calculation within the `retries_remaining` function. " + "It looks like the implementation adds attempts to the limit instead of subtracting them, leading " + "to a logical error. This directly impacts the `test_retries_remaining` test and, I presume, causes " + "issues in the `src/worker.py` module. To fix this, the return value needs to be changed to " + "`max(0, limit - attempts)` to properly calculate the remaining retries. This correction should " + "resolve the test failure and ensure the correct behavior of the retry logic called from the worker.\n\n\n" +) + +GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT: Final = ( + "The bug is that `retries_remaining` incorrectly adds `attempts` to `limit` instead of subtracting " + "it, causing `test_retries_remaining` to fail by returning 4 instead of 2. Fixing the implementation " + "to return `max(0, limit - attempts)` will resolve the test failure and correct the behavior for its " + "caller in `src/worker.py`." +) + +GEMINI_3_8_FLASH_AGENT_PROVIDER_RESPONSE: Final = { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_4323393", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + { + "functionCall": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_4323394", + } + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 557, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "yyHWapj7Br7sz7IPte2L2QU", +} + +GEMINI_3_1_PRO_PREVIEW_AGENT_PROVIDER_RESPONSE: Final = { + "candidates": [ + { + "content": { + "parts": [ + {"text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, "thought": True}, + { + "functionCall": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_4206648", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + { + "functionCall": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_4206649", + } + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + "finishMessage": "Model generated function call(s).", + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 602, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "thoughtsTokenCount": 45, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "0CHWas7iDZLqz7IPsLnekAs", +} + +GEMINI_3_8_FLASH_AGENT_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_TEST_CASE, + scenario="agent_tools", + litellm_request={**CODING_AGENT_REQUEST, "model": "gemini/gemini-3.8-flash"}, + expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", + expected_provider_request=VERTEX_AI_AGENT_REQUEST, + mock_provider_response=GEMINI_3_8_FLASH_AGENT_PROVIDER_RESPONSE, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "gemini/gemini-3.8-flash", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + "function": { + "arguments": '{"query": "retries_remaining"}', + "name": "search_code", + }, + "id": "call_4323393__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, + "type": "function", + }, + { + "index": 1, + "function": { + "arguments": '{"path": "tests/test_retry.py"}', + "name": "run_tests", + }, + "id": "call_4323394", + "type": "function", + }, + ], + "images": [], + "thinking_blocks": [], + "provider_specific_fields": { + "thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + } + ], + "usage": { + "completion_tokens": 41, + "prompt_tokens": 516, + "total_tokens": 557, + "completion_tokens_details": {"text_tokens": 41}, + "prompt_tokens_details": {"text_tokens": 516}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + +GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE: Final = replace( + GEMINI_3_1_PRO_PREVIEW_TEST_CASE, + scenario="agent_tools", + litellm_request={**CODING_AGENT_REQUEST, "model": "gemini/gemini-3.1-pro-preview"}, + expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", + expected_provider_request=VERTEX_AI_AGENT_REQUEST, + mock_provider_response=GEMINI_3_1_PRO_PREVIEW_AGENT_PROVIDER_RESPONSE, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "gemini/gemini-3.1-pro-preview", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "provider_specific_fields": { + "thought_signature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + "function": { + "arguments": '{"query": "retries_remaining"}', + "name": "search_code", + }, + "id": "call_4206648__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, + "type": "function", + }, + { + "index": 1, + "function": { + "arguments": '{"path": "tests/test_retry.py"}', + "name": "run_tests", + }, + "id": "call_4206649", + "type": "function", + }, + ], + "images": [], + "reasoning_content": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": GEMINI_3_1_PRO_PREVIEW_THOUGHT, + } + ], + "provider_specific_fields": { + "thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + } + ], + "usage": { + "completion_tokens": 86, + "prompt_tokens": 516, + "total_tokens": 602, + "completion_tokens_details": {"reasoning_tokens": 45, "text_tokens": 41}, + "prompt_tokens_details": {"text_tokens": 516}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + +GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = { + **VERTEX_AI_AGENT_REQUEST, + "contents": [ + *list_value(VERTEX_AI_AGENT_REQUEST["contents"]), + { + "role": "model", + "parts": [ + { + "function_call": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_4323393", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + { + "function_call": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_4323394", + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "search_code", + "response": { + "matches": [ + { + "path": "src/worker.py", + "line": 12, + "text": "remaining = retries_remaining(3, attempts)", + } + ] + }, + "id": "call_4323393", + } + }, + { + "function_response": { + "name": "run_tests", + "response": { + "exit_code": 1, + "output": "FAILED tests/test_retry.py::test_retries_remaining - assert 4 == 2", + }, + "id": "call_4323394", + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "text": ( + "Stop after these diagnostics. Do not call more tools or change files. " + "Summarize the bug, the caller found and the failed test in two short sentences." + ) + } + ], + }, + ], +} + +GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_PROVIDER_REQUEST: Final = { + **GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST, + "contents": [ + *list_value(VERTEX_AI_AGENT_REQUEST["contents"]), + { + "role": "model", + "parts": [ + {"text": GEMINI_3_1_PRO_PREVIEW_THOUGHT, "thought": True}, + { + "function_call": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_4206648", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + { + "function_call": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_4206649", + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "search_code", + "response": { + "matches": [ + { + "path": "src/worker.py", + "line": 12, + "text": "remaining = retries_remaining(3, attempts)", + } + ] + }, + "id": "call_4206648", + } + }, + { + "function_response": { + "name": "run_tests", + "response": { + "exit_code": 1, + "output": "FAILED tests/test_retry.py::test_retries_remaining - assert 4 == 2", + }, + "id": "call_4206649", + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "text": ( + "Stop after these diagnostics. Do not call more tools or change files. " + "Summarize the bug, the caller found and the failed test in two short sentences." + ) + } + ], + }, + ], +} + +GEMINI_3_8_FLASH_AGENT_FOLLOWUP_TEST_CASE: Final = replace( + GEMINI_3_8_FLASH_AGENT_TEST_CASE, + scenario="agent_followup", + litellm_request=followup_request( + GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_request, + GEMINI_3_8_FLASH_AGENT_TEST_CASE.expected_litellm_response, + ), + expected_provider_endpoint="/models/gemini-3.8-flash:generateContent", + expected_provider_request=GEMINI_3_8_FLASH_AGENT_FOLLOWUP_PROVIDER_REQUEST, + mock_provider_response={ + "candidates": [ + { + "content": { + "parts": [ + { + "text": GEMINI_3_8_FLASH_FOLLOWUP_TEXT, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + } + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 670, + "candidatesTokenCount": 89, + "totalTokenCount": 759, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 670}], + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "nyLWatOHEKHoz7IPnICMuAs", + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "gemini/gemini-3.8-flash", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": GEMINI_3_8_FLASH_FOLLOWUP_TEXT, + "role": "assistant", + "images": [], + "thinking_blocks": [], + "provider_specific_fields": { + "thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + } + ], + "usage": { + "completion_tokens": 89, + "prompt_tokens": 670, + "total_tokens": 759, + "completion_tokens_details": {"text_tokens": 89}, + "prompt_tokens_details": {"text_tokens": 670}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + +GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_TEST_CASE: Final = replace( + GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE, + scenario="agent_followup", + litellm_request=followup_request( + GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_request, + GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.expected_litellm_response, + ), + expected_provider_endpoint="/models/gemini-3.1-pro-preview:generateContent", + expected_provider_request=GEMINI_3_1_PRO_PREVIEW_AGENT_FOLLOWUP_PROVIDER_REQUEST, + mock_provider_response={ + "candidates": [ + { + "content": { + "parts": [ + {"text": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT, "thought": True}, + { + "text": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 887, + "candidatesTokenCount": 78, + "totalTokenCount": 1329, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 887}], + "thoughtsTokenCount": 364, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "oiLWauy3Kr2Ez7IPse_RwAs", + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "gemini/gemini-3.1-pro-preview", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_TEXT, + "role": "assistant", + "images": [], + "reasoning_content": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT, + "thinking_blocks": [ + { + "type": "thinking", + "thinking": GEMINI_3_1_PRO_PREVIEW_FOLLOWUP_THOUGHT, + } + ], + "provider_specific_fields": { + "thought_signatures": [GEMINI_SKIP_VALIDATION_SIGNATURE], + }, + }, + "provider_specific_fields": {"native_finish_reason": "STOP"}, + } + ], + "usage": { + "completion_tokens": 442, + "prompt_tokens": 887, + "total_tokens": 1329, + "completion_tokens_details": {"reasoning_tokens": 364, "text_tokens": 78}, + "prompt_tokens_details": {"text_tokens": 887}, + }, + "vertex_ai_grounding_metadata": [], + "vertex_ai_url_context_metadata": [], + "vertex_ai_safety_results": [], + "vertex_ai_citation_metadata": [], + }, +) + +GEMINI_3_8_FLASH_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="agent_streaming", + litellm_endpoint=GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_endpoint, + litellm_request={ + **GEMINI_3_8_FLASH_AGENT_TEST_CASE.litellm_request, + "stream": True, + "stream_options": {"include_usage": True}, + }, + expected_provider_headers=GEMINI_3_8_FLASH_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_endpoint="/models/gemini-3.8-flash:streamGenerateContent?alt=sse", + expected_provider_request=VERTEX_AI_AGENT_REQUEST, + mock_provider_frames=( + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_5122749", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + } + ], + "role": "model", + }, + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 19, + "totalTokenCount": 535, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "zSHWaoPJKvPrz7IP7dqb-AM", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_5122752", + } + } + ], + "role": "model", + }, + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 557, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "zSHWaoPJKvPrz7IP7dqb-AM", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": {"parts": [{"text": ""}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 557, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.8-flash", + "responseId": "zSHWaoPJKvPrz7IP7dqb-AM", + } + ), + ), + expected_litellm_message=coding_agent_stream_message( + ("call_5122749__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, "call_5122752"), + ), + expected_litellm_usage={ + "prompt_tokens": 516, + "completion_tokens": 41, + "total_tokens": 557, + "prompt_tokens_details": {"text_tokens": 516}, + "completion_tokens_details": {"text_tokens": 41}, + }, +) + +GEMINI_3_1_PRO_PREVIEW_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="agent_streaming", + litellm_endpoint=GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_endpoint, + litellm_request={ + **GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.litellm_request, + "stream": True, + "stream_options": {"include_usage": True}, + }, + expected_provider_headers=GEMINI_3_1_PRO_PREVIEW_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_endpoint="/models/gemini-3.1-pro-preview:streamGenerateContent?alt=sse", + expected_provider_request=VERTEX_AI_AGENT_REQUEST, + mock_provider_frames=( + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "search_code", + "args": {"query": "retries_remaining"}, + "id": "call_6112941", + }, + "thoughtSignature": GEMINI_SKIP_VALIDATION_SIGNATURE, + } + ], + "role": "model", + }, + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 19, + "totalTokenCount": 599, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "thoughtsTokenCount": 64, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "1SHWaoD1Ode1qtsPo6rR4AI", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "run_tests", + "args": {"path": "tests/test_retry.py"}, + "id": "call_6112947", + } + } + ], + "role": "model", + }, + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 621, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "thoughtsTokenCount": 64, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "1SHWaoD1Ode1qtsPo6rR4AI", + } + ), + SSEFrame( + data={ + "candidates": [ + { + "content": {"parts": [{"text": ""}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 516, + "candidatesTokenCount": 41, + "totalTokenCount": 621, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 516}], + "thoughtsTokenCount": 64, + "serviceTier": "standard", + }, + "modelVersion": "gemini-3.1-pro-preview", + "responseId": "1SHWaoD1Ode1qtsPo6rR4AI", + } + ), + ), + expected_litellm_message=coding_agent_stream_message( + ("call_6112941__thought__" + GEMINI_SKIP_VALIDATION_SIGNATURE, "call_6112947"), + ), + expected_litellm_usage={ + "prompt_tokens": 516, + "completion_tokens": 105, + "total_tokens": 621, + "prompt_tokens_details": {"text_tokens": 516}, + "completion_tokens_details": {"reasoning_tokens": 64, "text_tokens": 41}, + }, +) From 3b2cc34c092d5a6b2a587392234995d16e299c8e Mon Sep 17 00:00:00 2001 From: limdeng <42972202+limdeng@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:54:35 +0800 Subject: [PATCH 066/124] feat(dashscope): add separate explicit and implicit cache pricing (#45567) * feat(dashscope): price Qwen implicit context-cache reads separately * fix(dashscope): normalize implicit cache price strings * fix(dashscope): price explicit cache hits through an explicit_ override key Keep cache_read_input_token_cost as the default (implicit) Qwen cache-read rate, which the cost map already holds, and add explicit_cache_read_input_token_cost for hits DashScope tags with cache_type "ephemeral". Untyped and unknown cache types keep the base rate, so older releases reading the remote cost map keep billing unchanged. Add explicit cache-read and cache-creation rates to the qwen3.7-max rows. * fix(types): accept explicit_cache_read_input_token_cost as a constructor keyword * fix(dashscope): inherit the explicit cache price on custom deployments and keep tier rates all-or-nothing Deployments that set only input and output prices now inherit explicit_cache_read_input_token_cost from the cost map, so explicit hits bill at the explicit rate. A pricing tier without its own explicit rate bills its own cache_read rate instead of the model-level explicit one, matching how an off-peak block falls back. The DashScope calculator now delegates the explicit rate to the shared resolver, and the register_model warning is back to main's wording. * fix(dashscope): keep a deployment's own cache-read price for explicit hits and ignore a non-string cache_type * fix(dashscope): keep cache_type through Responses usage and let a registered cache-read price cover explicit hits --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 99 +++- litellm/llms/dashscope/cost_calculator.py | 13 +- ...odel_prices_and_context_window_backup.json | 6 + .../transformation.py | 5 + litellm/responses/utils.py | 1 + litellm/router.py | 20 +- litellm/types/utils.py | 12 + litellm/utils.py | 15 +- model_prices_and_context_window.json | 6 + model_prices_and_context_window.schema.json | 4 + .../test_dashscope_cost_calculator.py | 506 ++++++++++++++++++ .../test_spend_tracking_utils.py | 1 + tests/unit/responses/test_responses_utils.py | 46 +- .../unit/test_router_model_cost_isolation.py | 61 +++ tests/unit/test_utils.py | 47 ++ tests/unit/types/test_types_utils.py | 40 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 17 files changed, 872 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 1b836e935d7..31da260588c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -356,7 +356,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide ) -def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None: +def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None: tiered_pricing: Final = model_info.get("tiered_pricing") if not isinstance(tiered_pricing, list) or not tiered_pricing: return None @@ -545,6 +545,71 @@ class TokenRates: return self.output_rate if self.reasoning_rate is None else self.reasoning_rate +def resolve_explicit_cache_read_cost_rate( + model_info: ModelInfo, + usage: Usage, + current_time: datetime | None, + service_tier: str | None = None, +) -> float | None: + cache_read_cost_key: Final = "explicit_cache_read_input_token_cost" + service_cache_read_cost_key: Final = _get_service_tier_cost_key(cache_read_cost_key, service_tier) + if ( + model_info.get(service_cache_read_cost_key) is None + and model_info.get(cache_read_cost_key) is None + and model_info.get("tiered_pricing") is None + and model_info.get("off_peak_pricing") is None + ): + return None + + prompt_tokens_details: Final = usage.prompt_tokens_details + cache_type: Final = ( + getattr(prompt_tokens_details, "cache_type", None) if prompt_tokens_details is not None else None + ) + if parse_prompt_tokens_details(usage)["cache_hit_tokens"] <= 0 or cache_type != "ephemeral": + return None + + tier: Final = _select_priced_tier(model_info=model_info, usage=usage) + off_peak: Final = _open_off_peak_block(model_info, current_time) + model_cache_read_cost_rate: Final = _get_cost_per_unit( + model_info, + service_cache_read_cost_key, + default_value=None, + ) + tier_has_explicit_rate: Final = tier is not None and cache_read_cost_key in tier + off_peak_has_explicit_rate: Final = ( + off_peak is not None and _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) is not None + ) + if model_cache_read_cost_rate is None and not tier_has_explicit_rate and not off_peak_has_explicit_rate: + return None + + cache_read_cost_rate: Final = ( + tier_rate(tier, cache_read_cost_key) + if tier is not None and tier_has_explicit_rate + else tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token") + if tier is not None + else model_cache_read_cost_rate + if model_cache_read_cost_rate is not None + else _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_token_cost", service_tier), + default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0, + ) + or 0.0 + ) + + if off_peak is None: + return cache_read_cost_rate + mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) + generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost")) + return ( + mode_specific_off_peak_rate + if mode_specific_off_peak_rate is not None + else generic_off_peak_rate + if generic_off_peak_rate is not None + else cache_read_cost_rate + ) + + def _parse_off_peak_rate(value: object) -> float | None: if isinstance(value, bool): return None @@ -1458,12 +1523,19 @@ def generic_cost_per_token( current_time=billing_time, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) + resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( + model_info=resolved_model_info, + usage=usage, + current_time=billing_time, + service_tier=service_tier, + ) + effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost prompt_cost = _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, model_info=resolved_model_info, prompt_base_cost=prompt_base_cost, - cache_read_cost=cache_read_cost, + cache_read_cost=effective_cache_read_cost, cache_creation_cost=cache_creation_cost, cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, service_tier=service_tier, @@ -1658,6 +1730,16 @@ def _cost_map_billed_rates( current_time=billing_time, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) + provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate( + model_info=model_info, + usage=usage, + current_time=billing_time, + service_tier=service_tier, + ) + effective_cache_read_cost_rate: Final = ( + cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate + ) + reasoning_rate: Final = _resolve_billed_reasoning_rate( model_info=model_info, usage=usage, @@ -1679,9 +1761,9 @@ def _cost_map_billed_rates( return BilledTokenRates( input_cost_per_token=prompt_base_cost, output_cost_per_token=completion_base_cost, - cache_read_input_token_cost=cache_read_cost_rate, + cache_read_input_token_cost=effective_cache_read_cost_rate, cache_read_input_audio_token_cost=( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate + audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate ), cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, @@ -1806,6 +1888,13 @@ def calculate_prompt_caching_savings( current_time=billed_at, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) + provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( + model_info=model_info, + usage=usage, + current_time=billed_at, + service_tier=service_tier, + ) + effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost write_rate: Final = cache_creation_cost or prompt_base_cost write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate prompt_tokens_details: Final = parse_prompt_tokens_details(usage) @@ -1820,7 +1909,7 @@ def calculate_prompt_caching_savings( if details is not None else None ) - read_discount: Final = cache_read_tokens * max(prompt_base_cost - cache_read_cost, 0.0) + read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0) write_premium: Final = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_details, diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 17f70ec5db7..98504f54533 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -7,7 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate. See https://help.aliyun.com/zh/model-studio/billing-for-model-studio """ -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime from typing import Final @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( apply_off_peak_pricing, parse_completion_tokens_details, parse_prompt_tokens_details, + resolve_explicit_cache_read_cost_rate, ) from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -137,5 +138,13 @@ def cost_per_token( ) standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier) rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates) + explicit_cache_read_rate: Final = resolve_explicit_cache_read_cost_rate( + model_info=model_info, + usage=usage, + current_time=current_time, + ) + billed_rates: Final = ( + rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate) + ) - return _bill(breakdown, rates) + return _bill(breakdown, billed_rates) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7f8a2c2ae93..30229f2a73f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18018,7 +18018,9 @@ ] }, "dashscope/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "dashscope", "max_input_tokens": 991808, @@ -19285,7 +19287,9 @@ ] }, "qwencloud/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "qwencloud", "max_input_tokens": 991808, @@ -20512,7 +20516,9 @@ ] }, "qwen_ai_platform/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a2acbc09784..b613b817225 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3406,6 +3406,10 @@ class LiteLLMCompletionResponsesConfig: if cache_write_tokens is not None else MappingProxyType({}) ) + cache_type: Final = getattr(prompt_details, "cache_type", None) + cache_type_extra: Final[Mapping[str, str]] = ( + MappingProxyType({"cache_type": cache_type}) if cache_type is not None else MappingProxyType({}) + ) # The cost path reads the grounding counters off the input details, and a realtime # session's usage is rebuilt from its own response.done, so dropping them here bills # no per-query grounding fee at all. @@ -3432,6 +3436,7 @@ class LiteLLMCompletionResponsesConfig: cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None ), **cache_write_extra, + **cache_type_extra, **grounding_request_counts, ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index be8397c9270..503d85aaed9 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1259,6 +1259,7 @@ class ResponseAPILoggingUtils: ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), + cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), google_maps_grounding_requests=getattr( response_api_usage.input_tokens_details, "google_maps_grounding_requests", None diff --git a/litellm/router.py b/litellm/router.py index 142b2019e6e..94755d0d5ba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9815,7 +9815,8 @@ class Router: does not silently bill cache_read/cache_creation at 0. User-specified cache fields always win; only ``None``/missing entries - are inherited. No-op when the backend model has no canonical entry. + are inherited, and a user-set cache_read price also covers explicit + cache hits. No-op when the backend model has no canonical entry. """ cache_fields: Final = ( "cache_creation_input_token_cost", @@ -9824,17 +9825,22 @@ class Router: "cache_read_input_token_cost", "cache_read_input_token_cost_above_200k_tokens", ) - if all(model_info.get(f) is not None for f in cache_fields): + inherited_fields: Final = ( + cache_fields + if model_info.get("cache_read_input_token_cost") is not None + else (*cache_fields, "explicit_cache_read_input_token_cost") + ) + missing_fields: Final = tuple(f for f in inherited_fields if model_info.get(f) is None) + if not missing_fields: return try: backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) except Exception: return - for field in cache_fields: - if model_info.get(field) is None: - backend_value = backend_info.get(field) - if backend_value is not None: - model_info[field] = backend_value + for field in missing_fields: + backend_value = backend_info.get(field) + if backend_value is not None: + model_info[field] = backend_value @staticmethod def _cost_map_backend_model(deployment: Deployment) -> str: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 99788dad4e1..576c4f093d3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -291,6 +291,7 @@ class OffPeakPricing(TypedDict, total=False): output_cost_per_token: ReadOnly[float] output_cost_per_reasoning_token: ReadOnly[float] cache_read_input_token_cost: ReadOnly[float] + explicit_cache_read_input_token_cost: ReadOnly[float] cache_creation_input_token_cost: ReadOnly[float] @@ -320,6 +321,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost: float | None cache_read_input_audio_token_cost: ReadOnly[float | None] cache_read_input_image_token_cost: ReadOnly[float | None] + explicit_cache_read_input_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_read_input_token_cost_balanced: ReadOnly[float | None] @@ -1840,6 +1842,13 @@ class PromptTokensDetailsWrapper( cached_tokens_details: CachedTokensDetails | None = None """Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split.""" + cache_type: str | None = None + """Provider-reported cache mode. DashScope returns ``ephemeral`` for explicit Qwen Context Cache requests.""" + + @field_validator("cache_type", mode="before") + @classmethod + def _drop_non_string_cache_type(cls, value: object) -> str | None: + return value if isinstance(value, str) else None def __setattr__(self, name: str, value: object) -> None: super().__setattr__(name, value) @@ -1889,6 +1898,8 @@ class PromptTokensDetailsWrapper( del self.cache_creation_token_details if self.cached_tokens_details is None: del self.cached_tokens_details + if self.cache_type is None: + del self.cache_type class ServerToolUse(LiteLLMBaseModel): @@ -3921,6 +3932,7 @@ class MirroredPricingParams(LiteLLMBaseModel): input_cost_per_character: float | None = None output_cost_per_character: float | None = None cache_read_input_token_cost: float | None = None + explicit_cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None tiered_pricing: list[dict[str, Any]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 3990a870484..fc47a4dba2e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3670,6 +3670,15 @@ def _registration_base( return _RegistrationWrite(key=key, entry=dict(fallback.entry)) +def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: + return ( + frozenset({"explicit_cache_read_input_token_cost"}) + if value.get("cache_read_input_token_cost") is not None + and value.get("explicit_cache_read_input_token_cost") is None + else frozenset() + ) + + def _registration_write( key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None ) -> _RegistrationWrite: @@ -3684,7 +3693,10 @@ def _registration_write( for name, field_value in base.entry.items() if name != "litellm_provider" or field_value is not None } - return _RegistrationWrite(key=base.key, entry=update_dictionary(existing_model, value)) + return _RegistrationWrite( + key=base.key, + entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))}, + ) def _registration_attempt( @@ -6614,6 +6626,7 @@ def get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7f8a2c2ae93..30229f2a73f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18018,7 +18018,9 @@ ] }, "dashscope/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "dashscope", "max_input_tokens": 991808, @@ -19285,7 +19287,9 @@ ] }, "qwencloud/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "qwencloud", "max_input_tokens": 991808, @@ -20512,7 +20516,9 @@ ] }, "qwen_ai_platform/qwen3.7-max": { + "cache_creation_input_token_cost": 3.125e-06, "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, "input_cost_per_token": 2.5e-06, "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index ce9af67eb94..0e8f4f596c0 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -297,6 +297,10 @@ "format": "date", "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$" }, + "explicit_cache_read_input_token_cost": { + "type": "number", + "minimum": 0 + }, "gemini_audio_only_live": { "type": "boolean" }, diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index a30d35d46f2..551caab7153 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -16,11 +16,18 @@ import pytest # Add the project root to Python path import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + resolve_explicit_cache_read_cost_rate, + calculate_prompt_caching_savings, + generic_cost_per_token, + get_token_type_cost_breakdown, +) from litellm.llms.dashscope.cost_calculator import ( cost_per_token as dashscope_cost_per_token, ) from litellm.types.utils import ( CompletionTokensDetailsWrapper, + ModelInfo, PromptTokensDetailsWrapper, Usage, ) @@ -35,6 +42,196 @@ class TestDashscopeCostCalculator: os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") + @pytest.mark.parametrize( + "prompt_tokens_details", + [ + {"cached_tokens": 0, "cache_type": "ephemeral"}, + {"cached_tokens": 600}, + {"cached_tokens": 600, "cache_type": "unknown-future-type"}, + ], + ) + def test_explicit_cache_rate_requires_an_ephemeral_cache_hit( + self, + prompt_tokens_details: dict[str, int | str], + ): + assert ( + resolve_explicit_cache_read_cost_rate( + model_info={ + "key": "qwen-cache-rate-guard-test", + "explicit_cache_read_input_token_cost": 1e-07, + }, + usage=Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details=prompt_tokens_details, + ), + current_time=None, + ) + is None + ) + + @pytest.mark.parametrize( + ("model_info", "current_time", "expected_cache_rate"), + [ + ( + { + "key": "qwen-tier-mode-rate-test", + "tiered_pricing": [ + { + "range": [0, 2000], + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + } + ], + }, + None, + 1e-07, + ), + ( + { + "key": "qwen-tier-generic-fallback-test", + "tiered_pricing": [ + { + "range": [0, 2000], + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + } + ], + }, + None, + None, + ), + ( + { + "key": "qwen-flat-generic-fallback-test", + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + }, + None, + None, + ), + ( + { + "key": "qwen-invalid-explicit-rate-test", + "input_cost_per_token": 1e-06, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "explicit_cache_read_input_token_cost": True, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + None, + ), + ( + { + "key": "qwen-tier-off-peak-mode-rate-test", + "tiered_pricing": [ + { + "range": [0, 2000], + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + } + ], + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "explicit_cache_read_input_token_cost": 5e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 5e-08, + ), + ( + { + "key": "qwen-flat-off-peak-mode-rate-test", + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "explicit_cache_read_input_token_cost": 5e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 5e-08, + ), + ( + { + "key": "qwen-off-peak-mode-rate-test", + "input_cost_per_token": 1e-06, + "explicit_cache_read_input_token_cost": 1e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "explicit_cache_read_input_token_cost": 5e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 5e-08, + ), + ( + { + "key": "qwen-off-peak-generic-rate-test", + "input_cost_per_token": 1e-06, + "explicit_cache_read_input_token_cost": 1e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "explicit_cache_read_input_token_cost": "invalid", + "cache_read_input_token_cost": 8e-08, + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 8e-08, + ), + ( + { + "key": "qwen-off-peak-standard-fallback-test", + "input_cost_per_token": 1e-06, + "explicit_cache_read_input_token_cost": 1e-07, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + }, + }, + datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), + 1e-07, + ), + ], + ) + def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks( + self, + model_info: ModelInfo, + current_time: datetime | None, + expected_cache_rate: float | None, + ): + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + cache_rate = resolve_explicit_cache_read_cost_rate( + model_info=model_info, + usage=usage, + current_time=current_time, + ) + + assert cache_rate == expected_cache_rate + + def test_provider_neutral_cache_rate_resolver_handles_service_tier_rate(self): + cache_rate = resolve_explicit_cache_read_cost_rate( + model_info={ + "key": "qwen-service-tier-mode-rate-test", + "explicit_cache_read_input_token_cost_priority": 3e-07, + }, + usage=Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ), + current_time=None, + service_tier="priority", + ) + + assert cache_rate == 3e-07 + def test_dashscope_flat_pricing_fallback(self): """ Tests that the dashscope calculator falls back to flat pricing for models @@ -53,6 +250,276 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + @pytest.mark.parametrize("explicit_cache_rate", [1e-07, "1e-7"]) + @pytest.mark.parametrize( + ("cache_type", "expected_cache_rate"), + [ + ("ephemeral", 1e-07), + (None, 2e-07), + ("unknown-future-type", 2e-07), + ], + ) + def test_dashscope_flat_pricing_distinguishes_explicit_and_implicit_cache_reads( + self, cache_type: str | None, expected_cache_rate: float, explicit_cache_rate: float | str + ) -> None: + litellm.model_cost["dashscope/qwen-context-cache-mode-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "explicit_cache_read_input_token_cost": explicit_cache_rate, + "cache_read_input_token_cost": 2e-07, + } + prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600} + if cache_type is not None: + prompt_tokens_details["cache_type"] = cache_type + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details=prompt_tokens_details, + ) + + prompt_cost, _ = dashscope_cost_per_token(model="qwen-context-cache-mode-test", usage=usage) + + assert math.isclose( + prompt_cost, + (400 * 1e-06) + (600 * expected_cache_rate), + rel_tol=1e-10, + ) + + def test_dashscope_explicit_cache_read_falls_back_to_existing_cache_read_rate(self): + litellm.model_cost["dashscope/qwen-explicit-cache-fallback-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + } + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-explicit-cache-fallback-test", + usage=usage, + ) + + assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) + + @pytest.mark.parametrize( + ("prompt_tokens_details", "expected_cache_rate"), + [ + ({"cached_tokens": 600, "cache_type": "ephemeral"}, 1e-07), + ({"cached_tokens": 600}, 2e-07), + ], + ) + def test_completion_cost_routes_dashscope_cache_mode_pricing( + self, + prompt_tokens_details: dict[str, int | str], + expected_cache_rate: float, + ): + litellm.model_cost["dashscope/qwen-completion-cache-mode-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + } + response = litellm.ModelResponse( + model="qwen-completion-cache-mode-test", + usage=Usage( + prompt_tokens=1000, + completion_tokens=100, + total_tokens=1100, + prompt_tokens_details=prompt_tokens_details, + ), + ) + + cost = litellm.completion_cost( + completion_response=response, + model="qwen-completion-cache-mode-test", + custom_llm_provider="dashscope", + ) + + assert math.isclose( + cost, + (400 * 1e-06) + (600 * expected_cache_rate) + (100 * 4e-06), + rel_tol=1e-10, + ) + + def test_dashscope_tiered_pricing_uses_mode_specific_cache_read_rate(self): + self._register_tiered_model( + "dashscope/qwen-tiered-cache-mode-test", + [ + { + "range": [0, 1000], + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "explicit_cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost": 2e-07, + } + ], + ) + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + prompt_cost, _ = dashscope_cost_per_token(model="qwen-tiered-cache-mode-test", usage=usage) + + assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) + + @pytest.mark.parametrize( + ("explicit_cache_rate", "expected_cache_rate"), + [(1e-07, 1e-07), ("1e-7", 1e-07), (0.0, 0.0), ("0", 0.0)], + ) + def test_dashscope_flat_explicit_cache_read_rate_wins_even_at_zero( + self, explicit_cache_rate: float | str, expected_cache_rate: float + ) -> None: + litellm.model_cost["dashscope/qwen-flat-explicit-zero-rate-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": explicit_cache_rate, + } + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-flat-explicit-zero-rate-test", + usage=usage, + ) + + assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * expected_cache_rate), rel_tol=1e-10) + + def test_dashscope_tier_without_an_explicit_rate_bills_that_tiers_cache_read_rate(self): + litellm.model_cost["dashscope/qwen-two-tier-explicit-fallback-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "explicit_cache_read_input_token_cost": 4e-08, + "tiered_pricing": [ + { + "range": [0, 1000], + "input_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 8e-08, + }, + { + "range": [1000, 10000], + "input_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + }, + ], + } + usage = Usage( + prompt_tokens=3000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 2000, "cache_type": "ephemeral"}, + ) + + prompt_cost, _ = dashscope_cost_per_token(model="qwen-two-tier-explicit-fallback-test", usage=usage) + breakdown = get_token_type_cost_breakdown( + model="qwen-two-tier-explicit-fallback-test", + custom_llm_provider="dashscope", + usage=usage, + ) + + assert math.isclose(breakdown.cache_read_cost, 2000 * 2.4e-07, rel_tol=1e-10) + assert math.isclose(prompt_cost, (1000 * 1.2e-06) + breakdown.cache_read_cost, rel_tol=1e-10) + + def test_dashscope_prompt_caching_savings_uses_explicit_cache_read_rate(self): + model_info: ModelInfo = { + "key": "qwen-cache-savings-mode-test", + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + } + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + savings = calculate_prompt_caching_savings( + model_info=model_info, + usage=usage, + custom_llm_provider="dashscope", + ) + + assert math.isclose(savings, 600 * (1e-06 - 1e-07), rel_tol=1e-10) + + def test_openai_compatible_qwen_total_uses_explicit_cache_read_rate(self): + model_info: ModelInfo = { + "key": "qwen-generic-cache-mode-test", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + } + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, + ) + + prompt_cost, _ = generic_cost_per_token( + model="custom-qwen", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + + assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * 1e-07), rel_tol=1e-10) + + @pytest.mark.parametrize( + ("cache_type", "expected_cache_rate"), + [("ephemeral", 1e-07), (None, 2e-07)], + ) + def test_dashscope_cost_breakdown_matches_mode_specific_total( + self, + cache_type: str | None, + expected_cache_rate: float, + ): + litellm.model_cost["dashscope/qwen-cache-breakdown-mode-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + } + prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600} + if cache_type is not None: + prompt_tokens_details["cache_type"] = cache_type + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details=prompt_tokens_details, + ) + + prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-breakdown-mode-test", usage=usage) + breakdown = get_token_type_cost_breakdown( + model="qwen-cache-breakdown-mode-test", + custom_llm_provider="dashscope", + usage=usage, + ) + + assert math.isclose(breakdown.cache_read_cost, 600 * expected_cache_rate, rel_tol=1e-10) + assert math.isclose(prompt_cost, (400 * 1e-06) + breakdown.cache_read_cost, rel_tol=1e-10) + def test_dashscope_tiered_pricing_within_first_tier(self): """ Tests the dashscope tiered pricing when the request's input falls in the first tier. @@ -577,6 +1044,45 @@ class TestDashscopeCostCalculator: assert math.isclose(peak_prompt_cost, (600 * 2.4e-06) + (300 * 2e-07) + (100 * 3e-06), rel_tol=1e-10) assert math.isclose(peak_completion_cost, 200 * 4.8e-06, rel_tol=1e-10) + @pytest.mark.parametrize( + ("cache_type", "expected_cache_rate"), + [("ephemeral", 5e-08), (None, 8e-08)], + ) + def test_dashscope_off_peak_uses_mode_specific_cache_read_rate( + self, cache_type: str | None, expected_cache_rate: float + ): + model_key = "dashscope/qwen-cache-mode-off-peak-test" + litellm.model_cost[model_key] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + "off_peak_pricing": { + "hours_utc": self.OFF_PEAK_WINDOW, + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8e-08, + "explicit_cache_read_input_token_cost": 5e-08, + }, + } + prompt_tokens_details: dict[str, int | str] = {"cached_tokens": 600} + if cache_type is not None: + prompt_tokens_details["cache_type"] = cache_type + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details=prompt_tokens_details, + ) + + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-cache-mode-off-peak-test", + usage=usage, + current_time=self.INSIDE_WINDOW, + ) + + assert math.isclose(prompt_cost, (400 * 1e-06) + (600 * expected_cache_rate), rel_tol=1e-10) + def test_dashscope_off_peak_window_overrides_the_selected_tier(self): """An open off-peak window bills the whole request at the flat off-peak rates, whichever tier the input volume selected.""" diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 11a881527e6..55358a3b0de 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -556,6 +556,7 @@ def test_get_logging_payload_maps_nested_cache_creation_input_tokens(): assert additional_usage_values["cache_creation_input_tokens"] == 2048 assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048 + assert additional_usage_values["prompt_tokens_details"]["cache_type"] == "ephemeral" def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens(): diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 55d90acaaee..6ee2c495705 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -7,9 +7,11 @@ import pytest import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.utils import Usage +from litellm.utils import _invalidate_model_cost_lowercase_map class TestResponsesAPIRequestUtils: @@ -890,3 +892,45 @@ class TestResponsesInputToChatMessages: assert ResponsesAPIRequestUtils.responses_input_to_chat_messages( [reasoning_item, user_message, "stray"] ) == [user_message] + + +def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): + model = "dashscope/qwen-responses-explicit-cache-test" + previous_row = litellm.model_cost.get(model) + litellm.model_cost[model] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, + } + _invalidate_model_cost_lowercase_map() + try: + chat_usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"}, + ) + responses_object = ResponsesAPIResponse( + id="resp_explicit_cache", + created_at=0, + model="qwen-responses-explicit-cache-test", + object="response", + status="completed", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=LiteLLMCompletionResponsesConfig.transform_chat_completion_usage_to_responses_usage(chat_usage), + ) + + assert litellm.completion_cost(completion_response=responses_object, model=model) == pytest.approx( + 1000 * 2.5e-07 + ) + finally: + if previous_row is None: + litellm.model_cost.pop(model, None) + else: + litellm.model_cost[model] = previous_row + _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 4cce5b50f0b..ffb71e86303 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -953,6 +953,67 @@ def test_partial_pricing_does_not_overwrite_explicit_cache_fields(): _restore_model_cost_entries(model_keys) +@pytest.mark.parametrize( + ("deployment_cache_prices", "billed_cache_read_rate"), + [({}, 2.5e-07), ({"cache_read_input_token_cost": 1e-07}, 1e-07)], + ids=["inherits-builtin-explicit-rate", "own-cache-read-price-wins"], +) +def test_partial_custom_pricing_bills_explicit_cache_hits( + deployment_cache_prices: dict[str, float], billed_cache_read_rate: float +): + backend_model = "dashscope/qwen-explicit-cache-inherit-backend" + deploy_id = "qwen-deploy-partial-pricing" + model_keys = { + deploy_id: litellm.model_cost.get(deploy_id), + backend_model: litellm.model_cost.get(backend_model), + } + litellm.model_cost[backend_model] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_creation_input_token_cost": 3.125e-06, + "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, + } + _invalidate_model_cost_lowercase_map() + try: + Router( + model_list=[ + { + "model_name": "qwen-custom", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 6e-06, + **deployment_cache_prices, + }, + "model_info": {"id": deploy_id}, + } + ], + ) + response = litellm.ModelResponse( + model=backend_model, + usage=litellm.Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"}, + ), + ) + + cost = litellm.completion_cost( + completion_response=response, + model=backend_model, + custom_pricing=True, + router_model_id=deploy_id, + ) + + assert cost == pytest.approx(1000 * billed_cache_read_rate) + finally: + _restore_model_cost_entries(model_keys) + + def test_inherit_builtin_cache_pricing_fills_only_missing_fields(): """Direct unit test of the helper: missing cache fields are filled from the backend model's built-in entry, while an explicitly set cache field and the diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7df4910c24f..cf0bf9b5d79 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -866,6 +866,7 @@ def validate_model_cost_values(model_data, exceptions=None): "cache_creation_input_token_cost", "cache_creation_input_audio_token_cost", "cache_read_input_token_cost", + "explicit_cache_read_input_token_cost", "cache_read_input_audio_token_cost", "cache_read_input_image_token_cost", "input_dbu_cost_per_token", @@ -964,6 +965,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_creation_input_token_cost_ultrafast": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, + "explicit_cache_read_input_token_cost": {"type": "number"}, "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"}, @@ -7777,6 +7779,51 @@ def test_register_model_without_a_provider_never_inherits_another_providers_entr _invalidate_model_cost_lowercase_map() +@pytest.mark.parametrize( + "registered_prices, billed_cache_read_rate", + [ + pytest.param({"input_cost_per_token": 2e-06}, 2.5e-07, id="inherits-builtin-explicit-rate"), + pytest.param({"cache_read_input_token_cost": 1e-07}, 1e-07, id="own-cache-read-price-wins"), + pytest.param( + {"cache_read_input_token_cost": 1e-07, "explicit_cache_read_input_token_cost": 5e-08}, + 5e-08, + id="own-explicit-price-wins", + ), + ], +) +def test_register_model_over_a_builtin_row_bills_explicit_cache_hits( + monkeypatch: pytest.MonkeyPatch, registered_prices: dict[str, float], billed_cache_read_rate: float +) -> None: + from litellm.utils import _invalidate_model_cost_lowercase_map + + key: Final = "dashscope/qwen-register-explicit-cache-test" + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + litellm.model_cost[key] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "cache_read_input_token_cost": 5e-07, + "explicit_cache_read_input_token_cost": 2.5e-07, + } + _invalidate_model_cost_lowercase_map() + try: + litellm.register_model({key: registered_prices}, persist_across_reloads=False) + explicit_hit: Final = ModelResponse( + usage=Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 1000, "cache_type": "ephemeral"}, + ) + ) + + assert litellm.completion_cost(completion_response=explicit_hit, model=key) == pytest.approx( + 1000 * billed_cache_read_rate + ) + finally: + _invalidate_model_cost_lowercase_map() + + @pytest.mark.parametrize( "model, litellm_params, expects_native_config", [ diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index fbe00ca260b..d85e48540f9 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -209,6 +209,46 @@ def test_prompt_tokens_details_maps_nested_cache_creation_input_tokens(): assert not hasattr(non_int, "cache_write_tokens") +def test_prompt_tokens_details_preserves_qwen_cache_type(): + from litellm.types.utils import PromptTokensDetailsWrapper + + details = PromptTokensDetailsWrapper( + cached_tokens=1889, + cache_creation_input_tokens=0, + cache_type="ephemeral", + ) + + assert details.cache_type == "ephemeral" + assert details.model_dump()["cache_type"] == "ephemeral" + + +@pytest.mark.parametrize("cache_type", [1, {"mode": "explicit"}, ["ephemeral"]]) +def test_usage_ignores_a_non_string_cache_type(cache_type: object): + from litellm.types.utils import Usage + + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details={"cached_tokens": 600, "cache_type": cache_type}, + ) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 600 + assert getattr(usage.prompt_tokens_details, "cache_type", None) is None + + +def test_custom_pricing_accepts_qwen_explicit_cache_rate(): + from litellm.types.utils import CustomPricingLiteLLMParams + + pricing = CustomPricingLiteLLMParams( + cache_read_input_token_cost=2e-07, + explicit_cache_read_input_token_cost=1e-07, + ) + + assert pricing.cache_read_input_token_cost == 2e-07 + assert pricing.explicit_cache_read_input_token_cost == 1e-07 + + def test_usage_server_tool_use_dict_is_coerced_and_round_trips(): from litellm.types.utils import ServerToolUse, Usage diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ac096862b56..88d54d3731a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35162,6 +35162,8 @@ export interface components { default_api_key_tpm_limit?: number | null; /** Drop Params */ drop_params?: boolean | string | null; + /** Explicit Cache Read Input Token Cost */ + explicit_cache_read_input_token_cost?: number | null; /** * Fireworks Forward User Id * @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests. @@ -50058,6 +50060,8 @@ export interface components { discoverable?: boolean | null; /** Enable Tag Filtering */ enable_tag_filtering?: boolean | null; + /** Explicit Cache Read Input Token Cost */ + explicit_cache_read_input_token_cost?: number | null; /** Id */ id: string | null; /** Input Cost Per Character */ @@ -50412,6 +50416,8 @@ export interface components { default_api_key_tpm_limit?: number | null; /** Drop Params */ drop_params?: boolean | string | null; + /** Explicit Cache Read Input Token Cost */ + explicit_cache_read_input_token_cost?: number | null; /** * Fireworks Forward User Id * @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests. From fad31403dbdee7afc95e9d10347433259394781d Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:55:33 -0700 Subject: [PATCH 067/124] chore(pricing): add github_copilot gpt-4.1 deprecation dates (#47492) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++++ model_prices_and_context_window.json | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 30229f2a73f..cc284c0ce4c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33002,22 +33002,26 @@ "supports_parallel_function_calling": true }, "github_copilot/gpt-4.1": { + "deprecation_date": "2026-06-01", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", + "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, "supports_vision": true }, "github_copilot/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-06-01", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", + "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 30229f2a73f..cc284c0ce4c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33002,22 +33002,26 @@ "supports_parallel_function_calling": true }, "github_copilot/gpt-4.1": { + "deprecation_date": "2026-06-01", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", + "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, "supports_vision": true }, "github_copilot/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-06-01", "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", + "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, From e34c779c670d21b14aeaed2d1c9f05ce2ed528ad Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:11:30 +0000 Subject: [PATCH 068/124] fix(proxy): decode zstd and gzip request bodies before auth reads them (#47418) * test(proxy): cover gzip and zstd request bodies on /v1/systemone * fix(proxy): decode zstd and gzip request bodies before auth reads them * fix(proxy): bound request body decompression and keep it inside CORS Limit the decoded size by an expansion ratio and the operator's max_request_size_mb, pass bodies without the coding's magic number through, answer 415 when zstd is unavailable, and register the middleware innermost so errors carry CORS and security headers * fix(proxy): budget decoded request bodies per worker and forward plugin bodies as sent Each worker holds at most MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB of decoded bodies and answers 503 past it. /plugin-proxy/ bodies reach the plugin as sent, a zstd skippable frame counts as zstd, the decoded-size 413 names the decoded limit, and an integer max_request_size_mb stays an integer in the 413 text. * fix(proxy): read the magic number before size limits and reserve declared decoded sizes * test(proxy): pin gzip mtime so parametrized test ids match across xdist workers * fix(proxy): parse max_request_size_mb the same way in the auth size check * fix(proxy): pass Claude Code OTLP export bodies through undecoded and skip the size parse without premium * test(proxy): cover client disconnects, chunked bodies past the cap, and body-less request messages in the decompression middleware * test(proxy): name the decompression test cases so CI logs stay small --- litellm/constants.py | 5 + litellm/proxy/auth/auth_utils.py | 17 +- .../request_body_decompression_middleware.py | 418 +++++++++++ .../request_size_limit_middleware.py | 6 +- litellm/proxy/proxy_server.py | 12 +- pyproject.toml | 2 + tests/code_coverage_tests/liccheck.ini | 1 + tests/unit/proxy/auth/test_auth_utils.py | 44 +- .../decisions_endpoints/test_endpoints.py | 132 +++- ...t_request_body_decompression_middleware.py | 667 ++++++++++++++++++ uv.lock | 103 +++ 11 files changed, 1394 insertions(+), 13 deletions(-) create mode 100644 litellm/proxy/middleware/request_body_decompression_middleware.py create mode 100644 tests/unit/proxy/middleware/test_request_body_decompression_middleware.py diff --git a/litellm/constants.py b/litellm/constants.py index 7d8311eb915..c7673db6976 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -191,6 +191,11 @@ DEFAULT_IMAGE_HEIGHT: Final = get_env_int("DEFAULT_IMAGE_HEIGHT", 300) # Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected # Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = get_env_float("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50.0) +MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB: Final = get_env_float("MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB", 50.0) +MAX_REQUEST_BODY_DECOMPRESSION_RATIO: Final = get_env_int("MAX_REQUEST_BODY_DECOMPRESSION_RATIO", 200) +MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB: Final = get_env_float( + "MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB", 200.0 +) MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = get_env_int( "MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024 ) # 1MB = 1024KB diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a05121ff2b..96142d75403 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1047,6 +1047,9 @@ def normalize_request_route(route: str) -> str: return route +MAX_REQUEST_SIZE_MB_SETTING: Final = TypeAdapter[int | float | None](int | float | None) + + async def check_if_request_size_is_safe(request: Request) -> bool: """ Enterprise Only: @@ -1062,19 +1065,19 @@ async def check_if_request_size_is_safe(request: Request) -> bool: ProxyException: If the request size is too large """ - from litellm.proxy.proxy_server import general_settings, premium_user + from litellm.proxy.proxy_server import configured_max_request_size_mb, general_settings, premium_user - max_request_size_mb: Final = general_settings.get("max_request_size_mb", None) - - if max_request_size_mb is not None: - # Check if premium user - if premium_user is not True: + if premium_user is not True: + if general_settings.get("max_request_size_mb") is not None: verbose_proxy_logger.warning( "using max_request_size_mb - not checking - this is an enterprise only feature. %s", CommonProxyErrors.not_premium_user.value, ) - return True + return True + max_request_size_mb: Final = configured_max_request_size_mb() + + if max_request_size_mb is not None: # Get the request body content_length: Final = request.headers.get("content-length") diff --git a/litellm/proxy/middleware/request_body_decompression_middleware.py b/litellm/proxy/middleware/request_body_decompression_middleware.py new file mode 100644 index 00000000000..5eb8a9bfc91 --- /dev/null +++ b/litellm/proxy/middleware/request_body_decompression_middleware.py @@ -0,0 +1,418 @@ +import io +import sys +import zlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Protocol, TypeAlias + +import anyio.to_thread +from starlette.datastructures import Headers +from starlette.requests import ClientDisconnect, Request +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from litellm.constants import ( + MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB, + MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB, + MAX_REQUEST_BODY_DECOMPRESSION_RATIO, +) +from litellm.proxy.auth.auth_utils import get_request_route +from litellm.proxy.middleware.request_size_limit_middleware import MaxRequestSizeGetter, send_request_too_large + +RFC_9659_ZSTD_WINDOW_LOG_MAX: Final = 23 +GZIP_MAGIC: Final = b"\x1f\x8b" +ZSTD_FRAME_MAGIC: Final = b"\x28\xb5\x2f\xfd" +ZSTD_SKIPPABLE_FRAME_MAGIC_HIGH_BYTES: Final = b"\x2a\x4d\x18" +SIGNATURE_LENGTH: Final = len(ZSTD_FRAME_MAGIC) +GZIP_HEADER_AND_TRAILER_LENGTH: Final = 18 +RAW_BODY_ROUTE_PREFIXES: Final = ( + "/plugin-proxy/", + "/claude_code_gateway/v1/metrics", + "/claude_code_gateway/v1/logs", + "/claude_code_gateway/v1/traces", +) +CODING_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"x-gzip": "gzip"}) +_REPLACED_REQUEST_HEADERS: Final = frozenset({b"content-encoding", b"content-length", b"transfer-encoding"}) +_CONTENT_ENCODING_HEADER: Final = frozenset({b"content-encoding"}) + + +class _Decompressor(Protocol): + @property + def eof(self) -> bool: ... + + @property + def unused_data(self) -> bytes: ... + + def decompress(self, data: bytes, /, max_length: int) -> bytes: ... + + +@dataclass(frozen=True, slots=True) +class Codec: + name: str + new_decompressor: Callable[[], _Decompressor] + error: type[Exception] + declared_size: Callable[[bytes], int | None] + + +def _new_gzip_decompressor() -> _Decompressor: + return zlib.decompressobj(wbits=31) + + +def _gzip_declared_size(data: bytes) -> int | None: + if len(data) < GZIP_HEADER_AND_TRAILER_LENGTH: + return None + return int.from_bytes(data[-4:], "little") + + +GZIP: Final = Codec( + name="gzip", + new_decompressor=_new_gzip_decompressor, + error=zlib.error, + declared_size=_gzip_declared_size, +) + + +def _starts_a_gzip_member(data: bytes) -> bool: + return data.startswith(GZIP_MAGIC) + + +def _starts_a_zstd_frame(data: bytes) -> bool: + if data.startswith(ZSTD_FRAME_MAGIC): + return True + return len(data) >= 4 and data[0] & 0xF0 == 0x50 and data[1:4] == ZSTD_SKIPPABLE_FRAME_MAGIC_HIGH_BYTES + + +def load_zstd_codec() -> Codec | None: + try: + if sys.version_info >= (3, 14): + from compression.zstd import DecompressionParameter, ZstdDecompressor, ZstdError, get_frame_info + else: + from backports.zstd import DecompressionParameter, ZstdDecompressor, ZstdError, get_frame_info + except ImportError: + return None + + def new_zstd_decompressor() -> _Decompressor: + return ZstdDecompressor(options={DecompressionParameter.window_log_max: RFC_9659_ZSTD_WINDOW_LOG_MAX}) + + def zstd_declared_size(data: bytes) -> int | None: + try: + return get_frame_info(data).decompressed_size + except ZstdError: + return None + + return Codec( + name="zstd", + new_decompressor=new_zstd_decompressor, + error=ZstdError, + declared_size=zstd_declared_size, + ) + + +_ZSTD: Final = load_zstd_codec() +CODING_SIGNATURES: Final[Mapping[str, Callable[[bytes], bool]]] = MappingProxyType( + {"gzip": _starts_a_gzip_member, "zstd": _starts_a_zstd_frame} +) +AVAILABLE_CODECS: Final[Mapping[str, Codec]] = MappingProxyType( + {codec.name: codec for codec in (GZIP, _ZSTD) if codec is not None} +) + + +@dataclass(frozen=True, slots=True) +class _Prefix: + data: bytes + more_body: bool + + +@dataclass(frozen=True, slots=True) +class _Body: + data: bytes + + +@dataclass(frozen=True, slots=True) +class _TooLarge: + pass + + +@dataclass(frozen=True, slots=True) +class _DecodedTooLarge: + pass + + +@dataclass(frozen=True, slots=True) +class _OverRatio: + pass + + +@dataclass(frozen=True, slots=True) +class _ClientDisconnected: + pass + + +@dataclass(frozen=True, slots=True) +class _Undecodable: + message: str + + +@dataclass(frozen=True, slots=True) +class _Limits: + max_mb: float + max_ratio: int + + @property + def max_bytes(self) -> int: + return int(self.max_mb * 1024 * 1024) + + +_ReceivedBody: TypeAlias = _Body | _TooLarge | _ClientDisconnected +_DecodedBody: TypeAlias = _Body | _DecodedTooLarge | _OverRatio | _Undecodable + + +class _DecodedBodyBudget: + def __init__(self, max_bytes: int) -> None: + self.max_bytes: Final = max_bytes + self.reserved_bytes = 0 + + def try_reserve(self, size: int) -> bool: + if self.reserved_bytes > 0 and self.reserved_bytes + size > self.max_bytes: + return False + self.hold(size) + return True + + def hold(self, size: int) -> None: + self.reserved_bytes += size + + def release(self, size: int) -> None: + self.reserved_bytes -= size + + +def _coding_for(scope: Scope) -> str | None: + if scope["type"] != "http": + return None + content_encodings: Final = Headers(scope=scope).getlist("content-encoding") + if len(content_encodings) != 1: + return None + named_coding: Final = content_encodings[0].strip().lower() + coding: Final = CODING_ALIASES.get(named_coding, named_coding) + if coding not in CODING_SIGNATURES: + return None + if get_request_route(Request(scope)).startswith(RAW_BODY_ROUTE_PREFIXES): + return None + return coding + + +def _request_chunk(message: Message) -> _Prefix | _ClientDisconnected: + match message: + case {"type": "http.disconnect"}: + return _ClientDisconnected() + case {"body": bytes() as body, "more_body": True}: + return _Prefix(body, more_body=True) + case {"body": bytes() as body}: + return _Prefix(body, more_body=False) + case {"more_body": True}: + return _Prefix(b"", more_body=True) + case _: + return _Prefix(b"", more_body=False) + + +async def _receive_prefix(receive: Receive, received: bytes = b"") -> _Prefix | _ClientDisconnected: + match _request_chunk(await receive()): + case _ClientDisconnected() as disconnected: + return disconnected + case _Prefix(data=data, more_body=more_body): + prefix: Final = _Prefix(received + data, more_body) + if len(prefix.data) >= SIGNATURE_LENGTH or not more_body: + return prefix + return await _receive_prefix(receive, prefix.data) + + +async def _receive_body(scope: Scope, receive: Receive, prefix: _Prefix, max_bytes: int) -> _ReceivedBody: + if len(prefix.data) > max_bytes: + return _TooLarge() + if not prefix.more_body: + return _Body(prefix.data) + buffer: Final = io.BytesIO() + buffer.write(prefix.data) + try: + async for chunk in Request(scope, receive).stream(): + buffer.write(chunk) + if buffer.tell() > max_bytes: + return _TooLarge() + except ClientDisconnect: + return _ClientDisconnected() + return _Body(buffer.getvalue()) + + +def _max_decoded_length(codec: Codec, data: bytes, limits: _Limits) -> int: + max_by_limits: Final = min(limits.max_bytes, len(data) * limits.max_ratio) + declared: Final = codec.declared_size(data) + return max_by_limits if declared is None else min(max_by_limits, declared) + + +def _decode(codec: Codec, data: bytes, limits: _Limits) -> _DecodedBody: + max_by_ratio: Final = len(data) * limits.max_ratio + declared: Final = codec.declared_size(data) + max_length: Final = _max_decoded_length(codec, data, limits) + decompressor: Final = codec.new_decompressor() + try: + decoded: Final = decompressor.decompress(data, max_length=max_length + 1) + except codec.error as error: + return _Undecodable(f"Request body is not valid {codec.name} data: {error}") + if len(decoded) > limits.max_bytes: + return _DecodedTooLarge() + if len(decoded) > max_by_ratio: + return _OverRatio() + if declared is not None and len(decoded) > declared: + return _Undecodable(f"Request body decodes past the size its {codec.name} data declares") + if not decompressor.eof: + return _Undecodable(f"Request body ends before the end of its {codec.name} stream") + if decompressor.unused_data: + return _Undecodable(f"Request body has data after the end of its {codec.name} stream") + return _Body(decoded) + + +def _headers_without(scope: Scope, dropped: frozenset[bytes]) -> tuple[tuple[bytes, bytes], ...]: + return tuple((name, value) for name, value in Headers(scope=scope).raw if name.lower() not in dropped) + + +def _identity_scope(scope: Scope, body_length: int) -> Scope: + kept_headers: Final = _headers_without(scope, _REPLACED_REQUEST_HEADERS) + return {**scope, "headers": [*kept_headers, (b"content-length", str(body_length).encode("latin-1"))]} + + +def _replay_body(body: bytes, receive: Receive, more_body: bool = False) -> Receive: + body_message: Final[Message] = {"type": "http.request", "body": body, "more_body": more_body} + pending: Final = iter((body_message,)) + + async def replay() -> Message: + return next(pending, None) or await receive() + + return replay + + +def _error_response( + message: str, + status_code: int, + error_type: str = "invalid_request_error", + headers: Mapping[str, str] | None = None, +) -> JSONResponse: + return JSONResponse( + {"error": {"message": message, "type": error_type, "param": None, "code": str(status_code)}}, + status_code=status_code, + headers=headers, + ) + + +def _held_length(decoded: _DecodedBody) -> int: + return len(decoded.data) if isinstance(decoded, _Body) else 0 + + +def _effective_max_mb(max_decompressed_body_size_mb: float, operator_max_mb: float | None) -> float: + if operator_max_mb is None or operator_max_mb <= 0: + return max_decompressed_body_size_mb + return min(max_decompressed_body_size_mb, operator_max_mb) + + +def _no_operator_limit() -> None: + return None + + +class RequestBodyDecompressionMiddleware: + def __init__( + self, + app: ASGIApp, + get_max_request_size_mb: MaxRequestSizeGetter = _no_operator_limit, + max_decompressed_body_size_mb: float = MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB, + max_decompression_ratio: int = MAX_REQUEST_BODY_DECOMPRESSION_RATIO, + max_decoded_bodies_in_flight_mb: float = MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB, + codecs: Mapping[str, Codec] = AVAILABLE_CODECS, + ) -> None: + self.app = app + self.get_max_request_size_mb: Final = get_max_request_size_mb + self.max_decompressed_body_size_mb: Final = max_decompressed_body_size_mb + self.max_decompression_ratio: Final = max_decompression_ratio + self.budget: Final = _DecodedBodyBudget(int(max_decoded_bodies_in_flight_mb * 1024 * 1024)) + self.codecs: Final = codecs + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + coding: Final = _coding_for(scope) + if coding is None: + await self.app(scope, receive, send) + return + match await _receive_prefix(receive): + case _ClientDisconnected(): + return + case _Prefix() as prefix: + await self._decode_if_encoded(scope, receive, send, coding, prefix) + + async def _decode_if_encoded( + self, scope: Scope, receive: Receive, send: Send, coding: str, prefix: _Prefix + ) -> None: + if not CODING_SIGNATURES[coding](prefix.data): + sent_as_is: Final = {**scope, "headers": [*_headers_without(scope, _CONTENT_ENCODING_HEADER)]} + await self.app(sent_as_is, _replay_body(prefix.data, receive, more_body=prefix.more_body), send) + return + codec: Final = self.codecs.get(coding) + if codec is None: + supported: Final = ", ".join(sorted(self.codecs)) + await _error_response( + f"This proxy cannot decode {coding} request bodies. Supported encodings: {supported}", + 415, + headers={"accept-encoding": supported}, + )(scope, receive, send) + return + await self._decode_then_call(scope, receive, send, codec, prefix) + + async def _decode_then_call( + self, scope: Scope, receive: Receive, send: Send, codec: Codec, prefix: _Prefix + ) -> None: + max_mb: Final = _effective_max_mb(self.max_decompressed_body_size_mb, self.get_max_request_size_mb()) + limits: Final = _Limits(max_mb=max_mb, max_ratio=self.max_decompression_ratio) + match await _receive_body(scope, receive, prefix, limits.max_bytes): + case _Body(data=data): + await self._decode_within_budget(scope, receive, send, codec, data, limits) + case _TooLarge(): + await send_request_too_large(send=send, max_request_size_mb=max_mb) + case _ClientDisconnected(): + return + + async def _decode_within_budget( + self, scope: Scope, receive: Receive, send: Send, codec: Codec, data: bytes, limits: _Limits + ) -> None: + reservation: Final = _max_decoded_length(codec, data, limits) + if not self.budget.try_reserve(reservation): + await _error_response( + "The proxy is decoding too many compressed request bodies right now. " + "Retry later, or send this body uncompressed", + 503, + error_type="overloaded_error", + headers={"retry-after": "1"}, + )(scope, receive, send) + return + try: + decoded: Final = await anyio.to_thread.run_sync(_decode, codec, data, limits) + finally: + self.budget.release(reservation) + held: Final = _held_length(decoded) + self.budget.hold(held) + try: + await self._respond(scope, receive, send, decoded, limits) + finally: + self.budget.release(held) + + async def _respond( + self, scope: Scope, receive: Receive, send: Send, decoded: _DecodedBody, limits: _Limits + ) -> None: + match decoded: + case _Body(data=body): + await self.app(_identity_scope(scope, len(body)), _replay_body(body, receive), send) + case _DecodedTooLarge(): + await _error_response(f"Request body is larger than {limits.max_mb:g} MB when decoded", 413)( + scope, receive, send + ) + case _OverRatio(): + await _error_response(f"Request body expands more than {limits.max_ratio} times when decoded", 413)( + scope, receive, send + ) + case _Undecodable(message=message): + await _error_response(message, 400)(scope, receive, send) diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py index 1715752d145..cbbc79cc2ca 100644 --- a/litellm/proxy/middleware/request_size_limit_middleware.py +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -44,7 +44,7 @@ class RequestSizeLimitMiddleware: content_length: Final = _get_content_length(scope=scope) if content_length is not None and content_length > max_request_size_bytes: - await _send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) + await send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) return received_body_bytes = 0 @@ -74,7 +74,7 @@ class RequestSizeLimitMiddleware: except RequestEntityTooLarge: if response_started: raise - await _send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) + await send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) def _mb_to_bytes(max_request_size_mb: float | None) -> int | None: @@ -97,7 +97,7 @@ def _get_content_length(scope: Scope) -> int | None: return None -async def _send_request_too_large( +async def send_request_too_large( send: Send, max_request_size_mb: float | None, ) -> None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 811d4b8601e..341f12ba720 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -401,6 +401,7 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET from litellm.proxy.auth.auth_utils import ( + MAX_REQUEST_SIZE_MB_SETTING, check_response_size_is_safe, is_request_body_safe, log_model_access_denial, @@ -840,6 +841,7 @@ from litellm.proxy.middleware.per_request_root_path_middleware import ( get_server_root_paths, ) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware +from litellm.proxy.middleware.request_body_decompression_middleware import RequestBodyDecompressionMiddleware from litellm.proxy.middleware.request_size_limit_middleware import ( RequestSizeLimitMiddleware, ) @@ -2750,6 +2752,14 @@ current_dir = os.path.dirname(os.path.abspath(__file__)) # app.mount("/ui", StaticFiles(directory=ui_path, html=True), name="ui") +def configured_max_request_size_mb() -> int | float | None: + return MAX_REQUEST_SIZE_MB_SETTING.validate_python(general_settings.get("max_request_size_mb")) + + +app.add_middleware( + RequestBodyDecompressionMiddleware, + get_max_request_size_mb=lambda: configured_max_request_size_mb() if premium_user is True else None, +) app.add_middleware( CORSMiddleware, allow_origins=origins, @@ -21507,7 +21517,7 @@ attach_lazy_features(app) app.router.routes = hot_routes_first(app.router.routes) app.add_middleware( RequestSizeLimitMiddleware, - get_max_request_size_mb=lambda: general_settings.get("max_request_size_mb"), + get_max_request_size_mb=configured_max_request_size_mb, is_request_size_limit_enabled=lambda: premium_user is True, ) app.add_middleware( diff --git a/pyproject.toml b/pyproject.toml index 6c71140e6b8..9871a83b421 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ proxy = [ "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "expression>=5.6.0,<6.0", + "backports-zstd>=1.0.0,<2.0; python_version < '3.14'", ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports are all guarded, so it runs on the base SDK plus these packages, and @@ -259,6 +260,7 @@ proxy-dev = [ "opentelemetry-instrumentation-fastapi==0.54b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", + "backports-zstd==1.8.0; python_version < '3.14'", ] ci = [ "psutil==7.2.2", diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index b66493386c1..e533907659b 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -179,3 +179,4 @@ pytest-rerunfailures: >=15.1 # MPL 2.0 license pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE +backports-zstd: >=1.0.0 # PSF-2.0 license - https://github.com/Rogdham/backports.zstd/blob/master/LICENSE.txt diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 02be27d921e..7b1c895483d 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -12,12 +12,13 @@ import pytest from fastapi import HTTPException, Request import litellm -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( _allow_model_level_clientside_configurable_parameters, _get_customer_id_from_standard_headers, abbreviate_api_key, check_complete_credentials, + check_if_request_size_is_safe, custom_auth_common_checks_warning, get_customer_user_header_from_mapping, log_once_if_budget_reservation_disabled, @@ -5033,3 +5034,44 @@ def test_get_end_user_id_from_request_body_backwards_compatibility(): ) for request_body, expected_end_user_id in cases: assert get_end_user_id_from_request_body(request_body) == expected_end_user_id + + +def _request_with_content_length(content_length: int) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [(b"content-length", str(content_length).encode("latin-1"))], + } + ) + + +@pytest.mark.asyncio +async def test_a_quoted_max_request_size_mb_lets_a_smaller_keyed_request_through(): + with ( + patch("litellm.proxy.proxy_server.general_settings", {"max_request_size_mb": "10"}), + patch("litellm.proxy.proxy_server.premium_user", True), + ): + assert await check_if_request_size_is_safe(request=_request_with_content_length(1024)) is True + + +@pytest.mark.asyncio +async def test_a_quoted_max_request_size_mb_rejects_a_larger_keyed_request_with_400(): + with ( + patch("litellm.proxy.proxy_server.general_settings", {"max_request_size_mb": "10"}), + patch("litellm.proxy.proxy_server.premium_user", True), + pytest.raises(ProxyException) as raised, + ): + await check_if_request_size_is_safe(request=_request_with_content_length(11 * 1024 * 1024)) + assert raised.value.code == "400" + assert "Max size is 10 MB" in raised.value.message + + +@pytest.mark.asyncio +async def test_a_non_numeric_max_request_size_mb_is_ignored_without_premium(): + with ( + patch("litellm.proxy.proxy_server.general_settings", {"max_request_size_mb": "10MB"}), + patch("litellm.proxy.proxy_server.premium_user", False), + ): + assert await check_if_request_size_is_safe(request=_request_with_content_length(1024)) is True diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 6ca8c723b06..fcf56eaea9a 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1,8 +1,11 @@ from __future__ import annotations import asyncio +import gzip import json -from collections.abc import AsyncGenerator, Iterator, Mapping +import random +import sys +from collections.abc import AsyncGenerator, Callable, Iterator, Mapping from contextlib import asynccontextmanager from pathlib import Path from typing import Final @@ -18,14 +21,23 @@ from starlette.routing import Match import litellm from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, + hash_token, initialize, ) from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from tests._master_key import MASTER_KEY + +if sys.version_info >= (3, 14): + from compression import zstd +else: + from backports import zstd _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -1282,3 +1294,121 @@ def test_the_two_decisions_routes_document_different_request_formats(decisions_o assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") + + +def test_proxy_systemone_reads_a_zstd_compressed_request_body( + client: TestClient, + respx_mock: respx.MockRouter, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/systemone", + content=zstd.compress(json.dumps(_REQUEST).encode()), + headers={"content-type": "application/json", "content-encoding": "zstd"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == _RESPONSE["answers"] + assert json.loads(upstream.calls[0].request.content)["questions"] == _REQUEST["questions"] + + +@pytest.mark.parametrize( + ("content_encoding", "compress"), + (("gzip", gzip.compress), ("zstd", zstd.compress)), +) +def test_a_compressed_body_naming_a_model_outside_the_key_is_refused_by_auth( + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + content_encoding: str, + compress: Callable[[bytes], bytes], +) -> None: + user_key: Final = "sk-decider-only" + user_api_key_cache: Final = UserApiKeyCache() + user_api_key_cache.set_cache( + key=hash_token(user_key), value=UserAPIKeyAuth(token=hash_token(user_key), models=["decider"]) + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", "no-db-needed-for-a-cached-key") + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post( + "/v1/systemone", + content=compress(json.dumps({**_REQUEST, "model": "not-on-this-key"}).encode()), + headers={ + "content-type": "application/json", + "content-encoding": content_encoding, + "authorization": f"Bearer {user_key}", + }, + ) + + assert response.status_code == 403, response.text + assert "not-on-this-key" in response.text + assert not upstream.called + + +@pytest.mark.parametrize( + ("content_encoding", "undecodable_body"), + (("gzip", gzip.compress(b"{}", mtime=0)[:-8]), ("zstd", zstd.compress(b"{}") + b"trailing")), +) +def test_a_browser_can_read_the_error_for_a_compressed_body_the_proxy_cannot_decode( + client: TestClient, + respx_mock: respx.MockRouter, + content_encoding: str, + undecodable_body: bytes, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + origin: Final = "https://app.example.test" + + response: Final = client.post( + "/v1/systemone", + content=undecodable_body, + headers={"content-type": "application/json", "content-encoding": content_encoding, "origin": origin}, + ) + + assert response.status_code == 400, response.text + assert response.headers["access-control-allow-origin"] in ("*", origin) + assert response.headers["x-content-type-options"] == "nosniff" + assert not upstream.called + + +def test_the_operator_request_size_limit_applies_to_the_decoded_body( + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + max_request_size_mb: Final = 0.01 + monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) + monkeypatch.setitem(litellm.proxy.proxy_server.general_settings, "max_request_size_mb", max_request_size_mb) + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + state_text: Final = "".join(random.Random(0).choices("abcd", k=20_000)) + compressed: Final = zstd.compress(json.dumps({**_REQUEST, "state": {"source": state_text}}).encode()) + + response: Final = client.post( + "/v1/systemone", + content=compressed, + headers={"content-type": "application/json", "content-encoding": "zstd"}, + ) + + assert len(compressed) < max_request_size_mb * 1024 * 1024 + assert response.status_code == 413, response.text + assert response.json()["error"]["message"] == f"Request body is larger than {max_request_size_mb} MB when decoded" + assert not upstream.called + + +def test_a_whole_number_request_size_limit_is_named_as_configured_in_the_413( + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) + monkeypatch.setitem(litellm.proxy.proxy_server.general_settings, "max_request_size_mb", 1) + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/systemone", json={**_REQUEST, "state": {"source": "a" * 1024 * 1024}}) + + assert response.status_code == 413, response.text + assert response.json() == {"error": "Request size is too large. Max size is 1 MB"} + assert not upstream.called diff --git a/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py new file mode 100644 index 00000000000..ca5774b6be0 --- /dev/null +++ b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py @@ -0,0 +1,667 @@ +import asyncio +import base64 +import gzip +import json +import math +import random +import struct +import sys +import tracemalloc +import zlib +from collections.abc import Callable +from types import MappingProxyType +from typing import Final + +import anyio +import httpx +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.routing import Route +from starlette.testclient import TestClient +from starlette.types import Message, Receive, Scope, Send + +from litellm.proxy.middleware.request_body_decompression_middleware import ( + GZIP, + GZIP_MAGIC, + RFC_9659_ZSTD_WINDOW_LOG_MAX, + ZSTD_FRAME_MAGIC, + RequestBodyDecompressionMiddleware, + load_zstd_codec, +) + +if sys.version_info >= (3, 14): + from compression import zstd +else: + from backports import zstd + +MIB: Final = 1024 * 1024 +CAP_MB: Final = 1.0 +REQUEST_JSON: Final = json.dumps( + {"model": "jev", "state": "order history " * 200, "questions": {"is_refund_request": {"type": "noul"}}} +).encode() + + +async def _echo_request(request: Request) -> Response: + body: Final = await request.body() + return JSONResponse( + { + "body_hex": body.hex(), + "content_encoding": request.headers.getlist("content-encoding"), + "content_length": request.headers.get("content-length"), + } + ) + + +def _client(max_decompressed_body_size_mb: float = CAP_MB, **options: object) -> TestClient: + echo_app: Final = Starlette( + routes=[ + Route("/v1/systemone", _echo_request, methods=["POST"]), + Route("/plugin-proxy/{path:path}", _echo_request, methods=["POST"]), + Route("/claude_code_gateway/v1/{signal}", _echo_request, methods=["POST"]), + ] + ) + return TestClient( + RequestBodyDecompressionMiddleware( + echo_app, + max_decompressed_body_size_mb=max_decompressed_body_size_mb, + **options, # pyright: ignore[reportArgumentType] # each test passes one typed keyword + ) + ) + + +def _low_entropy_text(length: int) -> bytes: + return bytes(random.Random(0).choices(b"abcd", k=length)) + + +def _post(client: TestClient, body: bytes, content_encodings: tuple[str, ...], path: str = "/v1/systemone") -> Response: + headers: Final = [("content-type", "application/json"), *(("content-encoding", v) for v in content_encodings)] + return client.post(path, content=body, headers=headers) + + +def _decoded_too_large(max_mb: float) -> dict[str, dict[str, str | None]]: + return { + "error": { + "message": f"Request body is larger than {max_mb:g} MB when decoded", + "type": "invalid_request_error", + "param": None, + "code": "413", + } + } + + +def _zeros_compressed(compress_chunk: Callable[[bytes], bytes], flush: Callable[[], bytes], decoded_mib: int) -> bytes: + zero_mib: Final = bytes(MIB) + return b"".join([*(compress_chunk(zero_mib) for _ in range(decoded_mib)), flush()]) + + +def _gzip_bomb(decoded_mib: int) -> bytes: + compressor: Final = zlib.compressobj(9, zlib.DEFLATED, 31) + return _zeros_compressed(compressor.compress, compressor.flush, decoded_mib) + + +def _zstd_bomb(decoded_mib: int) -> bytes: + compressor: Final = zstd.ZstdCompressor() + return _zeros_compressed(compressor.compress, compressor.flush, decoded_mib) + + +@pytest.mark.parametrize( + ("compressed", "content_encoding"), + [ + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), "gzip", id="gzip"), + pytest.param(zstd.compress(REQUEST_JSON), "zstd", id="zstd"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), " GZip ", id="gzip-mixed-case-padded"), + pytest.param(zstd.compress(REQUEST_JSON), "ZSTD", id="zstd-upper-case"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), "x-gzip", id="x-gzip"), + ], +) +def test_supported_encoding_reaches_the_app_decoded(compressed: bytes, content_encoding: str): + response: Final = _post(_client(), compressed, (content_encoding,)) + + assert response.status_code == 200 + assert response.json() == { + "body_hex": REQUEST_JSON.hex(), + "content_encoding": [], + "content_length": str(len(REQUEST_JSON)), + } + + +@pytest.mark.parametrize( + ("body", "content_encodings"), + [ + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), (), id="no-header"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), ("identity",), id="identity"), + pytest.param(b"\x1b\x00brotli-ish", ("br",), id="br"), + pytest.param(REQUEST_JSON, ("compress",), id="compress"), + pytest.param(zlib.compress(REQUEST_JSON), ("deflate",), id="deflate"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), ("gzip, zstd",), id="stacked-in-one-header"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), ("gzip", "gzip"), id="two-headers"), + ], +) +def test_other_encodings_reach_the_app_untouched(body: bytes, content_encodings: tuple[str, ...]): + response: Final = _post(_client(), body, content_encodings) + + assert response.status_code == 200 + assert response.json() == { + "body_hex": body.hex(), + "content_encoding": list(content_encodings), + "content_length": str(len(body)), + } + + +@pytest.mark.parametrize( + ("body", "content_encoding"), + [ + pytest.param(REQUEST_JSON, "gzip", id="plain-labeled-gzip"), + pytest.param(REQUEST_JSON, "zstd", id="plain-labeled-zstd"), + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), "zstd", id="gzip-labeled-zstd"), + pytest.param(zstd.compress(REQUEST_JSON), "gzip", id="zstd-labeled-gzip"), + pytest.param(_low_entropy_text(2 * int(CAP_MB * MIB)), "gzip", id="plain-past-the-cap-labeled-gzip"), + pytest.param(_low_entropy_text(2 * int(CAP_MB * MIB)), "zstd", id="plain-past-the-cap-labeled-zstd"), + pytest.param(b"{}", "zstd", id="tiny-plain-labeled-zstd"), + ], +) +def test_body_that_is_not_in_its_labeled_encoding_reaches_the_app_as_sent(body: bytes, content_encoding: str): + response: Final = _post(_client(), body, (content_encoding,)) + + assert response.status_code == 200 + assert response.json() == {"body_hex": body.hex(), "content_encoding": [], "content_length": str(len(body))} + + +@pytest.mark.parametrize("content_encoding", ["gzip", "zstd"]) +def test_empty_body_with_supported_encoding_reaches_the_app_empty(content_encoding: str): + response: Final = _post(_client(), b"", (content_encoding,)) + + assert response.status_code == 200 + assert response.json() == {"body_hex": "", "content_encoding": [], "content_length": "0"} + + +@pytest.mark.parametrize(("bomb", "content_encoding"), [(_gzip_bomb, "gzip"), (_zstd_bomb, "zstd")]) +def test_decompression_bomb_is_rejected_without_inflating_past_the_ratio_limit( + bomb: Callable[[int], bytes], content_encoding: str +): + decoded_mib: Final = 200 + compressed: Final = bomb(decoded_mib) + client: Final = _client(max_decompressed_body_size_mb=decoded_mib * 2) + tracemalloc.start() + try: + response = _post(client, compressed, (content_encoding,)) + _, peak_bytes = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert response.status_code == 413 + assert response.json()["error"]["message"] == "Request body expands more than 200 times when decoded" + assert response.json()["error"]["type"] == "invalid_request_error" + assert peak_bytes < decoded_mib * MIB / 2 + + +def test_ratio_limit_rejects_only_a_body_that_expands_past_it(): + compressed: Final = zstd.compress(REQUEST_JSON) + ratio: Final = math.ceil(len(REQUEST_JSON) / len(compressed)) + + at_limit: Final = _post(_client(max_decompression_ratio=ratio), compressed, ("zstd",)) + over_limit: Final = _post(_client(max_decompression_ratio=ratio - 1), compressed, ("zstd",)) + + assert at_limit.status_code == 200 + assert over_limit.status_code == 413 + + +def test_body_decoding_past_the_cap_is_rejected(): + compressed: Final = zstd.compress(_low_entropy_text(2 * int(CAP_MB * MIB))) + + response: Final = _post(_client(), compressed, ("zstd",)) + + assert len(compressed) < CAP_MB * MIB + assert response.status_code == 413 + assert response.json() == _decoded_too_large(CAP_MB) + + +@pytest.mark.parametrize( + ("operator_max_mb", "effective_max_mb"), + [(0.25, 0.25), (CAP_MB * 4, CAP_MB), (None, CAP_MB), (0, CAP_MB)], +) +def test_operator_request_size_limit_caps_the_decoded_body_when_it_is_lower( + operator_max_mb: float | None, effective_max_mb: float +): + client: Final = _client(get_max_request_size_mb=lambda: operator_max_mb) + within: Final = zstd.compress(_low_entropy_text(int(effective_max_mb * MIB))) + over: Final = zstd.compress(_low_entropy_text(int(effective_max_mb * MIB) + 1)) + + assert _post(client, within, ("zstd",)).status_code == 200 + over_response: Final = _post(client, over, ("zstd",)) + assert over_response.status_code == 413 + assert over_response.json() == _decoded_too_large(effective_max_mb) + + +@pytest.mark.parametrize(("signature", "content_encoding"), [(GZIP_MAGIC, "gzip"), (ZSTD_FRAME_MAGIC, "zstd")]) +def test_compressed_body_over_the_cap_is_rejected_before_decoding(signature: bytes, content_encoding: str): + over_cap_on_the_wire: Final = signature + bytes(int(CAP_MB * MIB)) + + response: Final = _post(_client(), over_cap_on_the_wire, (content_encoding,)) + + assert response.status_code == 413 + assert response.json() == {"error": f"Request size is too large. Max size is {CAP_MB} MB"} + + +def test_body_decoding_to_exactly_the_cap_reaches_the_app(): + exactly_cap: Final = _low_entropy_text(int(CAP_MB * MIB)) + + response: Final = _post(_client(), zstd.compress(exactly_cap), ("zstd",)) + + assert response.status_code == 200 + assert response.json()["content_length"] == str(len(exactly_cap)) + + +def _zstd_frame_needing_a_window_over_the_rfc_9659_limit() -> bytes: + window_log: Final = RFC_9659_ZSTD_WINDOW_LOG_MAX + 1 + content: Final = bytes(2**RFC_9659_ZSTD_WINDOW_LOG_MAX + MIB) + return zstd.compress(content, options={zstd.CompressionParameter.window_log: window_log}) + + +def _zstd_skippable_frame(content: bytes) -> bytes: + return struct.pack(" bytes: + return gzip.compress(content, mtime=0)[:-4] + (len(content) // 2).to_bytes(4, "little") + + +@pytest.mark.parametrize( + ("body", "content_encoding", "message"), + [ + pytest.param( + GZIP_MAGIC + b"not gzip at all", "gzip", "Request body is not valid gzip data: ", id="corrupt-gzip" + ), + pytest.param( + ZSTD_FRAME_MAGIC + b"not zstd at all", "zstd", "Request body is not valid zstd data: ", id="corrupt-zstd" + ), + pytest.param( + gzip.compress(REQUEST_JSON, mtime=0)[:-8], + "gzip", + "Request body ends before the end of its gzip stream", + id="truncated-gzip", + ), + pytest.param( + zstd.compress(REQUEST_JSON)[:-8], + "zstd", + "Request body ends before the end of its zstd stream", + id="truncated-zstd", + ), + pytest.param( + gzip.compress(REQUEST_JSON, mtime=0) + b"{}", + "gzip", + "Request body has data after the end of its gzip stream", + id="gzip-trailing-data", + ), + pytest.param( + zstd.compress(REQUEST_JSON) + b"{}", + "zstd", + "Request body has data after the end of its zstd stream", + id="zstd-trailing-data", + ), + pytest.param( + _zstd_frame_needing_a_window_over_the_rfc_9659_limit(), + "zstd", + "Request body is not valid zstd data: ", + id="zstd-window-over-the-rfc-limit", + ), + pytest.param( + _zstd_skippable_frame(b"meta") + zstd.compress(REQUEST_JSON), + "zstd", + "Request body has data after the end of its zstd stream", + id="zstd-skippable-frame-first", + ), + pytest.param( + _gzip_declaring_a_smaller_size(REQUEST_JSON), + "gzip", + "Request body decodes past the size its gzip data declares", + id="gzip-declaring-a-smaller-size", + ), + ], +) +def test_undecodable_body_is_rejected_with_an_openai_style_400(body: bytes, content_encoding: str, message: str): + response: Final = _post(_client(max_decompressed_body_size_mb=50.0), body, (content_encoding,)) + + assert response.status_code == 400 + error: Final = response.json()["error"] + assert error["message"].startswith(message) + assert {key: error[key] for key in ("type", "param", "code")} == { + "type": "invalid_request_error", + "param": None, + "code": "400", + } + + +def test_zstd_body_gets_a_415_naming_gzip_when_this_python_cannot_decode_zstd(): + client: Final = _client(codecs=MappingProxyType({"gzip": GZIP})) + + zstd_response: Final = _post(client, zstd.compress(REQUEST_JSON), ("zstd",)) + gzip_response: Final = _post(client, gzip.compress(REQUEST_JSON, mtime=0), ("gzip",)) + plain_labeled_zstd_response: Final = _post(client, REQUEST_JSON, ("zstd",)) + + assert zstd_response.status_code == 415 + assert zstd_response.headers["accept-encoding"] == "gzip" + assert zstd_response.json()["error"]["message"] == ( + "This proxy cannot decode zstd request bodies. Supported encodings: gzip" + ) + assert gzip_response.status_code == 200 + assert plain_labeled_zstd_response.json()["body_hex"] == REQUEST_JSON.hex() + + +def test_zstd_is_unavailable_without_failing_the_import_on_a_python_without_zstd(monkeypatch: pytest.MonkeyPatch): + for module in ("_zstd", "compression.zstd", "backports.zstd"): + monkeypatch.setitem(sys.modules, module, None) + + assert load_zstd_codec() is None + + +@pytest.mark.parametrize( + "path", + [ + "/plugin-proxy/notes/v1/run", + "/claude_code_gateway/v1/metrics", + "/claude_code_gateway/v1/logs", + "/claude_code_gateway/v1/traces", + ], +) +@pytest.mark.parametrize( + ("body", "content_encoding"), + [ + pytest.param(gzip.compress(REQUEST_JSON, mtime=0), "gzip", id="gzip"), + pytest.param(zstd.compress(REQUEST_JSON), "zstd", id="zstd"), + pytest.param(zstd.compress(_low_entropy_text(int(CAP_MB * MIB) * 2)), "zstd", id="zstd-past-the-cap"), + pytest.param(_gzip_bomb(2), "gzip", id="gzip-past-the-ratio-limit"), + pytest.param( + gzip.compress(REQUEST_JSON, mtime=0) + gzip.compress(REQUEST_JSON, mtime=0), "gzip", id="gzip-two-members" + ), + ], +) +def test_raw_body_routes_get_the_body_as_sent(path: str, body: bytes, content_encoding: str): + response: Final = _post(_client(codecs=MappingProxyType({"gzip": GZIP})), body, (content_encoding,), path=path) + + assert response.status_code == 200 + assert response.json() == { + "body_hex": body.hex(), + "content_encoding": [content_encoding], + "content_length": str(len(body)), + } + + +@pytest.mark.asyncio +async def test_compressed_bodies_past_the_in_flight_budget_get_a_503_until_earlier_ones_finish(): + compressed: Final = zstd.compress(REQUEST_JSON) + app_has_the_first_body: Final = asyncio.Event() + let_the_first_request_finish: Final = asyncio.Event() + + async def route_holding_the_first_request(request: Request) -> Response: + body: Final = await request.body() + if not app_has_the_first_body.is_set(): + app_has_the_first_body.set() + await let_the_first_request_finish.wait() + return JSONResponse({"decoded_bytes": len(body)}) + + middleware: Final = RequestBodyDecompressionMiddleware( + Starlette(routes=[Route("/v1/systemone", route_holding_the_first_request, methods=["POST"])]), + max_decompressed_body_size_mb=CAP_MB, + max_decoded_bodies_in_flight_mb=1.5 * len(REQUEST_JSON) / MIB, + ) + headers: Final = {"content-type": "application/json", "content-encoding": "zstd"} + with anyio.fail_after(30): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=middleware), base_url="http://proxy") as client: + first: Final = asyncio.create_task(client.post("/v1/systemone", content=compressed, headers=headers)) + await app_has_the_first_body.wait() + while_first_is_in_flight: Final = await client.post("/v1/systemone", content=compressed, headers=headers) + let_the_first_request_finish.set() + first_response: Final = await first + after_first_finished: Final = await client.post("/v1/systemone", content=compressed, headers=headers) + + assert first_response.json() == {"decoded_bytes": len(REQUEST_JSON)} + assert while_first_is_in_flight.status_code == 503 + assert while_first_is_in_flight.headers["retry-after"] == "1" + assert while_first_is_in_flight.json()["error"]["type"] == "overloaded_error" + assert after_first_finished.json() == {"decoded_bytes": len(REQUEST_JSON)} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("compress", [gzip.compress, zstd.compress]) +async def test_concurrent_compressed_bodies_count_their_declared_size_against_the_budget( + compress: Callable[[bytes], bytes], +): + hard_to_compress: Final = json.dumps( + {"model": "jev", "image": base64.b64encode(random.Random(0).randbytes(48 * 1024)).decode()} + ).encode() + compressed: Final = compress(hard_to_compress) + all_bodies_arrived: Final = asyncio.Event() + bodies_seen: Final[list[int]] = [] + + async def route_holding_every_request(request: Request) -> Response: + bodies_seen.append(len(await request.body())) + if len(bodies_seen) == 3: + all_bodies_arrived.set() + await all_bodies_arrived.wait() + return JSONResponse({"decoded_bytes": bodies_seen[-1]}) + + middleware: Final = RequestBodyDecompressionMiddleware( + Starlette(routes=[Route("/v1/systemone", route_holding_every_request, methods=["POST"])]), + max_decompressed_body_size_mb=CAP_MB, + max_decoded_bodies_in_flight_mb=3.5 * len(hard_to_compress) / MIB, + ) + headers: Final = { + "content-type": "application/json", + "content-encoding": "zstd" if compress is zstd.compress else "gzip", + } + with anyio.fail_after(30): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=middleware), base_url="http://proxy") as client: + responses: Final = await asyncio.gather( + *(client.post("/v1/systemone", content=compressed, headers=headers) for _ in range(3)) + ) + + assert len(compressed) * 200 > CAP_MB * MIB + assert [response.status_code for response in responses] == [200, 200, 200] + assert bodies_seen == [len(hard_to_compress)] * 3 + + +def _http_scope(headers: list[tuple[bytes, bytes]]) -> Scope: + return {"type": "http", "method": "POST", "path": "/v1/systemone", "headers": headers} + + +def _scripted_receive(messages: list[Message]) -> Receive: + async def receive() -> Message: + return messages.pop(0) + + return receive + + +@pytest.mark.asyncio +async def test_chunked_body_is_decoded_whole_and_later_receives_reach_the_client(): + compressed: Final = zstd.compress(REQUEST_JSON) + third: Final = len(compressed) // 3 + client_messages: Final[list[Message]] = [ + {"type": "http.request", "body": compressed[:2], "more_body": True}, + {"type": "http.request", "body": compressed[2:third], "more_body": True}, + {"type": "http.request", "body": compressed[third : 2 * third], "more_body": True}, + {"type": "http.request", "body": compressed[2 * third :], "more_body": False}, + {"type": "http.disconnect"}, + ] + seen: Final[list[tuple[Scope, list[Message]]]] = [] + + async def app(scope: Scope, receive: Receive, send: Send) -> None: + seen.append((scope, [await receive(), await receive()])) + + async def send(message: Message) -> None: + raise AssertionError(f"middleware sent {message}") + + headers: Final = [ + (b"content-type", b"application/json"), + (b"content-encoding", b"zstd"), + (b"transfer-encoding", b"chunked"), + ] + await RequestBodyDecompressionMiddleware(app)(_http_scope(headers), _scripted_receive(client_messages), send) + + [(app_scope, app_messages)] = seen + assert app_scope["headers"] == [ + (b"content-type", b"application/json"), + (b"content-length", str(len(REQUEST_JSON)).encode()), + ] + assert app_messages == [ + {"type": "http.request", "body": REQUEST_JSON, "more_body": False}, + {"type": "http.disconnect"}, + ] + + +@pytest.mark.asyncio +async def test_chunked_plain_body_labeled_gzip_reaches_the_app_in_the_chunks_sent(): + client_messages: Final[list[Message]] = [ + {"type": "http.request", "body": REQUEST_JSON[:1], "more_body": True}, + {"type": "http.request", "body": REQUEST_JSON[1:9], "more_body": True}, + {"type": "http.request", "body": REQUEST_JSON[9:], "more_body": False}, + {"type": "http.disconnect"}, + ] + seen: Final[list[tuple[Scope, list[Message]]]] = [] + + async def app(scope: Scope, receive: Receive, send: Send) -> None: + seen.append((scope, [await receive(), await receive(), await receive()])) + + async def send(message: Message) -> None: + raise AssertionError(f"middleware sent {message}") + + headers: Final = [(b"content-encoding", b"gzip"), (b"transfer-encoding", b"chunked")] + await RequestBodyDecompressionMiddleware(app)(_http_scope(headers), _scripted_receive(client_messages), send) + + [(app_scope, app_messages)] = seen + assert app_scope["headers"] == [(b"transfer-encoding", b"chunked")] + assert app_messages == [ + {"type": "http.request", "body": REQUEST_JSON[:9], "more_body": True}, + {"type": "http.request", "body": REQUEST_JSON[9:], "more_body": False}, + {"type": "http.disconnect"}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_messages", + [ + pytest.param([{"type": "http.disconnect"}], id="before-the-body"), + pytest.param( + [{"type": "http.request", "body": GZIP_MAGIC, "more_body": True}, {"type": "http.disconnect"}], + id="after-a-short-first-chunk", + ), + pytest.param( + [ + {"type": "http.request", "body": gzip.compress(REQUEST_JSON, mtime=0)[:10], "more_body": True}, + {"type": "http.disconnect"}, + ], + id="mid-body", + ), + ], +) +async def test_client_disconnect_ends_the_request_without_a_response(client_messages: list[Message]): + sent: Final[list[Message]] = [] + app_calls: Final[list[Scope]] = [] + + async def app(scope: Scope, receive: Receive, send: Send) -> None: + app_calls.append(scope) + + async def send(message: Message) -> None: + sent.append(message) + + await RequestBodyDecompressionMiddleware(app)( + _http_scope([(b"content-encoding", b"gzip")]), _scripted_receive(client_messages), send + ) + + assert (app_calls, sent) == ([], []) + + +@pytest.mark.asyncio +async def test_chunked_compressed_body_past_the_cap_gets_a_413_without_reading_the_rest(): + compressed: Final = gzip.compress(random.Random(0).randbytes(2 * MIB), mtime=0) + chunk_size: Final = 64 * 1024 + chunks: Final = [compressed[start : start + chunk_size] for start in range(0, len(compressed), chunk_size)] + client_messages: Final[list[Message]] = [ + *({"type": "http.request", "body": chunk, "more_body": True} for chunk in chunks[:-1]), + {"type": "http.request", "body": chunks[-1], "more_body": False}, + ] + sent: Final[list[Message]] = [] + app_calls: Final[list[Scope]] = [] + + async def app(scope: Scope, receive: Receive, send: Send) -> None: + app_calls.append(scope) + + async def send(message: Message) -> None: + sent.append(message) + + await RequestBodyDecompressionMiddleware(app, max_decompressed_body_size_mb=CAP_MB)( + _http_scope([(b"content-encoding", b"gzip"), (b"transfer-encoding", b"chunked")]), + _scripted_receive(client_messages), + send, + ) + + assert app_calls == [] + assert sent[0]["status"] == 413 + assert json.loads(sent[1]["body"]) == {"error": f"Request size is too large. Max size is {CAP_MB} MB"} + assert client_messages, "the middleware read the whole body past the cap" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("client_messages", "body_the_app_gets"), + [ + pytest.param( + [ + {"type": "http.request", "more_body": True}, + {"type": "http.request", "body": gzip.compress(REQUEST_JSON, mtime=0), "more_body": False}, + {"type": "http.disconnect"}, + ], + REQUEST_JSON, + id="first-chunk-without-a-body", + ), + pytest.param([{"type": "http.request"}, {"type": "http.disconnect"}], b"", id="empty-request"), + ], +) +async def test_request_messages_without_a_body_key_count_as_empty_chunks( + client_messages: list[Message], body_the_app_gets: bytes +): + seen: Final[list[tuple[Scope, Message]]] = [] + + async def app(scope: Scope, receive: Receive, send: Send) -> None: + seen.append((scope, await receive())) + + async def send(message: Message) -> None: + raise AssertionError(f"middleware sent {message}") + + await RequestBodyDecompressionMiddleware(app)( + _http_scope([(b"content-encoding", b"gzip")]), _scripted_receive(client_messages), send + ) + + [(app_scope, first_message)] = seen + assert (b"content-encoding", b"gzip") not in app_scope["headers"] + assert first_message == {"type": "http.request", "body": body_the_app_gets, "more_body": False} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "scope", + [ + {"type": "lifespan"}, + {"type": "websocket", "path": "/v1/realtime", "headers": [(b"content-encoding", b"gzip")]}, + ], +) +async def test_non_http_scopes_reach_the_app_untouched(scope: Scope): + seen: Final[list[tuple[Scope, Receive]]] = [] + + async def receive() -> Message: + raise AssertionError("middleware read from a non-http scope") + + async def app(app_scope: Scope, app_receive: Receive, send: Send) -> None: + seen.append((app_scope, app_receive)) + + async def send(message: Message) -> None: + raise AssertionError(f"middleware sent {message}") + + await RequestBodyDecompressionMiddleware(app)(scope, receive, send) + + [(app_scope, app_receive)] = seen + assert app_scope is scope + assert app_receive is receive diff --git a/uv.lock b/uv.lock index 69d9f777687..9b490cf95f4 100644 --- a/uv.lock +++ b/uv.lock @@ -725,6 +725,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, ] +[[package]] +name = "backports-zstd" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/9c/13569626440e88f09d16f43ec1c2aa0d10a523be2811414580d1cfb7c9f3/backports_zstd-1.8.0.tar.gz", hash = "sha256:9dae4f4c481716e3db473d667457b4f508ff7459c0931b567a5c9677fb3db316", size = 1006566, upload-time = "2026-10-10T16:36:40.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/a4/3178d6941ebb397b5241ec635e3b116c8203b314578853bf4f02b1c5c9a4/backports_zstd-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5173afe530ca59bba8938a19edcb875c70f78bf9fee01cb3614a97876d112962", size = 439453, upload-time = "2026-10-10T16:33:57.245Z" }, + { url = "https://files.pythonhosted.org/packages/63/62/5ce79a4f9433537e9b233c2c3e946863322150c9c988e7effc0db319e5d4/backports_zstd-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e213317db53e787ef7bf13c5a2070bd98a888ca7603bbd1904ede443c197f3cc", size = 368265, upload-time = "2026-10-10T16:33:59.037Z" }, + { url = "https://files.pythonhosted.org/packages/69/36/30c6aa8155a72959275fe840b9c84efe3bbb075e03d2717d8b9b0bc1c465/backports_zstd-1.8.0-cp310-cp310-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:d1c0902770bfcee67b5ff4a5ec69b7ceaf230816e5cd9cc3654a03dd584eead9", size = 508607, upload-time = "2026-10-10T16:34:00.762Z" }, + { url = "https://files.pythonhosted.org/packages/f0/75/bd269392aa8bd5f0f44ca9503f4f1daf1df084141efad6934a83a0b53b06/backports_zstd-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bb99f835f6d1e6ad0bc1c1ac430baf6d39a9183e37c4f295fb876214ac4c7e28", size = 478536, upload-time = "2026-10-10T16:34:02.825Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/9b6f674e439da130cce94029c6cfd343a2caf78f6d50e6b50e142befa21c/backports_zstd-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:62f633740f25f383b0a3edc7e8bbdc18d38d62a3db7167e77fc715f75e6f233c", size = 583901, upload-time = "2026-10-10T16:34:04.965Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e2/6e3e3333ca22f16fc500b4982d4a441bbd6433db5fb23c0dfe3ee7be0fc3/backports_zstd-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38ffdc14e37a0e94eff3b771fc071903b25caa48b092ed59662246970ef01e99", size = 643609, upload-time = "2026-10-10T16:34:07.103Z" }, + { url = "https://files.pythonhosted.org/packages/0f/0f/32cf11767d5db2f508d1e01029ae509b92232628504c3c6c0112871f6627/backports_zstd-1.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b58cd328afcb538f3ca5dc2ac47f8dfb68635d5b906d5efcb59054bc86219214", size = 493825, upload-time = "2026-10-10T16:34:08.735Z" }, + { url = "https://files.pythonhosted.org/packages/80/bf/16e5a0af75f2461e4c518796099d5a297803656049e64c0207a88de11a82/backports_zstd-1.8.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f43a0247b7daeea20e792627ec929b995fc290484b11ab314d4c58cc5f5558d8", size = 567704, upload-time = "2026-10-10T16:34:10.423Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9c/e761f5eeb780303af2e9be4748bf484621e4e36b59ae85611109ae8587b3/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1c11797f5129872ca0278d7a1628ff254cf773d9cae337cf30efce5646f8ccd7", size = 484541, upload-time = "2026-10-10T16:34:12.118Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b8/5e528163601cb3324840346695fdad98463ac01f657b68e61e865568a342/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b37a2189c2be170369dfb083a2ab4793b510e9d0f207cd047ca47f97e8995ba5", size = 512124, upload-time = "2026-10-10T16:34:13.741Z" }, + { url = "https://files.pythonhosted.org/packages/dc/16/84b807b56425a1821318b15775706c2549cec1c52d2cf48efafc62beec00/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70da152b5cf4a75459fb87abc00d263b2012653646372a03904bed67897938be", size = 588257, upload-time = "2026-10-10T16:34:15.513Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/1f3bbc063f78285ea17e9f12022c22f6b46fc6db4746c6466f093e29c88e/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fc9ee08e6a17f388f670a421b36a5d3a9417a404c2f39ac0bf5e6ad958ac853c", size = 565562, upload-time = "2026-10-10T16:34:17.163Z" }, + { url = "https://files.pythonhosted.org/packages/0f/14/a2f8a2eb880a57402cf527ccfaa4fe41edeb0c21428305873f0ce7fb244a/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52ccf581406f4610570d5e411d5eee9cf0fdde9ee5cd9fc95ae9b12edd150e6c", size = 634170, upload-time = "2026-10-10T16:34:19.056Z" }, + { url = "https://files.pythonhosted.org/packages/9f/12/8c1d9e475815d24cf0508bd7cc1811a3b536174b77adeed88a63419c642c/backports_zstd-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d23957b8067e04b15cf59a41098d75855e15e66699dd2b81259316cbe86a3df", size = 497815, upload-time = "2026-10-10T16:34:21.411Z" }, + { url = "https://files.pythonhosted.org/packages/88/b4/3916d264038cc9a69693c16aa9dab0de93b8dd418fb6486926a4823aae7d/backports_zstd-1.8.0-cp310-cp310-win32.whl", hash = "sha256:6a73b782aba89d45e2c19c1b6491eed2c90e5de9536c26173fc62be2d011486a", size = 292740, upload-time = "2026-10-10T16:34:22.961Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ac/9d56c553c7660a42862d4efdd1f499366ba0f31ff2090e7cb3fb4c56a767/backports_zstd-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:6202f9eb6b44301d3ab62c7d717a1becb530b6d09ccc4d2ff4a4b662220e05e2", size = 330377, upload-time = "2026-10-10T16:34:24.565Z" }, + { url = "https://files.pythonhosted.org/packages/f1/49/c659a40b3149f1756505c0c3f26378f0f658d446e04f87e52953f17b989c/backports_zstd-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:b66cfbd6ac3221624ea5088950f243187cb9e24a3e5ad0bc89d093fd143b0696", size = 321825, upload-time = "2026-10-10T16:34:26.292Z" }, + { url = "https://files.pythonhosted.org/packages/da/b2/43853a0c366f26b140c272adce74b3c280a2e28ee023c53af53ddd6d9d93/backports_zstd-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c4af1b9542bc6420d55ff47d7efe13c19f56a80cbdd1ffd0a29767801dab886", size = 439457, upload-time = "2026-10-10T16:34:28.048Z" }, + { url = "https://files.pythonhosted.org/packages/20/6d/ab02ba30a51fa9ec452ee0aaccee7e9c3feda8b3a1b0f7e6aeac0a8a5259/backports_zstd-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8efdb220f34418cef987da10d857cf95cdcffe431cc0e536efc25d7279abf118", size = 368264, upload-time = "2026-10-10T16:34:29.599Z" }, + { url = "https://files.pythonhosted.org/packages/cd/71/7632053324885d43fe9ad376607885462386a1de6ec6daad3eee291c6ac8/backports_zstd-1.8.0-cp311-cp311-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:e70eefb72358ae3c94eac62cf7fa3c392cc21f0a8221d6cdaf3d74aedb9775bf", size = 508606, upload-time = "2026-10-10T16:34:31.201Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/7743d8b0c0b28696b2b4757d90afe2844e8a91121d63951829ad9d27edb2/backports_zstd-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6f9ecc5a251fd9495ee717daa0dc87c195f50d6d3679ddb430eb58256a0ca53", size = 478535, upload-time = "2026-10-10T16:34:32.859Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a2/99a32b753e233f501287ee7df2011a9828242c9f0d1c6a5045a4fd587f2e/backports_zstd-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:84d7c45f063ee8cce1dc14cf382511554b0db19234094fa91214be68d185a5a8", size = 583901, upload-time = "2026-10-10T16:34:34.625Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fd/1812a60ed4943049accfd820d18eeca8ad79461eea9b0be6f52b29614851/backports_zstd-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:117e1ebc7224ea328c7fba82dfe6b76cead2a2b1f427dabcd8a5fa87c47abd15", size = 643571, upload-time = "2026-10-10T16:34:36.43Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/3eb6466013bbee7f12cf442507ca80d3e31ec1fd68156c57647518a47d27/backports_zstd-1.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7fe40a58dbe1fd358e0ceb5b6b3f50a9b328f8fff42dcb3bdaeb9a022c2506", size = 493818, upload-time = "2026-10-10T16:34:38.185Z" }, + { url = "https://files.pythonhosted.org/packages/66/c7/1c8fb5b9e97aa172d68e4bbfb808962a32e9c89b7f25f81cec47c16b5d6d/backports_zstd-1.8.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ba1f16c4196b8392e0adc1f201d0d1aadcc0b78dbe9049fc3d98633cbce565d9", size = 567704, upload-time = "2026-10-10T16:34:40.111Z" }, + { url = "https://files.pythonhosted.org/packages/ab/46/8ff2cca539dc1bc35e85c75772ce901ccaa4696cc0c32f8bd00f426595f9/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3568397b72546bab27054fb7526f90b2842a6978cda1224f37c061087ea15bb1", size = 484551, upload-time = "2026-10-10T16:34:41.776Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/2324e68cb8404b95bdd292575f52c8dd6567a23a4985e6e0322260ea6747/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d0a6cafbc18dd32832bd4c22a40348634d191afadf3e0b82fc5df225dfb94e3b", size = 512127, upload-time = "2026-10-10T16:34:43.418Z" }, + { url = "https://files.pythonhosted.org/packages/b7/06/a18156cd52d65f8186a4ee72ce6fe200a23dc3d366f43097d30d77b2cb5d/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e67b330874664e41cb03216e4e33fe79b91304269b329fca82f5bd9e0501a48d", size = 588258, upload-time = "2026-10-10T16:34:45.029Z" }, + { url = "https://files.pythonhosted.org/packages/de/ee/e70d81890364b508fde19979a728161ed836795eab83753c1fdd4e41b395/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:290b41aa11285c8e1eeba7450afb7e9fd61572373410110a2a06a23ae97937f9", size = 565565, upload-time = "2026-10-10T16:34:46.632Z" }, + { url = "https://files.pythonhosted.org/packages/31/72/843335eba25b83c6e1c4febca74cf0e8a80c1108876fef2fe2ebce80bc79/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:13c00e1c66c78a0d1e1c60d0806e9bd430d4c5c92cdce3fa8d087aea436bf449", size = 634218, upload-time = "2026-10-10T16:34:48.272Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/86a428aed44e8389e4436f9e913ba90563efd61779ad5caa360822154fe5/backports_zstd-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0f722107de223fe68efa83b1cc3a11d67d1888441073732f0d350ff8111d23df", size = 497825, upload-time = "2026-10-10T16:34:50.146Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0e/a8e246b4ef0e992cd764f7bc898de2878380c3af4b85d5c0e2bd6d22d0fe/backports_zstd-1.8.0-cp311-cp311-win32.whl", hash = "sha256:6b6c46d5d5932b7ad24f42069104919fa806fac0a02144aa8af0f9bb96705274", size = 292853, upload-time = "2026-10-10T16:34:51.927Z" }, + { url = "https://files.pythonhosted.org/packages/50/53/4e36af749d8c115659acfee2bcc6ebbf5cc34fdd30b467c205eae4925c6d/backports_zstd-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a11422c67c6295d36a7a30bac5df82e8a4fc82539d8def0d082ecf15cb24f538", size = 330411, upload-time = "2026-10-10T16:34:53.439Z" }, + { url = "https://files.pythonhosted.org/packages/43/13/9a027f33f95d2d4ab565e9d3655cb8f71e2a1e32e86a57195a787e00483b/backports_zstd-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:0a77b019b80038b1426a74849b0fb8f9b46f876cee74f6d59f26acd1559d4c01", size = 321855, upload-time = "2026-10-10T16:34:54.865Z" }, + { url = "https://files.pythonhosted.org/packages/d3/03/3c303d6f3066f84f2c52acfc38852546a836596dd9a2bc7add83bd96b527/backports_zstd-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6e024aee6bfd04094fce60133b0e6bd0f8027cdb2823157880bc87f1ffdfee21", size = 439718, upload-time = "2026-10-10T16:34:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/92/31/1e73b2835c78a9067ecba390b0eea032f827fc0b2f8bf2c8656992c30dc8/backports_zstd-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d810d83c8a703f424ed2a49aa271078c91b530da2d8c104bd88207e68d116de8", size = 368337, upload-time = "2026-10-10T16:34:58.287Z" }, + { url = "https://files.pythonhosted.org/packages/85/43/b0cc88c7d13a544f6d38f288fd96e1595395dad31f49fad2619f06b96d95/backports_zstd-1.8.0-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:d057948e8cffa19f0cc8668e06fd502ad8a69f398e91a426b39dcc5eeb197c2f", size = 509148, upload-time = "2026-10-10T16:34:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/81cc731a0408c3cba05a44ece00476305dbe1a52e27a4c323c98685f7015/backports_zstd-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aa762cf369d9bfca1e013eaad562f8e129d71b7a82f0c459870d6d21651bcb3", size = 478911, upload-time = "2026-10-10T16:35:01.791Z" }, + { url = "https://files.pythonhosted.org/packages/df/63/dc62779cabb725a8974a2d303bfe0d7cd5b8987fab79ab445c48efcfb2e4/backports_zstd-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b9d6c4ca7d927fd094badcf9174ee5c82ddb4855fe14658806c8c8a07d4a165", size = 584283, upload-time = "2026-10-10T16:35:03.666Z" }, + { url = "https://files.pythonhosted.org/packages/e5/12/5e8ce29119d78845cd3351bcd79baa16a30aa8c19f8c359a1719a15d97b3/backports_zstd-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d85b8ce50aea247289be183f853e67c106959c4048ce286b26c4663b06bb6d", size = 643167, upload-time = "2026-10-10T16:35:05.342Z" }, + { url = "https://files.pythonhosted.org/packages/3f/08/a9d59fb9e20215ede0c8ea4d729373dc0592aee45776cdd86c92c3c6242c/backports_zstd-1.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9e9aa28a44db1897fb637f037175566f3b75890d4bae6cae7ba34f1df1e0804", size = 496867, upload-time = "2026-10-10T16:35:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/e8/b8/abcd2be476a47dd236500c405df32aa81902c54750b26c626f190bbef6b9/backports_zstd-1.8.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2c431f3cdc7eb663a42574e27a8604a18181ea4e193504f222d8e61c6f5f8b78", size = 571623, upload-time = "2026-10-10T16:35:09.014Z" }, + { url = "https://files.pythonhosted.org/packages/03/ce/31e668dcdfe017b3240f49c3ef67b108224d3f66d90e9f26caecafc3c29c/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e0431230a67e8f07210efe654abda9844a55c3bf57d74e60425d9d65770b1de4", size = 484948, upload-time = "2026-10-10T16:35:10.974Z" }, + { url = "https://files.pythonhosted.org/packages/5a/98/d9122b7531830ceb0f62adb88694bb8cc414a27d1d03539c44dd96fa7a63/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9b62b6c8c5a43b294d4358c2016bfbc507cc574315ffa75346ccf0b621746461", size = 512635, upload-time = "2026-10-10T16:35:12.658Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/168c6d0c93a3ad6568d0b0ac2f732efc9132b2839d4e6759e61f5239107d/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:869ab7e5421873dfbdbf646d52b4e8d711093972819c06c6daf3249a1ec6e0e7", size = 588696, upload-time = "2026-10-10T16:35:14.595Z" }, + { url = "https://files.pythonhosted.org/packages/22/32/b8eacce542dae88df98f923e81c079a01b66b7fbdf103e319f6fb1df2dfa/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ec1a796429674ebc0e2d48feb3b6658bf49d3ae840b0c0e14ad50c4d6b7341fe", size = 568895, upload-time = "2026-10-10T16:35:16.287Z" }, + { url = "https://files.pythonhosted.org/packages/dd/16/8abede9513ec8fd584e36159b1dce82042a97214e69f53f08605b245999f/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:775b701a576769df053cfb7d9456b06223b40e329c010be6cc178fe9e404a3d2", size = 633610, upload-time = "2026-10-10T16:35:18.014Z" }, + { url = "https://files.pythonhosted.org/packages/6d/74/4e82ed15ae212b0fc0cd8f82c5bbf6a9dd584b6b37df0c3485663c6ad105/backports_zstd-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ab77a2e6e21c57e8341bb7656c71d1a1653151ebe787b3f092ce86a02543eb52", size = 501407, upload-time = "2026-10-10T16:35:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/bd/02/7e86774e0a3c2457d23939acbb32bdb019e6bdec48892986255faa262c3d/backports_zstd-1.8.0-cp312-cp312-win32.whl", hash = "sha256:f99b44c2c13fc60f65ad568bf7401d9540370f996b1040793a34988324e3b712", size = 293065, upload-time = "2026-10-10T16:35:21.309Z" }, + { url = "https://files.pythonhosted.org/packages/a5/78/2f497fd2bbf46099e46650f75467967d21f25bb921c894d28d493bbfb7e4/backports_zstd-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:1eddf59fedaf19dd3a8e9c597add7eb6f0d51d4467a0924b2dcd2c118ed18ff5", size = 330563, upload-time = "2026-10-10T16:35:22.968Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2c/3a1a91cea5b98e24cb54ecf142a72246d2e1efa5efe41504388188598951/backports_zstd-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:2b3247a7a916b90f155b4133eedaceadd0c37b4149ee32e4d74fe512a14be89b", size = 322189, upload-time = "2026-10-10T16:35:24.494Z" }, + { url = "https://files.pythonhosted.org/packages/66/a8/7a04f1daaa42936ec3d98f213b4698b18053d1154f2aee1d067c4121fe3a/backports_zstd-1.8.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4e92ff4ce96b3c61d25900875b6cf1ee249349b8e419abd80893ec9b8026444e", size = 401586, upload-time = "2026-10-10T16:35:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c2/d26216501b3e13583084e11106ade1779b280f3304c75d84d2dfb9e5d609/backports_zstd-1.8.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:0c2e652b4fbc2e6b7bd05a09b6eab3a51bfaed9e7fca1bc81d763dc47361e2ff", size = 455589, upload-time = "2026-10-10T16:35:28.174Z" }, + { url = "https://files.pythonhosted.org/packages/df/66/372b138fa7e7be4d6aff343a55dd77e492867cb5de701899b5aa01722836/backports_zstd-1.8.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915d3e7e57194b5cee33f10cf2d9f5c4f7658c8a167236f9ba5501520cf133e8", size = 358662, upload-time = "2026-10-10T16:35:29.819Z" }, + { url = "https://files.pythonhosted.org/packages/7a/26/0b89de2f83088f89e10ea3f4a5badef9bc95098bdd39a3031362da48dc60/backports_zstd-1.8.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e6f8483b795a09c0e0fbacca4fa844242bc6d5fc64b8a6ee99f88ad8af27b08", size = 367357, upload-time = "2026-10-10T16:35:31.649Z" }, + { url = "https://files.pythonhosted.org/packages/74/01/5239b39d3f65ba80e2129b9273bf736245e4a1c03b8a317ed399c4fe10dd/backports_zstd-1.8.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1fe4b06a019aa4cdf87af320eef56a4bdbdb924ead36a7a918645d72edece966", size = 447892, upload-time = "2026-10-10T16:35:33.534Z" }, + { url = "https://files.pythonhosted.org/packages/b5/13/e4eceee62d144f68944addb0179368d626f96d3644d965620774f1f5e463/backports_zstd-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:49c4006cdf41c15ffcc74f10d9a6485be841106cd4d5aa7ea7bf1075cc37fb83", size = 439240, upload-time = "2026-10-10T16:35:35.351Z" }, + { url = "https://files.pythonhosted.org/packages/1f/5f/996aceebbbc4eebc05d99fe1714b1b0930260eac5171e8ebc3a952390c0d/backports_zstd-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fa862d24b7fb392279a95bc9acc1f0ede8a25de9efbed03fb305ceac2f6abb0", size = 367710, upload-time = "2026-10-10T16:35:37.004Z" }, + { url = "https://files.pythonhosted.org/packages/93/0b/c373a7f92df9df1f9e0657ea0dd86c45444b8414db616b3d38b62f90075c/backports_zstd-1.8.0-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:9af83a6d7dc67896fd91bcd4c2cd182ba97d7cca2b09a94373a5fef154001d98", size = 508347, upload-time = "2026-10-10T16:35:38.683Z" }, + { url = "https://files.pythonhosted.org/packages/b4/36/07dca77032300047efd09808d49ab9d1fff8657553adbc8e0e6405aba864/backports_zstd-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a808ba1371231c00a2b71f03840a727088e287d0ee1dfb3230958950f21f421", size = 478416, upload-time = "2026-10-10T16:35:40.504Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a9/bb96724619a1dcc3a9e3138d15a6f7a2fc40b581926db4ac00e424af79c1/backports_zstd-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6cc15051c282ac2585a2425d22f416ae2deb5afb441b22831b349b02fd58a782", size = 583888, upload-time = "2026-10-10T16:35:42.159Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6d/65e6e437eb54b5be2ce7248ac236d82a771a672457c950e7f96849699274/backports_zstd-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a23d38d7b9ca93403acd3c2c306af6e547a24d150c25ac2d7a8acd751fbd968", size = 644796, upload-time = "2026-10-10T16:35:43.882Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6d/3c422b33d40aaca6e9d9fdd47f1a047ac499de749c887ab3dab62f731fb2/backports_zstd-1.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44a9004f9e809ea56910d326d21946650369db59eb86edc0c76840f21530704c", size = 493385, upload-time = "2026-10-10T16:35:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b9/ea08e2c2b8a7bfabff359852e4d7a9cbc2cde09715907250c0e53432fbe9/backports_zstd-1.8.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ff307f3f0ef3b7f40ccfce42c0704fddc99cd30bca451330f42466db1981be9", size = 568613, upload-time = "2026-10-10T16:35:47.394Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/775cb7317f1f693c7f3e96fa5cf5426b461616b52730a72f978f31b334b0/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6c8572e27c5f0b9d11020d3f597bf3c35fe0f5ae6f99156dc52b0bd937ba8908", size = 484237, upload-time = "2026-10-10T16:35:49.496Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f8/c31798a8911390fb0d4f058f65cba2e54141d6394c35430b1d495d121667/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cc1d9d3660c40abe4095de80f43ce4c955d08f7d9803d3da97176aa61b76d923", size = 511865, upload-time = "2026-10-10T16:35:51.223Z" }, + { url = "https://files.pythonhosted.org/packages/68/df/0ff79b6a2d7f5c10d3ebc7e23b5281f51130feb4db8afadac98ba5131c18/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:83cea5cdd70e1d74382be6deeeda1db79aedd1a06af4f8a8fbafba9eedae5230", size = 588422, upload-time = "2026-10-10T16:35:53.371Z" }, + { url = "https://files.pythonhosted.org/packages/19/a7/d5dbad63911fc3040253dc209a7aac8921e928fe64f3fcde051066aa5a75/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e74eb204b9d7798fc57393202c443fc2ec84283d82387168baeb763f8beb224d", size = 566480, upload-time = "2026-10-10T16:35:55.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b9/621e734eb144d56c7632b763c0ce3fa196839fc0f82830244206a9d37d8d/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:515497b3d49dd6d7a84fb16a0a0007bc460b4a7e1f55e70f33315c66d3844e8e", size = 635191, upload-time = "2026-10-10T16:35:57.307Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/1b6709f13f2a22a1d72e15f114ab62e852db33ba0f8840c7d102523bcdb6/backports_zstd-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6283c90997038abf46c8a0bb75afb4dc6cbf061421802fda0afc382fe4b348b3", size = 497769, upload-time = "2026-10-10T16:35:59.395Z" }, + { url = "https://files.pythonhosted.org/packages/de/52/cd0a82fd52ae159a0316d2257156968c356cab81062d6050af48a4e8a3d6/backports_zstd-1.8.0-cp313-cp313-win32.whl", hash = "sha256:9d76a3193a3a4a6b1249021e7ecf72e4cabc1dca611c6fb41db1c0b5d2faf741", size = 292645, upload-time = "2026-10-10T16:36:01.439Z" }, + { url = "https://files.pythonhosted.org/packages/12/0e/5c5a916cea73b455850083ccf76078de655face3dfe4126848570c57a6dd/backports_zstd-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:b583990d554cc6f6141c5c43b6db3c7da87a214253e08339d917ee3baa3021b6", size = 330247, upload-time = "2026-10-10T16:36:03.058Z" }, + { url = "https://files.pythonhosted.org/packages/86/3c/7297d87eed9254f6b4823c05b37aa07ec2a99bc5f195760dc574e925eecf/backports_zstd-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:0600e166cb00739a26de74ee1696221a53a4d5dc1f96a0bdeb6b307c1626c15c", size = 322066, upload-time = "2026-10-10T16:36:04.932Z" }, + { url = "https://files.pythonhosted.org/packages/56/c5/a48a8595d151903328ec68041862b42e06ba4cd44662009a5e23d2c912c6/backports_zstd-1.8.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:403985e468f1cccb87a7e9e4f1d78106ea8e77dcdda3038d645d052a8d8e1ce3", size = 414047, upload-time = "2026-10-10T16:36:06.652Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a4/6646415f884005001a1dd6b6067e2d4067188b874ff62b2e3313d7bf2f30/backports_zstd-1.8.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:045e15ed3b3ebd8816edaa7d66f024becf050d9aec09605f549ce33cfda01098", size = 344734, upload-time = "2026-10-10T16:36:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/a5/5afcdc74fd49eb8536f2db78b9d0ee066f5f68c87261fb2914aed79928fd/backports_zstd-1.8.0-pp310-pypy310_pp73-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:9da207eb5264a03d29d62169d3dfe0790dc47f85b1785f25e9b01763f227dcdd", size = 422892, upload-time = "2026-10-10T16:36:10.201Z" }, + { url = "https://files.pythonhosted.org/packages/68/66/d16f7be06b7bad41311b3d405782b0fcb26e61ed323ae3f2bef347517329/backports_zstd-1.8.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ebee106e5592549e3eca5d2cf2575de73a87b046f5d433f63ffbefcd6ab5e24", size = 396433, upload-time = "2026-10-10T16:36:12.075Z" }, + { url = "https://files.pythonhosted.org/packages/35/65/7c1dbc9a6cb9005001bc64a99a96eeb29f9dbdc82a558ced1ae652a37372/backports_zstd-1.8.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:200313a6aae64e7f54bdd703317b16560e195f37426bb308e9a495e27ec4efd0", size = 416398, upload-time = "2026-10-10T16:36:13.835Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a4/b45f63e146f69c3b7516410638c9832db3c254c7a7f6f63be18b3e98268d/backports_zstd-1.8.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7b48d33ef2446bd5f4922757451d8eefbae25cc08da7c216ba200ff1acdb4352", size = 316947, upload-time = "2026-10-10T16:36:15.682Z" }, + { url = "https://files.pythonhosted.org/packages/42/1c/74a4b8310af405f477b5278ae652d35f0609acae3f23c9fc472f79d11600/backports_zstd-1.8.0-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:900b357bbae805bb98672471ede748c80ccfc1212be0b4ef52a102750ef742a7", size = 413974, upload-time = "2026-10-10T16:36:17.615Z" }, + { url = "https://files.pythonhosted.org/packages/30/1c/3bb324f70aac60a4c5aad60b9d365af2dac81205b20ecf66e04947381228/backports_zstd-1.8.0-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:1eae18c682f7daf8d7b39c988516d7a123ec446beb77f709d0cb1475ab57f0cc", size = 344649, upload-time = "2026-10-10T16:36:19.602Z" }, + { url = "https://files.pythonhosted.org/packages/95/fc/a62c13e0498fb951a65caf8c979624fddd1085e388b067ec7b225b59c1e9/backports_zstd-1.8.0-pp311-pypy311_pp80-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:59d29e16273a440af6beb11965cfa84cd19207b38fb5302b2430bc8eabef4812", size = 422892, upload-time = "2026-10-10T16:36:21.375Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9b/6d8e6044eb6a829c075f2f1e59dc6a9789de606c4ef95fb66095efb3a47f/backports_zstd-1.8.0-pp311-pypy311_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:307badd18496d7c7c6adb91b524b120b4fd3ab5609ec794c36953b9a5f4f4728", size = 396431, upload-time = "2026-10-10T16:36:23.436Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/c5dd607ca0281509ce22b683d43ad801b68b36b9dd0429e5d34c50886f6f/backports_zstd-1.8.0-pp311-pypy311_pp80-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40966dc0a3d08d56f83a6b79239d3f294896c9aee453449064fc3627058448fb", size = 416398, upload-time = "2026-10-10T16:36:25.197Z" }, + { url = "https://files.pythonhosted.org/packages/24/9c/0210e539a290f64d1303afeae4f79f94ed97e8cf7171bd385fc373a4c414/backports_zstd-1.8.0-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:029bca2385ebb4355135bdb8559792d2768ae19707705eea84e68c42a30a0276", size = 404276, upload-time = "2026-10-10T16:36:27.003Z" }, + { url = "https://files.pythonhosted.org/packages/1f/c8/dba9e5905e83ac955c1c19b797f59f5335a351664a7b25a709929d63dfbc/backports_zstd-1.8.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:f710d03f84d74f11737735f846b44ef1545cadb73ef47bcd3d0e124f253dd763", size = 413972, upload-time = "2026-10-10T16:36:28.92Z" }, + { url = "https://files.pythonhosted.org/packages/93/11/8ee691bfd2c8292a573a0378a616372aa01ed9e6001d5778ae666a239265/backports_zstd-1.8.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:2b11fb8b9c798657c97ad3165893f146c300e2f7f800e9c54c0d2143052c1486", size = 344652, upload-time = "2026-10-10T16:36:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/19/33/86bb2cd5c6e827adba98fb091ccecb29dae3bb33e0406f8e08be7bdbe70b/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ec7351d3e6ea92338dc4e0e53c876d2e2092e07ad3a2083088e0160200efdd15", size = 422892, upload-time = "2026-10-10T16:36:32.708Z" }, + { url = "https://files.pythonhosted.org/packages/42/a2/629f5e9c3edd2a31f7dd65b8097241b5036f98105efac251a12c1a8f7cb5/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63ae348b629121eeb967244fecd254f41b4b3a63d074c252f4d7777f5d17c71c", size = 396431, upload-time = "2026-10-10T16:36:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f6/9c223e9cccc5a797c17475fde1a8a78ada0dcdd39be2302f4605e565c0ce/backports_zstd-1.8.0-pp312-pypy312_pp80-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:163b5c36321bf5652b6e4aeb04d3644ddbf9c1881a82322e376e5be3532af26b", size = 416398, upload-time = "2026-10-10T16:36:36.706Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e3/2eb6f517c9a6746a735b49ba4ab3ed3df6c4ec9072169805547ae590e296/backports_zstd-1.8.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:3f0288db18a64f4f4146f4526456ff62b2edb625b2d43956e764885edd3f1da2", size = 404272, upload-time = "2026-10-10T16:36:38.766Z" }, +] + [[package]] name = "basedpyright" version = "1.39.7" @@ -4576,6 +4675,7 @@ proxy = [ { name = "azure-identity" }, { name = "azure-storage-blob" }, { name = "backoff" }, + { name = "backports-zstd", marker = "python_full_version < '3.14'" }, { name = "boto3" }, { name = "cryptography" }, { name = "expression" }, @@ -4746,6 +4846,7 @@ healthcheck = [ proxy-dev = [ { name = "a2a-sdk" }, { name = "azure-identity" }, + { name = "backports-zstd", marker = "python_full_version < '3.14'" }, { name = "hypercorn" }, { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'" }, { name = "opentelemetry-api" }, @@ -4775,6 +4876,7 @@ requires-dist = [ { name = "azure-storage-blob", marker = "extra == 'proxy'", specifier = ">=12.28.0,<13.0" }, { name = "azure-storage-file-datalake", marker = "extra == 'proxy-runtime'", specifier = ">=12.20.0,<13.0" }, { name = "backoff", marker = "extra == 'proxy'", specifier = ">=2.2.1,<3.0" }, + { name = "backports-zstd", marker = "python_full_version < '3.14' and extra == 'proxy'", specifier = ">=1.0.0,<2.0" }, { name = "boto3", specifier = ">=1.43.1,<2.0" }, { name = "boto3", marker = "extra == 'proxy'", specifier = ">=1.43.1,<2.0" }, { name = "click", specifier = ">=8.0.0,<9.0" }, @@ -4965,6 +5067,7 @@ healthcheck = [ proxy-dev = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "azure-identity", specifier = "==1.25.2" }, + { name = "backports-zstd", marker = "python_full_version < '3.14'", specifier = "==1.8.0" }, { name = "hypercorn", specifier = "==0.17.3" }, { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }, { name = "opentelemetry-api", specifier = "==1.33.1" }, From bda4e60f9b5227943baed6b0a373e93ed87b35b8 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Mon, 19 Oct 2026 22:31:39 -0700 Subject: [PATCH 069/124] fix(streaming): keep finish_reason and provider usage when rebuilding streamed text completions (#44349) * fix(streaming): keep finish_reason and provider usage when rebuilding streamed text completions stream_chunk_builder_text_completion read finish_reason from chunks[-1] and recounted usage from messages. With include_usage the last chunk is a usage-only trailer, so finish_reason came back None (or raised IndexError when the trailer had no choices), and prompt_tokens was 0 because a text-completion prompt is not in messages. Scan for the last finish_reason and prefer the provider-reported usage, as the chat path already does. Co-Authored-By: Claude Opus 5.5 * fix(streaming): type the text completion chunk builder so its scans add no basedpyright debt Read chunk fields as typed attributes instead of untyped subscripts, which clears the reportUnknownArgumentType breach the lint gate flagged * fix(streaming): keep the provider's full usage, including token details, when rebuilding text completions Co-Authored-By: Claude Opus 5.5 * fix(streaming): rebuild dict-form text completion chunks again stream_chunk_builder routes dict chunks holding TextChoices to the text completion builder, which now reads attributes. Convert them to TextCompletionResponse at the routing point. Co-Authored-By: Claude Opus 5.5 * fix(streaming): accept dict chunks in the text rebuild and derive its usage like the chat path Moves the dict to TextCompletionResponse conversion into stream_chunk_builder_text_completion so direct callers keep working, and builds usage through ChunkProcessor.calculate_usage over the chunks that report counts, so a trailer without total_tokens keeps its prompt and completion counts and the result is a fresh Usage * fix(streaming): read text completion usage through the shared usage extractor so usage dicts on typed chunks rebuild * fix(streaming): merge text completion logprobs across chunks and rebuild dict text chunks as text completions * fix(streaming): rebuild one text completion choice per index and keep null logprobs fields null * fix(streaming): keep dumped chat chunks with a choice-level text on the chat rebuild path Route a dict chunk to the text completion builder only when its choice has `text` and no `delta`. The Responses bridge snapshots chat chunks with model_dump(), and Together's chat choices carry `text` next to `delta`, so streamed /v1/responses calls to Together ended without response.completed. Group text completion choices by index in one sorted pass, so rebuilding an n>1 stream costs about the same as rebuilding an n=1 stream of the same size. * fix(streaming): keep text completion rebuilds working on mixed chunks and unreadable logprobs A chat chunk inside a text completion stream now converts to a text chunk that keeps its finish_reason and adds no text. A dict chunk whose logprobs fail validation is read again with logprobs dropped, and logprobs that are not a Logprobs object keep the last value sent. Each of these inputs raised or lost data at the previous commit, and each now rebuilds without raising Also adds a Purview test showing a multi-choice text stream is blocked like the non-streamed response, and Final annotations on the new test locals * fix(streaming): rebuild text completion choices without an index at index 0 Text completion choices whose index is None now rebuild under index 0, as on the merge base, where the sort by index raised at the previous commit. A chat chunk inside a text completion stream now keeps the usage it reports, and a choice whose logprobs mix a Logprobs object with another shape keeps the last value sent, as on the merge base * fix(streaming): rebuild text completion choices with a string index under that int A text completion choice whose index is a string now rebuilds under the int it names, and one whose index cannot be read as an int rebuilds at index 0, as on the merge base. A string index mixed with an int index raised TypeError at the previous commit, and a string-only index came back as a string * test(streaming): drop a docstring that repeats its test * fix(streaming): fall back to choice 0 for an infinite text completion index A float infinity index made int() raise OverflowError, which the index coercion did not catch, so the rebuild failed where the merge base returned choice 0. --------- Co-authored-by: Lai Wei <8022184+roywei@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 180 ++++++--- litellm/main.py | 2 +- ...test_text_completion_stream_rebuild_sdk.py | 280 ++++++++++++++ ...ext_completion_stream_guardrail_rebuild.py | 286 +++++++++++++++ .../test_streaming_chunk_builder_utils.py | 341 +++++++++++++++++- .../guardrail_hooks/test_microsoft_purview.py | 57 +++ 6 files changed, 1093 insertions(+), 53 deletions(-) create mode 100644 tests/integration/sdk/test_text_completion_stream_rebuild_sdk.py create mode 100644 tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d505192184e..b42210e2793 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,13 +1,13 @@ import base64 import builtins import time -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace from itertools import chain, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast +from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ReadOnly, Required import litellm @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) from litellm.types.llms.openai import ( + AllMessageValues, ChatCompletionAssistantContentValue, ChatCompletionAudioDelta, ) @@ -36,11 +37,14 @@ from litellm.types.utils import ( Function, FunctionCall, LlmProviders, + Logprobs, + Message, ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, ServerToolUse, StreamingChoices, + TextChoices, TextCompletionResponse, Usage, ) @@ -857,7 +861,9 @@ class ChunkProcessor: return reasoning_tokens @staticmethod - def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None: + def extract_usage_chunk( + chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream | TextCompletionResponse", + ) -> Usage | None: usage_chunk: Usage | CompletionUsage | None = None if hasattr(chunk, "usage") and chunk.usage is not None: usage_chunk = chunk.usage @@ -916,7 +922,7 @@ class ChunkProcessor: cost: float | None = None for chunk in chunks: - usage_chunk = self._extract_usage_chunk(chunk) + usage_chunk = self.extract_usage_chunk(chunk) if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) @@ -1013,7 +1019,7 @@ class ChunkProcessor: values: Final = [ value for chunk in chunks - if (usage_chunk := self._extract_usage_chunk(chunk)) is not None + if (usage_chunk := self.extract_usage_chunk(chunk)) is not None and isinstance(value := getattr(usage_chunk, field, None), str) ] return values[-1] if values else None @@ -1086,7 +1092,7 @@ class ChunkProcessor: chunks: Sequence["_UsageBearingChunk | ModelResponse"], model: str, completion_output: str, - messages: Sequence | None = None, + messages: Sequence[object] | None = None, reasoning_tokens: int | None = None, count_prompt_tokens: Callable[[], int] | None = None, ) -> Usage: @@ -1199,7 +1205,7 @@ class ChunkProcessor: return returned_usage -def concatenate_base64_list(base64_strings: list[str]) -> str: +def concatenate_base64_list(base64_strings: Sequence[str]) -> str: """ Concatenates a list of base64-encoded strings. @@ -1216,35 +1222,115 @@ def concatenate_base64_list(base64_strings: list[str]) -> str: return base64.b64encode(combined_bytes).decode("utf-8") -def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse: - id: Final = chunks[0]["id"] - object: Final = chunks[0]["object"] - created: Final = chunks[0]["created"] - model: Final = chunks[0]["model"] - system_fingerprint: Final = chunks[0].get("system_fingerprint", None) - finish_reason: Final = chunks[-1]["choices"][0]["finish_reason"] - logprobs: Final = chunks[-1]["choices"][0]["logprobs"] +_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]]) - content_list: Final = [] - for chunk in chunks: - choices = chunk["choices"] - for choice in choices: - if choice is not None and hasattr(choice, "text") and choice.get("text") is not None: - _choice = choice.get("text") - content_list.append(_choice) - - # Combine the "content" strings into a single string || combine the 'function' strings into a single string - combined_content: Final = "".join(content_list) +def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse: try: - prompt_tokens = token_counter(model=model, messages=messages) - except Exception: # don't allow this failing to block a complete streaming response from being returned - print_verbose("token_counter failed, assuming prompt tokens is 0") - prompt_tokens = 0 - completion_tokens: Final = token_counter( - model=model, - text=combined_content, - count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages + return TextCompletionResponse.model_validate(chunk) + except ValidationError: + choices: Final = _CHOICE_DICTS.validate_python(chunk.get("choices", [])) + return TextCompletionResponse.model_validate( + {**chunk, "choices": [{**choice, "logprobs": None} for choice in choices]} + ) + + +def _as_text_completion_chunk( + chunk: TextCompletionResponse | ModelResponseStream | Mapping[str, object], +) -> TextCompletionResponse: + if isinstance(chunk, TextCompletionResponse): + return chunk + if isinstance(chunk, ModelResponseStream): + return TextCompletionResponse( + id=chunk.id, + created=chunk.created, + model=chunk.model, + choices=[TextChoices(index=choice.index, finish_reason=choice.finish_reason) for choice in chunk.choices], + usage=ChunkProcessor.extract_usage_chunk(chunk), + ) + return _text_completion_chunk_from_mapping(chunk) + + +_LogprobsValue: Final = TypeVar("_LogprobsValue") + + +def _joined_or_none(fields: Iterable[Sequence[_LogprobsValue] | None]) -> list[_LogprobsValue] | None: + sent: Final = tuple(field for field in fields if field is not None) + return list(chain.from_iterable(sent)) if sent else None + + +class _CarriesLogprobs(Protocol): + @property + def logprobs(self) -> object: ... + + +def _sent_logprobs(choice: _CarriesLogprobs) -> object: + return choice.logprobs + + +def _merged_logprobs(choices: Sequence[TextChoices]) -> object: + sent: Final = tuple(logprobs for logprobs in map(_sent_logprobs, choices) if logprobs is not None) + parts: Final = tuple(logprobs for logprobs in sent if isinstance(logprobs, Logprobs)) + if len(parts) < len(sent): + return sent[-1] + if not parts: + return None + return Logprobs( + text_offset=_joined_or_none(part.text_offset for part in parts), + token_logprobs=_joined_or_none(part.token_logprobs for part in parts), + tokens=_joined_or_none(part.tokens for part in parts), + top_logprobs=_joined_or_none(part.top_logprobs for part in parts), + ) + + +def _text_of(choice: TextChoices) -> str: + return choice.text if isinstance(choice.text, str) else "" + + +def _index_of(choice: TextChoices) -> int: + try: + return int(choice.index or 0) + except (TypeError, ValueError, OverflowError): + return 0 + + +def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoices: + return TextChoices( + text="".join(map(_text_of, choices)), + index=index, + logprobs=_merged_logprobs(choices), + finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None), + ) + + +def _reported_usage(chunk: TextCompletionResponse) -> Usage | None: + usage: Final = ChunkProcessor.extract_usage_chunk(chunk) + return usage if usage is not None and (usage.prompt_tokens or usage.completion_tokens) else None + + +def stream_chunk_builder_text_completion( + chunks: Sequence[TextCompletionResponse | ModelResponseStream | Mapping[str, object]], + messages: Sequence[AllMessageValues | Message] | None = None, +) -> TextCompletionResponse: + text_chunks: Final = tuple(_as_text_completion_chunk(chunk) for chunk in chunks) + id: Final = text_chunks[0].id + object: Final = text_chunks[0].object + created: Final = text_chunks[0].created + model: Final = text_chunks[0].model + system_fingerprint: Final = text_chunks[0].get("system_fingerprint", None) + all_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in text_chunks)) + choices: Final = [ + _rebuilt_text_choice(index, tuple(group)) + for index, group in groupby(sorted(all_choices, key=_index_of), key=_index_of) + ] or [_rebuilt_text_choice(0, ())] + combined_content: Final = "".join(map(_text_of, choices)) + + reported_usages: Final = tuple(reported for reported in map(_reported_usage, text_chunks) if reported is not None) + usage: Final = ChunkProcessor(list(text_chunks), messages).calculate_usage( + chunks=tuple(_UsageBearingChunk(usage=reported) for reported in reported_usages), + model=model or "", + completion_output=combined_content, + messages=messages, ) response: Final = { @@ -1253,19 +1339,8 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None "created": created, "model": model, "system_fingerprint": system_fingerprint, - "choices": [ - { - "text": combined_content, - "index": 0, - "logprobs": logprobs, - "finish_reason": finish_reason, - } - ], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, - }, + "choices": choices, + "usage": usage, } return TextCompletionResponse(**response) @@ -1416,9 +1491,14 @@ def stream_chunk_builder( return None ## Route to the text completion logic first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None) - if first_chunk_with_choices is not None and isinstance( - first_chunk_with_choices["choices"][0], litellm.utils.TextChoices - ): # route to the text completion logic + if first_chunk_with_choices is not None and ( + isinstance(first_chunk_with_choices["choices"][0], TextChoices) + or ( + isinstance(first_chunk_with_choices["choices"][0], Mapping) + and "text" in first_chunk_with_choices["choices"][0] + and "delta" not in first_chunk_with_choices["choices"][0] + ) + ): return stream_chunk_builder_text_completion(chunks=chunks, messages=messages) model: Final = chunks[0]["model"] diff --git a/litellm/main.py b/litellm/main.py index 8e9ff3cf910..65c504072a9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -9166,7 +9166,7 @@ from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( # noqa: stream_chunk_builder as stream_chunk_builder, # noqa: PLC0414 # pyright: ignore[reportUnknownVariableType] # chunks is an untyped list ) from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( # noqa: E402 # re-export after the move - stream_chunk_builder_text_completion as stream_chunk_builder_text_completion, # noqa: PLC0414 # pyright: ignore[reportUnknownVariableType] # chunks is an untyped list + stream_chunk_builder_text_completion as stream_chunk_builder_text_completion, # noqa: PLC0414 # re-export after the move ) ########## Token Counting API ########## diff --git a/tests/integration/sdk/test_text_completion_stream_rebuild_sdk.py b/tests/integration/sdk/test_text_completion_stream_rebuild_sdk.py new file mode 100644 index 00000000000..9ece946abca --- /dev/null +++ b/tests/integration/sdk/test_text_completion_stream_rebuild_sdk.py @@ -0,0 +1,280 @@ +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from itertools import chain +from typing import Final + +import pytest +from integration._support.timing import assert_within_factor_of_control, timed +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.types.utils import TextCompletionResponse + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PROVIDER_KEY: Final = "synthetic-provider-key" +_PIECES: Final = ("Hello", " streamed", " world") +_PROMPT_TOKENS: Final = 11 +_COMPLETION_TOKENS: Final = 3 +_CACHED_TOKENS: Final = 4 +_USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": _PROMPT_TOKENS, + "completion_tokens": _COMPLETION_TOKENS, + "total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS, + "prompt_tokens_details": {"cached_tokens": _CACHED_TOKENS}, +} +_CHOICE_PIECES: Final = (("Hi", " there"), ("Hey", " you")) +_GRID_CHOICES: Final = 128 +_GRID_CHUNKS: Final = 8192 +_GRID_ROUNDS: Final = 2 +_REBUILD_FACTOR: Final = 2 + + +@dataclass(frozen=True, slots=True) +class Member: + model: str + target: str + + +_MEMBERS: Final = { + "text_completion_openai": Member("text-completion-openai/gpt-3.5-turbo-instruct", "/v1/completions"), + "hosted_vllm": Member("hosted_vllm/Qwen/Qwen3-0.6B", "/v1/completions"), + "openai_chat_backed": Member("openai/gpt-5.4-nano", "/v1/chat/completions"), +} + + +def _frame(body: Mapping[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(body).encode() + b"\n\n" + + +def _text_choice(index: int, text: str, finish_reason: str | None) -> dict[str, JsonValue]: + return {"text": text, "index": index, "logprobs": None, "finish_reason": finish_reason} + + +def _chat_choice(index: int, text: str, finish_reason: str | None) -> dict[str, JsonValue]: + return {"index": index, "delta": {"role": "assistant", "content": text}, "finish_reason": finish_reason} + + +def _stream( + request: Request, identity: str, pieces: tuple[tuple[int, str, str | None], ...], include_usage: bool +) -> Reply: + chat: Final = request.target.endswith("/chat/completions") + head: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk" if chat else "text_completion", + "created": 1, + "model": "scripted", + } + choice: Final = _chat_choice if chat else _text_choice + usage: Final = (_frame({**head, "choices": [], "usage": _USAGE}),) if include_usage else () + return Reply( + content_type="text/event-stream", + chunks=( + *(_frame({**head, "choices": [choice(index, text, finish)]}) for index, text, finish in pieces), + *usage, + b"data: [DONE]\n\n", + ), + ) + + +def _asks_for_usage(body: Mapping[str, JsonValue]) -> bool: + options: Final = body.get("stream_options") + return isinstance(options, dict) and options.get("include_usage") is True + + +def _upstream(identity: str, pieces: tuple[tuple[int, str, str | None], ...]) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body.get("stream") is True, body + return _stream(request, identity, pieces, _asks_for_usage(body)) + + return respond + + +def _one_choice_pieces() -> tuple[tuple[int, str, str | None], ...]: + return ( + *((0, piece, None) for piece in _PIECES), + (0, "", "length"), + ) + + +def _two_choice_pieces() -> tuple[tuple[int, str, str | None], ...]: + (first, second) = _CHOICE_PIECES + return ((0, first[0], None), (1, second[0], None), (0, first[1], "stop"), (1, second[1], "length")) + + +@pytest.fixture +def identity() -> str: + return "cmpl-rebuild-" + uuid.uuid4().hex + + +@pytest.fixture +def one_choice_wire(identity: str) -> Iterator[Wire]: + with wire_server(_upstream(identity, _one_choice_pieces())) as served: + yield served + + +@pytest.fixture +def two_choice_wire(identity: str) -> Iterator[Wire]: + with wire_server(_upstream(identity, _two_choice_pieces())) as served: + yield served + + +def _stream_options(include_usage: bool) -> dict[str, object]: + return {"stream_options": {"include_usage": True}} if include_usage else {} + + +async def _async_chunks( + member: Member, wire: Wire, include_usage: bool, extra: Mapping[str, object] +) -> tuple[TextCompletionResponse, ...]: + stream: Final = await litellm.atext_completion( + model=member.model, + prompt="rebuild probe", + api_base=wire.url + "/v1", + api_key=_PROVIDER_KEY, + max_tokens=16, + stream=True, + num_retries=0, + **_stream_options(include_usage), + **extra, + ) + return tuple([chunk async for chunk in stream]) + + +def _sync_chunks(member: Member, wire: Wire, include_usage: bool) -> tuple[TextCompletionResponse, ...]: + stream: Final = litellm.text_completion( + model=member.model, + prompt="rebuild probe", + api_base=wire.url + "/v1", + api_key=_PROVIDER_KEY, + max_tokens=16, + stream=True, + num_retries=0, + **_stream_options(include_usage), + ) + return tuple(stream) + + +def _rebuilt(chunks: tuple[TextCompletionResponse, ...]) -> TextCompletionResponse: + rebuilt: Final = litellm.stream_chunk_builder(chunks=list(chunks)) + assert isinstance(rebuilt, TextCompletionResponse), rebuilt + return rebuilt + + +def _posted_targets(wire: Wire) -> tuple[str, ...]: + return tuple(request.target for request in wire.drain() if request.method == "POST") + + +@pytest.mark.parametrize( + ("member", "mode"), + [ + ("text_completion_openai", "async"), + ("text_completion_openai", "sync"), + ("hosted_vllm", "async"), + ("openai_chat_backed", "async"), + ], +) +async def test_rebuilt_text_completion_stream_keeps_the_reported_finish_reason_and_usage( + one_choice_wire: Wire, identity: str, member: str, mode: str +) -> None: + chosen: Final = _MEMBERS[member] + chunks: Final = ( + await _async_chunks(chosen, one_choice_wire, True, {}) + if mode == "async" + else _sync_chunks(chosen, one_choice_wire, True) + ) + + rebuilt: Final = _rebuilt(chunks) + + assert [(choice.index, choice.text) for choice in rebuilt.choices] == [(0, "".join(_PIECES))], rebuilt + assert rebuilt.choices[0].finish_reason == "length", rebuilt + assert (rebuilt.usage.prompt_tokens, rebuilt.usage.completion_tokens, rebuilt.usage.total_tokens) == ( + _PROMPT_TOKENS, + _COMPLETION_TOKENS, + _PROMPT_TOKENS + _COMPLETION_TOKENS, + ), rebuilt.usage + assert rebuilt.usage.prompt_tokens_details is not None, rebuilt.usage + assert rebuilt.usage.prompt_tokens_details.cached_tokens == _CACHED_TOKENS, rebuilt.usage + assert rebuilt.id == identity, rebuilt + assert _posted_targets(one_choice_wire) == (chosen.target,) + + +@pytest.mark.parametrize("member", ["text_completion_openai", "openai_chat_backed"]) +async def test_rebuilt_text_completion_stream_without_reported_usage_counts_the_streamed_text( + one_choice_wire: Wire, member: str +) -> None: + chosen: Final = _MEMBERS[member] + chunks: Final = await _async_chunks(chosen, one_choice_wire, False, {}) + + rebuilt: Final = _rebuilt(chunks) + + text: Final = "".join(_PIECES) + assert rebuilt.choices[0].text == text, rebuilt + assert rebuilt.choices[0].finish_reason == "length", rebuilt + expected_completion: Final = litellm.token_counter(model=chosen.model, text=text, count_response_tokens=True) + assert expected_completion > 0 + assert rebuilt.usage.completion_tokens == expected_completion, rebuilt.usage + assert rebuilt.usage.total_tokens == rebuilt.usage.prompt_tokens + rebuilt.usage.completion_tokens, rebuilt.usage + assert _posted_targets(one_choice_wire) == (chosen.target,) + + +async def test_rebuilt_chat_backed_text_completion_stream_keeps_each_choice_text_apart(two_choice_wire: Wire) -> None: + chosen: Final = _MEMBERS["openai_chat_backed"] + chunks: Final = await _async_chunks(chosen, two_choice_wire, True, {"n": 2}) + + rebuilt: Final = _rebuilt(chunks) + + assert [(choice.index, choice.text) for choice in rebuilt.choices] == [ + (index, "".join(pieces)) for index, pieces in enumerate(_CHOICE_PIECES) + ], rebuilt + assert (rebuilt.usage.prompt_tokens, rebuilt.usage.completion_tokens) == (_PROMPT_TOKENS, _COMPLETION_TOKENS) + assert _posted_targets(two_choice_wire) == (chosen.target,) + + +def _grid_row(step: int, last: int, choices: int) -> Iterator[tuple[int, str, str | None]]: + return ((index, "x", "length" if step == last else None) for index in range(choices)) + + +def _grid_pieces(choices: int) -> tuple[tuple[int, str, str | None], ...]: + steps: Final = _GRID_CHUNKS // choices + return tuple(chain.from_iterable(_grid_row(step, steps - 1, choices) for step in range(steps))) + + +def _grid_upstream(request: Request) -> Reply: + requested: Final = _JSON_OBJECT.validate_json(request.body).get("n") + choices: Final = requested if isinstance(requested, int) else 1 + return _stream(request, "cmpl-grid-" + uuid.uuid4().hex, _grid_pieces(choices), False) + + +@pytest.fixture +def grid_wire() -> Iterator[Wire]: + with wire_server(_grid_upstream) as served: + yield served + + +async def _rebuild_seconds(wire: Wire, choices: int) -> float: + chunks: Final = await _async_chunks(_MEMBERS["openai_chat_backed"], wire, False, {"n": choices}) + run: Final = timed(lambda: _rebuilt(chunks)) + assert sum(len(choice.text) for choice in run.value.choices) == _GRID_CHUNKS, run.value.usage + return run.seconds + + +@pytest.mark.timeout(600) +async def test_rebuilding_a_many_choice_text_completion_stream_costs_what_one_choice_of_the_same_size_does( + grid_wire: Wire, record_property: Callable[[str, object], None] +) -> None: + pairs: Final = [ + (await _rebuild_seconds(grid_wire, 1), await _rebuild_seconds(grid_wire, _GRID_CHOICES)) + for _ in range(_GRID_ROUNDS) + ] + record_property("one_choice_rebuild_seconds", [control for control, _ in pairs]) + record_property("many_choice_rebuild_seconds", [observed for _, observed in pairs]) + + assert_within_factor_of_control( + min(observed for _, observed in pairs), + control=min(control for control, _ in pairs), + factor=_REBUILD_FACTOR, + ) + assert len(_posted_targets(grid_wire)) == 2 * _GRID_ROUNDS diff --git a/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py b/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py new file mode 100644 index 00000000000..60a33d46b29 --- /dev/null +++ b/tests/integration/streaming/test_text_completion_stream_guardrail_rebuild.py @@ -0,0 +1,286 @@ +import json +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + eventually, + gateway_from_environment, + list_value, + object_value, +) +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120 +pytestmark: Final = pytest.mark.timeout(_CELL_SECONDS) + +_PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +_PROVIDER_KEY: Final = "synthetic-provider-key" +_GUARDRAIL: Final = "audit-text-rebuild-tool-permission" +_TEXT_MODEL: Final = f"audit-text-instruct-{uuid.uuid4().hex}" +_PIECES: Final = ("Hello", " streamed", " world") +_PROMPT_TOKENS: Final = 11 +_COMPLETION_TOKENS: Final = 3 +_USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": _PROMPT_TOKENS, + "completion_tokens": _COMPLETION_TOKENS, + "total_tokens": _PROMPT_TOKENS + _COMPLETION_TOKENS, +} +_BRIDGE_TEXT: Final = "bridge reply" +_TOGETHER_MODEL: Final = "together_ai/Qwen/Qwen3.5-9B" + + +@dataclass(frozen=True, slots=True) +class _Rig: + wire: Wire + proxy: OwnedProxy + + +@dataclass(frozen=True, slots=True) +class _Stream: + status: int + body: bytes + + @property + def frames(self) -> tuple[dict[str, JsonValue], ...]: + return tuple( + JSON_OBJECT.validate_json(line[6:]) + for line in self.body.decode().splitlines() + if line.startswith("data: {") + ) + + @property + def choices(self) -> tuple[Mapping[str, JsonValue], ...]: + return tuple(chain.from_iterable(_choices_of(frame) for frame in self.frames)) + + @property + def text(self) -> str: + return "".join(str(choice.get("text") or "") for choice in self.choices) + + @property + def finish_reasons(self) -> tuple[JsonValue, ...]: + return tuple(choice["finish_reason"] for choice in self.choices if choice.get("finish_reason") is not None) + + @property + def usages(self) -> tuple[Mapping[str, JsonValue], ...]: + return tuple(object_value(frame["usage"]) for frame in self.frames if frame.get("usage") is not None) + + @property + def ids(self) -> frozenset[JsonValue]: + return frozenset(frame["id"] for frame in self.frames) + + +def _choices_of(frame: Mapping[str, JsonValue]) -> Iterator[Mapping[str, JsonValue]]: + return (object_value(choice) for choice in list_value(frame.get("choices") or [])) + + +def _frame(body: Mapping[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(body).encode() + b"\n\n" + + +def _text_frame(identity: str, choices: list[JsonValue], **extra: JsonValue) -> bytes: + return _frame( + {"id": identity, "object": "text_completion", "created": 1, "model": "scripted", "choices": choices, **extra} + ) + + +def _text_choice(text: str, finish_reason: str | None) -> dict[str, JsonValue]: + return {"text": text, "index": 0, "logprobs": None, "finish_reason": finish_reason} + + +def _respond(request: Request) -> Reply: + body: Final = JSON_OBJECT.validate_json(request.body) + assert body.get("stream") is True, body + assert object_value(body["stream_options"]).get("include_usage") is True, body + identity: Final = body["prompt"] + assert isinstance(identity, str), body + return Reply( + content_type="text/event-stream", + chunks=( + *(_text_frame(identity, [_text_choice(piece, None)]) for piece in _PIECES), + _text_frame(identity, [_text_choice("", "length")]), + _text_frame(identity, [], usage=_USAGE), + b"data: [DONE]\n\n", + ), + ) + + +def _config(directory: Path, wire: Wire) -> Path: + base: Final = object_value(yaml.safe_load(_PROXY_CONFIG.read_text())) + config: Final = { + **base, + "model_list": [ + *list_value(base["model_list"]), + { + "model_name": _TEXT_MODEL, + "litellm_params": { + "model": "text-completion-openai/gpt-3.5-turbo-instruct", + "api_base": f"{wire.url}/v1", + "api_key": _PROVIDER_KEY, + }, + }, + ], + "guardrails": [ + { + "guardrail_name": _GUARDRAIL, + "litellm_params": { + "guardrail": "tool_permission", + "mode": "post_call", + "default_on": True, + "default_action": "allow", + "rules": [], + }, + } + ], + } + path: Final = directory / f"text-rebuild-{uuid.uuid4().hex}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("text-rebuild-rig") + with gateway_from_environment() as gateway, wire_server(_respond) as wire: + with owned_proxy_process( + gateway, + directory, + {}, + config=_config(directory, wire), + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + client_timeout=_CELL_SECONDS, + ) as owned: + yield _Rig(wire, owned) + + +def _stream(rig: _Rig, identity: str) -> _Stream: + gateway: Final = rig.proxy.gateway + with gateway.client.stream( + "POST", + "/v1/completions", + headers={"Authorization": f"Bearer {gateway.key}"}, + json={ + "model": _TEXT_MODEL, + "prompt": identity, + "max_tokens": 16, + "stream": True, + "stream_options": {"include_usage": True}, + "cache": {"no-cache": True}, + }, + ) as response: + return _Stream(response.status_code, response.read()) + + +def _posted_targets(wire: Wire) -> tuple[str, ...]: + return tuple(request.target for request in wire.drain() if request.method == "POST") + + +def test_guarded_text_completion_stream_reaches_the_caller_whole_with_its_usage(rig: _Rig) -> None: + rig.wire.drain() + identity: Final = f"cmpl-guarded-{uuid.uuid4().hex}" + + stream: Final = _stream(rig, identity) + + assert stream.status == 200, stream.body + assert stream.body.rstrip().endswith(b"data: [DONE]"), stream.body[-400:] + assert stream.ids == {identity}, stream.ids + assert stream.text == "".join(_PIECES), stream.frames + assert stream.finish_reasons == ("length",), stream.frames + assert [(usage["prompt_tokens"], usage["completion_tokens"]) for usage in stream.usages] == [ + (_PROMPT_TOKENS, _COMPLETION_TOKENS) + ], stream.frames + assert _posted_targets(rig.wire) == ("/v1/completions",) + (row,) = eventually( + lambda: read_rows( + 'SELECT prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + (identity,), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert (row["prompt_tokens"], row["completion_tokens"]) == (_PROMPT_TOKENS, _COMPLETION_TOKENS), row + assert _GUARDRAIL in list_value(object_value(row["metadata"])["applied_guardrails"]), row + + +def _together_chunk(choices: list[JsonValue], **extra: JsonValue) -> bytes: + return _frame( + { + "id": f"chatcmpl-together-{uuid.uuid4().hex}", + "object": "chat.completion.chunk", + "created": 1, + "model": "scripted", + "choices": choices, + **extra, + } + ) + + +def _together_chat_stream(request: Request) -> Reply: + assert request.method == "POST" and request.target.endswith("/chat/completions"), request + assert JSON_OBJECT.validate_json(request.body).get("stream") is True, request.body + return Reply( + content_type="text/event-stream", + chunks=( + _together_chunk( + [ + { + "index": 0, + "text": _BRIDGE_TEXT, + "delta": {"role": "assistant", "content": _BRIDGE_TEXT}, + "finish_reason": None, + } + ] + ), + _together_chunk([{"index": 0, "text": "", "delta": {}, "finish_reason": "stop"}]), + _together_chunk([], usage=_USAGE), + b"data: [DONE]\n\n", + ), + ) + + +def _completed_events(body: bytes) -> tuple[Mapping[str, JsonValue], ...]: + return tuple( + object_value(event["response"]) + for event in _Stream(200, body).frames + if event.get("type") == "response.completed" + ) + + +def _output_text(response: Mapping[str, JsonValue]) -> str: + parts: Final = chain.from_iterable( + _content_parts(object_value(item)) for item in list_value(response.get("output") or []) + ) + return "".join(str(part.get("text") or "") for part in parts) + + +def _content_parts(item: Mapping[str, JsonValue]) -> Iterator[Mapping[str, JsonValue]]: + return (object_value(part) for part in list_value(item.get("content") or [])) + + +def test_responses_bridge_rebuilds_a_chat_stream_whose_choices_also_carry_text(gateway: Gateway) -> None: + with wire_server(_together_chat_stream) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_TOGETHER_MODEL, api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY) + key: Final = scenario.key(models=[model]) + response: Final = gateway.client.post( + "/v1/responses", + json={"model": model, "input": f"bridge probe {uuid.uuid4().hex}", "stream": True}, + headers={"Authorization": f"Bearer {key}"}, + ) + + assert response.status_code == 200, response.text + (completed,) = _completed_events(response.content) + assert _output_text(completed) == _BRIDGE_TEXT, completed + usage: Final = object_value(completed["usage"]) + assert (usage["input_tokens"], usage["output_tokens"]) == (_PROMPT_TOKENS, _COMPLETION_TOKENS), usage + assert [request.target for request in wire.drain()] == ["/v1/chat/completions"] diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index f8cd0416b1c..5c56245cf33 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,6 +1,6 @@ import base64 import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Final from unittest.mock import MagicMock @@ -12,7 +12,7 @@ import respx import litellm from litellm import ChatCompletionUsageBlock, stream_chunk_builder from litellm.types.utils import GenericStreamingChunk -from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor, stream_chunk_builder_text_completion from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( ChatCompletionDeltaToolCall, @@ -23,6 +23,8 @@ from litellm.types.utils import ( PromptTokensDetails, ServerToolUse, StreamingChoices, + TextChoices, + TextCompletionResponse, Usage, ) @@ -2140,6 +2142,341 @@ def test_stream_chunk_builder_joins_openai_audio_output_pieces() -> None: } +@pytest.mark.parametrize("reported_total", [{"total_tokens": 13}, {}]) +@pytest.mark.parametrize("trailer_choices", [[], [{"text": None, "index": 0, "logprobs": None, "finish_reason": None}]]) +def test_stream_chunk_builder_text_completion_keeps_finish_reason_and_provider_usage( + trailer_choices: list[dict[str, object]], reported_total: dict[str, int] +) -> None: + from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion + from litellm.types.utils import TextCompletionResponse + + def chunk(choices: list[dict[str, object]], **extra: object) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices, **extra + ) + + chunks: Final = [ + chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]), + chunk( + trailer_choices, + usage={ + "prompt_tokens": 7, + "completion_tokens": 6, + **reported_total, + "prompt_tokens_details": {"cached_tokens": 4}, + }, + ), + ] + + response: Final = stream_chunk_builder_text_completion(chunks=chunks, messages=None) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].finish_reason == "length" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13) + assert response.usage.prompt_tokens_details.cached_tokens == 4 + + +def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_chunk() -> None: + def chunk(choices: list[dict[str, object]]) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices + ) + + trailer: Final = chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]) + trailer["usage"] = {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13} + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk([{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": None}]), trailer] + ) + + assert response.choices[0].finish_reason == "length" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13) + + +@pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion]) +def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None: + def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]: + return { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": choices, + **extra, + } + + chunks: Final = [ + chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}]), + chunk([], usage={"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}), + ] + + response: Final = build(chunks=chunks) + + assert response.id == "cmpl-1" + assert response.choices[0].text == "Hello world" + assert response.choices[0].finish_reason == "stop" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 2, 5) + + +@pytest.mark.parametrize("choice_text", ["Hi", None]) +def test_stream_chunk_builder_keeps_dumped_chat_chunks_with_a_choice_text_on_the_chat_path( + choice_text: str | None, +) -> None: + from litellm.types.utils import ModelResponseStream + + def dumped_chunk(content: str, finish_reason: str | None) -> dict[str, object]: + return ModelResponseStream( + id="chatcmpl-1", + object="chat.completion.chunk", + created=1, + model="my-model", + choices=[ + { + "index": 0, + "text": choice_text, + "delta": {"role": "assistant", "content": content}, + "finish_reason": finish_reason, + "logprobs": None, + } + ], + ).model_dump() + + response: Final = litellm.stream_chunk_builder(chunks=[dumped_chunk("Hi", None), dumped_chunk(" there", "stop")]) + + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "Hi there" + assert response.choices[0].finish_reason == "stop" + + +def test_stream_chunk_builder_keeps_logprobs_from_every_text_completion_chunk() -> None: + def chunk(text: str, offset: int, logprob: float, finish_reason: str | None) -> dict[str, object]: + return { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": [ + { + "text": text, + "index": 0, + "logprobs": { + "tokens": [text], + "token_logprobs": [logprob], + "top_logprobs": [{text: logprob}], + "text_offset": [offset], + }, + "finish_reason": finish_reason, + } + ], + } + + trailer: Final = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": [], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk("Hello", 5, -0.1, None), chunk(" world", 10, -0.2, "length"), trailer] + ) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].logprobs.model_dump() == { + "tokens": ["Hello", " world"], + "token_logprobs": [-0.1, -0.2], + "top_logprobs": [{"Hello": -0.1}, {" world": -0.2}], + "text_offset": [5, 10], + } + + +def _text_completion_chunk(*choices: dict[str, object]) -> dict[str, object]: + return {"id": "cmpl-1", "object": "text_completion", "created": 1, "model": "my-model", "choices": list(choices)} + + +def _text_choice( + index: int, text: str, offset: int, top_logprobs: list[dict[str, float]] | None, finish_reason: str | None = None +) -> dict[str, object]: + return { + "text": text, + "index": index, + "logprobs": {"tokens": [text], "token_logprobs": [-0.5], "top_logprobs": top_logprobs, "text_offset": [offset]}, + "finish_reason": finish_reason, + } + + +def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() -> None: + usage_trailer: Final = { + **_text_completion_chunk(), + "usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5}, + } + + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk(_text_choice(0, "Hi", 5, [{"Hi": -0.5}])), + _text_completion_chunk(_text_choice(1, "Hey", 5, [{"Hey": -0.5}])), + _text_completion_chunk(_text_choice(0, " there", 7, [{" there": -0.5}], "stop")), + _text_completion_chunk(_text_choice(1, " you", 8, [{" you": -0.5}], "length")), + usage_trailer, + ] + ) + + assert [ + (choice.index, choice.text, choice.finish_reason, choice.logprobs.tokens, choice.logprobs.text_offset) + for choice in response.choices + ] == [(0, "Hi there", "stop", ["Hi", " there"], [5, 7]), (1, "Hey you", "length", ["Hey", " you"], [5, 8])] + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4) + + +def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk(_text_choice(0, "Hello", 5, None)), + _text_completion_chunk(_text_choice(0, " world", 10, None, "length")), + ] + ) + + assert response.choices[0].logprobs.model_dump() == { + "tokens": ["Hello", " world"], + "token_logprobs": [-0.5, -0.5], + "top_logprobs": None, + "text_offset": [5, 10], + } + + +def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + TextCompletionResponse(id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello")]), + ModelResponseStream( + id="cmpl-1", + model="my-model", + choices=[StreamingChoices(index=0, delta=Delta(content=" world"), finish_reason="stop")], + ), + ] + ) + + assert isinstance(response, TextCompletionResponse) + assert [(choice.text, choice.finish_reason) for choice in response.choices] == [("Hello", "stop")] + + +def test_stream_chunk_builder_drops_only_the_text_completion_logprobs_it_cannot_read() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk({"text": "Hello", "index": 0, "logprobs": {"tokens": ["Hello"]}}), + _text_completion_chunk(_text_choice(0, " world", 10, [{" world": -0.5}], "stop")), + ] + ) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].logprobs.tokens == [" world"] + + +@pytest.mark.parametrize( + "first_logprobs", + [[-0.1], {"tokens": ["Hello"], "token_logprobs": [-0.1], "top_logprobs": None, "text_offset": [0]}], + ids=["list", "logprobs_object"], +) +def test_stream_chunk_builder_keeps_text_completion_logprobs_of_another_shape( + first_logprobs: Sequence[float] | Mapping[str, object], +) -> None: + def chunk( + text: str, logprobs: Sequence[float] | Mapping[str, object], finish_reason: str | None + ) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", + model="my-model", + choices=[TextChoices(index=0, text=text, logprobs=logprobs, finish_reason=finish_reason)], + ) + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk("Hello", first_logprobs, None), chunk(" world", [-0.2], "stop")] + ) + + assert (response.choices[0].text, response.choices[0].logprobs) == ("Hello world", [-0.2]) + + +def test_stream_chunk_builder_keeps_the_usage_a_chat_chunk_reports_in_a_text_completion_stream() -> None: + response: Final = stream_chunk_builder_text_completion( + chunks=[ + TextCompletionResponse( + id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello", finish_reason="stop")] + ), + ModelResponseStream( + id="cmpl-1", + model="my-model", + choices=[], + usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4), + ), + ] + ) + + assert response.usage is not None + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 1, 4) + + +@pytest.mark.parametrize( + ("first_index", "second_index", "expected"), + [ + (None, None, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ("0", 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ("0", "0", ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + (float("inf"), 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ( + 0, + "1", + ( + {"index": 0, "text": "Hello", "finish_reason": None}, + {"index": 1, "text": " world", "finish_reason": "stop"}, + ), + ), + ], + ids=["none", "string_then_int", "strings", "infinity_then_int", "int_then_string"], +) +def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_not_ints_as_ints( + first_index: float | str | None, + second_index: int | str | None, + expected: tuple[Mapping[str, object], ...], +) -> None: + response: Final = stream_chunk_builder_text_completion( + chunks=[ + _text_completion_chunk({"text": "Hello", "index": first_index}), + _text_completion_chunk({"text": " world", "index": second_index, "finish_reason": "stop"}), + ] + ) + + assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected + + +def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None: + from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion + from litellm.types.utils import TextCompletionResponse + + chunks: Final = [ + TextCompletionResponse( + id="cmpl-1", + object="text_completion", + created=1, + model="my-model", + choices=[{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": "stop"}], + ) + ] + + response: Final = stream_chunk_builder_text_completion( + chunks=chunks, messages=[{"role": "user", "content": [{"type": "not-a-content-type"}]}] + ) + + assert response.choices[0].text == "Hello world" + assert response.usage.prompt_tokens == 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens == response.usage.completion_tokens + + def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index 4a7a14fceaa..25d2556bf26 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -1,7 +1,9 @@ """Unit tests for the Microsoft Purview DLP guardrail.""" import asyncio +import json import time +from typing import Final from unittest.mock import AsyncMock, Mock, patch import httpx @@ -536,6 +538,61 @@ class TestTextCompletionHooks: assert "alpha" in combined assert "beta" in combined + @pytest.mark.asyncio + async def test_streamed_multi_choice_text_completion_is_blocked_like_the_non_streamed_one(self): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.types.utils import TextChoices, TextCompletionResponse + + sensitive: Final = "4111 1111 1111 1111" + choice_pieces: Final = ((sensitive[:9], sensitive[9:]), ("Hey", " you")) + user: Final = UserAPIKeyAuth(api_key="test", user_id="user-123") + + def graph(request: httpx.Request) -> httpx.Response: + if request.url.host == "login.microsoftonline.com": + return httpx.Response(200, json={"access_token": "graph-token", "expires_in": 3600}) + if not request.url.path.endswith("/processContent"): + return httpx.Response(200, json={"value": []}, headers={"ETag": "scope-etag"}) + entry: Final = json.loads(request.content)["contentToProcess"]["contentEntries"][0] + block: Final = {"@odata.type": "#microsoft.graph.restrictAccessAction", "restrictionAction": "block"} + actions: Final = [block] if sensitive in entry["content"]["data"] else [] + return httpx.Response(200, json={"protectionScopeState": "unchanged", "policyActions": actions}) + + def guardrail_on_graph() -> MicrosoftPurviewDLPGuardrail: + guardrail: Final = _make_guardrail() + guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(graph)) + return guardrail + + def chunk(index: int, text: str) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-n2", model="gpt-3.5-turbo-instruct", choices=[TextChoices(text=text, index=index)] + ) + + async def interleaved_stream(): + for step in range(2): + for index, pieces in enumerate(choice_pieces): + yield chunk(index, pieces[step]) + + non_streamed: Final = TextCompletionResponse( + id="cmpl-n2", + model="gpt-3.5-turbo-instruct", + choices=[TextChoices(text="".join(pieces), index=index) for index, pieces in enumerate(choice_pieces)], + ) + with pytest.raises(HTTPException) as non_streamed_block: + await guardrail_on_graph().async_post_call_success_hook( + data={}, user_api_key_dict=user, response=non_streamed + ) + + async def drain_guarded_stream() -> None: + async for _ in guardrail_on_graph().async_post_call_streaming_iterator_hook( + user_api_key_dict=user, response=interleaved_stream(), request_data={} + ): + pass + + with pytest.raises(HTTPException) as streamed_block: + await drain_guarded_stream() + + assert streamed_block.value.status_code == non_streamed_block.value.status_code == 400 + # --------------------------------------------------------------- # Responses API hooks From ba61ecdf570c91a2409560138170ff9ab1fe7ad7 Mon Sep 17 00:00:00 2001 From: ekpog200 <62500055+ekpog200@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:42:59 +0300 Subject: [PATCH 070/124] fix(videos)!: keep model_id in proxy video ids so per-model keys resolve on status/content (#44614) * fix(videos): keep model_id in proxy video ids so per-model keys resolve on status/content Provider transforms encode returned video ids with an empty model_id, so GET /v1/videos/{id} and /content cannot resolve the deployment (and its per-model api_key) and fail with invalid_api_key. Re-stamp the returned VideoObject id with the resolved model_id in video_generation, video_status, video_remix, video_edit and video_extension. Fixes #33423 * fix(videos): return a copy instead of mutating the response, read hidden params via helper * test(videos): fully type the new video id re-stamping tests Annotate every parameter and return added by this PR (hidden_params, harness, response, -> None), drop Any from the test helper, mark locals Final, and narrow the endpoint/helper results with isinstance(VideoObject) instead of reading .id off an untyped value. Route the mocked processor response through a typed Harness.respond_with, read NotRequired DecodedVideoId keys with .get, and name the rule and reason on the _hidden_params suppressions. Co-Authored-By: Claude Opus 5.5 * fix(videos): fill only an empty model in returned video ids Keep an id that already names a model as is, so the asked-for name, routing group and served group in it still drive access checks and routing. Fill an empty model from the model the caller asked for before the routed deployment id, and read the request model through a typed view so the type gate does not grow * fix(videos): route video calls to the deployment that created the video Video ids now carry the id of the deployment that created them. Status, content, remix, edit and extension calls go to that deployment when it belongs to the requested group, so a group spread over two provider accounts no longer polls the wrong one. A provider model shared by several groups resolves to the creating deployment's group, and the asked-for model is written into an id only when the router can route it. * fix(videos): pin video calls only to deployments the caller may use Pin a video id's creating deployment at deployment selection, and only when the caller may use that deployment in the requested group, so aliases, routing groups and access groups still apply and the spend label and fallbacks keep the requested group. A status poll keeps the deployment its id names and takes the deployment that answered when the id names none. An alias that routes to the creating deployment's group is kept on status and content. * test(router): reset module fallback globals between unit tests * fix(videos): check and route polls by the name the router serves A poll on a video created through a provider wildcard is checked against the name the wildcard serves, a team model is checked by its public name, a poll stamps the deployment that answered, and the sync Router pins video calls the same way the async path does * fix(videos): pin polls only among the deployments routing allows Video calls route by group again and the creating deployment is picked only from the deployments left after access, team, health, cooldown, budget and tag filters, so a pinned poll no longer skips them. Wildcard groups now pin too. Adds tests for cooldown, wildcard pinning and character lookups through a provider wildcard, and return types on the auth tests * fix(videos): scope the video pin to video calls, keep fallback-served videos reachable, skip cooldown on video 404s * fix(videos): stamp the requested group only when one of its router fallbacks served the create * fix(videos): follow chained router fallbacks for fallback-served videos, test with a real Router * test(router): list the video pin helper with its sibling in the router coverage exemptions * refactor(auth): walk chained router fallbacks without recursion --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/base_llm/videos/async_task_utils.py | 10 +- litellm/proxy/auth/auth_utils.py | 50 ++- litellm/proxy/video_endpoints/endpoints.py | 92 ++-- litellm/proxy/video_endpoints/utils.py | 115 ++++- litellm/router.py | 98 ++++- litellm/router_utils/cooldown_handlers.py | 6 + .../router_utils/pattern_match_deployments.py | 2 +- litellm/types/videos/main.py | 2 + litellm/types/videos/utils.py | 111 +++-- .../router_code_coverage.py | 2 + .../routing/test_video_poll_routing_wire.py | 4 +- tests/unit/conftest.py | 10 +- tests/unit/proxy/auth/test_auth_utils.py | 412 ++++++++++++----- .../proxy/video_endpoints/test_endpoints.py | 162 ++++++- .../unit/proxy/video_endpoints/test_utils.py | 413 +++++++++++++++++- .../test_router_provider_endpoints.py | 243 +++++++++++ tests/unit/test_video_generation.py | 30 ++ 17 files changed, 1530 insertions(+), 232 deletions(-) diff --git a/litellm/llms/base_llm/videos/async_task_utils.py b/litellm/llms/base_llm/videos/async_task_utils.py index d2fb3ed7606..507aefea6c6 100644 --- a/litellm/llms/base_llm/videos/async_task_utils.py +++ b/litellm/llms/base_llm/videos/async_task_utils.py @@ -8,7 +8,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from litellm.images.utils import ImageEditRequestUtils -from litellm.types.videos.utils import decode_video_id_with_provider +from litellm.types.videos.utils import decode_video_id_with_provider, id_safe_name _FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object]) _FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) @@ -25,16 +25,12 @@ def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]: return _fields(getattr(logging_obj, "litellm_params", None)) or {} -def _id_safe_name(name: object) -> str | None: - return name if isinstance(name, str) and name and ";" not in name else None - - def _stamped_models(metadata: object) -> tuple[str, str | None] | None: fields: Final = _fields(metadata) if fields is None: return None - requested: Final = _id_safe_name(fields.get("model_group")) - served: Final = _id_safe_name(fields.get("deployment_model_name")) + requested: Final = id_safe_name(fields.get("model_group")) + served: Final = id_safe_name(fields.get("deployment_model_name")) if requested is None: return None return requested, None if served is None or served == requested or "*" in served else served diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 96142d75403..73d20ae632c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -4,6 +4,7 @@ import re import sys from collections.abc import Collection, Iterator, Mapping from functools import lru_cache +from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType from typing import Any, Final, Protocol @@ -44,6 +45,7 @@ from litellm.types.router import ( ) from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params +from litellm.types.videos.main import DecodedVideoId def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: @@ -597,6 +599,31 @@ def fallback_target_model_name(target: object) -> str | None: return None +def _direct_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: + from litellm.proxy.auth.model_checks import get_all_fallbacks + + return frozenset( + name for target in get_all_fallbacks(model, llm_router) if (name := fallback_target_model_name(target)) + ) + + +def _fallback_hop( + frontier_and_reached: tuple[frozenset[str], frozenset[str]], llm_router: Router +) -> tuple[frozenset[str], frozenset[str]]: + frontier, reached = frontier_and_reached + next_hop: Final = ( + frozenset(chain.from_iterable(_direct_fallback_model_names(model, llm_router) for model in frontier)) - reached + ) + return next_hop, reached | next_hop + + +def router_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: + """The groups a failure on `model` can reach through the router's fallbacks, chained and generic (`*`) ones.""" + hops: Final = accumulate(repeat(llm_router), _fallback_hop, initial=(frozenset({model}), frozenset({model}))) + _, reached = next(hop for hop in hops if not hop[0]) + return reached - {model} + + def request_fallback_model_names(request_body: Mapping[str, object]) -> tuple[str, ...]: return tuple( dict.fromkeys( @@ -2042,10 +2069,10 @@ def _extract_models_from_managed_resource_id( ) if resource_id_field == "video_id": - model_id = decode_video_id_with_provider(resource_id).get("model_id") + decoded_video: Final = decode_video_id_with_provider(resource_id) _append_model_candidates( candidates=candidates, - value=_resolve_model_id_with_router(model_id, llm_router), + value=_video_access_group(decoded_video, llm_router) or decoded_video.get("model_id"), ) else: model_id = decode_character_id_with_provider(resource_id).get("model_id") @@ -2059,6 +2086,25 @@ def _extract_models_from_managed_resource_id( return _dedupe_model_candidates(candidates) +def _video_access_group(decoded_video: DecodedVideoId, llm_router: Router | None) -> str | None: + """ + The group a key must be allowed to call to use a video: the creating deployment's group, or, while router + fallbacks skip model access checks, the group whose router fallback created it. + """ + from litellm.proxy.auth.fallback_model_access import router_fallback_access_check + from litellm.proxy.video_endpoints.utils import video_model_group + + creating_group: Final = video_model_group(decoded_video, llm_router) + requested_model: Final = decoded_video.get("requested_model") + if llm_router is None or creating_group is None or requested_model is None: + return creating_group + if router_fallback_access_check.is_enforced(): + return creating_group + if creating_group in router_fallback_model_names(requested_model, llm_router): + return requested_model + return creating_group + + def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | None) -> str | None: if model_id is None or llm_router is None: return model_id diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index c4f33eae114..d4643994043 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -20,9 +20,14 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio from litellm.proxy.video_endpoints.utils import ( + VideoRequestRoute, encode_character_id_in_response, + encode_video_id_in_response, extract_model_from_target_model_names, + fallback_requested_model, get_custom_provider_from_data, + routable_model_name, + video_model_group, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -84,6 +89,7 @@ async def video_generation( # Read request body data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) if input_reference is not None: input_reference_file: Final = await batch_to_bytesio([input_reference]) if input_reference_file: @@ -118,7 +124,13 @@ async def video_generation( version=version, ) else: - return generated + requested_model: Final = routable_model_name(requested.model, llm_router, requested.custom_llm_provider) + return encode_video_id_in_response( + response=generated, + fallback_provider=requested.custom_llm_provider, + fallback_model_id=requested_model, + requested_model=fallback_requested_model(requested_model, generated, llm_router), + ) @router.get( @@ -269,12 +281,9 @@ async def video_status( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) @@ -305,7 +314,12 @@ async def video_status( version=version, ) else: - return status + return encode_video_id_in_response( + response=status, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded, + polled_video=decoded, + ) @router.get( @@ -358,7 +372,6 @@ async def video_content( decoded: Final = decode_video_id_with_provider(video_id) provider_from_id: Final = decoded.get("custom_llm_provider") - model_id_from_decoded: Final = decoded.get("model_id") custom_llm_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) @@ -369,12 +382,9 @@ async def video_content( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -462,6 +472,7 @@ async def video_remix( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) data["video_id"] = video_id decoded: Final = decode_video_id_with_provider(video_id) @@ -471,18 +482,15 @@ async def video_remix( custom_llm_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) - or data.get("custom_llm_provider") + or requested.custom_llm_provider or provider_from_id ) if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) @@ -513,7 +521,13 @@ async def video_remix( version=version, ) else: - return remixed + return encode_video_id_in_response( + response=remixed, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) @router.post( @@ -771,6 +785,7 @@ async def video_edit( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) uploaded_video: Final = data.pop("video", None) if isinstance(uploaded_video, StarletteUploadFile): video_files: Final = await batch_to_bytesio((uploaded_video,)) @@ -793,10 +808,9 @@ async def video_edit( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -826,7 +840,13 @@ async def video_edit( version=version, ) else: - return edited + return encode_video_id_in_response( + response=edited, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) @router.post( @@ -875,6 +895,7 @@ async def video_extension( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) data["video_id"] = video_reference_to_id(data.pop("video", None)) decoded: Final = decode_video_id_with_provider(data["video_id"]) @@ -890,10 +911,9 @@ async def video_extension( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -923,4 +943,10 @@ async def video_extension( version=version, ) else: - return extended + return encode_video_id_in_response( + response=extended, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 554f0f6ef53..a2be42f1b76 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,8 +1,18 @@ -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final import orjson +from pydantic import BaseModel, ConfigDict -from litellm.types.videos.utils import encode_character_id_with_provider +from litellm.types.videos.main import DecodedVideoId, VideoObject +from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_character_id_with_provider, + encode_video_id_with_provider, + id_safe_name, +) + +if TYPE_CHECKING: + from litellm.router import Router def extract_model_from_target_model_names(target_model_names: Any) -> str | None: @@ -65,3 +75,104 @@ def encode_character_id_in_response(response: Any, custom_llm_provider: str, mod model_id=model_id, ) return response + + +class VideoRequestRoute(BaseModel): + model_config = ConfigDict(frozen=True) + + model: object = None + custom_llm_provider: object = None + + +def routable_model_name(model: object, llm_router: "Router | None", custom_llm_provider: object) -> str | None: + name: Final = id_safe_name(model) + if name is None or llm_router is None: + return name + provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + return name if llm_router.resolve_model_name_from_model_id(name, provider) else None + + +def video_model_group(decoded: DecodedVideoId, llm_router: "Router | None") -> str | None: + """ + The group a video id names. A provider model that several groups share resolves to the group of the + deployment that created the video, under its team's public name for a team model, while a name the router + routes by itself (the id's model, bare or provider-prefixed), or one that routes to that deployment's group + (an alias of it), is kept as written. + """ + if llm_router is None: + return None + model_id: Final = decoded.get("model_id") + provider: Final = decoded.get("custom_llm_provider") + resolved: Final = llm_router.resolve_model_name_from_model_id(model_id, provider) if model_id else None + deployment_id: Final = decoded.get("deployment_id") + if not deployment_id or resolved in (model_id, f"{provider}/{model_id}"): + return resolved + deployment: Final = llm_router.get_deployment(model_id=deployment_id) + if deployment is None: + return resolved + creating_group: Final = deployment.model_info.team_public_model_name or deployment.model_name + if resolved is not None and llm_router.resolve_served_model_name(resolved, creating_group) == creating_group: + return resolved + return creating_group + + +def fallback_requested_model(requested_model: str | None, response: object, llm_router: "Router | None") -> str | None: + """The group a create asked for when one of that group's router fallbacks served it.""" + from litellm.proxy.auth.auth_utils import router_fallback_model_names + + if requested_model is None or llm_router is None or not isinstance(response, VideoObject): + return None + deployment_id: Final = id_safe_name(response.hidden_params.get("model_id")) + deployment: Final = llm_router.get_deployment(model_id=deployment_id) if deployment_id else None + if deployment is None: + return None + creating_group: Final = deployment.model_info.team_public_model_name or deployment.model_name + return requested_model if creating_group in router_fallback_model_names(requested_model, llm_router) else None + + +def encode_video_id_in_response( + response: object, + fallback_provider: object, + fallback_model_id: object, + polled_video: DecodedVideoId | None = None, + requested_model: str | None = None, +) -> object: + """ + Stamps the provider, model and the deployment that answered into a returned video id. A poll of an existing + video (`polled_video`) answered without a routed deployment keeps the deployment its id already named, and + every returned id keeps the group a router fallback was serving (`requested_model`). + """ + if not isinstance(response, VideoObject) or not response.id: + return response + + decoded: Final = decode_video_id_with_provider(response.id) + hidden_params: Final = response.hidden_params + routed_deployment_id: Final = id_safe_name(hidden_params.get("model_id")) + deployment_id: Final = routed_deployment_id or (polled_video or decoded).get("deployment_id") + kept_requested_model: Final = ( + id_safe_name(requested_model) or decoded.get("requested_model") or (polled_video or {}).get("requested_model") + ) + model_id: Final = decoded.get("model_id") or id_safe_name(fallback_model_id) or routed_deployment_id + provider: Final = ( + id_safe_name(decoded.get("custom_llm_provider")) + or id_safe_name(hidden_params.get("custom_llm_provider")) + or id_safe_name(fallback_provider) + ) + if not model_id or not provider: + return response + if ( + model_id == decoded.get("model_id") + and deployment_id == decoded.get("deployment_id") + and kept_requested_model == decoded.get("requested_model") + ): + return response + + restamped_id: Final = encode_video_id_with_provider( + video_id=decoded.get("video_id") or response.id, + provider=provider, + model_id=model_id, + served_model=decoded.get("served_model"), + deployment_id=deployment_id, + requested_model=kept_requested_model, + ) + return response.model_copy(update={"id": restamped_id}) diff --git a/litellm/router.py b/litellm/router.py index 94755d0d5ba..1ecff167126 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -34,6 +34,7 @@ from collections.abc import ( MutableMapping, Sequence, ) +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache, partial @@ -230,6 +231,7 @@ from litellm.router_utils.cooldown_handlers import ( is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_video_lookup_not_found, set_cooldown_deployments, ) from litellm.router_utils.fallback_event_handlers import ( @@ -400,6 +402,7 @@ from litellm.types.utils import ( shared_backend_model_info, ) from litellm.types.utils import ModelInfo as ModelMapInfo +from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES, decode_video_id_with_provider from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, @@ -516,6 +519,7 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _CallbackT = TypeVar("_CallbackT") +_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object]) _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( {"model", "api_base", "api_key", "api_version", "responses_api_version"} @@ -527,6 +531,7 @@ CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_b _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) +_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( { "prompt_caching": PromptCachingDeploymentCheck, @@ -5960,17 +5965,19 @@ class Router: passthrough_on_no_deployment: Final = kwargs.pop("passthrough_on_no_deployment", False) function_name: Final = "_ageneric_api_call_with_fallbacks" + handler_name: Final[str | None] = getattr(original_generic_function, "__name__", None) deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) try: - deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init above - model=model, - request_kwargs=kwargs, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - ) + with self._video_call_pin(handler_name, kwargs): + deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + request_kwargs=kwargs, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + ) except Exception as e: if passthrough_on_no_deployment: return await original_generic_function(model=model, **kwargs) @@ -5980,8 +5987,7 @@ class Router: deployment=deployment, kwargs=kwargs, function_name=function_name, - splits_stream_timeout=getattr(original_generic_function, "__name__", None) - in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, ) data: Final = deployment["litellm_params"].copy() @@ -6672,6 +6678,37 @@ class Router: ) return await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs) + @staticmethod + @contextlib.contextmanager + def _video_call_pin(handler_name: str | None, request_kwargs: Mapping[str, object]) -> Generator[None]: + """ + Scopes the deployment a video id names to the routing of one call on that video (status, content, remix, + edit, extension). Any other call routes as if no id were sent, whatever its body carries. + """ + video_id: Final = request_kwargs.get("video_id") + deployment_id: Final = ( + decode_video_id_with_provider(video_id).get("deployment_id") + if handler_name in VIDEO_ID_CALL_TYPES and isinstance(video_id, str) + else None + ) + token: Final = _video_call_deployment_id.set(deployment_id) + try: + yield + finally: + _video_call_deployment_id.reset(token) + + @staticmethod + def _narrow_to_video_deployment(healthy_deployments: Sequence[_DeploymentT]) -> Sequence[_DeploymentT]: + """ + A video lives on the provider account of the deployment that created it, so a call on an id naming a + deployment that every routing filter kept goes to that deployment, else to whichever one routing picks. + """ + deployment_id: Final = _video_call_deployment_id.get() + creating: Final = tuple( + deployment for deployment in healthy_deployments if deployment_id in Router._deployment_ids((deployment,)) + ) + return creating or healthy_deployments + def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs): """ Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router @@ -6693,13 +6730,14 @@ class Router: kwargs=kwargs, metadata_variable_name=metadata_variable_name, ) - deployment: Final = self.get_available_deployment( - model=model, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with self._video_call_pin(handler_name, kwargs): + deployment: Final = self.get_available_deployment( + model=model, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, @@ -9184,6 +9222,13 @@ class Router: ) return False + if is_video_lookup_not_found(exception, kwargs): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Provider 404 says the video is not on this deployment's account, not that the deployment is down." + ) + return False + exception_status: Final = getattr(exception, "status_code", "") if is_caller_timeout_408(kwargs, exception_status): @@ -12721,7 +12766,8 @@ class Router: 1. First, check if model_id directly matches a model_name or deployment ID 2. If not, search through router's model_list to find a match by litellm_params.model 3. If not, keep model_id when the router routes that name itself (an alias, a routing group, or a wildcard - whose deployment serves `custom_llm_provider`) + whose deployment serves `custom_llm_provider`), else name it `custom_llm_provider/model_id` when only that + form routes to such a wildcard 4. Return the model_name if found, None otherwise Args: @@ -12769,8 +12815,15 @@ class Router: if self.is_recognized_model(model_id): return model_id - if custom_llm_provider is not None and self.pattern_router.routes_to_provider(model_id, custom_llm_provider): + if custom_llm_provider is None: + return None + if self.pattern_router.routes_to_provider(model_id, custom_llm_provider): return model_id + provider_prefixed: Final = f"{custom_llm_provider}/{model_id}" + if not model_id.startswith(f"{custom_llm_provider}/") and self.pattern_router.routes_to_provider( + provider_prefixed, custom_llm_provider + ): + return provider_prefixed return None def resolve_served_model_name(self, model_name: str, served_model: str) -> str: @@ -14347,10 +14400,10 @@ class Router: def _filter_deployments_by_model_access_groups( self, model: str, - healthy_deployments: list, + healthy_deployments: list[DeploymentTypedDict], request_kwargs: Mapping[str, object] | None, request_team_id: str | None, - ) -> list: + ) -> list[DeploymentTypedDict]: """ Restrict candidate deployments to caller-authorized model access groups. @@ -14533,7 +14586,8 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils.get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order + list(self._narrow_to_video_deployment(cast(list[dict[str, object]], healthy_deployments))), + target_order=_target_order, ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in @@ -15735,7 +15789,7 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils.get_order_filtered_deployments( - healthy_deployments, target_order=_target_order + list(self._narrow_to_video_deployment(healthy_deployments)), target_order=_target_order ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 7ed4864ebc3..c362c6899db 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -38,6 +38,7 @@ from litellm.types.utils import ( CooldownDeploymentInfo, InternalCallOrigin, ) +from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES from .router_callbacks.track_deployment_metrics import ( get_deployment_failures_for_current_minute, @@ -92,6 +93,11 @@ def is_background_response_poll_not_found(exception: Exception, litellm_params: ) +def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool: + """Whether a call on an existing video id failed with a provider 404: the video is not on that account.""" + return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index f921f907f4c..4dd217cf59b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -174,7 +174,7 @@ class PatternMatchRouter: return None # No matching pattern found def routes_to_provider(self, request: str, custom_llm_provider: str) -> bool: - deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.route(request) or ()) + deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.get_pattern(request) or ()) return any( (deployment.litellm_params.custom_llm_provider or inferred_provider(deployment.litellm_params.model)) == custom_llm_provider diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 05444b1e7ec..4d6f8fc6a30 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -113,6 +113,8 @@ class DecodedVideoId(TypedDict, total=False): model_id: str | None video_id: str served_model: ReadOnly[str | None] + deployment_id: ReadOnly[str | None] + requested_model: ReadOnly[str | None] class CharacterObject(LiteLLMBaseModel): diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index 9a07c9ba802..0fc2b8ede1b 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -6,6 +6,7 @@ Format: vid_{base64_encoded_string} """ import base64 +from collections.abc import Sequence from typing import Final from litellm._logging import verbose_logger @@ -14,6 +15,22 @@ from litellm.types.videos.main import DecodedVideoId VIDEO_ID_PREFIX: Final = "video_" VIDEO_ID_SERVED_MODEL_PREFIX: Final = "served_model:" +VIDEO_ID_DEPLOYMENT_PREFIX: Final = "deployment_id:" +VIDEO_ID_REQUESTED_MODEL_PREFIX: Final = "requested_model:" +VIDEO_ID_CALL_TYPES: Final = frozenset( + { + "avideo_status", + "video_status", + "avideo_content", + "video_content", + "avideo_remix", + "video_remix", + "avideo_edit", + "video_edit", + "avideo_extension", + "video_extension", + } +) CHARACTER_ID_PREFIX: Final = "character_" CHARACTER_ID_TEMPLATE: Final = "litellm:custom_llm_provider:{};model_id:{};character_id:{}" @@ -37,7 +54,12 @@ def _add_base64_padding(value: str) -> str: def encode_video_id_with_provider( - video_id: str, provider: str, model_id: str | None = None, served_model: str | None = None + video_id: str, + provider: str, + model_id: str | None = None, + served_model: str | None = None, + deployment_id: str | None = None, + requested_model: str | None = None, ) -> str: """Encode provider and model_id into video_id using base64.""" if not provider or not video_id: @@ -54,12 +76,24 @@ def encode_video_id_with_provider( # ID is not encoded (even if it starts with video_), so encode it assembled_id = str(SpecialEnums.LITELLM_MANAGED_VIDEO_COMPLETE_STR.value).format(provider, model_id or "", video_id) served_part: Final = f";{VIDEO_ID_SERVED_MODEL_PREFIX}{served_model}" if served_model else "" + deployment_part: Final = f";{VIDEO_ID_DEPLOYMENT_PREFIX}{deployment_id}" if deployment_id else "" + requested_part: Final = f";{VIDEO_ID_REQUESTED_MODEL_PREFIX}{requested_model}" if requested_model else "" - base64_encoded_id: Final[str] = base64.b64encode((assembled_id + served_part).encode("utf-8")).decode("utf-8") + base64_encoded_id: Final[str] = base64.b64encode( + (assembled_id + served_part + deployment_part + requested_part).encode("utf-8") + ).decode("utf-8") return f"{VIDEO_ID_PREFIX}{base64_encoded_id}" +def id_safe_name(name: object) -> str | None: + return name if isinstance(name, str) and name and ";" not in name else None + + +def _trailing_part(parts: Sequence[str], prefix: str) -> str | None: + return next((part.removeprefix(prefix) or None for part in parts[3:] if part.startswith(prefix)), None) + + def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: """Decode provider and model_id from encoded video_id.""" if not encoded_video_id: @@ -80,43 +114,6 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: cleaned_id = encoded_video_id.replace(VIDEO_ID_PREFIX, "") cleaned_id = _add_base64_padding(cleaned_id) decoded_id: Final = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8") - - if ";" not in decoded_id: - return DecodedVideoId( - custom_llm_provider=None, - model_id=None, - video_id=encoded_video_id, - ) - - parts: Final = decoded_id.split(";") - - custom_llm_provider = None - model_id = None - decoded_video_id = encoded_video_id - - if len(parts) >= 3: - custom_llm_provider_part: Final = parts[0] - model_id_part: Final = parts[1] - video_id_part: Final = parts[2] - - custom_llm_provider = custom_llm_provider_part.replace("litellm:custom_llm_provider:", "") - model_id = model_id_part.replace("model_id:", "") - decoded_video_id = video_id_part.replace("video_id:", "") - - served_model: Final = ( - parts[3].removeprefix(VIDEO_ID_SERVED_MODEL_PREFIX) or None - if len(parts) >= 4 and parts[3].startswith(VIDEO_ID_SERVED_MODEL_PREFIX) - else None - ) - - if served_model is None: - return DecodedVideoId(custom_llm_provider=custom_llm_provider, model_id=model_id, video_id=decoded_video_id) - return DecodedVideoId( - custom_llm_provider=custom_llm_provider, - model_id=model_id, - video_id=decoded_video_id, - served_model=served_model, - ) except Exception as e: verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) return DecodedVideoId( @@ -125,6 +122,42 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: video_id=encoded_video_id, ) + if ";" not in decoded_id: + return DecodedVideoId( + custom_llm_provider=None, + model_id=None, + video_id=encoded_video_id, + ) + + parts: Final = decoded_id.split(";") + + custom_llm_provider = None + model_id = None + decoded_video_id = encoded_video_id + + if len(parts) >= 3: + custom_llm_provider_part: Final = parts[0] + model_id_part: Final = parts[1] + video_id_part: Final = parts[2] + + custom_llm_provider = custom_llm_provider_part.replace("litellm:custom_llm_provider:", "") + model_id = model_id_part.replace("model_id:", "") + decoded_video_id = video_id_part.replace("video_id:", "") + + served_model: Final = _trailing_part(parts, VIDEO_ID_SERVED_MODEL_PREFIX) + deployment_id: Final = _trailing_part(parts, VIDEO_ID_DEPLOYMENT_PREFIX) + requested_model: Final = _trailing_part(parts, VIDEO_ID_REQUESTED_MODEL_PREFIX) + decoded_parts: Final = DecodedVideoId( + custom_llm_provider=custom_llm_provider, model_id=model_id, video_id=decoded_video_id + ) + with_served: Final[DecodedVideoId] = ( + decoded_parts if served_model is None else {**decoded_parts, "served_model": served_model} + ) + with_deployment: Final[DecodedVideoId] = ( + with_served if deployment_id is None else {**with_served, "deployment_id": deployment_id} + ) + return with_deployment if requested_model is None else {**with_deployment, "requested_model": requested_model} + def extract_original_video_id(encoded_video_id: str) -> str: """Extract original video ID without encoding.""" diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 4ea242258f6..e72a85b7056 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -100,6 +100,8 @@ ignored_function_names = [ "_embedding", "_aembedding", "_stream_pre_content_error", # Tested through the non-retriable retry error tests in test_router.py + "_narrow_to_video_deployment", # Tested through video_status and avideo_status polls in test_router_provider_endpoints.py + "_video_call_pin", # Tested through the video_status pin tests and the chat video_id test in test_router_provider_endpoints.py "_deployment_num_retries", # Tested through the deployment num_retries mid-stream budget test in test_router.py "_request_fallback_list", # Tested through every mid-stream retry test in test_router.py "_request_model_group", # Tested through test_anthropic_messages_retry_budget_precedence_direct_call diff --git a/tests/integration/routing/test_video_poll_routing_wire.py b/tests/integration/routing/test_video_poll_routing_wire.py index e0d88a6b0ff..a3d767275a0 100644 --- a/tests/integration/routing/test_video_poll_routing_wire.py +++ b/tests/integration/routing/test_video_poll_routing_wire.py @@ -338,9 +338,9 @@ def test_an_id_without_a_served_group_still_polls_its_own_deployment(rig: _Rig) video_id: Final = created_id(create(rig.gateway, {"model": rig.member_a, "seconds": "5", "size": "1280x720"})) assert decoded(video_id).get("served_model") is None legacy: Final = forged_id("dashscope", rig.member_a, task_of(video_id), None) - assert legacy == video_id + assert {**decoded(legacy), "deployment_id": decoded(video_id).get("deployment_id")} == decoded(video_id) rig.drain() - _assert_served(_poll(rig, legacy), legacy) + _assert_served(_poll(rig, legacy), video_id) download: Final = _download(rig, legacy) assert download.status_code == 200 and download.content == MP4, download.text received: Final = rig.a.drain() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8331fb1937b..9028b8461b0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -93,7 +93,15 @@ CALLBACK_LISTS: Final = ( "_async_failure_callback", "_async_input_callback", ) -RESET_TO_NONE_GLOBALS: Final = ("model_fallbacks", "cache") +RESET_TO_NONE_GLOBALS: Final = ( + "model_fallbacks", + "cache", + "max_fallbacks", + "default_fallbacks", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", +) RESTORED_GLOBALS: Final = ( "disable_aiohttp_transport", "force_ipv4", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 7b1c895483d..f81ba155097 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -668,18 +668,11 @@ class TestLogOnceIfBudgetReservationDisabled: def test_logs_info_only_once_when_enabled(self, caplog): with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): log_once_if_budget_reservation_disabled(disabled=False) - assert not any( - "disable_budget_reservation is enabled" in record.message - for record in caplog.records - ) + assert not any("disable_budget_reservation is enabled" in record.message for record in caplog.records) for _ in range(3): log_once_if_budget_reservation_disabled(disabled=True) - records = [ - record - for record in caplog.records - if "disable_budget_reservation is enabled" in record.message - ] + records = [record for record in caplog.records if "disable_budget_reservation is enabled" in record.message] assert len(records) == 1 assert records[0].levelno == logging.INFO @@ -1007,19 +1000,28 @@ def test_get_model_from_request_no_request_extracts_model(): ) -@pytest.mark.parametrize("provider,model", [ - ("laya", "english"), ("laya", "multilingual"), ("laya", "typed-decisions"), - ("bespoke", "nimble-latest"), ("bespoke", "bespokelabs/Bespoke-Nimble-9B"), -]) +@pytest.mark.parametrize( + "provider,model", + [ + ("laya", "english"), + ("laya", "multilingual"), + ("laya", "typed-decisions"), + ("bespoke", "nimble-latest"), + ("bespoke", "bespokelabs/Bespoke-Nimble-9B"), + ], +) @pytest.mark.parametrize("suffix", ["", "/"]) def test_oss_native_model_uses_the_classifier_permission_identity(provider: str, model: str, suffix: str) -> None: - assert get_model_from_request( - request_data={"model": model}, route=f"/{provider}/v1/systemone{suffix}" - ) == f"{provider}/{model}" + assert ( + get_model_from_request(request_data={"model": model}, route=f"/{provider}/v1/systemone{suffix}") + == f"{provider}/{model}" + ) @pytest.mark.parametrize("provider", ["laya", "bespoke"]) -@pytest.mark.parametrize("model", [None, "", "auto", "laya/english", "bespoke/nimble-latest", "unknown", ["english"], 7]) +@pytest.mark.parametrize( + "model", [None, "", "auto", "laya/english", "bespoke/nimble-latest", "unknown", ["english"], 7] +) def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(provider: str, model: object) -> None: with pytest.raises(HTTPException) as denied: get_model_from_request(request_data={"model": model}, route=f"/{provider}/v1/systemone") @@ -1034,38 +1036,49 @@ def test_laya_model_normalization_does_not_change_other_provider_routes() -> Non def _cache_prediction_router(): from litellm.router import Router - return Router(model_list=[ - { - "model_name": group, - "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-provider-key"}, - "model_info": {"id": deployment_id, "team_id": team_id}, - } - for group, deployment_id, team_id in ( - ("current-group", "current-id", None), ("candidate-group", "candidate-id", None), - ("own-group", "own-id", "prediction-team"), ("foreign-group", "foreign-id", "foreign-team"), - ) - ]) + return Router( + model_list=[ + { + "model_name": group, + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-provider-key"}, + "model_info": {"id": deployment_id, "team_id": team_id}, + } + for group, deployment_id, team_id in ( + ("current-group", "current-id", None), + ("candidate-group", "candidate-id", None), + ("own-group", "own-id", "prediction-team"), + ("foreign-group", "foreign-id", "foreign-team"), + ) + ] + ) -@pytest.mark.parametrize("candidate,team_id,expected", [ - ("candidate-id", None, ["current-group", "candidate-group"]), - ("current-id", None, "current-group"), - ("missing-id", None, None), - ("candidate-group", None, None), - ("own-id", None, None), - ("own-id", "prediction-team", ["current-group", "own-group"]), - ("foreign-id", "prediction-team", None), -]) +@pytest.mark.parametrize( + "candidate,team_id,expected", + [ + ("candidate-id", None, ["current-group", "candidate-group"]), + ("current-id", None, "current-group"), + ("missing-id", None, None), + ("candidate-group", None, None), + ("own-id", None, None), + ("own-id", "prediction-team", ["current-group", "own-group"]), + ("foreign-id", "prediction-team", None), + ], +) def test_cache_prediction_auth_resolves_only_exact_deployment_ids(candidate, team_id, expected): - assert get_model_from_request( - request_data={ - "current_deployment_id": "current-id", "candidate_deployment_id": candidate, - "request": {"model": "caller-controlled-provider-model"}, - }, - route="/cost/predict-cache", - llm_router=_cache_prediction_router(), - team_id=team_id, - ) == expected + assert ( + get_model_from_request( + request_data={ + "current_deployment_id": "current-id", + "candidate_deployment_id": candidate, + "request": {"model": "caller-controlled-provider-model"}, + }, + route="/cost/predict-cache", + llm_router=_cache_prediction_router(), + team_id=team_id, + ) + == expected + ) def _cache_prediction_auth_app( @@ -1088,18 +1101,29 @@ def _cache_prediction_auth_app( router = _cache_prediction_router() allowed_models = ["current-group", "candidate-group", "own-group"] token = UserAPIKeyAuth( - api_key="test-proxy-key-hash", user_id="prediction-user", user_role=LitellmUserRoles.INTERNAL_USER, - models=allowed_models if key_models is None else key_models, team_id=team_id, + api_key="test-proxy-key-hash", + user_id="prediction-user", + user_role=LitellmUserRoles.INTERNAL_USER, + models=allowed_models if key_models is None else key_models, + team_id=team_id, team_models=allowed_models if team_models is None else team_models, - allowed_routes=allowed_routes, metadata=metadata or {}, + allowed_routes=allowed_routes, + metadata=metadata or {}, ) user = LiteLLM_UserTable( - user_id=token.user_id, user_role=LitellmUserRoles.INTERNAL_USER.value, models=user_models, + user_id=token.user_id, + user_role=LitellmUserRoles.INTERNAL_USER.value, + models=user_models, ) + async def authenticate(request, request_data, **_headers): await auth._enforce_key_and_fallback_model_access( - valid_token=token, request_data=request_data, route=request.url.path, request=request, - llm_model_list=router.get_model_list(), llm_router=router, + valid_token=token, + request_data=request_data, + route=request.url.path, + request=request, + llm_model_list=router.get_model_list(), + llm_router=router, ) return token @@ -1132,22 +1156,36 @@ def _cache_prediction_auth_app( def _cache_prediction_payload(candidate="candidate-id", current="current-id"): return { - "current_deployment_id": current, "candidate_deployment_id": candidate, - "request": {"messages": [{"role": "user", "content": [{ - "type": "text", "text": "Stable cached context", - "cache_control": {"type": "ephemeral"}, - }]}]}, + "current_deployment_id": current, + "candidate_deployment_id": candidate, + "request": { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Stable cached context", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + }, } @pytest.mark.asyncio -@pytest.mark.parametrize("allowed_routes,user_models,candidate,status_code", [ - (["/chat/completions"], ["current-group", "candidate-group"], "candidate-id", 403), - (["/cost/predict-cache"], ["current-group"], "candidate-id", 403), - (["/cost/*"], ["current-group", "candidate-group"], "candidate-id", 200), - (["/cost/predict-cache"], ["current-group"], "current-id", 200), - (["/cost/predict-cache"], ["current-group"], "missing-id", 404), -]) +@pytest.mark.parametrize( + "allowed_routes,user_models,candidate,status_code", + [ + (["/chat/completions"], ["current-group", "candidate-group"], "candidate-id", 403), + (["/cost/predict-cache"], ["current-group"], "candidate-id", 403), + (["/cost/*"], ["current-group", "candidate-group"], "candidate-id", 200), + (["/cost/predict-cache"], ["current-group"], "current-id", 200), + (["/cost/predict-cache"], ["current-group"], "missing-id", 404), + ], +) async def test_cache_prediction_authorizes_route_and_personal_models_before_provider_counts( monkeypatch, allowed_routes, user_models, candidate, status_code ): @@ -1166,20 +1204,27 @@ async def test_cache_prediction_authorizes_route_and_personal_models_before_prov @pytest.mark.asyncio @pytest.mark.parametrize("arm", ["current_deployment_id", "candidate_deployment_id"]) -@pytest.mark.parametrize("team_id,key_models,user_models,team_models", [ - (None, ["*"], ["*"], None), - (None, ["current-group", "candidate-group"], ["*"], None), - (None, ["*"], ["current-group", "candidate-group"], None), - ("prediction-team", ["*"], ["*"], ["current-group", "candidate-group"]), -]) +@pytest.mark.parametrize( + "team_id,key_models,user_models,team_models", + [ + (None, ["*"], ["*"], None), + (None, ["current-group", "candidate-group"], ["*"], None), + (None, ["*"], ["current-group", "candidate-group"], None), + ("prediction-team", ["*"], ["*"], ["current-group", "candidate-group"]), + ], +) async def test_cache_prediction_hides_foreign_and_missing_ids_before_model_authorization( monkeypatch, arm, team_id, key_models, user_models, team_models ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], user_models, - team_id=team_id, key_models=key_models, team_models=team_models, + monkeypatch, + ["/cost/predict-cache"], + user_models, + team_id=team_id, + key_models=key_models, + team_models=team_models, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: missing = await client.post("/cost/predict-cache", json={**_cache_prediction_payload(), arm: "missing-id"}) @@ -1192,19 +1237,26 @@ async def test_cache_prediction_hides_foreign_and_missing_ids_before_model_autho @pytest.mark.asyncio @pytest.mark.parametrize("arm", ["current_deployment_id", "candidate_deployment_id"]) -@pytest.mark.parametrize("key_models,team_models,status_code", [ - (["*"], ["*"], 200), - (["current-group", "candidate-group"], ["*"], 403), - (["*"], ["current-group", "candidate-group"], 403), -]) +@pytest.mark.parametrize( + "key_models,team_models,status_code", + [ + (["*"], ["*"], 200), + (["current-group", "candidate-group"], ["*"], 403), + (["*"], ["current-group", "candidate-group"], 403), + ], +) async def test_cache_prediction_checks_each_visible_team_deployment_model( monkeypatch, arm, key_models, team_models, status_code ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], ["*"], - team_id="prediction-team", key_models=key_models, team_models=team_models, + monkeypatch, + ["/cost/predict-cache"], + ["*"], + team_id="prediction-team", + key_models=key_models, + team_models=team_models, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: response = await client.post("/cost/predict-cache", json={**_cache_prediction_payload(), arm: "own-id"}) @@ -1230,19 +1282,24 @@ async def test_cache_prediction_checks_each_visible_personal_deployment_model(mo @pytest.mark.asyncio -@pytest.mark.parametrize("header_tag,key_tags,limit,status_code,provider_calls", [ - ("limited", [], 1, 429, 1), - (None, ["limited"], 1, 429, 1), - ("limited", ["limited"], 4, 200, 4), - ("unlimited", [], 1, 200, 4), -]) +@pytest.mark.parametrize( + "header_tag,key_tags,limit,status_code,provider_calls", + [ + ("limited", [], 1, 429, 1), + (None, ["limited"], 1, 429, 1), + ("limited", ["limited"], 4, 200, 4), + ("unlimited", [], 1, 200, 4), + ], +) async def test_cache_prediction_preserves_authenticated_header_and_key_tag_rpm( monkeypatch, header_tag, key_tags, limit, status_code, provider_calls ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], ["current-group", "candidate-group"], + monkeypatch, + ["/cost/predict-cache"], + ["current-group", "candidate-group"], metadata={"tag_rpm_limit": {"limited": limit}, "tags": key_tags}, ) headers = {"x-litellm-tags": header_tag} if header_tag else {} @@ -1396,7 +1453,11 @@ def _azure_relay_router(): model_list=[ { "model_name": "gpt", - "litellm_params": {"model": "azure_ai/gpt-5.4-mini", "api_base": "https://a.services.ai.azure.com", "api_key": "k"}, + "litellm_params": { + "model": "azure_ai/gpt-5.4-mini", + "api_base": "https://a.services.ai.azure.com", + "api_key": "k", + }, }, { "model_name": "other-group", @@ -1618,8 +1679,14 @@ def test_get_model_from_request_resolves_video_id_model_with_router(): provider="vertex_ai", model_id="veo-3.1-generate-001", ) - llm_router = MagicMock() - llm_router.resolve_model_name_from_model_id.return_value = "gcp/google/veo-3.1-generate-001" + llm_router = Router( + model_list=[ + { + "model_name": "gcp/google/veo-3.1-generate-001", + "litellm_params": {"model": "vertex_ai/veo-3.1-generate-001"}, + } + ] + ) assert ( get_model_from_request( @@ -1629,7 +1696,148 @@ def test_get_model_from_request_resolves_video_id_model_with_router(): ) == "gcp/google/veo-3.1-generate-001" ) - llm_router.resolve_model_name_from_model_id.assert_called_once_with("veo-3.1-generate-001") + + +def test_get_model_from_request_checks_the_group_of_the_deployment_that_created_the_video() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", provider="azure", model_id="sora-2", deployment_id="dep-sora-shared" + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "sora-shared" + ) + + +def test_get_model_from_request_checks_a_video_created_through_a_provider_wildcard_by_its_own_name() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", provider="openai", model_id="sora-2", deployment_id="dep-wildcard" + ) + routed_ids: Final = [ + deployment["model_info"]["id"] for deployment in llm_router.pattern_router.get_deployments_by_pattern("sora-2") + ] + + assert routed_ids == ["dep-wildcard"] + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "sora-2" + ) + + +def test_get_model_from_request_checks_a_wildcard_video_whose_bare_name_does_not_route_by_its_prefixed_name() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": "gemini/*", + "litellm_params": {"model": "gemini/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="operations/op_1", provider="gemini", model_id="unlisted-video-model", deployment_id="dep-wildcard" + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "gemini/unlisted-video-model" + ) + + +def _sora_fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-pair", "sora-backup", "sora-last", "sora-other") + ], + fallbacks=[{"sora-pair": ["sora-backup"]}, {"sora-backup": ["sora-last"]}], + ) + + +@pytest.mark.parametrize( + ("general_settings", "requested_model", "creating_group", "expected"), + [ + ({"enforce_fallback_model_access": False}, "sora-pair", "sora-backup", "sora-pair"), + ({"enforce_fallback_model_access": False}, "sora-pair", "sora-last", "sora-pair"), + ({}, "sora-pair", "sora-backup", "sora-backup"), + ({"enforce_fallback_model_access": False}, "sora-other", "sora-backup", "sora-backup"), + ({"enforce_fallback_model_access": False}, None, "sora-backup", "sora-backup"), + ], + ids=[ + "opt-out-fallback", + "opt-out-chained-fallback", + "enforced", + "opt-out-no-fallback-relation", + "opt-out-no-requested-model", + ], +) +def test_get_model_from_request_checks_a_fallback_served_video_by_the_requested_group_only_under_the_opt_out( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, object], + requested_model: str | None, + creating_group: str, + expected: str, +) -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", + provider="azure", + model_id="sora-2", + deployment_id=f"dep-{creating_group}", + requested_model=requested_model, + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=_sora_fallback_router(), + ) + == expected + ) _BATCH_DEPLOYMENT_ID = "8d0eaa7e6c6f54a425dfd0062cb6b0dc" @@ -4717,9 +4925,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: ), ], ) -def test_configurable_clientside_parameters( - allowed_param, input_value, should_return_true -): +def test_configurable_clientside_parameters(allowed_param, input_value, should_return_true): router = Router( model_list=[ { @@ -4753,9 +4959,7 @@ def test_get_customer_user_header_from_mapping_returns_customer_header_with_mixe def test_get_customer_user_header_from_mapping_no_customer_returns_none(): from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping - mappings = [ - {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"} - ] + mappings = [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}] result = get_customer_user_header_from_mapping(mappings) assert result is None @@ -4783,17 +4987,13 @@ def test_get_internal_user_header_from_mapping_returns_internal_header(): def test_get_internal_user_header_from_mapping_no_internal_returns_none(): from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - mappings = [ - {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"} - ] + mappings = [{"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}] result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings) assert result is None # Also support single mapping dict single_mapping = {"header_name": "X-Only-Customer", "litellm_user_role": "customer"} - result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping( - single_mapping - ) + result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(single_mapping) assert result is None @@ -4815,9 +5015,7 @@ def test_get_internal_user_header_from_mapping_no_internal_returns_none(): def test_get_model_from_request(request_data, expected_model): from litellm.proxy.auth.auth_utils import get_model_from_request - request_data = { - "target_model_names": "gpt-3.5-turbo, gpt-4o-mini-general-deployment" - } + request_data = {"target_model_names": "gpt-3.5-turbo, gpt-4o-mini-general-deployment"} route = "/openai/deployments/gpt-3.5-turbo" model = get_model_from_request(request_data, "/v1/files") assert model == ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"] @@ -4879,9 +5077,7 @@ def test_get_model_from_request(request_data, expected_model): ), ], ) -def test_get_model_from_request_vertex_ai_passthrough( - request_data, route, expected_model -): +def test_get_model_from_request_vertex_ai_passthrough(request_data, route, expected_model): """Test that get_model_from_request correctly extracts Vertex AI model from URL""" from litellm.proxy.auth.auth_utils import get_model_from_request diff --git a/tests/unit/proxy/video_endpoints/test_endpoints.py b/tests/unit/proxy/video_endpoints/test_endpoints.py index 5052d75f9f9..9809d26826b 100644 --- a/tests/unit/proxy/video_endpoints/test_endpoints.py +++ b/tests/unit/proxy/video_endpoints/test_endpoints.py @@ -28,7 +28,8 @@ mock's call args), and a brand-new kwarg added to this layer surfaces as a failu from contextlib import ExitStack from dataclasses import dataclass -from typing import Any, Dict, Optional +from collections.abc import Awaitable, Callable +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -41,7 +42,9 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( + decode_video_id_with_provider, encode_character_id_with_provider, encode_video_id_with_provider, ) @@ -70,6 +73,29 @@ RESOLVED_MODELS: Dict[str, str] = {VIDEO_MODEL_ID: "azure-sora"} SENTINEL = object() +def _video_with_empty_model_id( + hidden_params: dict[str, object] | None = None, +) -> VideoObject: + video: Final[VideoObject] = VideoObject( + id=encode_video_id_with_provider("video_new456", "azure", ""), + object="video", + status="queued", + ) + video.hidden_params = dict(hidden_params or {}) + return video + + +def _assert_id_resolves_to_deployment(response: object, *, deployment_id: str | None = None) -> None: + assert isinstance(response, VideoObject) + expected_deployment: Final = {} if deployment_id is None else {"deployment_id": deployment_id} + assert decode_video_id_with_provider(response.id) == { + "custom_llm_provider": "azure", + "model_id": VIDEO_MODEL_ID, + "video_id": "video_new456", + **expected_deployment, + } + + class FakeRequest: """Minimal stand-in. headers/query_params are read by the provider readers (mocked) and on the edit path the raw body is parsed for real via orjson.""" @@ -105,6 +131,9 @@ class Harness: assert self.base_process.call_count == 1 return dict(self.base_process.call_args.args[0].data) + def respond_with(self, response: object) -> None: + self.base_process.return_value = response # pyright: ignore[reportAny] # MagicMock subclasses Any in typeshed + def route_type(self) -> str: return self.base_process.call_args.kwargs["route_type"] @@ -118,6 +147,7 @@ def harness(): side_effect=lambda model_id, custom_llm_provider=None: RESOLVED_MODELS.get(model_id) ) router.resolve_model_name_from_model_id = resolve_model + router.get_deployment.return_value = None read_body = AsyncMock(return_value={}) batch_to_bytesio = AsyncMock(return_value=[b"filebytes"]) @@ -244,6 +274,36 @@ async def test_generation__input_reference_attached(harness): } +@pytest.mark.asyncio +async def test_generation__restamps_id_with_asked_for_model_over_router_deployment_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_generation(harness, body={"model": VIDEO_MODEL_ID, "prompt": "x"}) + + _assert_id_resolves_to_deployment(resp, deployment_id="deployment-uuid") + + +@pytest.mark.asyncio +async def test_generation__restamps_id_with_request_model_when_no_hidden_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_generation(harness, body={"model": VIDEO_MODEL_ID, "prompt": "x"}) + + _assert_id_resolves_to_deployment(resp) + + +@pytest.mark.asyncio +async def test_generation__unroutable_request_model_falls_back_to_router_deployment_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_generation(harness, body={"model": "team-public-sora", "prompt": "x"}) + + assert isinstance(resp, VideoObject) + decoded: Final = decode_video_id_with_provider(resp.id) + assert decoded.get("model_id") == "deployment-uuid" + assert decoded.get("deployment_id") == "deployment-uuid" + + @pytest.mark.asyncio async def test_generation__exception_routed_through_handler(harness): harness.base_process.side_effect = ValueError("provider boom") @@ -284,6 +344,24 @@ async def test_status__model_encoded_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_status__restamps_response_id_with_decoded_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_status(harness, AZURE_VIDEO_ID) + + _assert_id_resolves_to_deployment(resp) + + +@pytest.mark.asyncio +async def test_status__id_without_a_deployment_takes_the_deployment_that_answered_the_poll(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_status(harness, AZURE_VIDEO_ID) + + _assert_id_resolves_to_deployment(resp, deployment_id="deployment-uuid") + + @pytest.mark.asyncio async def test_status__plain_id_defaults_to_openai(harness): await call_status(harness, "video_plain") @@ -387,6 +465,30 @@ async def test_status_and_content__hand_the_served_member_to_the_request_process assert harness.base_process.call_args.args[0].served_model == "wan3-b" +@pytest.mark.asyncio +@pytest.mark.parametrize("call", [call_status, call_content]) +async def test_status_and_content__shared_provider_model_routes_to_the_creating_deployments_group( + harness: Harness, call: Callable[[Harness, str], Awaitable[object]] +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + harness.base_process.return_value = b"x" + video_id: Final = encode_video_id_with_provider("video_raw", "azure", "sora-2", deployment_id="dep-sora-shared") + + with patch.object(proxy_server, "llm_router", router): + await call(harness, video_id) + + assert harness.processor_data()["model"] == "sora-shared" + + # =========================================================================== # # POST /v1/videos/edits - video_edit # # =========================================================================== # @@ -422,6 +524,15 @@ async def test_edit__extracts_nested_video_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_edit__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_edit(harness, body={"prompt": "brighter", "video": {"id": AZURE_VIDEO_ID}}) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_edit__provider_from_body_data_for_plain_id(harness): """For a plain id, get_custom_provider_from_data (run for real) pulls the @@ -473,6 +584,23 @@ async def test_edit__json_string_video_reference_from_form_field(harness): assert harness.processor_data()["video_id"] == "video_plain" +@pytest.mark.asyncio +async def test_edit__uploaded_video_restamps_new_id_with_requested_model(harness: Harness) -> None: + import io + + upload: Final = StarletteUploadFile(file=io.BytesIO(b"rawmp4"), filename="clip.mp4") + harness.read_body.return_value = {"model": VIDEO_MODEL_ID, "prompt": "make it nighttime", "video": upload} + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await endpoints.video_edit( + request=FakeRequest(raw_body=b"multipart"), + fastapi_response=Response(), + user_api_key_dict=_user(), + ) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_edit__uploaded_video_file_is_forwarded_not_dropped(harness): """A multipart-uploaded source video must be converted to bytes and attached @@ -565,6 +693,15 @@ async def test_remix__model_encoded_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_remix__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_remix(harness, AZURE_VIDEO_ID, body={"prompt": "new colors"}) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_remix__provider_from_body_data_not_request_body_reader(harness): """remix resolves the provider from data.get('custom_llm_provider'), never @@ -694,6 +831,20 @@ async def test_get_character__plain_id_defaults_openai_no_encode(harness): assert resp["id"] == "char_raw3" +@pytest.mark.asyncio +async def test_get_character__model_served_only_by_a_provider_wildcard_routes_through_it(harness: Harness) -> None: + router: Final = Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "key-w"}}] + ) + harness.base_process.return_value = {"id": "char_raw4"} + character_id: Final = encode_character_id_with_provider("char_orig", "openai", "sora-2") + + with patch.object(proxy_server, "llm_router", router): + await call_get_character(harness, character_id) + + assert harness.processor_data()["model"] == "sora-2" + + # =========================================================================== # # POST /v1/videos/extensions - video_extension # # =========================================================================== # @@ -723,3 +874,12 @@ async def test_extension__extracts_nested_video_id_full_contract(harness): "custom_llm_provider": "azure", "model": "azure-sora", } + + +@pytest.mark.asyncio +async def test_extension__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_extension(harness, body={"prompt": "continue", "video": {"id": AZURE_VIDEO_ID}}) + + _assert_id_resolves_to_deployment(resp) diff --git a/tests/unit/proxy/video_endpoints/test_utils.py b/tests/unit/proxy/video_endpoints/test_utils.py index 9a2c208c075..01a2b7c2840 100644 --- a/tests/unit/proxy/video_endpoints/test_utils.py +++ b/tests/unit/proxy/video_endpoints/test_utils.py @@ -1,11 +1,12 @@ """ Pure-logic contract tests for litellm/proxy/video_endpoints/utils.py -Four helpers the video proxy endpoints lean on: +Five helpers the video proxy endpoints lean on: - extract_model_from_target_model_names: first model from a comma string / list - video_reference_to_id: normalize a video reference (dict / bare id / JSON string) to an id - get_custom_provider_from_data: provider precedence (top-level > extra_body) - encode_character_id_in_response: re-encode a response id in place + - encode_video_id_in_response: re-stamp a VideoObject id with the resolved model_id Every test asserts the exact result (or identity), so a mutation that flips a branch, drops a strip/filter, or changes precedence fails. The only collaborator @@ -13,19 +14,29 @@ is encode_character_id_with_provider, which runs for real; encoding assertions are checked by the genuine decode round-trip. """ +from typing import Final import pytest +from litellm.router import Router +from litellm.types.videos.main import DecodedVideoId from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, + encode_video_id_in_response, extract_model_from_target_model_names, + fallback_requested_model, get_custom_provider_from_data, + routable_model_name, + video_model_group, video_reference_to_id, ) +from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( decode_character_id_with_provider, + decode_video_id_with_provider, encode_character_id_with_provider, + encode_video_id_with_provider, ) # =========================================================================== # @@ -103,12 +114,7 @@ def test_provider__falsy_top_level_falls_through_to_extra_body(falsy): def test_provider__from_extra_body_dict(): - assert ( - get_custom_provider_from_data( - {"extra_body": {"custom_llm_provider": "bedrock"}} - ) - == "bedrock" - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": "bedrock"}}) == "bedrock" def test_provider__from_extra_body_json_string(): @@ -126,10 +132,7 @@ def test_provider__json_string_parsing_to_non_dict_is_none(): def test_provider__extra_body_provider_not_a_string_is_none(): - assert ( - get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) - is None - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) is None @pytest.mark.parametrize( @@ -161,9 +164,7 @@ def test_encode__dict_with_id_mutates_in_place_and_preserves_other_keys(): assert out is response # same dict, mutated in place assert out["object"] == "character" and out["name"] == "hero" - assert out["id"] == encode_character_id_with_provider( - "char_raw", "azure", "model-1" - ) + assert out["id"] == encode_character_id_with_provider("char_raw", "azure", "model-1") decoded = decode_character_id_with_provider(out["id"]) assert decoded["custom_llm_provider"] == "azure" assert decoded["model_id"] == "model-1" @@ -206,3 +207,387 @@ def test_encode__object_without_id_attr_returned_unchanged(): out = encode_character_id_in_response(resp, "azure", "model-1") assert out is resp assert not hasattr(resp, "id") + + +# =========================================================================== # +# encode_video_id_in_response +# =========================================================================== # + + +def _video(video_id: str, hidden_params: dict[str, object] | None = None) -> VideoObject: + video: Final[VideoObject] = VideoObject(id=video_id, object="video", status="queued") + video.hidden_params = dict(hidden_params or {}) + return video + + +def test_encode_video__fills_empty_model_id_and_deployment_from_hidden_params() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + assert out is not video + assert video.id == original_id + assert out.hidden_params == {"model_id": "deployment-1"} + assert out.status == "queued" + + +def test_encode_video__asked_for_model_wins_over_hidden_deployment_id() -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), hidden_params={"model_id": "deployment-1"} + ) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-alias") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora-alias" + assert decoded.get("deployment_id") == "deployment-1" + + +def test_encode_video__stamped_id_keeps_model_and_served_group_and_gains_routed_deployment() -> None: + original_id: Final = encode_video_id_with_provider("task_1", "volcengine", "seedance-group", "seedance-east") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="other") + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance-group", + "video_id": "task_1", + "served_model": "seedance-east", + "deployment_id": "deployment-1", + } + + +def test_encode_video__routed_deployment_replaces_the_one_in_the_id() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="deployment-old") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-new"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id).get("deployment_id") == "deployment-new" + + +@pytest.mark.parametrize( + ("polled_deployment", "answering_deployment", "returned_deployment"), + [ + (None, "deployment-routed", "deployment-routed"), + ("deployment-gone", "deployment-routed", "deployment-routed"), + ("deployment-creator", None, "deployment-creator"), + ], +) +def test_encode_video__a_poll_takes_the_deployment_that_answered_else_keeps_the_one_its_id_named( + polled_deployment: str | None, + answering_deployment: str | None, + returned_deployment: str, +) -> None: + polled_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=polled_deployment) + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), + hidden_params={} if answering_deployment is None else {"model_id": answering_deployment}, + ) + + out: Final = encode_video_id_in_response( + video, + fallback_provider=None, + fallback_model_id="sora", + polled_video=decode_video_id_with_provider(polled_id), + ) + + assert isinstance(out, VideoObject) + assert out.id == encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=returned_deployment) + + +def test_encode_video__a_poll_keeps_the_group_a_fallback_create_was_serving() -> None: + polled_id: Final = encode_video_id_with_provider( + "video_raw", "azure", "sora-2", deployment_id="dep-backup", requested_model="sora-pair" + ) + video: Final = _video(encode_video_id_with_provider("video_raw", "azure", "sora-2")) + + out: Final = encode_video_id_in_response( + video, fallback_provider=None, fallback_model_id=None, polled_video=decode_video_id_with_provider(polled_id) + ) + + assert isinstance(out, VideoObject) + assert out.id == polled_id + + +def test_encode_video__a_remix_of_a_fallback_video_keeps_its_requested_group() -> None: + video: Final = _video( + encode_video_id_with_provider("video_new", "azure", "sora-2"), hidden_params={"model_id": "dep-backup"} + ) + + out: Final = encode_video_id_in_response( + video, fallback_provider=None, fallback_model_id=None, requested_model="sora-pair" + ) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id).get("requested_model") == "sora-pair" + + +def test_encode_video__id_already_naming_the_routed_deployment_is_unchanged() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="deployment-1") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + assert encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="other") is video + + +def test_encode_video__unrouted_response_keeps_the_deployment_in_its_id() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "", deployment_id="deployment-1") + + out: Final = encode_video_id_in_response(_video(original_id), fallback_provider=None, fallback_model_id="sora") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora" + assert decoded.get("deployment_id") == "deployment-1" + + +def test_encode_video__name_with_id_separator_is_never_written_into_the_id() -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), hidden_params={"model_id": "deployment-1"} + ) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora;model_id:evil") + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + + +def test_encode_video__falls_back_to_given_model_id() -> None: + video: Final = _video(encode_video_id_with_provider("video_raw", "openai", "")) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-2") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora-2" + assert decoded.get("video_id") == "video_raw" + + +def test_encode_video__keeps_existing_model_id_over_fallback() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "azure", "deployment-1") + + out: Final = encode_video_id_in_response(_video(original_id), fallback_provider="openai", fallback_model_id="other") + + assert isinstance(out, VideoObject) + assert out.id == original_id + + +def test_encode_video__plain_id_encoded_with_fallback_provider() -> None: + video: Final = _video("video_raw", hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider="openai", fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + + +def test_encode_video__no_model_id_anywhere_unchanged() -> None: + video: Final = _video(encode_video_id_with_provider("video_raw", "openai", "")) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert out is video + + +def test_encode_video__no_provider_anywhere_unchanged() -> None: + video: Final = _video("video_raw") + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-2") + + assert out is video + + +@pytest.mark.parametrize("response", [b"video-bytes", {"id": "video_raw"}, None]) +def test_encode_video__non_video_object_passthrough(response: object) -> None: + assert encode_video_id_in_response(response, "openai", "sora-2") is response + + +def _router_with_groups(*names: str) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": "openai/sora-2", "api_key": "group-key"}} for name in names + ] + ) + + +def test_routable_model_name__keeps_a_name_the_router_resolves() -> None: + assert routable_model_name("sora-alias", _router_with_groups("sora-alias"), "openai") == "sora-alias" + + +def test_routable_model_name__drops_a_name_the_router_cannot_route() -> None: + assert routable_model_name("team-public-name", _router_with_groups("sora-alias"), "openai") is None + + +def test_routable_model_name__keeps_the_name_without_a_router() -> None: + assert routable_model_name("sora-2", None, None) == "sora-2" + + +@pytest.mark.parametrize("model", [None, "", 7, "sora;model_id:evil"]) +def test_routable_model_name__never_returns_a_name_unsafe_for_the_id(model: object) -> None: + assert routable_model_name(model, _router_with_groups("sora;model_id:evil"), "openai") is None + + +def _two_groups_sharing_sora() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + + +def _decoded(model_id: str, deployment_id: str | None) -> DecodedVideoId: + base: Final = DecodedVideoId(custom_llm_provider="azure", model_id=model_id, video_id="video_raw") + return base if deployment_id is None else {**base, "deployment_id": deployment_id} + + +@pytest.mark.parametrize("deployment_id", ["dep-sora-team", "dep-sora-shared"]) +def test_video_model_group__shared_provider_model_resolves_to_the_creating_deployments_group( + deployment_id: str, +) -> None: + router: Final = _two_groups_sharing_sora() + + assert video_model_group(_decoded("sora-2", deployment_id), router) == deployment_id.removeprefix("dep-") + + +@pytest.mark.parametrize("deployment_id", [None, "dep-gone"]) +def test_video_model_group__without_a_known_deployment_matches_plain_resolution(deployment_id: str | None) -> None: + router: Final = _two_groups_sharing_sora() + + assert video_model_group(_decoded("sora-2", deployment_id), router) == router.resolve_model_name_from_model_id( + "sora-2", "azure" + ) + + +def test_video_model_group__keeps_a_group_named_in_the_id_over_a_deployment_of_another_group() -> None: + assert video_model_group(_decoded("sora-team", "dep-sora-shared"), _two_groups_sharing_sora()) == "sora-team" + + +@pytest.mark.parametrize("deployment_id", ["dep-sora-team", "dep-sora-shared"]) +def test_video_model_group__keeps_an_alias_of_the_creating_deployments_group(deployment_id: str) -> None: + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/sora-{name}", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ], + model_group_alias={"sora-alias": deployment_id.removeprefix("dep-")}, + ) + provider_model: Final = f"sora-{deployment_id.removeprefix('dep-')}" + + assert video_model_group(_decoded(provider_model, deployment_id), router) == "sora-alias" + + +def test_video_model_group__names_a_team_model_by_its_public_name() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "model_name_team-1_internal", + "litellm_params": {"model": "azure/sora-2", "api_key": "team-key"}, + "model_info": {"id": "dep-team", "team_id": "team-1", "team_public_model_name": "team-sora"}, + } + ] + ) + + assert video_model_group(_decoded("sora-2", "dep-team"), router) == "team-sora" + + +def _fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-pair", "sora-backup", "sora-last") + ], + model_group_alias={"sora-alias": "sora-pair"}, + fallbacks=[{"sora-pair": ["sora-backup"]}, {"sora-backup": ["sora-last"]}], + ) + + +def _wildcard_router() -> Router: + return Router( + model_list=[ + { + "model_name": "gemini/*", + "litellm_params": {"model": "gemini/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + + +@pytest.mark.parametrize( + ("requested_model", "answering_deployment", "expected"), + [ + ("sora-pair", "dep-sora-backup", "sora-pair"), + ("sora-pair", "dep-sora-last", "sora-pair"), + ("sora-backup", "dep-sora-pair", None), + ("sora-pair", "dep-sora-pair", None), + ("sora-alias", "dep-sora-pair", None), + ("sora-pair", None, None), + (None, "dep-sora-backup", None), + ], + ids=[ + "fallback-served", + "chained-fallback-served", + "served-by-a-group-it-never-falls-back-to", + "served-by-own-group", + "served-by-alias-target", + "unrouted", + "no-requested-model", + ], +) +def test_fallback_requested_model__names_the_asked_group_only_when_another_group_served_the_create( + requested_model: str | None, answering_deployment: str | None, expected: str | None +) -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "azure", "sora-2"), + hidden_params={} if answering_deployment is None else {"model_id": answering_deployment}, + ) + + assert fallback_requested_model(requested_model, video, _fallback_router()) == expected + + +def test_fallback_requested_model__is_none_for_a_create_a_provider_wildcard_served() -> None: + video: Final = _video( + encode_video_id_with_provider("operations/op_1", "gemini", "veo-3.1-fast-generate-preview"), + hidden_params={"model_id": "dep-wildcard"}, + ) + + assert fallback_requested_model("gemini/veo-3.1-fast-generate-preview", video, _wildcard_router()) is None + + +def test_video_model_group__is_none_without_a_router() -> None: + assert video_model_group(_decoded("sora-2", "dep-sora-shared"), None) is None diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py index 708322f4fec..f8b57410872 100644 --- a/tests/unit/test_router/test_router_provider_endpoints.py +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -16,6 +16,7 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.router_callbacks.track_deployment_metrics import ( get_deployment_successes_for_current_minute, ) @@ -23,6 +24,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import HttpxBinaryResponseContent from litellm.types.router import RoutingStrategy from litellm.types.utils import ImageResponse, ModelResponse, RerankResponse +from litellm.types.videos.utils import encode_video_id_with_provider from openai import AsyncAzureOpenAI, AsyncOpenAI EVENT_TIMEOUT_SECONDS: Final = 5 @@ -730,3 +732,244 @@ async def test_router_moderation_endpoint_factory_invokes_default_model( assert route.call_count == 1 assert json.loads(route.calls[0].request.content) == {"input": "hello"} assert response.id == "modr-2" + + +def _sora_deployment( + model_name: str, api_key: str, deployment_id: str, access_groups: tuple[str, ...] = () +) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/sora-2", "api_key": api_key}, + "model_info": {"id": deployment_id, "access_groups": list(access_groups)}, + } + + +def _two_account_sora_router() -> Router: + return Router( + model_list=[ + _sora_deployment("sora", "key-a", "dep-a"), + _sora_deployment("sora", "key-b", "dep-b"), + _sora_deployment("other", "key-x", "dep-x"), + ], + model_group_alias={"sora-alias": "sora"}, + ) + + +async def _poll_status_keys( + router: Router, + model: str, + deployment_id: str, + respx_mock: respx.MockRouter, + caller: UserAPIKeyAuth | None = None, +) -> set[str]: + route: Final = respx_mock.get("https://api.openai.com/v1/videos/video_raw").respond( + 200, json={"id": "video_raw", "object": "video", "status": "in_progress"} + ) + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=deployment_id) + metadata: Final = {} if caller is None else {"user_api_key_auth": caller} + for _ in range(10): + await router.avideo_status(model=model, video_id=video_id, litellm_metadata=dict(metadata)) + return {call.request.headers["authorization"] for call in route.calls} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["sora", "sora-alias"]) +async def test_video_status_polls_the_deployment_that_created_the_video( + model: str, respx_mock: respx.MockRouter +) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), model, "dep-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +def test_sync_video_status_polls_the_deployment_that_created_the_video(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get("https://api.openai.com/v1/videos/video_raw").respond( + 200, json={"id": "video_raw", "object": "video", "status": "in_progress"} + ) + router: Final = _two_account_sora_router() + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="dep-b") + + for _ in range(10): + router.video_status(model="sora", video_id=video_id) + + assert {call.request.headers["authorization"] for call in route.calls} == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_on_a_wildcard_polls_the_deployment_that_created_the_video( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": key}, "model_info": {"id": dep}} + for dep, key in (("wc-a", "key-a"), ("wc-b", "key-b")) + ] + ) + + keys: Final = await _poll_status_keys(router, "openai/sora-2", "wc-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_never_pins_to_a_deployment_in_cooldown(respx_mock: respx.MockRouter) -> None: + router: Final = _two_account_sora_router() + router.cooldown_cache.add_deployment_to_cooldown( + model_id="dep-b", + original_exception=litellm.RateLimitError(message="upstream 429", llm_provider="openai", model="sora-2"), + exception_status=429, + cooldown_time=60, + ) + + keys: Final = await _poll_status_keys(router, "sora", "dep-b", respx_mock) + + assert keys == {"Bearer key-a"} + + +@pytest.mark.asyncio +async def test_video_status_on_a_routing_group_polls_the_member_deployment_that_created_the_video( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[_sora_deployment("sora-a", "key-a", "dep-a"), _sora_deployment("sora-b", "key-b", "dep-b")], + routing_groups=[ + {"group_name": "sora-group", "models": ["sora-a", "sora-b"], "routing_strategy": "simple-shuffle"} + ], + ) + + keys: Final = await _poll_status_keys(router, "sora-group", "dep-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_never_pins_to_a_deployment_outside_the_requested_group( + respx_mock: respx.MockRouter, +) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), "sora", "dep-x", respx_mock) + + assert keys <= {"Bearer key-a", "Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_with_a_removed_deployment_routes_across_the_group(respx_mock: respx.MockRouter) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), "sora", "dep-gone", respx_mock) + + assert keys <= {"Bearer key-a", "Bearer key-b"} + assert keys + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("key_access_group", "expected_key"), [("ag-a", "Bearer key-a"), ("ag-b", "Bearer key-b")]) +async def test_video_status_pins_only_to_a_deployment_in_the_callers_access_groups( + key_access_group: str, expected_key: str, respx_mock: respx.MockRouter +) -> None: + router: Final = Router( + model_list=[ + _sora_deployment("sora", "key-a", "dep-a", access_groups=("ag-a",)), + _sora_deployment("sora", "key-b", "dep-b", access_groups=("ag-b",)), + ] + ) + + keys: Final = await _poll_status_keys( + router, "sora", "dep-b", respx_mock, caller=UserAPIKeyAuth(models=[key_access_group]) + ) + + assert keys == {expected_key} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["sora", "sora-alias"]) +async def test_pinned_video_status_logs_the_requested_model_group( + model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + capture: Final = _RouterLoggingCapture("dep-b") + monkeypatch.setattr(litellm, "callbacks", [capture]) + + keys: Final = await _poll_status_keys(_two_account_sora_router(), model, "dep-b", respx_mock) + _, standard_logging_object = await capture.next_event() + + assert keys == {"Bearer key-b"} + assert isinstance(standard_logging_object, dict) + assert standard_logging_object["model_group"] == model + + +def _ordered_chat_router() -> Router: + return Router( + model_list=[ + { + "model_name": "chat-ordered", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": key, "order": order}, + "model_info": {"id": deployment_id}, + } + for deployment_id, key, order in (("dep-primary", "key-a", 1), ("dep-backup", "key-b", 2)) + ] + ) + + +@pytest.mark.asyncio +async def test_a_video_id_in_a_chat_body_never_overrides_deployment_order() -> None: + router: Final = _ordered_chat_router() + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "chat-ordered", deployment_id="dep-backup") + + chats: Final = [ + await router.acompletion( + model="chat-ordered", messages=list(ROUTING_MESSAGES), mock_response="ok", video_id=video_id + ) + for _ in range(4) + ] + responses: Final = [ + await router.aresponses(model="chat-ordered", input="route this request", mock_response="ok", video_id=video_id) + for _ in range(4) + ] + + assert {chat._hidden_params["model_id"] for chat in chats} == {"dep-primary"} + assert {response._hidden_params["model_id"] for response in responses} == {"dep-primary"} + + +@pytest.mark.asyncio +async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://api.openai.com/v1/videos/video_gone").respond( + 404, json={"error": {"code": "NotFound", "message": "Video not found"}} + ) + router: Final = Router( + model_list=[_sora_deployment("sora", "key-a", "dep-a"), _sora_deployment("sora", "key-b", "dep-b")], + num_retries=0, + allowed_fails=1, + ) + video_id: Final = encode_video_id_with_provider("video_gone", "openai", "sora", deployment_id="dep-b") + + for _ in range(3): + with pytest.raises(litellm.NotFoundError): + await router.avideo_status(model="sora", video_id=video_id) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] + + +@pytest.mark.asyncio +async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.openai.com/v1/chat/completions").respond( + 404, json={"error": {"code": "NotFound", "message": "Deployment not found"}} + ) + router: Final = Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "key-a"}, + "model_info": {"id": "dep-chat"}, + }, + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "key-b"}, + "model_info": {"id": "dep-chat-b"}, + }, + ], + num_retries=0, + allowed_fails=1, + ) + + for _ in range(3): + with contextlib.suppress(litellm.NotFoundError, litellm.RateLimitError): + await router.acompletion(model="chat", messages=list(ROUTING_MESSAGES)) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-chat", "dep-chat-b"], parent_otel_span=None) diff --git a/tests/unit/test_video_generation.py b/tests/unit/test_video_generation.py index 5c4bc8ea652..df7e9c68b90 100644 --- a/tests/unit/test_video_generation.py +++ b/tests/unit/test_video_generation.py @@ -2,6 +2,7 @@ import asyncio import io import json import os +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -1269,6 +1270,35 @@ def test_encode_video_id_with_provider_handles_azure_video_prefix(): assert encoded_twice == encoded_id # Should return the same encoded ID +@pytest.mark.parametrize( + "served_model, deployment_id, expected_extras", + [ + ("seedance-east", "dep-1", {"served_model": "seedance-east", "deployment_id": "dep-1"}), + (None, "dep-1", {"deployment_id": "dep-1"}), + ("seedance-east", None, {"served_model": "seedance-east"}), + (None, None, {}), + ], +) +def test_video_id_round_trips_served_model_and_deployment_id( + served_model: str | None, deployment_id: str | None, expected_extras: dict[str, str] +) -> None: + from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_video_id_with_provider, + ) + + encoded: Final = encode_video_id_with_provider( + "task_1", "volcengine", "seedance-group", served_model=served_model, deployment_id=deployment_id + ) + + assert decode_video_id_with_provider(encoded) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance-group", + "video_id": "task_1", + **expected_extras, + } + + class TestVideoListTransformation: """Tests for video list request/response transformation with provider ID encoding.""" From f03f98ac227f4b90bb99049823fa9baf5b7a8469 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 23:32:31 -0700 Subject: [PATCH 071/124] fix(search): price Tavily advanced and Linkup deep searches at their own cost map entries (#47411) * fix(search): price Tavily and Linkup searches at their per-mode cost map entries * test(search): read the reported cost through the public hidden_params property * fix(search): keep the type gate flat on the cost map membership check * test(search): drop vendor price guards and a redundant case, type helper params as mappings * fix(search): keep a registered search model's own price, cover every interception endpoint, hostile modes and a failing burst * fix(search): keep a runtime-registered base search price for modes that have no runtime price of their own --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/search/cost_calculator.py | 39 +- .../pricing/test_search_mode_pricing.py | 411 ++++++++++++++++++ tests/unit/search/test_main.py | 193 +++++++- 3 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 tests/integration/pricing/test_search_mode_pricing.py diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 21f27075e0f..15755128136 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -8,12 +8,17 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.utils import get_model_info PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) +SEARCH_MODE_PARAM_BY_PROVIDER: Final[Mapping[str, str]] = MappingProxyType( + {"tavily": "search_depth", "linkup": "depth"} +) def _provider_usage( @@ -28,6 +33,32 @@ def _provider_usage( return None +def _registered_at_runtime(entry: str) -> bool: + """Whether `entry`'s live price differs from the catalog as loaded, i.e. `register_model` changed it.""" + live: Final[object] = litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed + return live != GetModelCostMap.loaded_model_cost_map().get(entry) # pyright: ignore[reportUnknownVariableType] # `live` comes from the untyped registry + + +def _pricing_model( + model: str, + custom_llm_provider: str | None, + optional_params: Mapping[str, object] | None, +) -> str: + """`/search-` when `model` is the generic `/search` entry (bare or provider-prefixed) and + the request's mode parameter names a priced entry, else `model`. A price registered at runtime on the generic entry + keeps applying until the mode entry is registered too.""" + mode_param: Final = SEARCH_MODE_PARAM_BY_PROVIDER.get(custom_llm_provider or "") + generic_model: Final = f"{custom_llm_provider}/search" + if mode_param is None or model not in (generic_model, f"{custom_llm_provider}/{generic_model}"): + return model + mode: Final = (optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS).get(mode_param) + mode_model: Final = f"{custom_llm_provider}/search-{mode}" + priced: Final = mode_model in litellm.model_cost # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + if not priced or (_registered_at_runtime(generic_model) and not _registered_at_runtime(mode_model)): + return model + return mode_model + + def search_provider_cost_per_query( model: str, custom_llm_provider: str | None = None, @@ -38,7 +69,8 @@ def search_provider_cost_per_query( Calculate cost for search-only providers. Returns (input_cost, output_cost) where input_cost = queries * cost_per_query - Supports tiered pricing based on max_results parameter. + Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the + provider's mode parameter (Tavily `search_depth`, Linkup `depth`). Args: model: Model name (e.g., "exa_ai/search", "tavily/search") @@ -61,7 +93,10 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) - model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info( + model=_pricing_model(model, custom_llm_provider, optional_params), + custom_llm_provider=custom_llm_provider, + ) # Check for tiered pricing (e.g., Exa AI based on max_results) tiered_pricing: Final = model_info.get("tiered_pricing") diff --git a/tests/integration/pricing/test_search_mode_pricing.py b/tests/integration/pricing/test_search_mode_pricing.py new file mode 100644 index 00000000000..02b1ce75dd6 --- /dev/null +++ b/tests/integration/pricing/test_search_mode_pricing.py @@ -0,0 +1,411 @@ +import json +import uuid +from collections.abc import Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from integration._support import interception_vendor as iv +from integration._support.client import Gateway, Scenario, eventually, string_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_API_KEY: Final = "synthetic-search-key" +_QUERY: Final = "integration search pricing" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_PACKAGED_PRICES: Final = _JSON_OBJECT.validate_json( + (Path(__file__).resolve().parents[3] / "litellm" / "model_prices_and_context_window_backup.json").read_bytes() +) +_RESULTS: Final = { + "tavily": {"results": [{"title": "Priced result", "url": "https://result.invalid/a", "content": "snippet"}]}, + "linkup": { + "results": [{"type": "text", "name": "Priced result", "url": "https://result.invalid/a", "content": "snippet"}] + }, +} +_MODE_PARAM: Final = {"tavily": "search_depth", "linkup": "depth"} +_QUERY_FIELD: Final = {"tavily": "query", "linkup": "q"} +_MODE_ENTRY: Final = {"tavily": "tavily/search-advanced", "linkup": "linkup/search-deep"} +_MODE_VALUE: Final = {"tavily": "advanced", "linkup": "deep"} +_BASE_MODE: Final = {"tavily": "basic", "linkup": "standard"} +_NO_PARAMS: Final[Mapping[str, JsonValue]] = MappingProxyType({}) + + +def _price(entry: str) -> float: + prices: Final = _PACKAGED_PRICES[entry] + assert isinstance(prices, dict), entry + rate: Final = prices["input_cost_per_query"] + assert isinstance(rate, int | float), (entry, rate) + return float(rate) + + +def _search_tool( + gateway: Gateway, + scenario: Scenario, + provider: str, + api_base: str, + litellm_params: Mapping[str, JsonValue] = _NO_PARAMS, +) -> str: + name: Final = f"{provider}-priced-{uuid.uuid4().hex}" + created: Final = gateway.post( + "/search_tools", + { + "search_tool": { + "search_tool_name": name, + "litellm_params": { + "search_provider": provider, + "api_key": _API_KEY, + "api_base": api_base, + **litellm_params, + }, + } + }, + ) + scenario.cleanups.callback(gateway.request, "DELETE", f"/search_tools/{string_value(created['search_tool_id'])}") + return name + + +def _spend(gateway: Gateway, call_id: str) -> float: + rows: Final = eventually( + lambda: gateway.rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), + lambda found: len(found) == 1, + seconds=70, + ) + return float(str(rows[0]["spend"])) + + +@pytest.mark.parametrize( + ("provider", "mode", "wire_mode", "priced_entry"), + [ + ("tavily", "advanced", "advanced", "tavily/search-advanced"), + ("tavily", "basic", "basic", "tavily/search"), + ("tavily", None, None, "tavily/search"), + ("linkup", "deep", "deep", "linkup/search-deep"), + ("linkup", "standard", "standard", "linkup/search"), + ("linkup", None, "standard", "linkup/search"), + ], + ids=["tavily-advanced", "tavily-basic", "tavily-default", "linkup-deep", "linkup-standard", "linkup-default"], +) +def test_search_is_charged_at_the_price_entry_for_the_mode_the_provider_ran( + gateway: Gateway, provider: str, mode: str | None, wire_mode: str | None, priced_entry: str +) -> None: + mode_param: Final = _MODE_PARAM[provider] + + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", f"/{provider}/search"), request.target + assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers + assert json.loads(request.body).get(mode_param) == wire_mode, request.body + return Reply(body=json.dumps(_RESULTS[provider]).encode()) + + body: Final = {"query": _QUERY} if mode is None else {"query": _QUERY, mode_param: mode} + with wire_server(respond) as wire, gateway.scenario() as scenario: + tool: Final = _search_tool(gateway, scenario, provider, f"{wire.url}/{provider}") + spellings: Final = ( + (f"/v1/search/{tool}", body), + (f"/search/{tool}", body), + ("/v1/search", {"search_tool_name": tool, **body}), + ("/search", {"model": tool, **body}), + ) + responses: Final = tuple(gateway.request("POST", path, sent) for path, sent in spellings) + for (path, _), response in zip(spellings, responses, strict=True): + assert response.status_code == 200, f"{path}: {response.text}" + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(_price(priced_entry)), ( + path, + response.headers, + ) + assert [request.target for request in wire.drain()] == [f"/{provider}/search"] * len(spellings) + assert [_spend(gateway, response.headers["x-litellm-call-id"]) for response in responses] == pytest.approx( + [_price(priced_entry)] * len(spellings) + ) + + +@pytest.mark.parametrize( + ("provider", "modes"), + [ + ("tavily", ("ultra-fast", "ADVANCED", " advanced", "", 7, ["advanced"], {"depth": "advanced"})), + ("linkup", ("DEEP", "deeper", "", 7, ["deep"], {"depth": "deep"})), + ], + ids=["tavily", "linkup"], +) +def test_a_mode_with_no_price_entry_of_its_own_is_charged_at_the_base_price( + gateway: Gateway, provider: str, modes: tuple[JsonValue, ...] +) -> None: + mode_param: Final = _MODE_PARAM[provider] + + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", f"/{provider}/search"), request.target + return Reply(body=json.dumps(_RESULTS[provider]).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + tool: Final = _search_tool(gateway, scenario, provider, f"{wire.url}/{provider}") + responses: Final = tuple( + gateway.request("POST", f"/v1/search/{tool}", {"query": _QUERY, mode_param: mode}) for mode in modes + ) + assert [response.status_code for response in responses] == [200] * len(modes), [r.text for r in responses] + assert [json.loads(request.body)[mode_param] for request in wire.drain()] == list(modes) + assert [float(response.headers["x-litellm-response-cost"]) for response in responses] == pytest.approx( + [_price(f"{provider}/search")] * len(modes) + ) + assert [_spend(gateway, response.headers["x-litellm-call-id"]) for response in responses] == pytest.approx( + [_price(f"{provider}/search")] * len(modes) + ) + + +def _burst_cells(provider: str) -> tuple[tuple[str, str | None], ...]: + return tuple((provider, mode) for mode in (_MODE_VALUE[provider], _BASE_MODE[provider], None)) + + +def _burst_price(provider: str, mode: str | None) -> float: + return _price(_MODE_ENTRY[provider] if mode == _MODE_VALUE[provider] else f"{provider}/search") + + +def test_a_concurrent_burst_with_upstream_failures_charges_each_success_at_its_mode_price_and_no_failure( + gateway: Gateway, +) -> None: + plan: Final = tuple( + (provider, mode, index % 7 == 0) + for index, (provider, mode) in enumerate((*_burst_cells("tavily"), *_burst_cells("linkup")) * 4) + ) + + def respond(request: Request) -> Reply: + provider: Final = request.target.split("/")[1] + if json.loads(request.body)[_QUERY_FIELD[provider]].endswith("-down"): + return Reply(status=500, body=b'{"detail":{"error":"scripted search outage"}}') + return Reply(body=json.dumps(_RESULTS[provider]).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + tools: Final = { + provider: _search_tool(gateway, scenario, provider, f"{wire.url}/{provider}") + for provider in ("tavily", "linkup") + } + + def send(index: int) -> httpx.Response: + provider, mode, fails = plan[index] + query: Final = f"{_QUERY} {index}{'-down' if fails else ''}" + body: Final = {"query": query} if mode is None else {"query": query, _MODE_PARAM[provider]: mode} + return gateway.request("POST", f"/v1/search/{tools[provider]}", body) + + with ThreadPoolExecutor(max_workers=12) as pool: + responses: Final = tuple(pool.map(send, range(len(plan)))) + assert [response.status_code for response in responses] == [500 if fails else 200 for *_, fails in plan], [ + response.text for response in responses + ] + assert len(wire.drain()) == len(plan) + successes: Final = tuple( + (response, _burst_price(provider, mode)) + for (provider, mode, fails), response in zip(plan, responses, strict=True) + if not fails + ) + assert [float(response.headers["x-litellm-response-cost"]) for response, _ in successes] == pytest.approx( + [price for _, price in successes] + ) + rows: Final = eventually( + lambda: gateway.rows( + 'SELECT request_id, status, spend FROM "LiteLLM_SpendLogs" WHERE model_group = ANY(%s)', + (list(tools.values()),), + ), + lambda found: len(found) == len(plan), + seconds=70, + ) + assert {str(row["request_id"]): float(str(row["spend"])) for row in rows if row["status"] == "success"} == ( + pytest.approx({response.headers["x-litellm-call-id"]: price for response, price in successes}) + ) + assert [float(str(row["spend"])) for row in rows if row["status"] != "success"] == [0.0] * ( + len(plan) - len(successes) + ) + + +def _owned_config( + tmp_path: Path, search_tools: Sequence[JsonValue], litellm_settings: Mapping[str, JsonValue] = _NO_PARAMS +) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["search_tools"] = list(search_tools) + config["litellm_settings"].update(litellm_settings) + path: Final = tmp_path / f"search-mode-{uuid.uuid4().hex}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _config_tool(name: str, provider: str, api_base: str, litellm_params: Mapping[str, JsonValue]) -> JsonValue: + return { + "search_tool_name": name, + "litellm_params": {"search_provider": provider, "api_key": _API_KEY, "api_base": api_base, **litellm_params}, + } + + +def test_a_mode_set_on_a_config_search_tool_is_charged_at_that_modes_price_unless_the_request_overrides_it( + gateway: Gateway, tmp_path: Path +) -> None: + cases: Final = (("tavily", "advanced", "basic"), ("linkup", "deep", "standard")) + + def respond(request: Request) -> Reply: + provider: Final = request.target.split("/")[1] + assert (request.method, request.target) == ("POST", f"/{provider}/search"), request.target + return Reply(body=json.dumps(_RESULTS[provider]).encode()) + + with wire_server(respond) as wire: + tools: Final = [ + _config_tool(f"{provider}-configured", provider, f"{wire.url}/{provider}", {_MODE_PARAM[provider]: mode}) + for provider, mode, _ in cases + ] + with owned_proxy(gateway, tmp_path, {}, config=_owned_config(tmp_path, tools)) as candidate: + for provider, configured_mode, base_mode in cases: + mode_param = _MODE_PARAM[provider] + configured = candidate.request("POST", f"/v1/search/{provider}-configured", {"query": _QUERY}) + overridden = candidate.request( + "POST", f"/v1/search/{provider}-configured", {"query": _QUERY, mode_param: base_mode} + ) + assert (configured.status_code, overridden.status_code) == (200, 200), ( + configured.text, + overridden.text, + ) + assert [json.loads(request.body)[mode_param] for request in wire.drain()] == [ + configured_mode, + base_mode, + ] + assert ( + float(configured.headers["x-litellm-response-cost"]), + float(overridden.headers["x-litellm-response-cost"]), + _spend(candidate, configured.headers["x-litellm-call-id"]), + _spend(candidate, overridden.headers["x-litellm-call-id"]), + ) == pytest.approx((_price(_MODE_ENTRY[provider]), _price(f"{provider}/search")) * 2), provider + + +def test_a_price_set_on_the_search_tool_still_wins_over_the_mode_price_entry(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert json.loads(request.body)["search_depth"] == "advanced", request.body + return Reply(body=json.dumps(_RESULTS["tavily"]).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + tool: Final = _search_tool(gateway, scenario, "tavily", f"{wire.url}/tavily", {"input_cost_per_query": 0.02}) + response: Final = gateway.request("POST", f"/v1/search/{tool}", {"query": _QUERY, "search_depth": "advanced"}) + assert response.status_code == 200, response.text + assert [request.target for request in wire.drain()] == ["/tavily/search"] + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(0.02), response.headers + assert _spend(gateway, response.headers["x-litellm-call-id"]) == pytest.approx(0.02) + + +def _anthropic_reply(identity: str, content: Sequence[JsonValue], stop_reason: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + ).encode() + ) + + +def test_a_web_search_intercepted_on_every_llm_endpoint_is_charged_at_the_search_tools_mode_price( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = f"search-mode-{uuid.uuid4().hex}" + tool: Final = f"tavily-intercepted-{uuid.uuid4().hex}" + chat_deployment: Final = iv.deployment("search") + responses_deployment: Final = iv.deployment("search") + + def respond(request: Request) -> Reply: + if request.target == "/tavily/search": + assert json.loads(request.body)["search_depth"] == "advanced", request.body + return Reply(body=json.dumps(_RESULTS["tavily"]).encode()) + if request.target != "/v1/messages": + return iv.respond(request) + body: Final = json.loads(request.body) + if any(entry.get("type") == "web_search_20250305" for entry in body["tools"]): + return _anthropic_reply(identity, [{"type": "text", "text": "not intercepted"}], "end_turn") + answered: Final = any( + isinstance(message["content"], list) and any(part["type"] == "tool_result" for part in message["content"]) + for message in body["messages"] + ) + if answered: + return _anthropic_reply(identity, [{"type": "text", "text": "answered"}], "end_turn") + tool_use: Final[JsonValue] = { + "type": "tool_use", + "id": identity, + "name": iv.SEARCH_TOOL, + "input": {"query": _QUERY}, + } + return _anthropic_reply(identity, [tool_use], "tool_use") + + with wire_server(respond) as wire: + path: Final = _owned_config( + tmp_path, + [_config_tool(tool, "tavily", f"{wire.url}/tavily", {"search_depth": "advanced"})], + { + "callbacks": ["websearch_interception"], + "websearch_interception_params": { + "enabled": True, + "enabled_providers": ["anthropic", "azure"], + "search_tool_name": tool, + }, + }, + ) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as models: + azure: Final = {"api_base": wire.url, "api_key": iv.AZURE_KEY, "api_version": iv.API_VERSION} + sends: Final = ( + ( + "/v1/messages", + { + "model": models.model( + model="anthropic/claude-sonnet-4-5-20250929", + api_base=wire.url, + api_key="synthetic-anthropic-key", + ), + "max_tokens": 64, + "messages": [{"role": "user", "content": identity}], + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 1}], + }, + "answered", + ), + ( + "/v1/chat/completions", + { + "model": models.model(model=f"azure/{chat_deployment}", **azure), + "messages": [{"role": "user", "content": f"Search for {chat_deployment}"}], + "tools": [{"type": "web_search"}], + }, + iv.answer(chat_deployment), + ), + ( + "/v1/responses", + { + "model": models.model(model=f"azure/{responses_deployment}", **azure), + "input": f"Search for {responses_deployment}", + "tools": [{"type": "web_search"}], + }, + iv.answer(responses_deployment), + ), + ) + responses: Final = tuple( + eventually( + lambda route=route, body=body: candidate.request("POST", route, body), + lambda response, answer=answer: answer in response.text, + seconds=40, + ) + for route, body, answer in sends + ) + assert [response.status_code for response in responses] == [200, 200, 200], [r.text for r in responses] + searches: Final = tuple( + json.loads(request.body)["query"] for request in wire.drain() if request.target == "/tavily/search" + ) + assert sorted(searches) == sorted((_QUERY, iv.query(chat_deployment), iv.query(responses_deployment))), ( + searches + ) + rows: Final = eventually( + lambda: candidate.rows( + """SELECT spend FROM "LiteLLM_SpendLogs" WHERE call_type IN ('search', 'asearch')""", () + ), + lambda found: len(found) == len(searches), + seconds=70, + ) + assert [float(str(row["spend"])) for row in rows] == pytest.approx([_price("tavily/search-advanced")] * 3), rows diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index 72266321715..e30fd2eb6f8 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -1,4 +1,6 @@ import datetime +import json +from collections.abc import Mapping from typing import Final import httpx @@ -6,19 +8,40 @@ import pytest import respx import litellm +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.litellm_logging import Logging from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" +TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" +LINKUP_SEARCH_URL: Final = "https://api.linkup.so/v1/search" +SEARCH_MODE_PRICES: Final = { + "tavily/search": 0.002, + "tavily/search-advanced": 0.007, + "tavily/search-fast": 0.001, + "linkup/search": 0.003, + "linkup/search-deep": 0.011, + "perplexity/search": 0.004, + "perplexity/search-advanced": 0.009, + "tavily/negotiated-search": 0.004, +} @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) +@pytest.fixture +def _search_mode_prices(_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: + for model, price in SEARCH_MODE_PRICES.items(): + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_query": price, "litellm_provider": model.split("/", 1)[0], "mode": "search"}, + ) + + def _proxy_style_logging_obj() -> Logging: """The proxy builds the logging object from the request body before the router adds the search tool's litellm_params, so the per-tool price only reaches cost tracking if the search call itself records it.""" @@ -90,3 +113,169 @@ def test_unpriced_fallback_search_tool_is_billed_at_its_own_price_not_the_failed ) assert fallback._hidden_params["response_cost"] == pytest.approx(map_price) + + +@pytest.mark.usefixtures("_search_mode_prices") +@pytest.mark.parametrize( + ("provider", "optional_params", "expected"), + [ + ("tavily", {"search_depth": "advanced"}, 0.007), + ("tavily", {"search_depth": "fast"}, 0.001), + ("tavily", {"search_depth": "basic"}, 0.002), + ("tavily", {"search_depth": "ultra-fast"}, 0.002), + ("tavily", {}, 0.002), + ("tavily", None, 0.002), + ("tavily", {"search_depth": ["advanced"]}, 0.002), + ("tavily", {"depth": "advanced"}, 0.002), + ("linkup", {"depth": "deep"}, 0.011), + ("linkup", {"depth": "standard"}, 0.003), + ("linkup", {}, 0.003), + ("linkup", {"search_depth": "deep"}, 0.003), + ("perplexity", {"search_depth": "advanced", "depth": "advanced"}, 0.004), + ], + ids=[ + "tavily-advanced", + "tavily-fast", + "tavily-basic-has-no-entry", + "tavily-mode-without-entry", + "tavily-no-mode", + "tavily-no-params", + "tavily-non-string-mode", + "tavily-ignores-linkup-param", + "linkup-deep", + "linkup-standard-has-no-entry", + "linkup-no-mode", + "linkup-ignores-tavily-param", + "provider-without-mode-param-keeps-base-entry", + ], +) +def test_search_cost_is_the_price_entry_of_the_requested_mode( + provider: str, optional_params: Mapping[str, object] | None, expected: float +) -> None: + cost: Final = search_provider_cost_per_query( + model=f"{provider}/search", custom_llm_provider=provider, optional_params=optional_params + ) + assert cost == pytest.approx((expected, 0.0)) + + +@pytest.mark.usefixtures("_search_mode_prices") +def test_a_registered_search_model_keeps_its_own_price_for_an_advanced_search() -> None: + cost: Final = search_provider_cost_per_query( + model="tavily/negotiated-search", custom_llm_provider="tavily", optional_params={"search_depth": "advanced"} + ) + assert cost == pytest.approx((0.004, 0.0)) + + +@pytest.fixture +def _loaded_catalog_prices(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, + "model_cost", + {key: dict(entry) for key, entry in GetModelCostMap.loaded_model_cost_map().items()}, + ) + + +MODE_CASES: Final = ( + pytest.param("tavily", "tavily/search", {"search_depth": "advanced"}, id="tavily"), + pytest.param("tavily", "tavily/tavily/search", {"search_depth": "advanced"}, id="tavily-sdk-spelling"), + pytest.param("linkup", "linkup/search", {"depth": "deep"}, id="linkup"), +) + + +def _register(monkeypatch: pytest.MonkeyPatch, entry: str, price: float) -> None: + monkeypatch.setitem( + litellm.model_cost, + entry, + {"input_cost_per_query": price, "litellm_provider": entry.split("/", 1)[0], "mode": "search"}, + ) + + +def _mode_entry(provider: str, optional_params: Mapping[str, object]) -> str: + return f"{provider}/search-{next(iter(optional_params.values()))}" + + +@pytest.mark.usefixtures("_loaded_catalog_prices") +@pytest.mark.parametrize(("provider", "model", "optional_params"), MODE_CASES) +def test_the_catalog_mode_price_applies_when_nothing_is_registered_at_runtime( + provider: str, model: str, optional_params: Mapping[str, object] +) -> None: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + mode_price: Final = catalog[_mode_entry(provider, optional_params)]["input_cost_per_query"] + assert mode_price != catalog[f"{provider}/search"]["input_cost_per_query"] + cost: Final = search_provider_cost_per_query( + model=model, custom_llm_provider=provider, optional_params=optional_params + ) + assert cost == pytest.approx((mode_price, 0.0)) + + +@pytest.mark.usefixtures("_loaded_catalog_prices") +@pytest.mark.parametrize( + ("provider", "optional_params"), + [ + ("tavily", {"search_depth": "ultra-fast"}), + ("tavily", {"search_depth": 7}), + ("tavily", {"search_depth": ["advanced"]}), + ("linkup", {"depth": "standard"}), + ("linkup", {"depth": "DEEP"}), + ], + ids=["tavily-ultra-fast", "tavily-int", "tavily-list", "linkup-standard", "linkup-uppercase"], +) +def test_a_mode_without_its_own_catalog_entry_gets_the_catalog_base_price( + provider: str, optional_params: Mapping[str, object] +) -> None: + cost: Final = search_provider_cost_per_query( + model=f"{provider}/search", custom_llm_provider=provider, optional_params=optional_params + ) + base_price: Final = GetModelCostMap.loaded_model_cost_map()[f"{provider}/search"]["input_cost_per_query"] + assert cost == pytest.approx((base_price, 0.0)) + + +@pytest.mark.usefixtures("_loaded_catalog_prices") +@pytest.mark.parametrize(("provider", "model", "optional_params"), MODE_CASES) +def test_a_runtime_price_on_the_generic_entry_keeps_applying_to_a_mode_without_a_runtime_price( + monkeypatch: pytest.MonkeyPatch, provider: str, model: str, optional_params: Mapping[str, object] +) -> None: + _register(monkeypatch, f"{provider}/search", 0.004) + cost: Final = search_provider_cost_per_query( + model=model, custom_llm_provider=provider, optional_params=optional_params + ) + assert cost == pytest.approx((0.004, 0.0)) + + +@pytest.mark.usefixtures("_loaded_catalog_prices") +@pytest.mark.parametrize(("provider", "model", "optional_params"), MODE_CASES) +def test_a_runtime_price_on_the_mode_entry_wins_over_a_runtime_price_on_the_generic_entry( + monkeypatch: pytest.MonkeyPatch, provider: str, model: str, optional_params: Mapping[str, object] +) -> None: + _register(monkeypatch, f"{provider}/search", 0.004) + _register(monkeypatch, _mode_entry(provider, optional_params), 0.011) + cost: Final = search_provider_cost_per_query( + model=model, custom_llm_provider=provider, optional_params=optional_params + ) + assert cost == pytest.approx((0.011, 0.0)) + + +@pytest.mark.usefixtures("_search_mode_prices") +def test_sync_tavily_advanced_search_reports_the_advanced_price(respx_mock: respx.MockRouter) -> None: + respx_mock.post(TAVILY_SEARCH_URL).respond(json={"results": [{"title": "t", "url": "https://example.com"}]}) + response: Final = litellm.search( + query="litellm release notes", search_provider="tavily", api_key="test-tavily-key", search_depth="advanced" + ) + assert json.loads(respx_mock.calls.last.request.content)["search_depth"] == "advanced" + assert response.hidden_params["response_cost"] == pytest.approx(0.007) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_search_mode_prices") +async def test_async_linkup_deep_search_reports_the_deep_price( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(LINKUP_SEARCH_URL).respond( + json={"results": [{"type": "text", "name": "t", "url": "https://example.com", "content": "c"}]} + ) + response: Final = await litellm.asearch( + query="litellm release notes", search_provider="linkup", api_key="test-linkup-key", depth="deep" + ) + assert json.loads(respx_mock.calls.last.request.content)["depth"] == "deep" + assert response.hidden_params["response_cost"] == pytest.approx(0.011) From f43f2d8c1598e640daa7345d1d46925fe61b9783 Mon Sep 17 00:00:00 2001 From: Max Azatian <66359507+HardMax71@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:35:47 +0200 Subject: [PATCH 072/124] fix(otel): put the error message on every failed span status (#44185) * fix(otel): put the error message on the failed span status The legacy OTel logger (used by otel, langfuse_otel, arize and others) ended a failed call with Status(StatusCode.ERROR) and no description, so backends that read the span status, such as Langfuse's status message, showed an empty error. The text was only in the error.message attribute. Use the same message as that attribute, as the OTel v2 emitter already does. * fix(otel): take the failure status message from _record_exception_on_span Instead of reading the logging payload a second time in _handle_failure, _record_exception_on_span returns the error message it already records as error.message, and _handle_failure puts it on the ERROR status. The other callers ignore the return value, so their spans are unchanged. Its input is now a typed view of what it reads (exception, error_information, error_str) instead of a bare dict. * fix(otel): put the error message on every failed span status Covers the proxy SERVER span, management endpoint spans, service spans, and Arize Phoenix spans, so no ERROR span ships with an empty status description * fix(otel): cut the error status to the span attribute length limit and describe dangling server spans The status description now uses the error text as stored on the span, so OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT applies to it too. A server span the proxy closes on a 4xx or 5xx outside the request path gets the same description, and an empty error message falls back to the error class * test(otel): type the validation-handler status test * test(otel): cover the Phoenix parent span and error_str length-limit status --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/arize/arize_phoenix.py | 14 +- litellm/integrations/opentelemetry.py | 115 +++++++---- litellm/proxy/proxy_server.py | 11 +- .../integrations/arize/test_arize_phoenix.py | 49 +++++ .../test_otel_exception_handler.py | 25 ++- tests/unit/integrations/test_opentelemetry.py | 184 +++++++++++++++++- 6 files changed, 343 insertions(+), 55 deletions(-) diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index ce585ad26c4..150d814f811 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -354,17 +354,16 @@ class ArizePhoenixLogger(OpenTelemetry): _project_name, tracer = self._resolve_tracer_for_kwargs(kwargs) ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer) - status: Final = Status(StatusCode.OK if success else StatusCode.ERROR) + status_code: Final = StatusCode.OK if success else StatusCode.ERROR span: Final = tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), context=ctx, ) - span.set_status(status) self.set_attributes(span, kwargs, response_obj) - if not success: - self._record_exception_on_span(span=span, kwargs=kwargs) + span_error_message: Final = None if success else self._record_exception_on_span(span=span, kwargs=kwargs) + span.set_status(Status(status_code, span_error_message)) if success: self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) @@ -373,10 +372,11 @@ class ArizePhoenixLogger(OpenTelemetry): self._create_guardrail_span(kwargs=kwargs, context=ctx) if parent_span is not None: - parent_span.set_status(status) self.set_attributes(parent_span, kwargs, response_obj) - if not success: - self._record_exception_on_span(span=parent_span, kwargs=kwargs) + parent_error_message: Final = ( + None if success else self._record_exception_on_span(span=parent_span, kwargs=kwargs) + ) + parent_span.set_status(Status(status_code, parent_error_message)) parent_span.end(end_time=self._to_ns(end_time)) if success: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 90392d9a45d..0d0e309034a 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -53,6 +53,7 @@ from litellm.types.utils import ( StandardCallbackDynamicParams, StandardLoggingMetadata, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, ) # OpenTelemetry imports moved to individual functions to avoid import errors when not installed @@ -104,8 +105,26 @@ class _ResponseWithUsageView(TypedDict, total=False): usage: "_UsageCompletionTokensView | None" +class _FailurePayloadView(TypedDict, total=False): + error_information: ReadOnly[StandardLoggingPayloadErrorInformation | None] + error_str: ReadOnly[str | None] + + +class _FailureKwargsView(TypedDict, total=False): + exception: ReadOnly[BaseException | None] + standard_logging_object: ReadOnly[_FailurePayloadView | None] + + _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) + +def _text_as_stored_on_span(span: Span, key: str, text: str) -> str: + from opentelemetry.sdk.trace import ReadableSpan + + stored: Final = span.attributes.get(key) if isinstance(span, ReadableSpan) and span.attributes else None + return stored if isinstance(stored, str) else text + + # Cap on credential-scoped providers held at once; each one owns an exporter thread. _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256 @@ -922,7 +941,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=value, ) - service_logging_span.set_status(Status(StatusCode.ERROR)) + service_logging_span.set_status( + Status( + StatusCode.ERROR, _text_as_stored_on_span(service_logging_span, "error", error) if error else None + ) + ) service_logging_span.end(end_time=_end_time_ns) async def async_post_call_failure_hook( @@ -937,8 +960,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): parent_otel_span: Final = user_api_key_dict.parent_otel_span if parent_otel_span is not None: - parent_otel_span.set_status(Status(StatusCode.ERROR)) - # Stamp team attributes onto the SERVER (root) span too, so the # trace root is team-filterable on the failure path like the # child exception span below. @@ -960,13 +981,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): original_exception=original_exception, traceback_str=traceback_str, ) - self._record_exception_on_span( + server_error_message: Final = self._record_exception_on_span( span=parent_otel_span, kwargs={ "exception": original_exception, "standard_logging_object": {"error_information": error_information}, }, ) + parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message)) # _record_exception_on_span only stamps when error_code is set; # bare TypeError etc. has none, and the span is about to be ended. @@ -995,7 +1017,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): team_id=user_api_key_dict.team_id, team_alias=user_api_key_dict.team_alias, ) - exception_logging_span.set_status(Status(StatusCode.ERROR)) + exception_logging_span.set_status(Status(StatusCode.ERROR, server_error_message)) exception_logging_span.end(end_time=self._to_ns(datetime.now())) # Emit guardrail spans for any guardrail invocations that @@ -2199,11 +2221,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if self._gen_ai_semconv_latest_experimental: span_kwargs["kind"] = self.span_kind.CLIENT span = otel_tracer.start_span(**span_kwargs) - span.set_status(Status(StatusCode.ERROR)) self.set_attributes(span, kwargs, response_obj) # Record exception information using OTEL standard method - self._record_exception_on_span(span=span, kwargs=kwargs) + span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs) + span.set_status(Status(StatusCode.ERROR, span_error_message)) span.end(end_time=self._to_ns(end_time)) else: @@ -2212,9 +2234,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Only set attributes if the span is still recording (not closed) # Note: parent_otel_span is guaranteed to be not None here if parent_otel_span.is_recording(): - parent_otel_span.set_status(Status(StatusCode.ERROR)) self.set_attributes(parent_otel_span, kwargs, response_obj) - self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) + parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) + parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message)) # Create span for guardrail information — ensure proper parenting (Issue #5) guardrail_ctx: Final = self._resolve_guardrail_context( @@ -2232,13 +2254,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ): parent_otel_span.end(end_time=self._to_ns(end_time)) - def _record_exception_on_span(self, span: Span, kwargs: dict): + def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None: """ Record exception information on the span using OTEL standard methods. This extracts error information from StandardLoggingPayload and: 1. Uses span.record_exception() for the actual exception object (OTEL standard) 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation + + Returns the error message it recorded, if any, for the span status description. """ try: from litellm.integrations._types.open_inference import ( @@ -2253,10 +2277,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span.record_exception(exception) # Get StandardLoggingPayload for structured error information - standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") + standard_logging_payload: Final = kwargs.get("standard_logging_object") if standard_logging_payload is None: - return + return None # Extract error_information from StandardLoggingPayload error_information: Final = standard_logging_payload.get("error_information") @@ -2270,59 +2294,67 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key=ErrorAttributes.ERROR_MESSAGE, value=error_str, ) - return + return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_str) if error_str else None # Set structured error attributes from StandardLoggingPayloadErrorInformation - if error_information.get("error_code"): + error_code: Final = error_information.get("error_code") + if error_code: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_CODE, - value=error_information["error_code"], + value=error_code, ) # Also expose under the OTel-standard name as an int # (error_code is a str, may be non-numeric). - _error_code_val: Final = error_information["error_code"] - if _error_code_val is not None: - try: - self.safe_set_attribute( - span=span, - key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, - value=int(_error_code_val), - ) - except (ValueError, TypeError): - pass + try: + self.safe_set_attribute( + span=span, + key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + value=int(error_code), + ) + except (ValueError, TypeError): + pass - if error_information.get("error_class"): + error_class: Final = error_information.get("error_class") + if error_class: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_TYPE, - value=error_information["error_class"], + value=error_class, ) - if error_information.get("error_message"): + error_message: Final = error_information.get("error_message") + if error_message: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_MESSAGE, - value=error_information["error_message"], + value=error_message, ) - if error_information.get("llm_provider"): + llm_provider: Final = error_information.get("llm_provider") + if llm_provider: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_LLM_PROVIDER, - value=error_information["llm_provider"], + value=llm_provider, ) - if error_information.get("traceback"): + traceback_text: Final = error_information.get("traceback") + if traceback_text: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_STACK_TRACE, - value=error_information["traceback"], + value=traceback_text, ) except Exception as e: verbose_logger.exception("OpenTelemetry: Error recording exception on span: %s", str(e)) + return None + else: + if error_message: + return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_message) + return error_class or None def set_tools_attributes(self, span: Span, tools): import json @@ -3518,8 +3550,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key="exception", value=str(_exception), ) - management_endpoint_span.set_status(Status(StatusCode.ERROR)) - management_endpoint_span.end(end_time=_end_time_ns) # The management wrapper has no other hook that closes the SERVER span. from litellm.litellm_core_utils.litellm_logging import ( @@ -3529,14 +3559,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): error_information: Final = StandardLoggingPayloadSetup.get_error_information( original_exception=_exception, ) - parent_otel_span.set_status(Status(StatusCode.ERROR)) - self._record_exception_on_span( + server_error_message: Final = self._record_exception_on_span( span=parent_otel_span, kwargs={ "exception": _exception, "standard_logging_object": {"error_information": error_information}, }, ) + management_endpoint_span.set_status(Status(StatusCode.ERROR, server_error_message)) + management_endpoint_span.end(end_time=_end_time_ns) + parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message)) parent_otel_span.end(end_time=_end_time_ns) def create_litellm_proxy_request_started_span( @@ -3596,19 +3628,20 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span: Span | None, exception: Exception | None, status_code: int, - ) -> None: + ) -> str | None: """Stamp structured ``error.*`` attributes on the SERVER span from the exception returned to the client, with ``error.code`` pinned to the real - response status. Idempotent (overwrites); emits no exception event.""" + response status. Idempotent (overwrites); emits no exception event. + Returns the error text for the span status description.""" if span is None or exception is None: - return + return None from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception) error_information["error_code"] = str(status_code) - self._record_exception_on_span( + return self._record_exception_on_span( span=span, kwargs={"standard_logging_object": {"error_information": error_information}}, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 341f12ba720..295f4380d7d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2240,9 +2240,14 @@ def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Ex open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code) return open_telemetry_logger.set_response_status_code_attribute(parent_otel_span, status_code) - if status_code >= 400: - open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code) - parent_otel_span.set_status(Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK)) + parent_otel_span.set_status( + Status( + StatusCode.ERROR, + open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code), + ) + if status_code >= 400 + else Status(StatusCode.OK) + ) parent_otel_span.end() except Exception as e: verbose_proxy_logger.debug("Error closing dangling OTEL SERVER span: %s", str(e)) diff --git a/tests/unit/integrations/arize/test_arize_phoenix.py b/tests/unit/integrations/arize/test_arize_phoenix.py index ca32a9cf9e3..eb9e1326ced 100644 --- a/tests/unit/integrations/arize/test_arize_phoenix.py +++ b/tests/unit/integrations/arize/test_arize_phoenix.py @@ -2,7 +2,15 @@ import asyncio, importlib, litellm, logging, os, unittest from unittest.mock import MagicMock, patch import pytest +from datetime import datetime, timedelta, timezone +from typing import Final +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from litellm.integrations.opentelemetry import LITELLM_REQUEST_SPAN_NAME from litellm.integrations.arize.arize_phoenix import ( ArizePhoenixConfig, ArizePhoenixLogger, @@ -852,6 +860,47 @@ class TestPhoenixTraceHandling: assert span.resource.attributes.get("model_id") == "unified-proj" +@pytest.mark.parametrize(("max_attribute_length", "expected_description"), [(None, "model not found"), (5, "model")]) +def test_failed_proxy_request_puts_the_error_message_on_every_phoenix_span_status( + max_attribute_length: int | None, expected_description: str +) -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + logger: Final = ArizePhoenixLogger(callback_name="arize_phoenix", tracer_provider=provider) + start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + logger._handle_failure( + { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": { + "custom_llm_provider": "openai", + "proxy_server_request": {"url": "/chat/completions", "method": "POST", "headers": {}}, + }, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "error_information": {"error_class": "NotFoundError", "error_message": "model not found"}, + }, + "exception": Exception("model not found"), + }, + None, + start, + start + timedelta(seconds=1), + ) + + statuses: Final = { + span.name: (span.status.status_code, span.status.description) for span in exporter.get_finished_spans() + } + assert statuses == { + "litellm_proxy_request": (StatusCode.ERROR, expected_description), + LITELLM_REQUEST_SPAN_NAME: (StatusCode.ERROR, expected_description), + } + + class TestGetArizePhoenixConfigProjectName: @patch.dict( "os.environ", {"PHOENIX_PROJECT_NAME": "phoenix-config-proj"}, clear=True diff --git a/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py b/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py index 56d067b16cf..08e7bda1350 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py +++ b/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py @@ -3,10 +3,13 @@ that fail after auth but before the route handler runs (e.g. /model/new TypeError or RequestValidationError).""" import asyncio +from collections.abc import Callable +from typing import Final import pytest from fastapi import HTTPException, Request from fastapi.exceptions import RequestValidationError +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import ProxyException @@ -133,10 +136,30 @@ def test_openai_exception_handler_stamps_structured_error_on_span( expected_url_path=path, where=f"openai_exception_handler ({path})", ) - attrs = get_server_span(wired_otel).attributes + server_span = get_server_span(wired_otel) + attrs = server_span.attributes assert attrs.get(ErrorAttributes.ERROR_MESSAGE) == msg assert attrs.get(ErrorAttributes.ERROR_TYPE) == "ProxyException" assert attrs.get(ErrorAttributes.ERROR_CODE) == "401" + assert server_span.status.description == msg + + +def test_validation_handler_puts_the_error_message_on_the_span_status( + wired_otel: InMemorySpanExporter, server_span_factory: Callable[[str], object] +) -> None: + path: Final = "/team/new" + request: Final = _fake_request(parent_otel_span=server_span_factory(path), path=path) + exc: Final = RequestValidationError( + errors=[{"type": "float_parsing", "loc": ("body", "max_budget"), "msg": "Input should be a valid number"}] + ) + + response: Final = asyncio.run(otel_request_validation_exception_handler(request, exc)) + + assert response.status_code == 422 + server_span: Final = get_server_span(wired_otel) + description: Final = server_span.status.description or "" + assert "Input should be a valid number" in description + assert description == server_span.attributes.get(ErrorAttributes.ERROR_MESSAGE) def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory): diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 9cc06ccaafc..d14703e48a1 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -25,7 +25,7 @@ from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData -from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -48,9 +48,10 @@ from litellm.integrations.opentelemetry import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy import proxy_server -from litellm.proxy._types import SpanAttributes +from litellm.proxy._types import ManagementEndpointLoggingPayload, ProxyException, SpanAttributes +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.services import ServiceLoggerPayload, ServiceTypes -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Mapping from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -3214,6 +3215,9 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase): ) +_FIXED_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + class TestOpenTelemetrySemanticConventions138(unittest.TestCase): """ Test suite for OpenTelemetry 1.38 Semantic Conventions compliance. @@ -3731,6 +3735,91 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase): error_spans = [s for s in spans if s.status.status_code == StatusCode.ERROR] self.assertTrue(error_spans, "Expected at least one span with ERROR status") + @parameterized.expand( + [ + ( + "own_span_error_information", + False, + { + "error_str": "litellm.ServiceUnavailableError: upstream returned 503", + "error_information": { + "error_class": "ServiceUnavailableError", + "error_message": "upstream returned 503", + }, + }, + "upstream returned 503", + ), + ( + "own_span_error_str_only", + False, + {"error_str": "litellm.ServiceUnavailableError: upstream returned 503"}, + "litellm.ServiceUnavailableError: upstream returned 503", + ), + ( + "own_span_error_str_only_cut_to_the_attribute_length_limit", + False, + {"error_str": "litellm.ServiceUnavailableError: upstream returned 503"}, + "litellm.Serv", + 12, + ), + ( + "own_span_empty_error_message", + False, + {"error_information": {"error_class": "ServiceUnavailableError", "error_message": ""}}, + "ServiceUnavailableError", + ), + ( + "parent_span_error_information", + True, + { + "error_information": { + "error_class": "ServiceUnavailableError", + "error_message": "upstream returned 503", + } + }, + "upstream returned 503", + ), + ] + ) + def test_handle_failure_error_status_carries_the_error_message( + self, + _name: str, + has_parent_span: bool, + error_fields: Mapping[str, object], + expected_description: str, + max_attribute_length: int | None = None, + ) -> None: + span_exporter: Final = InMemorySpanExporter() + tracer_provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length)) + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + otel: Final = OpenTelemetry(tracer_provider=tracer_provider) + otel.tracer = tracer_provider.get_tracer("litellm") + active_parent_span: Final = ( + trace.use_span(tracer_provider.get_tracer("proxy").start_span("parent_span"), end_on_exit=True) + if has_parent_span + else contextlib.nullcontext() + ) + start: Final = _FIXED_START + end: Final = start + timedelta(seconds=1) + kwargs: Final = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": {"id": "test-id", "call_type": "completion", "metadata": {}, **error_fields}, + "exception": Exception("upstream returned 503"), + } + + with patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}), active_parent_span: + otel._handle_failure(kwargs, None, start, end) + + error_spans: Final = [ + s for s in span_exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR + ] + self.assertEqual(len(error_spans), 1) + self.assertEqual(error_spans[0].name, "parent_span" if has_parent_span else "litellm_request") + self.assertEqual(error_spans[0].status.description, expected_description) + class TestRawSpanAttributeIsolation(unittest.TestCase): """Issue #3: raw_gen_ai_request span should only contain provider-specific @@ -5439,6 +5528,95 @@ class TestOpenTelemetryFailureHookStampsServerSpan(unittest.TestCase): ) +_PROXY_KEY_ERROR: Final = "Authentication Error, invalid proxy key" + + +def _proxy_key_exception() -> ProxyException: + return ProxyException(message=_PROXY_KEY_ERROR, type="auth_error", param="None", code=401) + + +def _fail_proxy_request(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + asyncio.run( + otel.async_post_call_failure_hook( + request_data={}, + original_exception=_proxy_key_exception(), + user_api_key_dict=UserAPIKeyAuth(parent_otel_span=otel.tracer.start_span("Received Proxy Server Request")), + ) + ) + + +def _fail_management_request(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + now: Final = _FIXED_START + asyncio.run( + otel.async_management_endpoint_failure_hook( + logging_payload=ManagementEndpointLoggingPayload( + route="/key/generate", request_data={}, exception=_proxy_key_exception(), start_time=now, end_time=now + ), + parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"), + ) + ) + + +def _fail_service_call(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + now: Final = _FIXED_START + asyncio.run( + otel.async_service_failure_hook( + payload=ServiceLoggerPayload( + is_error=True, + error="redis connection refused", + service=ServiceTypes.REDIS, + duration=0.1, + call_type="async_get_cache", + event_metadata=None, + ), + error="redis connection refused", + parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"), + start_time=now, + end_time=now, + ) + ) + + +_FAILED_SPANS: Final = pytest.mark.parametrize( + ("fail", "expected_description"), + [ + (_fail_proxy_request, _PROXY_KEY_ERROR), + (_fail_management_request, _PROXY_KEY_ERROR), + (_fail_service_call, "redis connection refused"), + ], + ids=["proxy_request", "management_request", "service_call"], +) + + +def _error_status_descriptions(fail: Callable[[TracerProvider], None], provider: TracerProvider) -> set[str | None]: + exporter: Final = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + fail(provider) + error_spans: Final = [s for s in exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR] + assert error_spans + return {s.status.description for s in error_spans} + + +@_FAILED_SPANS +def test_every_error_span_carries_the_error_message_in_its_status( + fail: Callable[[TracerProvider], None], expected_description: str +) -> None: + assert _error_status_descriptions(fail, TracerProvider()) == {expected_description} + + +@_FAILED_SPANS +def test_error_status_is_cut_to_the_span_attribute_length_limit( + fail: Callable[[TracerProvider], None], expected_description: str +) -> None: + limit: Final = 12 + provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=limit)) + + assert _error_status_descriptions(fail, provider) == {expected_description[:limit]} + + class TestOpenTelemetrySetProxyRequestRouteAttributes(unittest.TestCase): """http.route (template) + url.path (literal) must land on the SERVER span. The logging handlers write the litellm_request child span, so From 6b387e84b5fc487a2c4ed9e2eeef0c6dc008a98b Mon Sep 17 00:00:00 2001 From: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:41:41 +0800 Subject: [PATCH 073/124] fix(prometheus): preserve requested model after fallback (#45555) * fix(prometheus): preserve requested model after fallback Co-authored-by: Bytechoreographer * fix(prometheus): read fallback origin from both metadata buckets Co-authored-by: Bytechoreographer --------- Co-authored-by: Bytechoreographer --- litellm/integrations/prometheus.py | 18 ++++++- .../integrations/test_prometheus_labels.py | 50 ++++++++++++++++--- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 073e545a93f..19d161b8aa4 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -258,6 +258,7 @@ class _LabeledMetric: _MetricLike: TypeAlias = "NoOpMetric | _LabeledMetric | MetricWrapperBase" _SeriesLimitT: Final = TypeVar("_SeriesLimitT", int, float) +_REQUEST_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field(gt=0)]) _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)]) _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)]) @@ -1748,12 +1749,25 @@ class PrometheusLogger(CustomLogger): # increment litellm_proxy_total_requests_metric for all successful requests # (both streaming and non-streaming) in this single location to prevent # double-counting that occurs when async_post_call_success_hook also increments + request_metadata: Final = { + **_REQUEST_METADATA_ADAPTER.validate_python(get_litellm_metadata_from_kwargs(kwargs)), + **_REQUEST_METADATA_ADAPTER.validate_python(_metadata), + } + original_model_group: Final = request_metadata.get("original_model_group") + request_enum_values: Final = replace( + enum_values, + requested_model=( + original_model_group + if isinstance(original_model_group, str) and original_model_group + else standard_logging_payload["model_group"] + ), + ) PrometheusLogger._inc_labeled_counter( self, self.litellm_proxy_total_requests_metric, "litellm_proxy_total_requests_metric", - enum_values, - label_context=label_context, + request_enum_values, + label_context=PrometheusLabelFactoryContext(request_enum_values), ) def _increment_token_metrics( diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index d49ca57a3de..bff89b6cbf1 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -2,6 +2,8 @@ Unit tests for prometheus metric labels configuration """ +from typing import Final, Literal + import pytest import litellm @@ -612,7 +614,16 @@ def test_prometheus_label_value_sanitization_non_string_types(): @pytest.mark.asyncio -async def test_success_hook_emits_api_provider_value_on_token_metric(): +@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("populate_other_bucket", (False, True)) +@pytest.mark.parametrize("original_model_group", (None, "", 123, "requested-source")) +@pytest.mark.parametrize("stream", (False, True)) +async def test_success_hook_preserves_requested_model_and_deployment_labels( + metadata_bucket: Literal["metadata", "litellm_metadata"], + populate_other_bucket: bool, + original_model_group: str | int | None, + stream: bool, +) -> None: """ End-to-end emit wiring for the success path. @@ -628,11 +639,12 @@ async def test_success_hook_emits_api_provider_value_on_token_metric(): from litellm.integrations.prometheus import PrometheusLogger - payload = { + payload: Final = { "id": "t", "call_type": "completion", "response_cost": 0.001, "status": "success", + "stream": stream, "total_tokens": 30, "prompt_tokens": 20, "completion_tokens": 10, @@ -662,26 +674,52 @@ async def test_success_hook_emits_api_provider_value_on_token_metric(): "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None}, } + other_bucket: Final = "litellm_metadata" if metadata_bucket == "metadata" else "metadata" + litellm_params: Final = { + metadata_bucket: { + "request_tag": "router", + **({"original_model_group": original_model_group} if original_model_group is not None else {}), + }, + **({other_bucket: {"request_tag": "client"}} if populate_other_bucket else {}), + } _clear_prometheus_registry() try: - logger = PrometheusLogger() - now = datetime.datetime.now() + logger: Final = PrometheusLogger() + now: Final = datetime.datetime.fromtimestamp(1, tz=datetime.timezone.utc) await logger.async_log_success_event( { "model": "gpt-4o-mini", - "litellm_params": {"metadata": {}}, + "litellm_params": litellm_params, "standard_logging_object": payload, }, None, now, now, ) - samples = _collected_samples("litellm_total_tokens_metric_total") + request_samples: Final = _collected_samples("litellm_proxy_total_requests_metric_total") + assert len(request_samples) == 1 + assert request_samples[0].value == 1 + expected_requested_model: Final = ( + original_model_group + if isinstance(original_model_group, str) and original_model_group + else payload["model_group"] + ) + assert request_samples[0].labels["requested_model"] == expected_requested_model + assert request_samples[0].labels["model_id"] == payload["model_id"] + assert request_samples[0].labels["api_provider"] == payload["custom_llm_provider"] + assert request_samples[0].labels["status_code"] == "200" + samples: Final = _collected_samples("litellm_total_tokens_metric_total") assert samples, "expected litellm_total_tokens_metric to be emitted" assert all(s.labels.get("api_provider") == "openai" for s in samples), ( "collected token metric must carry api_provider=openai, got " f"{[s.labels.get('api_provider') for s in samples]}" ) + assert all(sample.labels["requested_model"] == payload["model_group"] for sample in samples) + spend_samples: Final = _collected_samples("litellm_spend_metric_total") + assert len(spend_samples) == 1 + assert spend_samples[0].value == payload["response_cost"] + assert spend_samples[0].labels["requested_model"] == payload["model_group"] + assert spend_samples[0].labels["model"] == payload["model"] finally: _clear_prometheus_registry() From 8a2782e41ce5a40d535769388e008f9b1f91b74c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:01:12 +0000 Subject: [PATCH 074/124] feat(router): route batch creates to mode batch deployments and expose supported_endpoints on /model_group/info (#47376) * feat(router): expose supported_endpoints on /model_group/info * feat(router): route batch creates to mode batch deployments in mixed model groups * fix(router): prefer batch deployments after eligibility filters and in group-name credential lookups * fix(router): keep group-name credential lookups on the first deployment and prefer batch deployments before cooldown Model-encoded ids carry only the group name, so preferring the batch deployment there moved ids created before the upgrade onto its credentials. Batch creates now pick batch-capable deployments before health checks and cooldown, routing and group info share one batch-capable check, the cost map's /v1/batch is dropped only for Azure, and /model/info applies the same per-deployment rule * test(router): map _listed_endpoints to its /model/info tests in the router coverage check * fix(router): skip only Azure chat deployments for batch creates and list batch support per caller team Narrow the batch create filter to Azure and azure_ai deployments that are not mode batch, so OpenAI groups keep their chat deployments during cooldown and under tag filters. Mark batch deployments by mode batch only, since stored endpoint lists from older Admin UI saves copy the cost map. Leave /model/info unchanged so the Admin UI never saves a computed endpoint list, and pass the caller's team into /model_group/info. * test(proxy): let the model hub router fakes take the caller's team_id * fix(router): gate batch-create routing on deployments the caller's team may use and list only /v1/batch for Azure batch deployments * fix(router): skip a malformed model_info.supported_endpoints instead of failing group info and chat * fix(router): apply the batch create filter after cooldown, managed files, and tags The batch filter ran before health checks and cooldown, so a batch create on a mixed Azure group answered with a router 429 or 401 when cooldown, tags, or a managed file's targets left no batch deployment, where main sends the create on and returns Azure's 400. The filter now runs right before order filtering, after every eligibility filter, so those cases answer as main does and creates still skip Azure chat deployments whenever a batch deployment is left * test(router): annotate the batch routing test assignments with Final --- litellm/proxy/proxy_server.py | 9 +- litellm/router.py | 70 ++- litellm/router_utils/batch_utils.py | 84 ++- litellm/router_utils/common_utils.py | 17 + litellm/types/router.py | 1 + .../router_code_coverage.py | 10 + .../proxy_server/test_routes_model_info.py | 49 +- .../public_v1/test_model_hub.py | 4 +- tests/unit/test_router/test_router.py | 517 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 10 files changed, 742 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 295f4380d7d..c17e0c67d68 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17757,7 +17757,7 @@ async def model_deprecations( def get_model_group_info( - llm_router: Router, all_models_str: Sequence[str], model_group: str | None + llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None ) -> list[ModelGroupInfoProxy]: model_groups: Final[list[ModelGroupInfoProxy]] = [] @@ -17770,7 +17770,7 @@ def get_model_group_info( if model_group is not None and model_group != model: continue - _model_group_info = llm_router.get_model_group_info(model_group=model) + _model_group_info = llm_router.get_model_group_info(model_group=model, team_id=team_id) if _model_group_info is not None: model_groups.append(ModelGroupInfoProxy.model_validate(_model_group_info.model_dump())) @@ -18002,7 +18002,10 @@ async def model_group_info( model_groups: Final = await append_agents_to_model_group( model_groups=get_model_group_info( - llm_router=llm_router, all_models_str=listed_group_names, model_group=model_group + llm_router=llm_router, + all_models_str=listed_group_names, + model_group=model_group, + team_id=user_api_key_dict.team_id, ), user_api_key_dict=user_api_key_dict, ) diff --git a/litellm/router.py b/litellm/router.py index 1ecff167126..54d4cce4656 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -190,10 +190,13 @@ from litellm.router_utils.auto_router_model_naming import ( count_capability_routers, ) from litellm.router_utils.batch_utils import ( + batch_create_routing, + deployment_supported_endpoints, get_router_metadata_variable_name, is_batch_retrieve_call_type, replace_model_in_jsonl, should_replace_model_in_jsonl, + without_deployments_that_cannot_batch, ) from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit from litellm.router_utils.clientside_credential_handler import ( @@ -202,6 +205,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, format_fallback_outcome_message, @@ -7184,12 +7188,13 @@ class Router: try: verbose_router_logger.debug("Inside _acreate_batch()- model: %s; kwargs: %s", model, kwargs) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( - model=model, - messages=[{"role": "user", "content": "files-api-fake-text"}], - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with batch_create_routing(): + deployment: Final = await self.async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "files-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) data: Final = deployment["litellm_params"].copy() model_name: Final = data["model"] @@ -12056,7 +12061,9 @@ class Router: ) return model_info - def _set_model_group_info(self, model_group: str, user_facing_model_group_name: str) -> ModelGroupInfo | None: + def _set_model_group_info( + self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None + ) -> ModelGroupInfo | None: """ For a given model group name, return the combined model info @@ -12187,6 +12194,7 @@ class Router: **model_info, "supports_fast_mode": True, "supported_reasoning_efforts": None, + "supported_endpoints": None, } ) else: @@ -12264,6 +12272,23 @@ class Router: if model_info.get("rpm", None) is not None and _deployment_rpm is None: _deployment_rpm = model_info.get("rpm") + model_group_info.supported_endpoints = ( + tuple( + dict.fromkeys( + ( + *(model_group_info.supported_endpoints or ()), + *( + deployment_supported_endpoints( + model_info.get("supported_endpoints"), model, llm_provider + ) + if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id) + else () + ), + ) + ) + ) + or None + ) model_group_info.supports_fast_mode = model_group_info.supports_fast_mode and ( AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider) ) @@ -12322,7 +12347,9 @@ class Router: return model_group_info - def get_model_group_info(self, model_group: str, *, include_hidden: bool = False) -> ModelGroupInfo | None: + def get_model_group_info( + self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None + ) -> ModelGroupInfo | None: """ For a given model group name, return the combined model info @@ -12346,10 +12373,13 @@ class Router: return self._set_model_group_info( model_group=_router_model_group, user_facing_model_group_name=model_group, + team_id=team_id, ) ## Check if actual model - return self._set_model_group_info(model_group=model_group, user_facing_model_group_name=model_group) + return self._set_model_group_info( + model_group=model_group, user_facing_model_group_name=model_group, team_id=team_id + ) async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: """ @@ -14585,8 +14615,15 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts + list[dict[str, object]], healthy_deployments + ) healthy_deployments = litellm.utils.get_order_filtered_deployments( - list(self._narrow_to_video_deployment(cast(list[dict[str, object]], healthy_deployments))), + list( + self._narrow_to_video_deployment( + without_deployments_that_cannot_batch(batch_candidates, caller_deployments=lambda: batch_candidates) + ) + ), target_order=_target_order, ) @@ -15788,8 +15825,19 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts + list[dict[str, object]], healthy_deployments + ) healthy_deployments = litellm.utils.get_order_filtered_deployments( - list(self._narrow_to_video_deployment(healthy_deployments)), target_order=_target_order + list( + self._narrow_to_video_deployment( + without_deployments_that_cannot_batch( + batch_candidates, + caller_deployments=lambda: deployments_the_caller_may_use(batch_candidates, routing_kwargs), + ) + ) + ), + target_order=_target_order, ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in diff --git a/litellm/router_utils/batch_utils.py b/litellm/router_utils/batch_utils.py index d71696a66b1..88c58412e01 100644 --- a/litellm/router_utils/batch_utils.py +++ b/litellm/router_utils/batch_utils.py @@ -1,16 +1,26 @@ import io import json -from collections.abc import Iterable +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar from os import PathLike -from typing import Final +from typing import Final, TypeVar -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose -from litellm.types.utils import CallTypes +from litellm.types.utils import CallTypes, LlmProviders _DECODED_JSONL_ROW: Final = TypeAdapter(object) +_DEPLOYMENT_FIELD: Final = TypeAdapter(Mapping[str, object]) +_ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...]) + +BATCH_ENDPOINT: Final = "/v1/batch" +_AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value}) + +_batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False) +_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object]) class InMemoryFile(io.BytesIO): @@ -65,6 +75,72 @@ def parse_jsonl_with_embedded_newlines(content: str) -> list[dict]: return json_objects +@contextmanager +def batch_create_routing() -> Generator[None, None, None]: + token: Final = _batch_create_routing.set(True) + try: + yield + finally: + _batch_create_routing.reset(token) + + +def is_batch_create_routing() -> bool: + return _batch_create_routing.get() + + +def _deployment_field(deployment: Mapping[str, object], field: str) -> Mapping[str, object]: + return _DEPLOYMENT_FIELD.validate_python(deployment.get(field) or {}) + + +def is_batch_capable(deployment: Mapping[str, object]) -> bool: + return _deployment_field(deployment, "model_info").get("mode") == "batch" + + +def _deployment_provider(deployment: Mapping[str, object]) -> str | None: + litellm_params: Final = _deployment_field(deployment, "litellm_params") + configured_provider: Final = litellm_params.get("custom_llm_provider") + if isinstance(configured_provider, str): + return configured_provider + model: Final = litellm_params.get("model") + return model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None + + +def _cannot_run_batches(deployment: Mapping[str, object], provider: str | None) -> bool: + return provider in _AZURE_PROVIDERS and not is_batch_capable(deployment) + + +def without_deployments_that_cannot_batch( + deployments: Sequence[_DeploymentT], caller_deployments: Callable[[], Iterable[Mapping[str, object]]] +) -> Sequence[_DeploymentT]: + if not is_batch_create_routing() or not any(is_batch_capable(deployment) for deployment in caller_deployments()): + return deployments + return [ + deployment + for deployment in deployments + if not _cannot_run_batches(deployment, _deployment_provider(deployment)) + ] + + +def _endpoint_paths(configured: object) -> tuple[str, ...]: + try: + return _ENDPOINT_PATHS.validate_python(configured) + except ValidationError: + return () + + +def deployment_supported_endpoints( + resolved_endpoints: object, deployment: Mapping[str, object], provider: str | None +) -> tuple[str, ...]: + endpoints: Final = _endpoint_paths(resolved_endpoints) + if is_batch_capable(deployment) and provider in _AZURE_PROVIDERS: + return (BATCH_ENDPOINT,) + if is_batch_capable(deployment): + return endpoints if BATCH_ENDPOINT in endpoints else (*endpoints, BATCH_ENDPOINT) + if _cannot_run_batches(deployment, provider): + return tuple(endpoint for endpoint in endpoints if endpoint != BATCH_ENDPOINT) + return endpoints + + def should_replace_model_in_jsonl( purpose: OpenAIFilesPurpose, passthrough: bool = False, diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index caab6e572de..a00d84babf3 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -35,6 +35,9 @@ _V = TypeVar("_V") MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +_DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment: @@ -327,6 +330,20 @@ def filter_team_based_models( ] +def deployments_the_caller_may_use( + deployments: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] | None +) -> tuple[Mapping[str, object], ...]: + try: + return _DEPLOYMENT_ROWS_ADAPTER.validate_python( + filter_team_based_models( + [dict(deployment) for deployment in deployments], + None if request_kwargs is None else dict(request_kwargs), + ) + ) + except BadRequestError: + return () + + _WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) diff --git a/litellm/types/router.py b/litellm/types/router.py index 36d298b2bc2..47dafcf275d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -982,6 +982,7 @@ class ModelGroupInfo(LiteLLMBaseModel): supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None description: str | None = None + supported_endpoints: tuple[str, ...] | None = None def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index e72a85b7056..3f0ad07a876 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -141,6 +141,16 @@ ignored_function_names = [ "async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py "_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py "_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py + "without_deployments_that_cannot_batch", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "batch_create_routing", # Tested through the mixed-group acreate_batch tests in test_router.py + "is_batch_create_routing", # Tested through the chat-after-batch-create test in test_router.py + "is_batch_capable", # Tested through the mixed-group acreate_batch and model group info tests in test_router.py + "deployments_the_caller_may_use", # Tested through the usage-based-routing team batch create test in test_router.py + "deployment_supported_endpoints", # Tested through the model group info batch endpoint tests in test_router.py + "_endpoint_paths", # Read by deployment_supported_endpoints, tested through the malformed endpoint list group info test in test_router.py + "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py + "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py + "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py ] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index cb7b5b13fa3..2c0e0944edb 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -801,7 +801,7 @@ def model_group_info_router(monkeypatch): router.get_model_access_groups.return_value = {} router.get_model_list.return_value = [] - def model_group_info(*, llm_router, all_models_str, model_group): + def model_group_info(*, llm_router, all_models_str, model_group, team_id=None): return [ModelGroupInfoProxy(model_group=name, providers=[]) for name in all_models_str] async def append_agents_to_model_group(*, model_groups, user_api_key_dict): @@ -866,6 +866,53 @@ def test_model_group_info_internal_user_key_model_restriction_applies(client, au assert [model["model_group"] for model in response.json()["data"]] == ["gpt-4"] +def _azure_group_deployment(sku_host: str, mode: str, **model_info: object) -> dict[str, object]: + return { + "model_name": "gpt-mixed", + "litellm_params": { + "model": f"azure/{sku_host}", + "api_base": f"https://{sku_host}.openai.azure.com", + "api_key": "dummy-key", + "api_version": "2025-04-01-preview", + }, + "model_info": {"mode": mode, **model_info}, + } + + +@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)]) +def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment( + client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool +): + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + _azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]), + _azure_group_deployment("batch-sku", "batch", team_id="team-a"), + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN, team_id=team_id): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + groups: Final = {group["model_group"]: group for group in response.json()["data"]} + assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch + + # --------------------------------------------------------------------------- # GET /v2/model/info?exclude_auto_routers # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 4267cdbbcc9..07886745222 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -25,7 +25,7 @@ class _FakeRouter: infos: Mapping[str, ModelGroupInfo] - def get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None: return self.infos.get(model_group) @@ -325,7 +325,7 @@ def test_no_router_answers_with_a_problem_rather_than_the_openai_error_shape(mon def test_an_unexpected_router_failure_answers_as_a_problem_not_the_openai_error_shape(monkeypatch): class _Exploding: - def get_model_group_info(self, model_group: str) -> ModelGroupInfo: + def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo: raise RuntimeError("router blew up") monkeypatch.setattr(litellm, "public_model_groups", ["boom"]) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9b1bdbf702e..90674dcb3f9 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2375,6 +2375,140 @@ def test_model_group_info_description_from_model_info(): assert result.description == "State-of-the-art language model." +@pytest.mark.parametrize("chat_deployment_first", [True, False]) +def test_model_group_info_lists_batch_for_a_mixed_chat_and_batch_group(chat_deployment_first: bool): + chat: Final = _azure_sku_deployment( + "gpt-mixed", "unmapped-chat-sku", "chat", supported_endpoints=["/v1/chat/completions"] + ) + batch: Final = _azure_sku_deployment("gpt-mixed", "unmapped-batch-sku", "batch") + router: Final = Router(model_list=[chat, batch] if chat_deployment_first else [batch, chat]) + + info: Final = router.get_model_group_info("gpt-mixed") + + assert info is not None + assert sorted(info.supported_endpoints or ()) == ["/v1/batch", "/v1/chat/completions"] + + +def test_model_group_info_lists_no_batch_for_a_chat_only_group(): + router: Final = Router( + model_list=[ + _azure_sku_deployment( + "gpt-declared", "unmapped-chat-sku", "chat", supported_endpoints=["/v1/chat/completions"] + ), + _azure_sku_deployment("gpt-undeclared", "unmapped-undeclared-sku", "chat"), + ] + ) + + declared: Final = router.get_model_group_info("gpt-declared") + undeclared: Final = router.get_model_group_info("gpt-undeclared") + + assert declared is not None and declared.supported_endpoints == ("/v1/chat/completions",) + assert undeclared is not None and undeclared.supported_endpoints is None + + +@pytest.mark.parametrize(("provider", "lists_batch"), [("azure", False), ("azure_ai", False), ("openai", True)]) +def test_model_group_info_lists_cost_map_batch_only_for_providers_that_batch_on_any_deployment( + provider: str, lists_batch: bool +): + """Azure runs batches only on a separate batch deployment, though the cost map lists /v1/batch for its chat + models, so an Azure chat group must not advertise it. OpenAI batches through the deployment that serves chat, + so its cost map listing stands.""" + cost_map_name, cost_map_endpoints = next( + (name, tuple(info["supported_endpoints"])) + for name, info in litellm.model_cost.items() + if info.get("litellm_provider") == provider + and info.get("mode") == "chat" + and "/v1/batch" in (info.get("supported_endpoints") or ()) + ) + router: Final = Router( + model_list=[ + { + "model_name": "gpt-chat", + "litellm_params": {"model": f"{provider}/{cost_map_name.removeprefix(f'{provider}/')}"}, + } + ] + ) + + info: Final = router.get_model_group_info("gpt-chat") + + assert info is not None + assert info.supported_endpoints == tuple( + endpoint for endpoint in cost_map_endpoints if lists_batch or endpoint != "/v1/batch" + ) + + +@pytest.mark.parametrize( + "malformed_endpoints", + ["/v1/chat/completions", [{"path": "/v1/chat/completions"}], [1], {"/v1/batch": True}], +) +def test_model_group_info_skips_a_malformed_endpoint_list_instead_of_failing(malformed_endpoints: object): + router: Final = Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]), + _azure_sku_deployment("gpt-mixed", "odd-sku", "chat", supported_endpoints=malformed_endpoints), + ] + ) + + info: Final = router.get_model_group_info("gpt-mixed") + + assert info is not None and info.supported_endpoints == ("/v1/chat/completions",) + + +def test_model_group_info_ignores_a_stored_batch_endpoint_on_an_azure_chat_deployment(): + """Admin UI saves before v1.104.0 stored the cost map's endpoint list, /v1/batch included, on Azure chat rows, + so a stored list cannot mark a deployment as one that runs batches.""" + router: Final = Router( + model_list=[ + _azure_sku_deployment( + "gpt-stored-endpoints", + "unmapped-chat-sku", + "chat", + supported_endpoints=["/v1/chat/completions", "/v1/batch"], + ) + ] + ) + + info: Final = router.get_model_group_info("gpt-stored-endpoints") + + assert info is not None and info.supported_endpoints == ("/v1/chat/completions",) + + +@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-b", False), ("team-a", True)]) +def test_model_group_info_lists_batch_only_to_callers_that_may_use_the_batch_deployment( + team_id: str | None, lists_batch: bool +): + router: Final = Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", team_id="team-a"), + ] + ) + + info: Final = router.get_model_group_info("gpt-mixed", team_id=team_id) + + assert info is not None + assert ("/v1/batch" in (info.supported_endpoints or ())) is lists_batch + + +def test_model_group_info_lists_only_batch_for_an_azure_batch_deployment(): + """An Azure batch deployment answers chat, Responses, and Messages calls with 400 "The current operation is not + allowed in this deployment", so a group of them must advertise only /v1/batch whatever the endpoint list says.""" + router: Final = Router( + model_list=[ + _azure_sku_deployment( + "gpt-batch-only", + "batch-sku", + "batch", + supported_endpoints=["/v1/chat/completions", "/v1/responses", "/v1/batch"], + ) + ] + ) + + info: Final = router.get_model_group_info("gpt-batch-only") + + assert info is not None and info.supported_endpoints == ("/v1/batch",) + + def _alias_cost_router() -> Router: return Router( model_list=[ @@ -12914,6 +13048,389 @@ async def test_acreate_batch_still_falls_back_within_the_owning_model_group(): assert attempted_models == ["openai/gpt-4o-mini", "openai/gpt-4o-mini-backup"] +def _azure_sku_deployment(model_group: str, sku_host: str, mode: str, **model_info: object) -> dict[str, object]: + return { + "model_name": model_group, + "litellm_params": { + "model": f"azure/{sku_host}", + "api_base": f"https://{sku_host}.openai.azure.com", + "api_key": "dummy-key", + "api_version": "2025-04-01-preview", + }, + "model_info": {"mode": mode, **model_info}, + } + + +_AZURE_BATCH_CREATED: Final = { + "id": "batch_created", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-batch-input", + "completion_window": "24h", + "status": "validating", + "created_at": 0, +} + + +async def _create_batches(router: Router, model_group: str, attempts: int) -> None: + for _ in range(attempts): + await router.acreate_batch( + model=model_group, + input_file_id="file-batch-input", + endpoint="/v1/chat/completions", + completion_window="24h", + ) + + +def _azure_chat_row_with_provider_param() -> dict[str, object]: + row: Final = _azure_sku_deployment("gpt-mixed", "chat-sku", "chat") + return {**row, "litellm_params": {**row["litellm_params"], "model": "chat-sku", "custom_llm_provider": "azure"}} + + +_AZURE_CHAT_ROWS: Final = { + "no-stored-list": lambda: _azure_sku_deployment("gpt-mixed", "chat-sku", "chat"), + "stored-cost-map-list": lambda: _azure_sku_deployment( + "gpt-mixed", "chat-sku", "chat", supported_endpoints=["/v1/chat/completions", "/v1/batch"] + ), + "custom-llm-provider-param": _azure_chat_row_with_provider_param, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("build_chat_row", _AZURE_CHAT_ROWS.values(), ids=_AZURE_CHAT_ROWS.keys()) +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "latency-based-routing", "usage-based-routing"]) +async def test_acreate_batch_on_a_mixed_group_only_reaches_batch_deployments( + monkeypatch: pytest.MonkeyPatch, routing_strategy: str, build_chat_row: Callable[[], dict[str, object]] +): + """Azure answers 400 invalid_deployment_type when a batch is created on a GlobalStandard + (chat) deployment, so a group mixing chat and batch deployments must send every batch + create to the deployment that /model_group/info names as the one listing /v1/batch.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + build_chat_row(), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch"), + ], + routing_strategy=routing_strategy, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response( + 400, + json={"error": {"code": "invalid_deployment_type", "message": "SKU 'GlobalStandard' is not supported"}}, + ) + ) + batch_route: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + await _create_batches(router, "gpt-mixed", attempts=20) + + assert chat_route.call_count == 0 + assert batch_route.call_count == 20 + + +def _azure_wildcard_deployment(sku_host: str, mode: str) -> dict[str, object]: + return { + "model_name": "mixed-wild/*", + "litellm_params": { + "model": "azure/*", + "api_base": f"https://{sku_host}.openai.azure.com", + "api_key": "dummy-key", + "api_version": "2025-04-01-preview", + }, + "model_info": {"mode": mode}, + } + + +@pytest.mark.asyncio +async def test_acreate_batch_on_a_mixed_wildcard_group_only_reaches_batch_deployments( + monkeypatch: pytest.MonkeyPatch, +): + """A wildcard group resolves its deployments before the named-group path does, and its batch + creates must still only reach the deployments declared with model_info.mode batch.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[_azure_wildcard_deployment("chat-sku", "chat"), _azure_wildcard_deployment("batch-sku", "batch")], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response( + 400, + json={"error": {"code": "invalid_deployment_type", "message": "SKU 'GlobalStandard' is not supported"}}, + ) + ) + batch_route: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + await _create_batches(router, "mixed-wild/gpt-5.4-mini", attempts=20) + + assert chat_route.call_count == 0 + assert batch_route.call_count == 20 + + +@pytest.mark.asyncio +async def test_chat_calls_after_a_batch_create_still_use_every_deployment_in_the_group( + monkeypatch: pytest.MonkeyPatch, +): + """Only the batch create itself is narrowed to mode batch deployments. Chat calls the same task makes + afterwards keep load balancing across the whole group.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat", id="chat-deployment"), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", id="batch-deployment"), + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + await _create_batches(router, "gpt-mixed", attempts=2) + + chat_deployments: Final = { + ( + await router.acompletion( + model="gpt-mixed", messages=[{"role": "user", "content": "hi"}], mock_response="ok" + ) + )._hidden_params["model_id"] + for _ in range(40) + } + + assert chat_deployments == {"chat-deployment", "batch-deployment"} + + +@pytest.mark.asyncio +async def test_acreate_batch_on_a_group_without_batch_deployments_still_uses_every_deployment( + monkeypatch: pytest.MonkeyPatch, +): + """Bedrock and OpenAI batch through the same deployment that serves chat, so a group with + no mode batch deployment keeps load balancing batch creates across all of them.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-chat-only", "first-chat-sku", "chat"), + _azure_sku_deployment("gpt-chat-only", "second-chat-sku", "chat"), + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + first_route: Final = respx_mock.post(host="first-chat-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + second_route: Final = respx_mock.post(host="second-chat-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + await _create_batches(router, "gpt-chat-only", attempts=40) + + assert first_route.call_count > 0 + assert second_route.call_count > 0 + assert first_route.call_count + second_route.call_count == 40 + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("team_id", "reached_host"), [(None, "chat-sku"), ("team-a", "batch-sku")]) +async def test_acreate_batch_prefers_only_the_batch_deployments_the_caller_may_use( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, reached_host: str +): + """A batch deployment owned by one team must not take a mixed group's batch creates away from callers + outside that team, who still reach the group's shared deployments.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat"), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", team_id="team-a"), + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + routes: Final = { + host: respx_mock.post(host=f"{host}.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + for host in ("chat-sku", "batch-sku") + } + await router.acreate_batch( + model="gpt-mixed", + input_file_id="file-batch-input", + endpoint="/v1/chat/completions", + completion_window="24h", + metadata={"user_api_key_team_id": team_id}, + ) + + assert {host: route.call_count for host, route in routes.items()} == { + host: int(host == reached_host) for host in routes + } + + +@pytest.mark.asyncio +async def test_acreate_batch_on_a_legacy_routing_strategy_keeps_shared_deployments_for_callers_outside_the_batch_team( + monkeypatch: pytest.MonkeyPatch, +): + """usage-based-routing picks through the sync path, which keeps every team's deployments, so a batch deployment + owned by one team must not pull the creates of callers outside that team onto its credentials.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat"), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch", team_id="team-a"), + ], + routing_strategy="usage-based-routing", + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + for _ in range(30): + await router.acreate_batch( + model="gpt-mixed", + input_file_id="file-batch-input", + endpoint="/v1/chat/completions", + completion_window="24h", + metadata={"user_api_key_team_id": None}, + ) + + assert chat_route.call_count > 0 + + +@pytest.mark.parametrize( + ("build_deployment", "model_name"), + [ + (functools.partial(_azure_sku_deployment, "gpt-mixed"), "gpt-mixed"), + (_azure_wildcard_deployment, "mixed-wild/gpt-5.4-mini"), + ], + ids=["named-group", "wildcard-group"], +) +def test_credentials_for_a_mixed_group_name_keep_resolving_to_its_first_deployment( + build_deployment: Callable[[str, str], dict[str, object]], model_name: str +): + """Model-encoded file, batch, and vector store ids name only the group, and every read resolves their + credentials by that name, so adding a batch deployment to a group must not move ids created earlier + onto it.""" + router: Final = Router(model_list=[build_deployment("chat-sku", "chat"), build_deployment("batch-sku", "batch")]) + + credentials: Final = router.get_deployment_credentials_with_provider(model_id=model_name) + + assert credentials is not None and credentials["api_base"] == "https://chat-sku.openai.azure.com" + + +_AZURE_CHAT_SKU_REJECTS_BATCH: Final = httpx.Response( + 400, + json={"error": {"code": "invalid_deployment_type", "message": "The current deployment is not supported for batch jobs"}}, +) + + +@pytest.mark.asyncio +async def test_acreate_batch_on_a_mixed_group_reaches_chat_as_on_main_while_its_batch_deployment_cools_down( + monkeypatch: pytest.MonkeyPatch, +): + """While the batch deployment cools down, a create goes where main sends it, so the caller keeps getting + Azure's 400 instead of a router 429 that main never returned for this request.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _azure_sku_deployment("gpt-mixed", "chat-sku", "chat"), + _azure_sku_deployment("gpt-mixed", "batch-sku", "batch"), + ], + num_retries=0, + ) + _cool_down(router, *(d["model_info"]["id"] for d in router.model_list if d["model_info"]["mode"] == "batch")) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock(return_value=_AZURE_CHAT_SKU_REJECTS_BATCH) + with pytest.raises(openai.BadRequestError): + await _create_batches(router, "gpt-mixed", attempts=1) + + assert chat_route.call_count == 1 + + +@pytest.mark.asyncio +async def test_acreate_batch_with_tags_that_match_only_the_chat_deployment_reaches_it_as_on_main( + monkeypatch: pytest.MonkeyPatch, +): + """Tag routing runs before the batch filter, so tags that leave only the chat deployment get Azure's 400 as + on main instead of a router error saying no deployment matches the tags.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chat: Final = _azure_sku_deployment("gpt-mixed", "chat-sku", "chat") + batch: Final = _azure_sku_deployment("gpt-mixed", "batch-sku", "batch") + router: Final = litellm.Router( + model_list=[ + {**chat, "litellm_params": {**chat["litellm_params"], "tags": ["chat-pin"]}}, + {**batch, "litellm_params": {**batch["litellm_params"], "tags": ["batch-pin"]}}, + ], + enable_tag_filtering=True, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat-sku.openai.azure.com").mock(return_value=_AZURE_CHAT_SKU_REJECTS_BATCH) + batch_route: Final = respx_mock.post(host="batch-sku.openai.azure.com").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + with pytest.raises(openai.BadRequestError): + await router.acreate_batch( + model="gpt-mixed", + input_file_id="file-batch-input", + endpoint="/v1/chat/completions", + completion_window="24h", + litellm_metadata={"tags": ["chat-pin"]}, + ) + await router.acreate_batch( + model="gpt-mixed", + input_file_id="file-batch-input", + endpoint="/v1/chat/completions", + completion_window="24h", + litellm_metadata={"tags": ["batch-pin"]}, + ) + + assert (chat_route.call_count, batch_route.call_count) == (1, 1) + + +def _openai_deployment(model_group: str, host: str, mode: str, **model_info: object) -> dict[str, object]: + return { + "model_name": model_group, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_base": f"https://{host}/v1", "api_key": "dummy-key"}, + "model_info": {"mode": mode, **model_info}, + } + + +@pytest.mark.asyncio +async def test_acreate_batch_on_an_openai_group_falls_back_to_chat_while_its_batch_deployment_cools_down( + monkeypatch: pytest.MonkeyPatch, +): + """OpenAI runs batches on the deployment that serves chat, so only Azure groups skip chat deployments for + batch creates and an OpenAI group keeps serving them while its batch deployment cools down.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("gpt-mixed", "chat.openai.test", "chat"), + _openai_deployment("gpt-mixed", "batch.openai.test", "batch"), + ], + num_retries=0, + ) + _cool_down(router, *(d["model_info"]["id"] for d in router.model_list if d["model_info"]["mode"] == "batch")) + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post(host="chat.openai.test").mock( + return_value=httpx.Response(200, json=_AZURE_BATCH_CREATED) + ) + await _create_batches(router, "gpt-mixed", attempts=5) + + assert chat_route.call_count == 5 + + @pytest.mark.asyncio async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): import httpx diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 88d54d3731a..bf2fe4300a3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38261,6 +38261,8 @@ export interface components { providers: string[]; /** Rpm */ rpm?: number | null; + /** Supported Endpoints */ + supported_endpoints?: string[] | null; /** * Supported Openai Params * @default [] From 312479739914ad46d37696308ec26bcdb652b7a8 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 20 Oct 2026 00:10:49 -0700 Subject: [PATCH 075/124] fix(bedrock): resolve profile pricing before applying custom rates (#47419) * fix(bedrock): price Converse calls using the invoked model profile * fix(pricing): retain registered rates across metadata updates * fix(pricing): honor temporary registered prices for Bedrock profiles * fix(bedrock): record invocation model ID for each completion attempt * test(bedrock): recalculate fallback cost after metadata updates * fix(pricing): preserve cache-only registered price overrides * fix(bedrock): preserve priced overrides without widening shared pricing * fix(bedrock): retain explicit pricing through token cost lookup * fix(bedrock): layer cache overrides over invoked profile rates * fix(bedrock): keep effective prices consistent across billing * fix(bedrock): share profile pricing and custom rate resolution * fix(router): remove unused pricing suppression * fix(cost): preserve rates across mixed token thresholds * fix(router): share deployment pricing during startup * fix(bedrock): preserve Router pricing inheritance for profiles * fix(bedrock): retain configured pricing for unmapped profiles --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 88 ++- .../litellm_core_utils/get_model_cost_map.py | 7 +- .../litellm_core_utils/llm_cost_calc/utils.py | 25 +- litellm/main.py | 1 + litellm/router.py | 83 +-- litellm/utils.py | 89 ++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 82 +++ .../chat/test_bedrock_converse_handler.py | 41 ++ tests/unit/test_cost_calculator.py | 559 ++++++++++++++++++ tests/unit/test_main.py | 89 +++ .../unit/test_router_model_cost_isolation.py | 181 +++++- tests/unit/test_utils.py | 39 +- 12 files changed, 1202 insertions(+), 82 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4c6d36c06db..20914df8fe2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -138,6 +138,7 @@ from litellm.types.utils import ( TranscriptionUsageTokensObject, Usage, VectorStoreSearchResponse, + is_server_derived_pricing_key, ) from litellm.utils import ( CallTypes, @@ -150,6 +151,8 @@ from litellm.utils import ( TextCompletionResponse, TranscriptionResponse, cached_get_model_info_helper, + model_info_with_pricing_overrides, + runtime_model_cost_registrations, token_counter, unmapped_model_generalization, ) @@ -438,6 +441,7 @@ def cost_per_token( ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection custom_model_info: OCRPricing | None = None, + bedrock_model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -740,6 +744,15 @@ def cost_per_token( elif custom_llm_provider == "anthropic": return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) elif custom_llm_provider == "bedrock": + resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False) + if resolved_bedrock_info is not None: + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider="bedrock", + service_tier=service_tier, + model_info=resolved_bedrock_info, + ) return bedrock_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) elif custom_llm_provider == "openai": return openai_cost_per_token( @@ -862,6 +875,37 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool: ) +def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool: + return _cost_map_entry_prices_anything(entry) or any( + value is not None and field.startswith("cache_") and is_server_derived_pricing_key(field) + for field, value in entry.items() + ) + + +def _bedrock_pricing_info( + model: str | None, + invoked_model_id: str | None, + custom_llm_provider: str | None, + explicit_pricing: bool, +) -> ModelInfo | None: + if explicit_pricing or custom_llm_provider != "bedrock" or not model: + return None + registration: Final = next( + ( + (key, entry) + for key, entry in runtime_model_cost_registrations(model, "bedrock") + if _bedrock_registration_has_prices(entry) + ), + None, + ) + profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None + if profile is None or not _declares_rates(profile): + return _cost_map_model_info(registration[0], None) if registration is not None else None + if registration is None or registration[0] == profile["key"]: + return profile + return model_info_with_pricing_overrides(profile, registration[1]) + + def _declares_rates(model_info: ModelInfo | None) -> bool: entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None return entry is not None and _cost_map_entry_prices_anything(entry) @@ -888,6 +932,16 @@ def _pricing_provider_for_cost_candidate( return current_provider if isinstance(current_provider, str) else None +def _invoked_model_id_for_cost( + optional_params: object, + logging_obj: LitellmLoggingObject | None, +) -> str | None: + recorded: Final[object] = getattr(logging_obj, "model_call_details", None) + return _get_hidden_str_for_cost_calc(optional_params, "model_id") or _get_hidden_str_for_cost_calc( + recorded, "model_id" + ) + + def select_model_name_for_cost_calc( model: str | None, completion_response: object | None, @@ -896,6 +950,7 @@ def select_model_name_for_cost_calc( custom_llm_provider: str | None = None, router_model_id: str | None = None, region_name: str | None = None, + invoked_model_id: str | None = None, ) -> str | None: """ 1. If custom pricing is true, return received model name @@ -916,6 +971,13 @@ def select_model_name_for_cost_calc( hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None) provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model") explicit_pricing: Final = custom_pricing is True or base_model is not None + configured_model: Final = model or ( + completion_response_model if isinstance(completion_response_model, str) else None + ) + bedrock_model_info: Final = _bedrock_pricing_info( + configured_model, invoked_model_id, custom_llm_provider, explicit_pricing + ) + bedrock_pricing_model: Final = bedrock_model_info["key"] if bedrock_model_info is not None else None priced_from_response: Final = provider_response_model is not None or completion_response_model is not None priced_region: Final = ( _get_hidden_str_for_cost_calc(hidden_params, "region_name") or region_name @@ -933,8 +995,8 @@ def select_model_name_for_cost_calc( else: return_model = model - elif base_model is not None or provider_response_model is not None: - return_model = base_model if base_model is not None else provider_response_model + elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None: + return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model elif completion_response_model is None and hidden_params is not None: if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0: @@ -1548,6 +1610,10 @@ def completion_cost( if base_model is not None and custom_pricing is not True else None ) + invoked_model_id: Final = _invoked_model_id_for_cost( + _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None, + litellm_logging_obj, + ) selected_model: Final = select_model_name_for_cost_calc( model=model, completion_response=completion_response, @@ -1556,8 +1622,15 @@ def completion_cost( base_model=base_model, router_model_id=router_model_id, region_name=region_name, + invoked_model_id=invoked_model_id, ) + bedrock_model_info: Final = _bedrock_pricing_info( + model or _get_response_model(completion_response), + invoked_model_id, + _get_provider_for_cost_calc(model, custom_llm_provider), + explicit_pricing, + ) requested_model: Final = model request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None raw_candidates: Final = ( @@ -1946,6 +2019,7 @@ def completion_cost( vertex_location=vertex_location, response=completion_response, custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id), + bedrock_model_info=bedrock_model_info, ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) @@ -2037,6 +2111,7 @@ def completion_cost( data_residency=data_residency, vertex_location=vertex_location, custom_cost_per_token=custom_cost_per_token, + model_info=bedrock_model_info, ) _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost @@ -2339,7 +2414,16 @@ def pricing_entry_for_cost_calc( custom_llm_provider=custom_llm_provider, router_model_id=router_model_id, region_name=region_name, + invoked_model_id=_invoked_model_id_for_cost(None, litellm_logging_obj), ) + bedrock_model_info: Final = _bedrock_pricing_info( + model or _get_response_model(completion_response), + _invoked_model_id_for_cost(None, litellm_logging_obj), + _get_provider_for_cost_calc(model, custom_llm_provider), + custom_pricing is True or base_model is not None, + ) + if bedrock_model_info is not None: + return bedrock_model_info["key"], bedrock_model_info candidates: Final = (selected_model, _get_response_model(completion_response), model) resolved: Final = next( (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info), diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 073e3a3455e..7b3835dc1ac 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -558,11 +558,14 @@ def get_model_cost_map_loaded_at() -> "datetime | None": class AliasedCostEntry(dict[str, object]): - __slots__ = ("names",) + __slots__ = ("names", "registered_fields") - def __init__(self, entry: Mapping[str, object], names: tuple[str, ...]) -> None: + def __init__( + self, entry: Mapping[str, object], names: tuple[str, ...], registered_fields: frozenset[str] = frozenset() + ) -> None: super().__init__(entry) self.names: tuple[str, ...] = names + self.registered_fields: frozenset[str] = registered_fields def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 31da260588c..a874459798d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -285,7 +285,7 @@ def _parse_token_threshold(threshold: str) -> float: def _parse_above_token_threshold(key: str) -> float: - return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0]) + return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0]) def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: @@ -748,10 +748,12 @@ def _get_token_base_cost( # Optimization: collect threshold keys first to avoid sorting all model_info keys. # Standard thresholds and thresholds suffixed for this request's service tier both count. tier_key_suffix: Final = get_service_tier_cost_key("", service_tier) + threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES) threshold_keys: Final = [ k for k in model_info - if k.startswith("input_cost_per_token_above_") + if k.startswith(threshold_prefixes) + and "_tokens" in k and ( not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix)) ) @@ -759,12 +761,12 @@ def _get_token_base_cost( # Only sort the threshold keys (typically 1-2 keys instead of 66+) threshold: float | None = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True): + for key in sorted(threshold_keys, key=_parse_above_token_threshold): value = model_info.get(key) if value is not None: try: # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.split("_above_")[1].split("_tokens")[0] + threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0] threshold = _parse_above_token_threshold(key) if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): # Prefer a service_tier-specific above-threshold key when available, @@ -777,7 +779,7 @@ def _get_token_base_cost( service_tier, ) if service_tier - else key + else f"input_cost_per_token_above_{threshold_str}_tokens" ) prompt_base_cost = cast( float, @@ -833,8 +835,6 @@ def _get_token_base_cost( ) cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) - - break except (IndexError, ValueError): continue except Exception: @@ -1780,20 +1780,23 @@ def get_billed_token_rates( vertex_location: str | None = None, current_time: datetime | None = None, custom_cost_per_token: CostPerToken | None = None, + model_info: ModelInfo | None = None, ) -> BilledTokenRates | None: """Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type breakdown resolve them. None when the model's pricing cannot be resolved.""" if custom_cost_per_token is not None: return _custom_pricing_rates(custom_cost_per_token) try: - model_info: Final = apply_provider_cache_read_default( - get_model_info(model=model, custom_llm_provider=custom_llm_provider), + resolved_model_info: Final = apply_provider_cache_read_default( + model_info + if model_info is not None + else get_model_info(model=model, custom_llm_provider=custom_llm_provider), custom_llm_provider, ) except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates return None return _cost_map_billed_rates( - model_info=model_info, + model_info=resolved_model_info, usage=usage, custom_llm_provider=custom_llm_provider, service_tier=service_tier, @@ -1812,6 +1815,7 @@ def get_token_type_cost_breakdown( vertex_location: str | None = None, current_time: datetime | None = None, custom_cost_per_token: CostPerToken | None = None, + model_info: ModelInfo | None = None, ) -> TokenTypeCostBreakdown: """ Provider-agnostic cost of reasoning and cache tokens, derived from the usage @@ -1836,6 +1840,7 @@ def get_token_type_cost_breakdown( vertex_location=vertex_location, current_time=current_time, custom_cost_per_token=custom_cost_per_token, + model_info=model_info, ) if rates is None: return TokenTypeCostBreakdown(0.0, 0.0, 0.0) diff --git a/litellm/main.py b/litellm/main.py index 65c504072a9..71cb7af1fbb 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6012,6 +6012,7 @@ def completion( optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, + model_id=optional_params.get("model_id"), ) if mock_response or mock_tool_calls or mock_timeout: kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered diff --git a/litellm/router.py b/litellm/router.py index 54d4cce4656..e3c1e56b7e8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -403,6 +403,7 @@ from litellm.types.utils import ( StandardLoggingRoutingDecision, Usage, all_litellm_params, + is_server_derived_pricing_key, shared_backend_model_info, ) from litellm.types.utils import ModelInfo as ModelMapInfo @@ -410,6 +411,7 @@ from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES, decode_video_id_with from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, + ModelInfoLookupMiss, ModelResponse, Rules, TextCompletionResponse, @@ -420,6 +422,8 @@ from litellm.utils import ( get_utc_datetime, is_generalized_model_info, is_region_allowed, + lookup_model_info_for_registration, + model_info_with_pricing_overrides, provider_rejectable_params, resolve_api_key_secret_reference, set_live_deployment_replay, @@ -9900,6 +9904,20 @@ class Router: params_base_model: Final = deployment.litellm_params.get("base_model") if isinstance(params_base_model, str) and params_base_model: return params_base_model + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + invoked_model: Final = params.get("model_id") + if ( + isinstance(invoked_model, str) + and invoked_model + and ( + deployment.litellm_params.custom_llm_provider == "bedrock" + or deployment.litellm_params.model.startswith("bedrock/") + ) + ): + profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}" + profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock") + if not isinstance(profile_info, ModelInfoLookupMiss): + return profile_model return deployment.litellm_params.model @staticmethod @@ -10077,39 +10095,7 @@ class Router: model_info=_model_info, ) ) - for field in CustomPricingLiteLLMParams.model_fields: - if deployment.litellm_params.get(field) is not None: - _model_info[field] = deployment.litellm_params[field] - - Router._inherit_builtin_base_rates_for_off_peak( - model_info=_model_info, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - if _model_info.get("input_cost_per_token") is not None: - Router._inherit_builtin_cache_pricing( - model_info=_model_info, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - Router._inherit_builtin_service_tier_pricing( - model_info=_model_info, - backend_model=Router._cost_map_backend_model(deployment), - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - Router._inherit_builtin_tiered_output_rate( - model_info=_model_info, - backend_model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) - - ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP - Router._register_deployment_in_model_cost( - model_id=deployment.model_info.id, - model_info=_model_info, - model=deployment.litellm_params.model, - custom_llm_provider=deployment.litellm_params.custom_llm_provider, - ) + Router._register_deployment_pricing(deployment) ## Check if LLM Deployment is allowed for this deployment if self.deployment_is_active_for_environment(deployment=deployment) is not True: @@ -11139,7 +11125,7 @@ class Router: return (backend_key,) @staticmethod - def _deployment_model_cost_payload(deployment: Deployment) -> dict: # mutable-ok: cost-map entry + def _deployment_model_cost_payload(deployment: Deployment) -> dict[str, object]: # mutable-ok: cost-map entry """The ``model_info`` a deployment contributes to ``litellm.model_cost``. Custom pricing lives on ``litellm_params`` rather than ``model_info``, and @@ -11147,30 +11133,49 @@ class Router: both are folded back in here. That keeps this reproducible from a deployment alone, which is what lets a refresh rebuild the same entries. """ - model_info: Final[dict] = deployment.model_info.model_dump(exclude_none=True) # mutable-ok: built in place + model_info: Final = dict( + _MODEL_INFO_ADAPTER.validate_python(deployment.model_info.model_dump(exclude_none=True)) + ) for field in CustomPricingLiteLLMParams.model_fields: field_value = deployment.litellm_params.get(field) if field_value is not None: model_info[field] = field_value + backend_model: Final = Router._cost_map_backend_model(deployment) + params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True)) + if ( + params.get("model_id") + and backend_model != deployment.litellm_params.model + and ( + deployment.litellm_params.custom_llm_provider == "bedrock" + or deployment.litellm_params.model.startswith("bedrock/") + ) + ): + profile_info: Final = lookup_model_info_for_registration(model=backend_model, custom_llm_provider="bedrock") + if not isinstance(profile_info, ModelInfoLookupMiss): + explicit_info: Final = _MODEL_INFO_ADAPTER.validate_python(model_info) + priced_info: Final = model_info_with_pricing_overrides(profile_info, explicit_info) + model_info.update( + {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} + ) Router._inherit_builtin_base_rates_for_off_peak( model_info=model_info, - backend_model=deployment.litellm_params.model, + backend_model=backend_model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) if model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( model_info=model_info, - backend_model=deployment.litellm_params.model, + backend_model=backend_model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) Router._inherit_builtin_service_tier_pricing( model_info=model_info, - backend_model=Router._cost_map_backend_model(deployment), + backend_model=backend_model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) Router._inherit_builtin_tiered_output_rate( model_info=model_info, - backend_model=deployment.litellm_params.model, + backend_model=backend_model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) return model_info diff --git a/litellm/utils.py b/litellm/utils.py index fc47a4dba2e..27c9667f032 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3411,14 +3411,20 @@ def _builtin_cost_map() -> dict[str, Mapping[str, object]]: return cast(dict[str, Mapping[str, object]], litellm.model_cost) -def _replace_cost_map_entry(cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object]) -> None: +def _replace_cost_map_entry( + cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object], registration: Mapping[str, object] +) -> None: current: Final = cost_map.get(key) merged: Final = {**(current or {}), **entry} - if not isinstance(current, AliasedCostEntry): - cost_map.update({key: merged}) - return - sharing_names: Final = tuple(name for name in current.names if cost_map.get(name) is current) - cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, current.names))) + names: Final = current.names if isinstance(current, AliasedCostEntry) else () + previous_fields: Final[frozenset[str]] = ( + current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset() + ) + registered_fields: Final = previous_fields | frozenset( + name for name, value in registration.items() if value is not None + ) + sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current) + cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields))) def _without_synthesized_costs( @@ -3499,6 +3505,25 @@ def _without_overridden_price_variants( ) +@with_config(ConfigDict(extra="allow")) +class _PricingModelInfo(ModelInfo): + pass + + +_PRICING_MODEL_INFO: Final = TypeAdapter(_PricingModelInfo) + + +def model_info_with_pricing_overrides(model_info: ModelInfo, registration: Mapping[str, object]) -> ModelInfo: + from litellm.types.utils import is_server_derived_pricing_key + + overrides: Final = { + name: value for name, value in registration.items() if value is not None and is_server_derived_pricing_key(name) + } + return _PRICING_MODEL_INFO.validate_python( + update_dictionary(dict(_without_overridden_price_variants(model_info, overrides)), overrides) + ) + + @dataclass(frozen=True, slots=True) class _BuiltinCostMatch: key: str @@ -3507,7 +3532,7 @@ class _BuiltinCostMatch: @dataclass(frozen=True, slots=True) -class _LookupMiss: +class ModelInfoLookupMiss: tripped: bool @@ -3594,17 +3619,19 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 -def _lookup_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | _LookupMiss: +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: try: info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) except ModelNotMappedError: - return _LookupMiss(tripped=False) + return ModelInfoLookupMiss(tripped=False) except Exception: - return _LookupMiss(tripped=True) - return _LookupMiss(tripped=False) if is_generalized_model_info(info) else info + return ModelInfoLookupMiss(tripped=True) + return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info -def _get_builtin_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | _LookupMiss: +def _get_builtin_model_info_for_registration( + model: str, custom_llm_provider: str | None +) -> ModelInfo | ModelInfoLookupMiss: """Resolve ``model`` to its built-in cost-map entry for registration merging. Misses when the lookup cannot map the model or resolved it via a @@ -3613,10 +3640,12 @@ def _get_builtin_model_info_for_registration(model: str, custom_llm_provider: st changing the map mid-walk, is retried, so a transient failure registers exactly where the untripped lookup would. """ - lookups: Final = (_lookup_for_registration(model, custom_llm_provider) for _ in range(_REGISTRATION_ATTEMPTS)) + lookups: Final = ( + lookup_model_info_for_registration(model, custom_llm_provider) for _ in range(_REGISTRATION_ATTEMPTS) + ) return next( - (lookup for lookup in lookups if not (isinstance(lookup, _LookupMiss) and lookup.tripped)), - _LookupMiss(tripped=True), + (lookup for lookup in lookups if not (isinstance(lookup, ModelInfoLookupMiss) and lookup.tripped)), + ModelInfoLookupMiss(tripped=True), ) @@ -3651,7 +3680,7 @@ def _registration_base( builtin_model_info: Final = _get_builtin_model_info_for_registration( model=key, custom_llm_provider=custom_llm_provider ) - if not isinstance(builtin_model_info, _LookupMiss): + if not isinstance(builtin_model_info, ModelInfoLookupMiss): return _RegistrationWrite( key=builtin_model_info["key"], entry=dict(_without_synthesized_costs(builtin_model_info, key=key, value=value)), @@ -3708,7 +3737,7 @@ def _registration_attempt( ) -> _RegistrationWrite | None: source: Final = _builtin_cost_map() write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name) - _replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry) + _replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry, value) return write if final_attempt or litellm.model_cost is source else None @@ -3731,6 +3760,27 @@ def _register_on_current_map( _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload +def runtime_model_cost_registrations( + model: str, custom_llm_provider: str +) -> tuple[tuple[str, Mapping[str, object]], ...]: + names: Final = get_potential_model_names(model, custom_llm_provider) + candidates: Final = ( + model, + names["combined_model_name"], + f"{custom_llm_provider}/{names['split_model']}", + names["split_model"], + names["stripped_model_name"], + ) + entries: Final = ((key, _builtin_cost_map().get(key)) for key in dict.fromkeys(candidates)) + registered: Final = ( + (key, entry) for key, entry in entries if isinstance(entry, AliasedCostEntry) and entry.registered_fields + ) + return tuple( + (key, MappingProxyType({name: value for name, value in entry.items() if name in entry.registered_fields})) + for key, entry in registered + ) + + class _LiveDeploymentReplay: """Single-slot holder for the callback that rebuilds live router deployments. @@ -3843,7 +3893,10 @@ def register_model( if persist_across_reloads: _registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost for _registered_key, _registered_value in _registrations.items(): - _runtime_registered_model_cost[_registered_key] = dict(_registered_value) + _runtime_registered_model_cost[_registered_key] = update_dictionary( + dict(_runtime_registered_model_cost.get(_registered_key, {})), + _RATE_TABLE.validate_python(_registered_value), + ) for key, value in loaded_model_cost.items(): write = _register_on_current_map(str(key), value, custom_llm_provider, warning_display_name) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f14f3c5f7d1..c99147d2e63 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2548,6 +2548,63 @@ def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base(): assert get_cost_per_unit(model_info_direct, "input_cost_per_token_priority") == 5e-6 +@pytest.mark.parametrize( + "field", ("output_cost_per_token", "cache_read_input_token_cost", "cache_creation_input_token_cost") +) +def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: + from typing import Final, cast + from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost + + info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "cache_read_input_token_cost": 0.003, + "cache_creation_input_token_cost": 0.004, + f"{field}_above_300k_tokens": 0.005, + }, + ) + rates: Final = _get_token_base_cost( + model_info=info, + usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), + ) + assert rates[0] == 0.001 + assert rates[1] == (0.005 if field == "output_cost_per_token" else 0.002) + assert rates[2] == (0.005 if field == "cache_creation_input_token_cost" else 0.004) + assert rates[4] == (0.005 if field == "cache_read_input_token_cost" else 0.003) + + +@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) +@pytest.mark.parametrize("higher_rate", (0.0, 0.009)) +def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, higher_rate: float) -> None: + from typing import Final, cast + from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost + + info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "cache_read_input_token_cost": 0.003, + "cache_creation_input_token_cost": 0.004, + "input_cost_per_token_above_200k_tokens": 0.005, + "output_cost_per_token_above_200k_tokens": 0.006, + "cache_creation_input_token_cost_above_200k_tokens": 0.007, + "cache_read_input_token_cost_above_512k_tokens": higher_rate, + "output_cost_per_token_above_512k_tokens": higher_rate, + }, + ) + rates: Final = _get_token_base_cost( + model_info=info, + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), + ) + assert rates[0] == (0.005 if prompt_tokens > 200_000 else 0.001) + assert rates[1] == (higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002) + assert rates[2] == (0.007 if prompt_tokens > 200_000 else 0.004) + assert rates[4] == (higher_rate if prompt_tokens > 512_000 else 0.003) + + def test_threshold_keys_exclude_service_tier_variants(): from typing import cast @@ -4138,3 +4195,28 @@ def test_vertex_gemini_4_argon_matches_gemini_api_global_pricing(_local_model_co key: gemini_row[key] for key in GEMINI_4_ARGON_TOKEN_PRICE_KEYS } assert vertex_row["litellm_provider"].startswith("vertex_ai") + + +@pytest.mark.parametrize("rate", (0.0, 0.001)) +def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: float) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "cache_read_input_token_cost": rate, + "cache_creation_input_token_cost": rate, + } + usage: Final = Usage( + prompt_tokens=20, completion_tokens=3, total_tokens=23, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5, + ) + prompt_cost, output_cost = generic_cost_per_token( + model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + ) + breakdown: Final = get_token_type_cost_breakdown( + model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + ) + assert breakdown.cache_read_cost == pytest.approx(5 * rate) + assert breakdown.cache_creation_cost == pytest.approx(5 * rate) + assert breakdown.rates is not None + assert prompt_cost == pytest.approx(10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost) + assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token) diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 1fa83254822..7d1b502df2b 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -918,3 +918,44 @@ def test_sync_converse_streaming_upstream_error_carries_the_upstream_body_as_tex assert raised.value.status_code == status_code assert raised.value.message == f"litellm.{error_class.__name__}: BedrockException - {_UPSTREAM_ERROR_BODY}" + + +@pytest.mark.parametrize("stream", (False, True)) +def test_converse_profile_override_records_cost_without_changing_response_model( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setenv("LITELLM_RUST", "0") + configuration.rust(False) + model: Final = "openai.gpt-5.6-sol" + profile: Final = f"us.{model}" + prices: Final = litellm.get_model_info(profile, custom_llm_provider="bedrock") + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == f"/model/{profile}/converse" + return httpx.Response(200, json=CONVERSE_RESPONSE) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + response: Final = litellm.completion( + model=f"bedrock/converse/{model}", + model_id=profile, + messages=[{"role": "user", "content": "hi"}], + api_key="test-bearer", + aws_region_name="us-east-1", + client=HTTPHandler(client=transport), + stream=stream, + fake_stream=stream, + max_tokens=16, + ) + result: Final = litellm.stream_chunk_builder(list(response)) if stream else response + + assert result is not None + assert result.model == model + assert result.choices[0].message.content == "hi" + cost: Final = ( + response.logging_obj.response_cost_calculator(result) if stream else result._hidden_params.get("response_cost") + ) + assert cost == pytest.approx( + CONVERSE_RESPONSE["usage"]["inputTokens"] * prices["input_cost_per_token"] + + CONVERSE_RESPONSE["usage"]["outputTokens"] * prices["output_cost_per_token"] + ) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 619d91e478c..99119799ac0 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -8868,6 +8868,37 @@ async def test_custom_pricing_matches_response_usage(async_mode: bool) -> None: assert response._hidden_params["response_cost"] == pytest.approx(_CUSTOM_PRICING_EXPECTED_COST) +@pytest.mark.parametrize("rate", (0.0, 0.001)) +@pytest.mark.parametrize("cache_field", ("cache_read_input_token_cost", "cache_creation_input_token_cost")) +@pytest.mark.usefixtures("isolated_pricing_registry") +@respx.mock +def test_router_cache_only_pricing_preserves_uncached_token_charges(rate: float, cache_field: str) -> None: + litellm.register_model( + {_CUSTOM_PRICING_MODEL: {**_CUSTOM_PRICING_RATES, "litellm_provider": "openai", "mode": "chat"}}, + persist_across_reloads=False, + ) + respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=_custom_pricing_upstream) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "cache-priced-deployment", + "litellm_params": { + "model": f"openai/{_CUSTOM_PRICING_MODEL}", + "api_key": "sk-test", + cache_field: rate, + }, + "model_info": {"id": "cache-only-deployment"}, + } + ] + ) + + response: Final = router.completion( + model="cache-priced-deployment", messages=[{"role": "user", "content": "How are you?"}] + ) + + assert response._hidden_params["response_cost"] == pytest.approx(_CUSTOM_PRICING_EXPECTED_COST) + + @pytest.mark.usefixtures("isolated_pricing_registry") @respx.mock def test_streamed_custom_pricing_cost_matches_non_streamed_cost() -> None: @@ -9743,3 +9774,531 @@ def test_responses_ws_logging_object_carries_the_completed_responses_id_tier_and assert (logged.id, logged.service_tier) == ("resp_a", "priority") assert (logged.usage.prompt_tokens, logged.usage.completion_tokens, logged.usage.total_tokens) == (10, 5, 15) assert ResponsesWebSocketTokenUsageProcessor.create_logging_object([{"type": "error", "error": {}}]).id is None + + +@pytest.mark.parametrize("profile_prefix", ("us", "global")) +@pytest.mark.parametrize( + "model_name", + ("gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-sol", "gpt-6-luna", "gpt-6-astra", "gpt-6.1-sol"), +) +@pytest.mark.parametrize("arn", (False, True)) +def test_bedrock_converse_cost_uses_invoked_profile( + _local_model_cost_map: None, model_name: str, profile_prefix: str, arn: bool +) -> None: + model: Final = f"openai.{model_name}" + profile: Final = f"{profile_prefix}.{model}" + invoked: Final = f"arn:aws:bedrock:us-east-1:123456789012:inference-profile/{profile}" if arn else profile + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + prices: Final = litellm.get_model_info(profile, custom_llm_provider="bedrock") + + cost: Final = completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": invoked}, + ) + + assert cost == pytest.approx(10 * prices["input_cost_per_token"] + 5 * prices["output_cost_per_token"]) + assert response.model == model + + +@pytest.mark.parametrize("custom_pricing", (False, True)) +def test_bedrock_converse_profile_preserves_explicit_base_and_custom_pricing( + _local_model_cost_map: None, custom_pricing: bool +) -> None: + model: Final = "openai.gpt-6-sol" + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + prices: Final = litellm.get_model_info(model, custom_llm_provider="bedrock") + + cost: Final = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="bedrock", + optional_params={"model_id": f"us.{model}"}, + custom_pricing=custom_pricing, + base_model=None if custom_pricing else model, + ) + + assert cost == pytest.approx(10 * prices["input_cost_per_token"] + 5 * prices["output_cost_per_token"]) + + +@pytest.mark.parametrize("stream", (False, True)) +def test_bedrock_converse_cost_retains_profile_after_dispatch_consumes_optional_params( + _local_model_cost_map: None, stream: bool +) -> None: + from litellm.cost_calculator import pricing_entry_for_cost_calc + + model: Final = "openai.gpt-5.6-sol" + profile: Final = f"us.{model}" + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="bedrock-profile-cost", + function_id="bedrock-profile-cost", + ) + logging_obj.update_environment_variables( + litellm_params={}, optional_params={"model_id": profile}, custom_llm_provider="bedrock" + ) + logging_obj.optional_params.pop("model_id") + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + prices: Final = litellm.get_model_info(profile, custom_llm_provider="bedrock") + + assert logging_obj.response_cost_calculator(response) == pytest.approx( + 10 * prices["input_cost_per_token"] + 5 * prices["output_cost_per_token"] + ) + entry: Final = pricing_entry_for_cost_calc(model, response, "bedrock", False, None, None, None, logging_obj) + assert entry is not None + assert entry[0] == prices["key"] + + +@pytest.mark.parametrize( + "invoked_model_id", + (None, "", 123, "unmapped-profile", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/opaque"), +) +def test_bedrock_converse_cost_keeps_configured_rates_when_target_has_no_prices( + _local_model_cost_map: None, invoked_model_id: object +) -> None: + model: Final = "openai.gpt-6-sol" + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + expected: Final = completion_cost(completion_response=response, model=model, custom_llm_provider="bedrock") + + assert ( + completion_cost( + completion_response=response, + model=model, + custom_llm_provider="bedrock", + optional_params={"model_id": invoked_model_id}, + ) + == expected + ) + + +def test_non_bedrock_cost_ignores_bedrock_model_id(_local_model_cost_map: None) -> None: + response: Final = ModelResponse( + model="gpt-6-sol", usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + ) + expected: Final = completion_cost(completion_response=response, custom_llm_provider="openai") + + assert ( + completion_cost( + completion_response=response, + custom_llm_provider="openai", + optional_params={"model_id": "us.openai.gpt-6-sol"}, + ) + == expected + ) + + +@pytest.mark.parametrize("rate", (0.0, 0.001)) +@pytest.mark.parametrize("persist_across_reloads", (True, False)) +@pytest.mark.parametrize("routed_response", (True, False)) +@pytest.mark.parametrize("metadata_prefix", ("bedrock/", "bedrock/converse/")) +def test_bedrock_converse_profile_preserves_registered_price_override( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + rate: float, + persist_across_reloads: bool, + routed_response: bool, + metadata_prefix: str, +) -> None: + import litellm.utils as model_utils + from litellm.cost_calculator import pricing_entry_for_cost_calc + + monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) + catalog: Final = dict(litellm.model_cost) + model: Final = "openai.gpt-5.6-sol" + litellm.register_model( + {f"{metadata_prefix}{model}": {"supports_function_calling": True}}, + persist_across_reloads=persist_across_reloads, + ) + litellm.register_model( + { + model: { + "input_cost_per_token": rate, + "output_cost_per_token": rate, + "litellm_provider": "bedrock_converse", + "mode": "chat", + } + }, + persist_across_reloads=persist_across_reloads, + ) + litellm.register_model( + {model: {"supports_function_calling": True, "input_cost_per_token": None}}, + persist_across_reloads=persist_across_reloads, + ) + response: Final = ModelResponse( + model=f"bedrock/converse/{model}" if routed_response else model, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="registered-pricing", + function_id="registered-pricing", + ) + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": f"us.{model}"}, + litellm_logging_obj=logging_obj, + ) == pytest.approx(15 * rate) + + assert logging_obj.billed_token_rates is not None + assert logging_obj.billed_token_rates.input_cost_per_token == rate + assert logging_obj.billed_token_rates.output_cost_per_token == rate + entry: Final = pricing_entry_for_cost_calc(model, response, "bedrock", False, None, None, None, None) + assert entry is not None + assert entry[0] == model + assert entry[1]["input_cost_per_token"] == rate + assert entry[1]["output_cost_per_token"] == rate + + updated_rate: Final = rate + 0.002 + litellm.register_model( + {model: {"input_cost_per_token": str(updated_rate)}}, persist_across_reloads=persist_across_reloads + ) + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": f"us.{model}"}, + ) == pytest.approx(10 * updated_rate + 5 * rate) + + monkeypatch.setattr(litellm, "model_cost", catalog) + model_utils._invalidate_model_cost_lowercase_map() + model_utils.reapply_runtime_model_cost_registrations() + profile_prices: Final = litellm.get_model_info(f"us.{model}", custom_llm_provider="bedrock") + expected_after_reload: Final = ( + 10 * updated_rate + 5 * rate + if persist_across_reloads + else 10 * profile_prices["input_cost_per_token"] + 5 * profile_prices["output_cost_per_token"] + ) + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": f"us.{model}"}, + ) == pytest.approx(expected_after_reload) + + +def test_bedrock_converse_profile_ignores_registered_capabilities_without_prices( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.utils as model_utils + + monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) + model: Final = "openai.gpt-6-sol" + litellm.register_model({model: {"supports_function_calling": True}}) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + prices: Final = litellm.get_model_info(f"us.{model}", custom_llm_provider="bedrock") + + assert completion_cost( + completion_response=response, + model=model, + custom_llm_provider="bedrock", + optional_params={"model_id": f"us.{model}"}, + ) == pytest.approx(10 * prices["input_cost_per_token"] + 5 * prices["output_cost_per_token"]) + + +@pytest.mark.parametrize("rate", (0.0, 0.001)) +@pytest.mark.parametrize("cache_field", ("cache_read_input_token_cost", "cache_creation_input_token_cost")) +@pytest.mark.parametrize("has_base_rates", (False, True)) +@pytest.mark.parametrize("persist_across_reloads", (False, True)) +@pytest.mark.parametrize("profile_prefix", ("us", "global")) +@pytest.mark.parametrize("provider", (None, "bedrock")) +def test_bedrock_converse_profile_preserves_cache_only_registered_prices( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + rate: float, + cache_field: str, + has_base_rates: bool, + persist_across_reloads: bool, + profile_prefix: str, + provider: str | None, +) -> None: + import litellm.utils as model_utils + from litellm.cost_calculator import pricing_entry_for_cost_calc + + monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) + model: Final = "openai.cache-overlay-test" + profile: Final = f"{profile_prefix}.{model}" + catalog: Final = { + **litellm.model_cost, + model: { + "litellm_provider": "bedrock", + "mode": "chat", + **({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} if has_base_rates else {}), + }, + profile: { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.02, + "output_cost_per_token": 0.04, + "cache_read_input_token_cost": 0.005, + "cache_creation_input_token_cost": 0.025, + }, + } + profile_before: Final = dict(catalog[profile]) + monkeypatch.setattr(litellm, "model_cost", dict(catalog)) + model_utils._invalidate_model_cost_lowercase_map() + litellm.register_model({model: {cache_field: rate}}, persist_across_reloads=persist_across_reloads) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=15, + completion_tokens=3, + total_tokens=18, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=5 if cache_field == "cache_read_input_token_cost" else 0 + ), + cache_creation_input_tokens=5 if cache_field == "cache_creation_input_token_cost" else 0, + ), + ) + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="cache-overlay", + function_id="cache-overlay", + ) + logging_obj.model_call_details["model_id"] = profile + expected: Final = 10 * 0.02 + 3 * 0.04 + 5 * rate + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider=provider, + optional_params={"model_id": profile}, + litellm_logging_obj=logging_obj, + ) == pytest.approx(expected) + breakdown_key: Final = "cache_read_cost" if cache_field == "cache_read_input_token_cost" else "cache_creation_cost" + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown.get(breakdown_key, 0.0) == pytest.approx(5 * rate) + assert logging_obj.billed_token_rates is not None + assert ( + logging_obj.billed_token_rates.cache_read_input_token_cost + if cache_field == "cache_read_input_token_cost" + else logging_obj.billed_token_rates.cache_creation_input_token_cost + ) == rate + entry: Final = pricing_entry_for_cost_calc( + f"bedrock/converse/{model}", response, provider, False, None, None, None, logging_obj + ) + assert entry is not None + assert entry[1]["input_cost_per_token"] == 0.02 + assert entry[1]["output_cost_per_token"] == 0.04 + assert entry[1][cache_field] == rate + assert litellm.model_cost[profile] == profile_before + + monkeypatch.setattr(litellm, "model_cost", dict(catalog)) + model_utils._invalidate_model_cost_lowercase_map() + model_utils.reapply_runtime_model_cost_registrations() + expected_after_reload: Final = ( + 10 * 0.02 + 3 * 0.04 + 5 * (rate if persist_across_reloads else catalog[profile][cache_field]) + ) + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider=provider, + optional_params={"model_id": profile}, + litellm_logging_obj=logging_obj, + ) == pytest.approx(expected_after_reload) + + +@pytest.mark.parametrize("profile", ("us.openai.missing-profile-test", "us.openai.unpriced-profile-test")) +def test_bedrock_cache_registration_with_unpriced_profile_keeps_registered_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, profile: str +) -> None: + import litellm.utils as model_utils + + model: Final = "openai.registered-cache-test" + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + model: { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + }, + "us.openai.unpriced-profile-test": {"litellm_provider": "bedrock", "mode": "chat"}, + }, + ) + model_utils._invalidate_model_cost_lowercase_map() + litellm.register_model({model: {"cache_read_input_token_cost": 0.001}}, persist_across_reloads=False) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=15, + completion_tokens=3, + total_tokens=18, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), + ), + ) + assert completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": profile}, + ) == pytest.approx(10 * 0.01 + 3 * 0.02 + 5 * 0.001) + + +@pytest.mark.parametrize("direction", ("input", "output")) +@pytest.mark.parametrize("rate", (0.0, 0.004)) +@pytest.mark.parametrize("prompt_tokens", (100, 300_001)) +def test_bedrock_partial_registration_overrides_only_its_profile_direction( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + direction: str, + rate: float, + prompt_tokens: int, +) -> None: + from litellm.cost_calculator import pricing_entry_for_cost_calc + + model: Final = "openai.partial-pricing-test" + profile: Final = f"us.{model}" + profile_entry: Final = { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_above_300k_tokens": 0.03, + "output_cost_per_token_above_300k_tokens": 0.06, + "cache_read_input_token_cost": 0.001, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, profile: dict(profile_entry)}) + litellm.register_model({model: {f"{direction}_cost_per_token": rate}}, persist_across_reloads=False) + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=3, + total_tokens=prompt_tokens + 3, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), + ), + ) + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="partial-pricing", + function_id="partial-pricing", + ) + logging_obj.model_call_details["model_id"] = profile + input_rate: Final = rate if direction == "input" else (0.03 if prompt_tokens > 300_000 else 0.01) + output_rate: Final = rate if direction == "output" else (0.06 if prompt_tokens > 300_000 else 0.02) + assert completion_cost( + completion_response=response, + model=model, + custom_llm_provider="bedrock", + litellm_logging_obj=logging_obj, + ) == pytest.approx((prompt_tokens - 5) * input_rate + 3 * output_rate + 5 * 0.001) + assert logging_obj.billed_token_rates is not None + assert logging_obj.billed_token_rates.input_cost_per_token == input_rate + assert logging_obj.billed_token_rates.output_cost_per_token == output_rate + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(5 * 0.001) + entry: Final = pricing_entry_for_cost_calc(model, response, "bedrock", False, None, None, None, logging_obj) + assert entry is not None + assert entry[1][f"{direction}_cost_per_token"] == rate + assert litellm.model_cost[profile] == profile_entry + + +@pytest.mark.parametrize("prompt_tokens", (299_999, 300_000, 300_001)) +@pytest.mark.parametrize("cache_field", ("cache_read_input_token_cost", "cache_creation_input_token_cost")) +@pytest.mark.parametrize("tier_override", (None, 0.0, 0.003)) +def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + prompt_tokens: int, + cache_field: str, + tier_override: float | None, +) -> None: + import litellm.utils as model_utils + from litellm.cost_calculator import pricing_entry_for_cost_calc + + model: Final = "openai.threshold-overlay-test" + profile: Final = f"us.{model}" + tier_field: Final = f"{cache_field}_above_300k_tokens" + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + profile: { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_above_300k_tokens": 0.03, + "output_cost_per_token_above_300k_tokens": 0.06, + tier_field: 0.007, + }, + }, + ) + model_utils._invalidate_model_cost_lowercase_map() + litellm.register_model( + { + model: { + cache_field: 0.001, + **({tier_field: tier_override} if tier_override is not None else {}), + } + }, + persist_across_reloads=False, + ) + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=3, + total_tokens=prompt_tokens + 3, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=5 if cache_field == "cache_read_input_token_cost" else 0 + ), + cache_creation_input_tokens=5 if cache_field == "cache_creation_input_token_cost" else 0, + ) + response: Final = ModelResponse(model=model, usage=usage) + logging_obj: Final = Logging( + model=model, + messages=[], + stream=False, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="threshold-overlay", + function_id="threshold-overlay", + ) + logging_obj.model_call_details["model_id"] = profile + expected_input_rate: Final = 0.03 if prompt_tokens > 300_000 else 0.01 + expected_output_rate: Final = 0.06 if prompt_tokens > 300_000 else 0.02 + expected_cache_rate: Final = ( + (tier_override if tier_override is not None else 0.007) if prompt_tokens > 300_000 else 0.001 + ) + total: Final = completion_cost( + completion_response=response, + model=f"bedrock/converse/{model}", + custom_llm_provider="bedrock", + optional_params={"model_id": profile}, + litellm_logging_obj=logging_obj, + ) + assert total == pytest.approx( + (prompt_tokens - 5) * expected_input_rate + 3 * expected_output_rate + 5 * expected_cache_rate + ) + breakdown_key: Final = "cache_read_cost" if cache_field == "cache_read_input_token_cost" else "cache_creation_cost" + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown.get(breakdown_key, 0.0) == pytest.approx(5 * expected_cache_rate) + assert logging_obj.billed_token_rates is not None + assert logging_obj.billed_token_rates.input_cost_per_token == expected_input_rate + assert logging_obj.billed_token_rates.output_cost_per_token == expected_output_rate + entry: Final = pricing_entry_for_cost_calc(model, response, "bedrock", False, None, None, None, logging_obj) + assert entry is not None + assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 + assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c3794ea7544..3e5cfeb455e 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -9626,3 +9626,92 @@ async def test_atranscription_ignores_a_non_string_base_url_when_api_base_is_set assert route.called assert response.text == "four score" + + +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize("asynchronous", (False, True)) +@pytest.mark.parametrize("profile,custom_price", ((None, False), ("us.openai.gpt-6-sol", False), (None, True))) +@pytest.mark.asyncio +async def test_converse_fallback_prices_the_current_attempt( + monkeypatch: pytest.MonkeyPatch, stream: bool, asynchronous: bool, profile: str | None, custom_price: bool +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.rust_bridge import configuration + from tests.unit.llms.bedrock.slow_upstream import CONVERSE_ANSWER + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setenv("LITELLM_RUST", "0") + configuration.rust(False) + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + failed_profile: Final = "us.amazon.nova-micro-v1:0" + model: Final = "openai.gpt-6-sol" + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-micro-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="acompletion" if asynchronous else "completion", + start_time=datetime(2026, 1, 1), + litellm_call_id="bedrock-fallback-pricing", + function_id="bedrock-fallback-pricing", + ) + + def respond(request: httpx.Request) -> httpx.Response: + if failed_profile in request.url.path: + return httpx.Response(400, json={"message": "Input is too long for requested model."}) + assert request.url.path == f"/model/{profile or model}/converse" + return httpx.Response(200, content=CONVERSE_ANSWER) + + client: Final = ( + AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + if asynchronous + else HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + ) + kwargs: Final = { + "messages": [{"role": "user", "content": "hi"}], + "api_key": "test-bearer", + "aws_region_name": "us-east-1", + "client": client, + "litellm_logging_obj": logging_obj, + "num_retries": 0, + "stream": stream, + "fake_stream": stream, + } + if asynchronous: + with pytest.raises(litellm.BadRequestError): + await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) + else: + with pytest.raises(litellm.BadRequestError): + litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) + + overrides: Final = { + **({"model_id": profile} if profile is not None else {}), + **({"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} if custom_price else {}), + } + response: Final = ( + await litellm.acompletion(model=f"bedrock/converse/{model}", **kwargs, **overrides) + if asynchronous + else litellm.completion(model=f"bedrock/converse/{model}", **kwargs, **overrides) + ) + result: Final = ( + litellm.stream_chunk_builder( + [chunk async for chunk in response] if asynchronous else list(response), logging_obj=response.logging_obj + ) + if stream + else response + ) + assert result is not None + prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock") + expected: Final = ( + 0.001 + 0.002 + if custom_price + else prices["input_cost_per_token"] + prices["output_cost_per_token"] + ) + active_logging: Final = response.logging_obj if stream else logging_obj + assert active_logging.response_cost_calculator(result) == pytest.approx(expected) + active_logging.update_environment_variables(litellm_params={}, optional_params={}) + assert active_logging.response_cost_calculator(result, ignore_stamped_cost=True) == pytest.approx(expected) + if asynchronous: + await client.close() + else: + client.close() + configuration.reset_rust_configuration() diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index ffb71e86303..1655dac6382 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -862,6 +862,178 @@ def test_should_not_downgrade_chatgpt_shared_key_mode_with_alias_override(): _restore_model_cost_entries(model_keys) +@pytest.mark.parametrize("service_tier", ("default", "priority", "flex")) +@pytest.mark.parametrize( + "field", + ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", "input_cost_per_token_priority"), +) +@pytest.mark.parametrize("rate", (0.0, 0.004)) +@pytest.mark.parametrize("explicit_base", (False, True)) +@pytest.mark.parametrize("provider", (None, "bedrock")) +@pytest.mark.parametrize("registration_path", ("direct", "startup", "reload")) +def test_bedrock_deployment_inherits_invoked_profile_around_custom_prices( + monkeypatch: pytest.MonkeyPatch, + field: str, + rate: float, + explicit_base: bool, + provider: str | None, + registration_path: str, + service_tier: str, +) -> None: + from litellm.cost_calculator import completion_cost + from litellm.types.utils import ModelResponse, Usage, PromptTokensDetailsWrapper + + model: Final = "openai.gpt-5.6-sol" + profile: Final = f"us.{model}" + base_model: Final = f"global.{model}" + profile_prices: Final = { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "cache_read_input_token_cost": 0.001, + "input_cost_per_token_priority": 0.05, + "output_cost_per_token_priority": 0.06, + "input_cost_per_token_flex": 0.007, + "output_cost_per_token_flex": 0.008, + } + base_prices: Final = {**profile_prices, "input_cost_per_token": 0.03, "output_cost_per_token": 0.04} + monkeypatch.setattr( + litellm, "model_cost", {**litellm.model_cost, profile: dict(profile_prices), base_model: dict(base_prices)} + ) + deployment: Final = Deployment( + model_name="profile-priced", + litellm_params=LiteLLM_Params( + model=f"converse/{model}" if provider else f"bedrock/converse/{model}", + model_id=profile, + custom_llm_provider=provider, + ), + model_info=ModelInfo( + **{ + "id": "profile-priced-deployment", + field: rate, + **({"base_model": f"bedrock/{base_model}"} if explicit_base else {}), + } + ), + ) + if registration_path == "direct": + Router._register_deployment_pricing(deployment) + else: + router: Final = Router(model_list=[deployment.to_json(exclude_none=True)]) + if registration_path == "reload": + router.replay_model_cost_registrations() + response: Final = ModelResponse( + model=model, + usage=Usage( + prompt_tokens=100, + completion_tokens=3, + total_tokens=103, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), + ), + ) + catalog: Final = base_prices if explicit_base else profile_prices + input_field: Final = "input_cost_per_token" + (f"_{service_tier}" if service_tier != "default" else "") + output_field: Final = "output_cost_per_token" + (f"_{service_tier}" if service_tier != "default" else "") + input_rate: Final = rate if field == input_field else catalog[input_field] + output_rate: Final = rate if field == output_field else catalog[output_field] + cache_rate: Final = rate if field == "cache_read_input_token_cost" else 0.001 + assert completion_cost( + model=model, + completion_response=response, + custom_llm_provider="bedrock", + custom_pricing=True, + router_model_id="profile-priced-deployment", + service_tier=service_tier, + ) == pytest.approx(95 * input_rate + 3 * output_rate + 5 * cache_rate) + assert litellm.model_cost[profile] == profile_prices + + +@pytest.mark.parametrize("catalog_entry", (None, {"litellm_provider": 1, "mode": "chat"})) +@pytest.mark.parametrize("known_model", (False, True)) +@pytest.mark.parametrize("explicit_base", (False, True)) +@pytest.mark.parametrize("service_tier", ("default", "priority", "flex")) +@pytest.mark.parametrize("override", (None, "cache_read_input_token_cost", "input_cost_per_token_priority")) +@pytest.mark.parametrize("reload", (False, True)) +def test_bedrock_deployment_keeps_custom_prices_when_profile_lookup_fails( + monkeypatch: pytest.MonkeyPatch, + catalog_entry: dict[str, object] | None, + known_model: bool, + explicit_base: bool, + service_tier: str, + override: str | None, + reload: bool, +) -> None: + from litellm.cost_calculator import completion_cost + from litellm.types.utils import ModelResponse, Usage, PromptTokensDetailsWrapper + + model: Final = "anthropic.unmapped-profile-pricing-test" + profile: Final = ( + "us.anthropic.invalid-profile-pricing-test" + if catalog_entry is not None + else "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/opaque-test" + ) + configured_prices: Final = { + "litellm_provider": "bedrock", + "mode": "chat", + "input_cost_per_token": 0.03, + "output_cost_per_token": 0.04, + "cache_read_input_token_cost": 0.001, + "input_cost_per_token_priority": 0.05, + "output_cost_per_token_priority": 0.06, + "input_cost_per_token_flex": 0.007, + "output_cost_per_token_flex": 0.008, + } + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + **({model: dict(configured_prices)} if known_model else {}), + **({profile: catalog_entry} if catalog_entry is not None else {}), + }, + ) + deployment: Final = Deployment( + model_name="custom-priced", + litellm_params=LiteLLM_Params(model=f"bedrock/converse/{model}", model_id=profile), + model_info=ModelInfo( + id="unmapped-profile-deployment", + input_cost_per_token=0.01, + output_cost_per_token=0.02, + **({override: 0.0} if override else {}), + **({"base_model": f"bedrock/{profile}"} if explicit_base else {}), + ), + ) + router: Final = Router(model_list=[deployment.to_json(exclude_none=True)]) + if reload: + router.replay_model_cost_registrations() + inherit_configured: Final = known_model and not explicit_base + tier_rates: Final = inherit_configured and service_tier != "default" + input_rate: Final = configured_prices[f"input_cost_per_token_{service_tier}"] if tier_rates else 0.01 + output_rate: Final = configured_prices[f"output_cost_per_token_{service_tier}"] if tier_rates else 0.02 + effective_input: Final = 0.0 if override == "input_cost_per_token_priority" and service_tier == "priority" else input_rate + cache_rate: Final = 0.0 if override == "cache_read_input_token_cost" else (0.001 if inherit_configured else effective_input) + assert completion_cost( + model=model, + completion_response=ModelResponse( + model=model, + usage=Usage( + prompt_tokens=100, + completion_tokens=3, + total_tokens=103, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=80), + ), + ), + custom_llm_provider="bedrock", + custom_pricing=True, + router_model_id="unmapped-profile-deployment", + service_tier=service_tier, + ) == pytest.approx(20 * effective_input + 80 * cache_rate + 3 * output_rate) + if known_model: + assert { + key: value for key, value in litellm.model_cost[model].items() if key in configured_prices + } == configured_prices + + def test_partial_custom_pricing_inherits_builtin_cache_pricing(): """A deployment that overrides only input/output cost on a cache-supporting model must still bill cache_read and cache_creation tokens. Before the @@ -2730,14 +2902,7 @@ def test_a_reload_rebuilds_exactly_what_a_fresh_boot_registered() -> None: ) rebuilt = litellm.model_cost["priced-id"] - assert at_boot.items() <= rebuilt.items(), ( - f"the rebuild changed or dropped a field the boot registration wrote: " - f"{ {k: (v, rebuilt.get(k)) for k, v in at_boot.items() if rebuilt.get(k) != v} }" - ) - assert {field: rebuilt[field] for field in set(rebuilt) - set(at_boot)} == { - "db_model": False, - "member_auto_router": False, - } + assert at_boot == rebuilt assert "member_auto_router" not in litellm.model_cost["gpt-4o"] assert router.model_list finally: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cf0bf9b5d79..44939ca7585 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4805,7 +4805,11 @@ async def test_builtin_string_callback_registers_when_subclass_already_active( @pytest.mark.usefixtures("isolated_openai_model_sets") @pytest.mark.usefixtures("isolated_openai_model_sets") -def test_reapply_runtime_registrations_replays_register_model_overrides(monkeypatch): +@pytest.mark.parametrize("rate", (0.0, 0.000123)) +@pytest.mark.parametrize("updated_rate", (None, 0.0, 0.000456)) +def test_reapply_runtime_registrations_replays_register_model_overrides( + monkeypatch: pytest.MonkeyPatch, rate: float, updated_rate: float | None +) -> None: """ register_model is the documented way to override pricing for a model. A price-data reload swaps litellm.model_cost for a freshly fetched catalog, @@ -4836,11 +4840,17 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(monkeypa "openai/gpt-4o": { "litellm_provider": "openai", "mode": "chat", - "input_cost_per_token": 0.000123, + "input_cost_per_token": rate, } } ) + litellm.register_model({"openai/gpt-4o": {"supports_function_calling": True, "input_cost_per_token": None}}) + litellm.register_model( + {"openai/gpt-4o": {"input_cost_per_token": str(updated_rate) if updated_rate is not None else None}} + ) + litellm.register_model({"openai/gpt-4o": {"supports_function_calling": False}}) + litellm.model_cost = { "openai/gpt-4o": { "litellm_provider": "openai", @@ -4852,7 +4862,9 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(monkeypa _invalidate_model_cost_lowercase_map() reapply_runtime_model_cost_registrations() - assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.000123 + expected_rate: Final = rate if updated_rate is None else updated_rate + assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == expected_rate + assert litellm.model_cost["openai/gpt-4o"]["supports_function_calling"] is False assert litellm.model_cost["openai/gpt-4o"]["max_input_tokens"] == 4242 finally: litellm.model_cost = saved_model_cost @@ -10657,6 +10669,27 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): assert prefixed_info["key"] == f"vertex_ai/{model}", model +@pytest.mark.parametrize( + "key", ("openai.test-converse", "bedrock/openai.test-converse", "bedrock/converse/openai.test-converse") +) +def test_runtime_model_cost_registrations_resolve_bedrock_routing(monkeypatch: pytest.MonkeyPatch, key: str) -> None: + from litellm import utils as model_utils + from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry + + rates: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} + alias: Final = "openai.test-converse-alias" + entry: Final = AliasedCostEntry({"litellm_provider": "bedrock_converse", "mode": "chat"}, (key, alias)) + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, key: entry, alias: entry}) + litellm.register_model({key: rates}, persist_across_reloads=False) + + registrations: Final = dict( + model_utils.runtime_model_cost_registrations("bedrock/converse/openai.test-converse", "bedrock") + ) + assert registrations[key] == rates + assert dict(model_utils.runtime_model_cost_registrations(alias, "bedrock"))[alias] == rates + assert model_utils.runtime_model_cost_registrations("bedrock/converse/openai.other-converse", "bedrock") == () + + def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_extra_body(): """LIT-9880: the hook hands a tool_config point on for the provider step, and OpenAI has no step that consumes it, so it must not ride along in extra_body and 400 at OpenAI.""" From dd243bdccba7d2ebe2efb0c634c5c1109c9cf718 Mon Sep 17 00:00:00 2001 From: Alon Sokolovski Date: Tue, 20 Oct 2026 10:11:59 +0300 Subject: [PATCH 076/124] feat(context_management)!: honor clear_at_least, exclude_tools and clear_tool_inputs in the clear_tool_uses polyfill (#47123) * feat(context_management): honor clear_at_least, exclude_tools and clear_tool_inputs in the clear_tool_uses polyfill The clear_tool_uses_20250919 polyfill honored only trigger and keep and ignored the other three fields with a warning. It now follows the Anthropic API, as measured against it: - keep counts clearable tool uses only; uses of exclude_tools are never cleared and count toward neither keep nor clear_at_least. - clear_at_least clears oldest first in batches of at least that many tokens and leaves an incomplete last batch, so the cleared prefix only grows between requests and each batch costs one prompt-cache miss. - clear_tool_inputs (true or a list of tool names) also empties the input of the cleared tool uses. Invalid values are ignored with a _invalid warning on the applied edit. * refactor(context_management): build the clear_tool_uses selection without mutable state * test(context_management): cover a history made only of excluded tools * fix(context_management): leave tiny tool results out of clear_tool_uses like the API The API neither counts nor clears a tool use whose result is 1 to 46 bytes of text, whatever the tool or error flag; empty results still count. Probed against the API with crafted histories. Test data under 46 bytes is lengthened so those tests keep exercising clearing * fix(context_management): count tiny tool results when clear_tool_inputs covers their tool The API leaves a use whose result is 1 to 46 bytes out of clear_tool_uses only while its input is kept. When clear_tool_inputs is true or names the use's tool, the API counts it toward keep and clears it like any other use. * fix(context_management): count cleared inputs and images toward clear_at_least, answer malformed fields with a 400 * fix(context_management): validate clear_tool_uses edits before any editor runs, drop only the malformed edit under drop_params * fix(context_management): drop only the malformed clear_tool_uses fields under drop_params A malformed clear_at_least, exclude_tools or clear_tool_inputs no longer drops the whole edit under drop_params. The edit still runs with its well-formed fields, so the clearing that worked before is kept. Also types the new tests and swaps the hand-rolled spy router for an AsyncMock * test(context_management): pin the 46 byte cutoff in UTF-8 bytes and a batch closing at exactly clear_at_least * fix(context_management): leave a non-string tool_use id for the provider, size lone surrogates, and read drop_params strings * fix(context_management): read a drop_params string the same way on count_tokens and both /v1/messages paths count_tokens still checked drop_params with is True, so a body drop_params of "true" answered 400 there while /v1/messages dropped the malformed field. Both routes and the sync path now use drops_unsupported_params. --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../pass_through/adapters/handler.py | 22 +- .../context_management/dispatcher.py | 25 +- .../context_management/editors/__init__.py | 4 +- .../editors/clear_tool_uses.py | 287 +++++++-- litellm/proxy/proxy_server.py | 3 +- .../test_handler_context_management.py | 14 +- .../test_clear_tool_uses.py | 561 +++++++++++++++++- .../context_management/test_compact.py | 16 +- .../context_management/test_dispatcher.py | 4 +- tests/unit/proxy/test_proxy_token_counter.py | 9 +- 10 files changed, 835 insertions(+), 110 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index cca00a8383b..01e781485a0 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -13,7 +13,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, @@ -331,8 +331,10 @@ async def _run_polyfill_if_enabled( does not disable a valid spec because context_management is a supported param). Raises ``AnthropicContextManagementError`` so the /v1/messages endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits`` - (``drop_params``) drops the invalid spec. All other exceptions are - best-effort swallowed (matches v0 behavior). + (``drop_params``) drops what is invalid: the malformed fields of a + ``clear_tool_uses`` edit alone, or the whole spec for an invalid ``compact`` + trigger. All other + exceptions are best-effort swallowed (matches v0 behavior). """ if not context_management_spec: return None @@ -353,6 +355,7 @@ async def _run_polyfill_if_enabled( sub_call_logging_kwargs=sub_call_logging_kwargs, parent_trace_id=parent_trace_id, count_only=count_only, + drop_invalid_edits=drop_invalid_edits, ) except AnthropicContextManagementError: if drop_invalid_edits: @@ -649,7 +652,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, - drop_params=litellm.drop_params is True or normalize_drop_params(extra_kwargs.get("drop_params")) is True, + drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) if openai_request is None: @@ -736,14 +739,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: """Handle non-Anthropic models asynchronously using the adapter""" context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None)) proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs) request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) reject_malformed_tool_result_text(messages, model, request_kwargs) + drop_invalid_edits: Final = drops_unsupported_params(request_kwargs.get("drop_params")) sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs) sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs) @@ -866,7 +867,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: **kwargs, ) - reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs)) + sync_request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs) + reject_malformed_tool_result_text(messages, model, sync_request_kwargs) # Run the context_management polyfill on the sync path too so that # ``litellm.messages.create()`` callers don't silently lose edits like @@ -875,9 +877,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: # bridge to it via ``run_async_function``. context_management: Final = kwargs.pop("context_management", None) additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None) - drop_invalid_edits: Final = litellm.drop_params is True or ( - "drop_params" in kwargs and kwargs["drop_params"] is True - ) + drop_invalid_edits: Final = drops_unsupported_params(sync_request_kwargs.get("drop_params")) # Deliberately do NOT auto-attach the proxy ``llm_router`` here: # ``run_async_function`` spawns a new event loop in a worker thread # to bridge to the async dispatcher, but the proxy router's httpx diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py index 3e1bab09d5a..bce653df6d1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py @@ -9,7 +9,12 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE -from .editors import apply_clear_tool_uses_20250919, apply_compact_20260112 +from .editors import ( + CheckedEdit, + apply_clear_tool_uses_20250919, + apply_compact_20260112, + check_clear_tool_uses_spec, +) from .result import PolyfillResult if TYPE_CHECKING: @@ -73,6 +78,12 @@ def _wrap_editor_return( ) +def _checked_edit(index: int, edit_spec: dict[str, object]) -> CheckedEdit: + if edit_spec.get("type") != CLEAR_TOOL_USES_EDIT_TYPE: + return CheckedEdit(edit=edit_spec, error=None) + return check_clear_tool_uses_spec(edit_spec, index) + + async def apply_context_management( *, model: str, @@ -86,9 +97,14 @@ async def apply_context_management( sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None, parent_trace_id: str | None = None, count_only: bool = False, + drop_invalid_edits: bool = False, ) -> PolyfillResult: """Run edits in order; return a single ``PolyfillResult``. + Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 + before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under + ``drop_invalid_edits``. + The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a worker thread so their token counts stay off the event loop; @@ -97,6 +113,11 @@ async def apply_context_management( edits: Final = normalize_spec(context_management_spec) if not edits: return PolyfillResult(messages=messages, system=system, applied_edits=[]) + checked: Final = tuple(_checked_edit(index, edit_spec) for index, edit_spec in enumerate(edits)) + first_error: Final = next((check.error for check in checked if check.error is not None), None) + if first_error is not None and not drop_invalid_edits: + raise first_error + well_formed_edits: Final = tuple(check.edit for check in checked) current_messages = messages current_system = system @@ -104,7 +125,7 @@ async def apply_context_management( aggregated_compaction_block = None aggregated_iterations_usage = None - for edit_spec in edits: + for edit_spec in well_formed_edits: edit_type = edit_spec.get("type") editor = _EDITOR_REGISTRY.get(edit_type) if isinstance(edit_type, str) else None if editor is None: diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py index 3e933a9880a..5fc8c553d4f 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py @@ -1,4 +1,4 @@ -from .clear_tool_uses import apply_clear_tool_uses_20250919 +from .clear_tool_uses import CheckedEdit, apply_clear_tool_uses_20250919, check_clear_tool_uses_spec from .compact import apply_compact_20260112 -__all__ = ["apply_clear_tool_uses_20250919", "apply_compact_20260112"] +__all__ = ["CheckedEdit", "apply_clear_tool_uses_20250919", "apply_compact_20260112", "check_clear_tool_uses_spec"] diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 00ecb315bf1..935ce77ef87 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -1,7 +1,23 @@ -"""``clear_tool_uses_20250919`` polyfill (v0: ``trigger`` and ``keep`` only).""" +"""``clear_tool_uses_20250919`` polyfill, matching the Anthropic API's selection. -from typing import Any, Final, cast +``keep`` counts clearable tool uses only. Uses of ``exclude_tools`` are never cleared and never count toward a +``tool_uses`` trigger, and neither are uses whose result is 1 to 46 bytes of text, which the API leaves out of its +count, unless ``clear_tool_inputs`` covers their tool: then their input can still be cleared, and the API counts and +clears them like any other. Clearing goes oldest first in batches that free at least ``clear_at_least`` tokens, +counting the result and any input it clears, and leaves an incomplete last batch, so the cleared prefix only grows +between requests and each batch costs one prompt-cache miss. ``check_clear_tool_uses_spec`` answers a malformed field +the way the API's 400 does, so the dispatcher can reject the edit, or drop only its malformed fields, before any editor +runs. +""" +from collections.abc import Mapping, Sequence, Set +from dataclasses import dataclass +from functools import reduce +from itertools import chain +from types import MappingProxyType +from typing import Annotated, Any, Final, Literal, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -13,8 +29,39 @@ from ..constants import ( DEFAULT_INPUT_TOKENS_TRIGGER, DEFAULT_KEEP_TOOL_USES, ) +from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +T = TypeVar("T") +# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep +_TINY_RESULT_BYTES: Final = 46 +ToolNames = Literal["all"] | frozenset[str] + + +class _ClearAtLeast(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["input_tokens"] + value: Annotated[StrictInt, Field(ge=0)] + + +class _ClearToolUsesFields(BaseModel): + model_config = ConfigDict(frozen=True) + + clear_at_least: _ClearAtLeast | None = None + exclude_tools: tuple[str, ...] | None = None + clear_tool_inputs: bool | tuple[str, ...] | None = None + + +_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +_API_MESSAGES: Final = MappingProxyType( + { + "tuple_type": "Input should be a valid array", + "model_type": "Input does not match the expected shape.", + } +) + class ClearToolUsesEditSpec(TypedDict, total=False): """The ``clear_tool_uses_20250919`` entry of a ``context_management`` spec.""" @@ -27,36 +74,125 @@ class ClearToolUsesEditSpec(TypedDict, total=False): clear_tool_inputs: ReadOnly[object] -def _count_tool_uses(messages: list[dict[str, object]]) -> int: - """Return the number of tool_use content blocks across all messages. - - Only counts blocks with a string ``id`` to stay consistent with - :func:`_collect_tool_use_ids_in_order`, which is the source of truth for - which blocks are clearable. - """ - count = 0 - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - if isinstance(block.get("id"), str): - count += 1 - return count +def _parse(adapter: TypeAdapter[T], value: object) -> T | None: + try: + return adapter.validate_python(value) + except ValidationError: + return None -def _collect_tool_use_ids_in_order(messages: list[dict[str, object]]) -> list[str]: - """Return tool_use ids in the chronological order they appear in messages.""" - ids: Final[list[str]] = [] - for msg in messages: - content = msg.get("content") - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "tool_use": - block_id = block.get("id") - if isinstance(block_id, str): - ids.append(block_id) - return ids +@dataclass(frozen=True, slots=True) +class CheckedEdit: + edit: dict[str, object] + error: AnthropicContextManagementError | None + + +def check_clear_tool_uses_spec(edit_spec: dict[str, object], index: int) -> CheckedEdit: + """``edits[index]`` without its malformed fields, with the API's 400 for the first one, or None when there is none.""" + try: + _ClearToolUsesFields.model_validate(edit_spec) + except ValidationError as error: + details: Final = error.errors(include_url=False) + malformed: Final = frozenset(detail["loc"][0] for detail in details) + first: Final = details[0] + location: Final = ".".join(map(str, ("edits", index, CLEAR_TOOL_USES_EDIT_TYPE, *first["loc"]))) + return CheckedEdit( + edit={key: value for key, value in edit_spec.items() if key not in malformed}, + error=AnthropicContextManagementError( + status_code=400, + message=f"context_management.{location}: {_API_MESSAGES.get(first['type'], first['msg'])}", + ), + ) + return CheckedEdit(edit=edit_spec, error=None) + + +def _input_tools(clear_tool_inputs: bool | tuple[str, ...] | None) -> ToolNames: + return "all" if clear_tool_inputs is True else frozenset(clear_tool_inputs or ()) + + +def _all_blocks(messages: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple(chain.from_iterable(_parse(_BLOCKS, message.get("content")) or () for message in messages)) + + +def _tool_use_ref(block: Mapping[str, object]) -> tuple[str, str | None] | None: + block_id: Final = block.get("id") + name: Final = block.get("name") + if block.get("type") != "tool_use" or not isinstance(block_id, str): + return None + return block_id, name if isinstance(name, str) else None + + +def _tool_uses_in_order(blocks: Sequence[Mapping[str, object]]) -> tuple[tuple[str, str | None], ...]: + refs: Final = (_tool_use_ref(block) for block in blocks) + return tuple(ref for ref in refs if ref is not None) + + +def _block_id(block: Mapping[str, object], block_type: str, id_key: str) -> str | None: + block_id: Final = block.get(id_key) + return block_id if block.get("type") == block_type and isinstance(block_id, str) else None + + +def _blocks_by_id( + blocks: Sequence[Mapping[str, object]], block_type: str, id_key: str +) -> Mapping[str, Mapping[str, object]]: + pairs: Final = ((_block_id(block, block_type, id_key), block) for block in blocks) + return {block_id: block for block_id, block in pairs if block_id is not None} + + +def _utf8_size(text: str) -> int: + return len(text.encode("utf-8", "surrogatepass")) + + +def _text_bytes(content: object) -> int | None: + """Return the UTF-8 size of a text-only result, or None when it holds anything but text.""" + if isinstance(content, str): + return _utf8_size(content) + parts: Final = _parse(_BLOCKS, content) or () + texts: Final = tuple(part.get("text") for part in parts) + if any(part.get("type") != "text" or not isinstance(text, str) for part, text in zip(parts, texts)): + return None + return sum(_utf8_size(text) for text in texts if isinstance(text, str)) + + +def _is_tiny(content: object) -> bool: + size: Final = _text_bytes(content) + return size is not None and 0 < size <= _TINY_RESULT_BYTES + + +def _tiny_result_ids(results: Mapping[str, Mapping[str, object]]) -> frozenset[str]: + """Return ids whose result is 1 to 46 bytes of text; the API neither counts nor clears those uses.""" + return frozenset(tool_use_id for tool_use_id, block in results.items() if _is_tiny(block.get("content"))) + + +def _clears_input(name: str | None, input_tools: ToolNames) -> bool: + return input_tools == "all" or (name is not None and name in input_tools) + + +def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: + return {**block, "content": build_cleared_tool_result_content(block.get("content"))} + + +def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: + """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" + before: Final = [result, use] if clears_input else [result] + after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + counts: Final = tuple( + litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + ) + return max(counts[0] - counts[1], 0) + + +def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], clear_at_least: int) -> tuple[str, ...]: + """Oldest first, close a batch once it holds ``clear_at_least`` tokens; drop the incomplete last batch.""" + + def step(state: tuple[int, int], item: tuple[int, str]) -> tuple[int, int]: + batch_end, batch_tokens = state + index, tool_use_id = item + total: Final = batch_tokens + tokens[tool_use_id] + return (index + 1, 0) if total >= clear_at_least else (batch_end, total) + + batch_end, _ = reduce(step, enumerate(candidates), (0, 0)) + return tuple(candidates[:batch_end]) def _trigger_met( @@ -64,6 +200,7 @@ def _trigger_met( model: str, messages: list[dict[str, object]], tools: list[dict[str, object]] | None, + counted_tool_uses: int, ) -> tuple[bool, int | None]: """Return (trigger_met, input_tokens if counted for reuse).""" trigger_type: Final = trigger.get("type", "input_tokens") @@ -72,7 +209,7 @@ def _trigger_met( if trigger_type == "tool_uses": if not isinstance(threshold, int): return False, None - return _count_tool_uses(messages) > threshold, None + return counted_tool_uses > threshold, None if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER @@ -132,11 +269,7 @@ def _clear_tool_results( and block.get("type") == "tool_result" and block.get("tool_use_id") in ids_to_clear ): - new_block = { - **block, - "content": build_cleared_tool_result_content(block.get("content")), - } - new_blocks.append(new_block) + new_blocks.append(_cleared_result(block)) mutated = True cleared += 1 else: @@ -150,6 +283,23 @@ def _clear_tool_results( return new_messages, cleared +def _without_input(block: Mapping[str, object], ids_to_clear: Set[str], tool_names: ToolNames) -> Mapping[str, object]: + match _tool_use_ref(block): + case (block_id, name) if block_id in ids_to_clear and _clears_input(name, tool_names): + return {**block, "input": dict[str, object]()} + case _: + return block + + +def _message_without_inputs( + message: dict[str, object], ids_to_clear: Set[str], tool_names: ToolNames +) -> dict[str, object]: # mutable-ok: the editor contract hands messages back as list[dict] + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return {**message, "content": [_without_input(block, ids_to_clear, tool_names) for block in blocks]} + + def apply_clear_tool_uses_20250919( *, model: str, @@ -158,15 +308,11 @@ def apply_clear_tool_uses_20250919( system: str | list[dict[str, object]] | None, edit_spec: ClearToolUsesEditSpec, ) -> tuple[list[dict[str, object]], AppliedEdit | None]: - """Apply clear_tool_uses; return (messages, AppliedEdit or None).""" - ignored_knobs = [knob for knob in ("clear_at_least", "exclude_tools", "clear_tool_inputs") if knob in edit_spec] - for ignored_knob in ignored_knobs: - verbose_logger.warning( - "context_management polyfill: ignoring '%s' on %s " - "(supported only on Anthropic-family forwarding path in v0)", - ignored_knob, - CLEAR_TOOL_USES_EDIT_TYPE, - ) + """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).""" + fields: Final = _ClearToolUsesFields.model_validate(edit_spec) + clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value + exclude_tools: Final = frozenset(fields.exclude_tools or ()) + input_tools: Final = _input_tools(fields.clear_tool_inputs) trigger: Final[dict[str, object]] = edit_spec.get("trigger") or { "type": "input_tokens", @@ -177,37 +323,56 @@ def apply_clear_tool_uses_20250919( "value": DEFAULT_KEEP_TOOL_USES, } - met, tokens_before = _trigger_met(trigger, model, messages, tools) + blocks: Final = _all_blocks(messages) + results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id") + uses: Final = _blocks_by_id(blocks, "tool_use", "id") + tiny: Final = _tiny_result_ids(results) + clearable: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in _tool_uses_in_order(blocks) + if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools)) + ) + met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable)) if not met: return messages, None keep_count: Final = _resolve_keep_count(keep) - tool_use_ids: Final = _collect_tool_use_ids_in_order(messages) - if len(tool_use_ids) <= keep_count: + if len(clearable) <= keep_count: return messages, None - ids_to_clear: Final = set(tool_use_ids[: len(tool_use_ids) - keep_count]) - - # Never clear the latest completed tool_result (reply context). last_completed_id: Final = _last_completed_tool_use_id(messages) - if last_completed_id is not None: - ids_to_clear.discard(last_completed_id) + candidates: Final = tuple( + (tool_use_id, name) + for tool_use_id, name in clearable[: len(clearable) - keep_count] + if tool_use_id in results and tool_use_id != last_completed_id + ) + candidate_ids: Final = tuple(tool_use_id for tool_use_id, _ in candidates) + tokens: Final = { + tool_use_id: _freed_tokens(model, results[tool_use_id], uses[tool_use_id], _clears_input(name, input_tools)) + for tool_use_id, name in (candidates if clear_at_least > 0 else ()) + } + ids_to_clear: Final = frozenset( + _select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids + ) - edited, cleared_count = _clear_tool_results(messages, ids_to_clear) + edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear)) verbose_logger.debug("context_management polyfill: edited: %s", edited) if cleared_count == 0: return messages, None + final_messages: Final = ( + [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited + ) - if tokens_before is None: - tokens_before = litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) - tokens_after: Final = litellm.token_counter(model=model, messages=edited, tools=cast(Any, tools)) + tokens_before: Final = ( + counted_tokens + if counted_tokens is not None + else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + ) + tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) - applied: Final[AppliedEdit] = { + return final_messages, { "type": CLEAR_TOOL_USES_EDIT_TYPE, "cleared_tool_uses": cleared_count, "cleared_input_tokens": cleared_input_tokens, } - if ignored_knobs: - applied["warnings"] = [f"{knob}_ignored" for knob in ignored_knobs] - return edited, applied diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c17e0c67d68..1840586a08e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -360,6 +360,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, get_parent_otel_span_from_kwargs, iter_items, + normalize_drop_params, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15208,7 +15209,7 @@ def _count_context_management( return CountContextManagement( spec=context_management, drop_invalid_edits=litellm.drop_params is True - or any(params.get("drop_params") is True for params in sent_params), + or any(normalize_drop_params(params.get("drop_params")) is True for params in sent_params), ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py index f906530c8d9..9b5477e3206 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py @@ -83,9 +83,16 @@ async def _send(router: Router, request_options: Mapping[str, object], backend: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: Mapping[str, object], @@ -113,7 +120,8 @@ async def test_messages_rejects_an_invalid_edit_without_drop_params() -> None: assert backend.calls == [] -def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit() -> None: +@pytest.mark.parametrize("drop_params", (True, "true"), ids=("drop-params", "drop-params-string")) +def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid_edit(drop_params: bool | str) -> None: backend: Final = _SyncBackend() with patch.object(litellm, "drop_params", False), patch("litellm.completion", new=backend): response: Final = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( @@ -122,7 +130,7 @@ def test_sync_messages_sends_the_unedited_body_when_drop_params_drops_an_invalid model="gemini/gemini-2.5-flash", api_key="test-key", context_management=_INVALID_COMPACT_EDIT, - drop_params=True, + drop_params=drop_params, ) assert isinstance(response, dict) diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py index ee50bdff461..2ffa73dda21 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py @@ -3,18 +3,32 @@ Unit tests for the in-gateway `clear_tool_uses_20250919` polyfill editor. """ from copy import deepcopy +from itertools import chain +from typing import Final +from unittest.mock import AsyncMock, MagicMock +import pytest + +import litellm +from litellm.llms.anthropic.pass_through.adapters.handler import _run_polyfill_if_enabled +from litellm.llms.anthropic.pass_through.context_management import ( + AnthropicContextManagementError, + PolyfillResult, + apply_context_management, +) from litellm.llms.anthropic.pass_through.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, + COMPACT_SUMMARY_MODEL_SETTING_KEY, ) from litellm.llms.anthropic.pass_through.context_management.editors.clear_tool_uses import ( apply_clear_tool_uses_20250919, ) +from litellm.types.llms.anthropic import AppliedEdit MODEL = "xai/grok-4" -def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): +def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai", name: str = "get_weather"): """Return an (assistant, user) message pair with one tool_use + tool_result.""" assistant_msg = { "role": "assistant", @@ -22,7 +36,7 @@ def _make_pair(tool_use_id: str, result_text: str, location: str = "Mumbai"): { "type": "tool_use", "id": tool_use_id, - "name": "get_weather", + "name": name, "input": {"location": location}, } ], @@ -203,10 +217,71 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 -def test_ignored_knobs_do_not_alter_behavior(): - """clear_at_least / exclude_tools / clear_tool_inputs are accepted but ignored in v0.""" +def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: + contents: Final = (message["content"] for message in messages) + blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) + return tuple(block for block in blocks if isinstance(block, dict) and block["type"] == block_type) + + +def _result_contents(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["tool_use_id"]: block["content"] for block in _blocks_of(messages, "tool_result")} + + +def _cleared_ids(messages: list[dict[str, object]]) -> set[object]: + placeholders: Final = (CLEARED_TOOL_RESULT_PLACEHOLDER, [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}]) + return {tool_use_id for tool_use_id, content in _result_contents(messages).items() if content in placeholders} + + +def _result_tokens(text: str) -> int: + return litellm.token_counter(model=MODEL, text=text) + + +def test_clear_at_least_clears_in_batches_and_leaves_an_incomplete_batch(): + """Oldest first, a batch closes once it reaches clear_at_least; the rest waits.""" + messages = _make_history(n_pairs=8) + one_result = _result_tokens(_result_contents(messages)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + # Three results fill a batch; two don't. + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=edit_spec + ) + # 7 eligible: two batches of 3; the 7th waits for its batch to fill. + assert applied is not None + assert applied["cleared_tool_uses"] == 6 + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(6)} + + +def test_clear_at_least_keeps_the_cleared_prefix_stable_between_requests(): + """A new tool use doesn't move the cleared prefix until the next batch fills (prompt-cache friendly).""" + history = _make_history(n_pairs=10) + one_result = _result_tokens(_result_contents(history)["toolu_00"]) + edit_spec = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one_result + 1}, + } + cleared: Final = tuple( + _cleared_ids( + apply_clear_tool_uses_20250919( + model=MODEL, messages=history[: 1 + 2 * n_pairs], tools=None, system=None, edit_spec=edit_spec + )[0] + ) + for n_pairs in (8, 9, 10) + ) + assert cleared[0] == cleared[1] == {f"toolu_{i:02d}" for i in range(6)} + assert cleared[2] == {f"toolu_{i:02d}" for i in range(9)} + + +def test_clear_at_least_out_of_reach_leaves_messages_unchanged(): messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -216,28 +291,46 @@ def test_ignored_knobs_do_not_alter_behavior(): "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, "clear_at_least": {"type": "input_tokens", "value": 999_999_999}, - "exclude_tools": ["get_weather"], - "clear_tool_inputs": True, }, ) - # Despite clear_at_least being huge, polyfill still applies (knob ignored). - # Despite clear_tool_inputs=True, inputs are NOT cleared (knob ignored). - assert applied is not None - assert applied["cleared_tool_uses"] == 2 - # Ignored knobs surface as warnings on the AppliedEdit so operators can - # see what was dropped (the v0 polyfill silently dropping them at debug - # log level made misconfiguration invisible from the response). - assert set(applied.get("warnings", [])) == { - "clear_at_least_ignored", - "exclude_tools_ignored", - "clear_tool_inputs_ignored", + assert applied is None + assert new_messages == original + + +def test_a_batch_closes_when_it_frees_exactly_clear_at_least(): + messages = _make_history(n_pairs=6) + clear_one_use = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 5}, } + _, one = apply_clear_tool_uses_20250919( + model=MODEL, messages=messages, tools=None, system=None, edit_spec=clear_one_use + ) + assert one is not None + assert one["cleared_tool_uses"] == 1 + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + **clear_one_use, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 2 * one["cleared_input_tokens"]}, + }, + ) + assert _cleared_ids(new_messages) == {f"toolu_{i:02d}" for i in range(4)} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 -def test_no_ignored_knobs_omits_warnings_field(): - """When the caller doesn't pass any unsupported knobs, no ``warnings`` are added.""" - messages = _make_history(n_pairs=3) - _, applied = apply_clear_tool_uses_20250919( +def test_exclude_tools_are_never_cleared_and_do_not_count_toward_keep(): + """keep counts clearable uses only: with keep=1 the newest get_weather result stays, not the newest search.""" + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather", "get_weather", "search"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, tools=None, @@ -246,10 +339,428 @@ def test_no_ignored_knobs_omits_warnings_field(): "type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 0}, "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], }, ) assert applied is not None - assert "warnings" not in applied + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_02"} + assert applied["cleared_tool_uses"] == 2 + + +def test_history_of_only_excluded_tools_is_left_unchanged(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i in range(4): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name="search")) + original = deepcopy(messages) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "exclude_tools": ["search"], + }, + ) + assert applied is None + assert new_messages == original + + +def _clear_with_keep_two(messages: list[dict[str, object]]) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + }, + ) + + +def test_tiny_results_are_never_cleared_and_do_not_count_toward_keep(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = [{"type": "text", "text": "\u9ce5" * 15}] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def _clear_with_keep_two_and_inputs( + messages: list[dict[str, object]], clear_tool_inputs: bool | list[str] +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 2}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + + +def test_tiny_results_count_and_clear_when_their_tool_inputs_are_cleared(): + for clear_tool_inputs in (True, ["get_weather"]): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, clear_tool_inputs) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02", "toolu_03"} + assert applied is not None + assert applied["cleared_tool_uses"] == 4 + + +def test_tiny_results_stay_out_when_clear_tool_inputs_names_other_tools(): + messages = _make_history(6) + messages[2]["content"][0]["content"] = "(Bash completed with no output)" + messages[12]["content"][0]["content"] = "ok" + new_messages, applied = _clear_with_keep_two_and_inputs(messages, ["read_file"]) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_results_over_46_bytes_and_empty_results_still_count(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 47 + messages[4]["content"][0]["content"] = "" + messages[6]["content"][0]["content"] = [ + {"type": "text", "text": "ok"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}, + ] + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} + assert new_messages[6]["content"][0]["content"] == [{"type": "text", "text": CLEARED_TOOL_RESULT_PLACEHOLDER}] + assert applied is not None + assert applied["cleared_tool_uses"] == 3 + + +def test_the_46_byte_cutoff_includes_46_and_counts_utf8_bytes(): + # The 46 byte cutoff was measured on api.anthropic.com on 2026-10-19. + messages = _make_history(5) + messages[2]["content"][0]["content"] = "x" * 46 + messages[4]["content"][0]["content"] = "鳥" * 16 + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + + +def test_a_result_holding_a_lone_surrogate_is_sized_like_any_other_text(): + messages = _make_history(5) + messages[2]["content"][0]["content"] = "\ud800" + new_messages, applied = _clear_with_keep_two(messages) + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert applied is not None + + +def _tool_inputs(messages: list[dict[str, object]]) -> dict[object, object]: + return {block["id"]: block["input"] for block in _blocks_of(messages, "tool_use")} + + +def test_clear_tool_inputs_true_empties_the_inputs_of_cleared_uses_only(): + messages = _make_history(n_pairs=3) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + # The caller's messages are not mutated. + assert _tool_inputs(messages)["toolu_00"] == {"location": "City0"} + + +def test_clear_tool_inputs_list_clears_only_the_named_tools(): + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, location=f"City{i}", name=name)) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": ["search"], + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(new_messages) == { + "toolu_00": {"location": "City0"}, + "toolu_01": {}, + "toolu_02": {"location": "City2"}, + } + + +def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): + messages = _make_history(n_pairs=4) + messages[1]["content"][0]["id"] = ["toolu_00"] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert new_messages[1] == messages[1] + assert _cleared_ids(new_messages) == {"toolu_01", "toolu_02"} + assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {}) + + +async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError: + with pytest.raises(AnthropicContextManagementError) as raised: + await apply_context_management( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": edits}, + ) + return raised.value + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("clear_at_least", {"type": "tool_uses", "value": 5}), + ("clear_at_least", {"value": 5}), + ("clear_at_least", {"type": "input_tokens", "value": -1}), + ("clear_at_least", {"type": "input_tokens", "value": 1.5}), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}), + ("clear_at_least", {"type": "input_tokens", "value": True}), + ("exclude_tools", "get_weather"), + ("exclude_tools", [1]), + ("clear_tool_inputs", "get_weather"), + ], +) +async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: + error = await _malformed_spec_error( + [{"type": "clear_tool_uses_20250919", "trigger": {"type": "tool_uses", "value": 99}, field: value}] + ) + assert error.status_code == 400 + assert error.message.startswith(f"context_management.edits.0.clear_tool_uses_20250919.{field}") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("exclude_tools", "memory", "exclude_tools: Input should be a valid array"), + ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), + ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), + ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ], +) +async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( + field: str, value: object, message: str +) -> None: + # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + error = await _malformed_spec_error( + [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] + ) + assert error.message.startswith("context_management.edits.1.clear_tool_uses_20250919.") + assert error.message.endswith(message) + + +_VALID_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, +} +_MALFORMED_EDIT: Final = {**_VALID_EDIT, "exclude_tools": "get_weather"} + + +@pytest.mark.parametrize( + ("field", "value"), + [("exclude_tools", "get_weather"), ("clear_at_least", {"type": "input_tokens", "value": "1200"})], +) +async def test_drop_params_drops_only_the_malformed_field(field: str, value: object) -> None: + async def run(drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + context_management_spec={"edits": [{**_VALID_EDIT, field: value, "clear_tool_inputs": True}]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + with pytest.raises( + AnthropicContextManagementError, match=rf"^context_management\.edits\.0\.clear_tool_uses_20250919\.{field}" + ): + await run(drop_invalid_edits=False) + result: Final = await run(drop_invalid_edits=True) + assert result is not None + assert _cleared_ids(result.messages) == {"toolu_00", "toolu_01"} + assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} + + +async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model") + router: Final = MagicMock() + router.acompletion = AsyncMock(side_effect=RuntimeError("summary model unavailable")) + with pytest.raises(AnthropicContextManagementError): + await apply_context_management( + model=MODEL, + messages=[{"role": "user", "content": "hello " * 60_000}], + tools=None, + system=None, + context_management_spec={ + "edits": [ + {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}, + _MALFORMED_EDIT, + ] + }, + llm_router=router, + ) + router.acompletion.assert_not_awaited() + + +@pytest.mark.parametrize(("clear_tool_inputs", "cleared"), [("yes", True), ("true", True), (1, True), ("no", False)]) +def test_clear_tool_inputs_reads_boolean_strings_and_numbers_like_the_api( + clear_tool_inputs: object, cleared: bool +) -> None: + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=_make_history(n_pairs=3), + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert applied is not None + assert _tool_inputs(new_messages)["toolu_00"] == ({} if cleared else {"location": "City0"}) + + +@pytest.mark.parametrize(("trigger_value", "triggers"), [(2, False), (1, True)]) +def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, triggers: bool) -> None: + messages = [{"role": "user", "content": "Plan a trip."}] + for i, name in enumerate(["search", "get_weather", "search", "get_weather"]): + messages.extend(_make_pair(f"toolu_{i:02d}", f"Result {i}: " + "x" * 200, name=name)) + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": trigger_value}, + "keep": {"type": "tool_uses", "value": 0}, + "exclude_tools": ["search"], + }, + ) + assert (applied is not None) == triggers + + +@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)]) +def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared( + clear_tool_inputs: bool, triggers: bool +) -> None: + messages = _make_history(n_pairs=3) + messages[2]["content"][0]["content"] = "ok" + _, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_tool_inputs": clear_tool_inputs, + }, + ) + assert (applied is not None) == triggers + + +def test_clear_at_least_counts_the_tool_inputs_it_clears(): + messages = [{"role": "user", "content": "Write the files."}] + for i in range(4): + assistant_msg, user_msg = _make_pair(f"toolu_{i:02d}", "ok", name="write_file") + assistant_msg["content"][0]["input"] = {"path": f"file{i}.txt", "content": "lorem ipsum " * 300} + messages.extend((assistant_msg, user_msg)) + one_input = litellm.token_counter(model=MODEL, text="lorem ipsum " * 300) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 3 * one_input // 2}, + "clear_tool_inputs": True, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01"} + assert [_tool_inputs(new_messages)[f"toolu_{i:02d}"] == {} for i in range(4)] == [True, True, False, False] + assert applied["cleared_input_tokens"] >= 3 * one_input // 2 + + +_ONE_PIXEL_PNG: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +) + + +def test_clear_at_least_counts_image_results(): + messages = _make_history(n_pairs=4) + image: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": _ONE_PIXEL_PNG}} + for user_msg in messages[2::2]: + user_msg["content"][0]["content"] = [image] + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + assert applied is not None + assert _cleared_ids(new_messages) == {"toolu_00", "toolu_01", "toolu_02"} def test_tool_result_list_content_shape_preserved(): @@ -266,7 +777,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_a", - "content": [{"type": "text", "text": "huge result"}], + "content": [{"type": "text", "text": "huge result: " + "x" * 200}], } ], }, @@ -280,7 +791,7 @@ def test_tool_result_list_content_shape_preserved(): { "type": "tool_result", "tool_use_id": "toolu_b", - "content": [{"type": "text", "text": "keep me"}], + "content": [{"type": "text", "text": "keep me: " + "y" * 200}], } ], }, diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py index 5e55c0ade32..9b3084f0e7c 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py @@ -2406,7 +2406,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "sunny in SF"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": "sunny in SF, 18 C, light wind from the west, no rain today", + } + ], }, { "role": "assistant", @@ -2414,7 +2420,13 @@ def _tool_use_messages() -> List[Dict[str, Any]]: }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_02", "content": "rainy in NY"}], + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "rainy in NY, 11 C, strong wind from the east, rain all day", + } + ], }, {"role": "user", "content": "now summarize both"}, ] diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py index e28d2ac5e65..66957456c95 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py @@ -22,7 +22,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t1", - "content": "first result", + "content": "first result: the forecast for the first city, sunny and mild", } ], }, @@ -36,7 +36,7 @@ def _history_with_two_tool_pairs(): { "type": "tool_result", "tool_use_id": "t2", - "content": "second result", + "content": "second result: the forecast for the second city, rainy and cold", } ], }, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index f1b8d3fd591..714d8bb77d7 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3236,9 +3236,16 @@ def _gemini_router(drop_params: bool = False) -> Router: ({"drop_params": True}, {}, {}, False), ({}, {"drop_params": True}, {}, False), ({}, {}, {"drop_params": True}, False), + ({}, {}, {"drop_params": "true"}, False), ({}, {}, {}, True), ), - ids=("deployment-drop-params", "router-default-drop-params", "request-drop-params", "litellm-drop-params"), + ids=( + "deployment-drop-params", + "router-default-drop-params", + "request-drop-params", + "request-drop-params-string", + "litellm-drop-params", + ), ) async def test_gemini_messages_count_tokens_counts_the_unedited_body_when_drop_params_drops_an_invalid_edit( deployment_params: dict[str, JsonValue], From 7ae9a1c350800b5dd058c1b4e0bd0d073fdc9f3a Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 20 Oct 2026 00:12:28 -0700 Subject: [PATCH 077/124] fix(router)!: only prefixed caller tags route once tag_routing_prefix is set (#47428) * fix(router): scope tag routing to prefixed tags once tag_routing_prefix is set With router_settings.tag_routing_prefix configured, an unprefixed caller tag (for example a spend tag such as user_id:234) no longer drives tag routing, so it can no longer refuse a model group whose deployments are untagged. Key/team/project tags (metadata.inherited_tags) keep applying with or without the prefix. Rows of batches the proxy runs itself now carry those tags as inherited_tags too, so they keep routing on them once the prefix is set. Takes over #39903. Co-authored-by: Deepanshu Lulla * test(integration): pin that proxy-run batch rows keep key/team tag protection * refactor(router): filter routing tags from the raw request tags; deterministic tests and key-tag cells * test(integration): audit cells for tag_routing_prefix: project tags, forged inherited_tags, odd tag values, outage, fallback, user-agent routing, concurrent burst * test(integration): proxy-run batch rows keep key and project tags too --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Co-authored-by: Deepanshu Lulla --- .../litellm_executed_batches.py | 1 + litellm/proxy/litellm_pre_call_utils.py | 19 +- litellm/router_strategy/tag_based_routing.py | 23 +- .../routing/test_tag_routing_prefix.py | 481 ++++++++++++++++++ .../test_litellm_executed_batches.py | 15 + .../test_router_tag_routing.py | 95 ++++ 6 files changed, 621 insertions(+), 13 deletions(-) create mode 100644 tests/integration/routing/test_tag_routing_prefix.py diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index e1a5fdd5a71..098dd2f9816 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -661,6 +661,7 @@ class LiteLLMExecutedBatchRunner: "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(run.user_api_key_dict), "user_api_end_user_max_budget": run.user_api_key_dict.end_user_max_budget, "tags": list(run.request_tags), + "inherited_tags": LiteLLMProxyRequestSetup.get_inherited_tags(run.user_api_key_dict), "batch_id": run.unified_batch_id, } diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 54882288b49..67a1d640103 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1782,6 +1782,14 @@ class LiteLLMProxyRequestSetup: ) return user_api_key_logged_metadata + @staticmethod + def get_inherited_tags(user_api_key_dict: UserAPIKeyAuth) -> tuple[str, ...]: + """Every tag key, team and project policy contributes, in that order, without repeats""" + key_tags: Final = (user_api_key_dict.metadata or MappingProxyType({})).get("tags") or () # pyright: ignore[reportUnknownMemberType] # key metadata is an untyped dict + team_tags: Final = (user_api_key_dict.team_metadata or {}).get("tags") or () + project_tags: Final = (user_api_key_dict.project_metadata or {}).get("tags") or () + return tuple(dict.fromkeys((*key_tags, *team_tags, *project_tags))) + @staticmethod def add_user_api_key_auth_to_request_metadata( data: dict, @@ -2506,7 +2514,6 @@ async def add_litellm_data_to_request( metadata=data[_metadata_variable_name], ) team_metadata: Final = user_api_key_dict.team_metadata or {} - project_metadata: Final = user_api_key_dict.project_metadata or {} # inherited_tags: every tag key/team/project policy contributed, read # directly from those three sources rather than snapshotted off the shared @@ -2518,12 +2525,10 @@ async def add_litellm_data_to_request( # allow_fail_open reads this (rather than subtracting caller_tags from the # final merged set) so a caller can't strip an inherited "!"/"&" # constraint's protection just by resubmitting its exact value alongside a - # conflicting one. - _key_tags: Final = (key_metadata or MappingProxyType({})).get("tags") or () - _team_tags: Final = team_metadata.get("tags") or () - _project_tags: Final = project_metadata.get("tags") or () - data[_metadata_variable_name]["inherited_tags"] = tuple( # rebind-ok: matches this file's data[...] mutation idiom - dict.fromkeys((*_key_tags, *_team_tags, *_project_tags)) + # conflicting one, and its tag_routing_prefix filter keeps these tags routing + # when they are unprefixed. + data[_metadata_variable_name]["inherited_tags"] = ( # rebind-ok: matches this file's data[...] mutation idiom + LiteLLMProxyRequestSetup.get_inherited_tags(user_api_key_dict) ) ## TEAM-LEVEL METADATA diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 541fdd19cff..73236294020 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -5,6 +5,7 @@ Use this to route requests between Teams - if deployments are set with default tags, return all default deployment - If no default_deployments are set, return all deployments - A "!tag" excludes deployments carrying that tag; a "&tag" requires it +- With tag_routing_prefix set, only prefixed request tags and key/team/project tags route """ import re @@ -432,6 +433,15 @@ def _inherited_constraint_sets( return frozenset(inherited_required), frozenset(inherited_excluded) +def _tags_that_route(request_tags: Sequence[str], metadata: object, routing_prefix: str) -> tuple[str, ...]: + if not routing_prefix: + return tuple(request_tags) + policy_tags: Final = frozenset(_tags_in_metadata(metadata, key="inherited_tags")) + return tuple( + tag.removeprefix(routing_prefix) for tag in request_tags if tag.startswith(routing_prefix) or tag in policy_tags + ) + + def _tag_known_to_group( llm_router_instance: LitellmRouter, model: str, @@ -525,12 +535,13 @@ async def get_deployments_for_tag( # A tag_routing_prefix-marked tag is stripped before matching -- everything # downstream (_split_tags, deployment matching) works off the unprefixed - # value, exactly as if the caller had sent it unprefixed -- and its - # post-strip value is remembered in routing_confirmed as an explicit, - # caller-declared routing directive, exempt from the "maybe foreign to this - # group" heuristics that unprefixed tags still go through unchanged below. - rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags or [], routing_prefix) - required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags) + # value -- and its post-strip value is remembered in routing_confirmed as an + # explicit, caller-declared routing directive, exempt from the "maybe + # foreign to this group" heuristics below. + routing_confirmed: Final = _strip_routing_prefix(request_tags or [], routing_prefix)[1] + required_tags, positive_tags, excluded_patterns = _split_tags( + _tags_that_route(request_tags or (), metadata, routing_prefix) + ) inherited_required_set, inherited_excluded_set = _inherited_constraint_sets( metadata.get("inherited_tags"), routing_prefix ) diff --git a/tests/integration/routing/test_tag_routing_prefix.py b/tests/integration/routing/test_tag_routing_prefix.py new file mode 100644 index 00000000000..d5745533fde --- /dev/null +++ b/tests/integration/routing/test_tag_routing_prefix.py @@ -0,0 +1,481 @@ +import json +import uuid +from collections.abc import Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + string_value, +) +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(240) + +Endpoint = Literal["chat", "messages", "responses"] + +ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") +UNTAGGED: Final = "prefix-untagged" +REGIONAL: Final = "prefix-regional" +BATCHED: Final = "prefix-batched" +FAIL_OPEN_BATCHED: Final = "prefix-fail-open-batched" +OUTAGE: Final = "prefix-outage" +DOWN: Final = "prefix-down" +USER_AGENT_ROUTED: Final = "prefix-user-agent" +REFUSED: Final = "Not allowed to access model due to tags configuration" + + +def _respond(request: Request) -> Reply: + if request.method != "POST": + if request.target.endswith("/files"): + return Reply(status=404, body=b'{"error": {"message": "no files api"}}') + return Reply(body=b'{"object": "list", "data": []}') + body: Final = JSON_OBJECT.validate_json(request.body) + backend: Final = string_value(body["model"]) + if backend.endswith("-down-backend"): + return Reply(status=500, body=b'{"error": {"message": "upstream down"}}') + stream: Final = body.get("stream") is True + if request.target.endswith("/responses"): + return responses_reply(f"resp_{uuid.uuid4().hex}", backend, "tag routed", stream=stream) + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", backend, "tag routed", stream=stream) + + +def _config(directory: Path, wire: Wire, prefix: str) -> Path: + def deployment( + group: str, + backend: str, + tags: Sequence[str], + provider: str = "openai", + fail_open: bool = False, + tag_regex: Sequence[str] = (), + ) -> dict[str, JsonValue]: + return { + "model_name": group, + "litellm_params": { + "model": f"{provider}/{backend}", + "api_base": f"{wire.url}/v1", + "api_key": "synthetic-tag-prefix-key", + **({"tags": list(tags)} if tags else {}), + **({"tag_regex": list(tag_regex)} if tag_regex else {}), + }, + "model_info": {"id": backend, **({"allow_fail_open": True} if fail_open else {})}, + } + + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + deployment(UNTAGGED, "untagged-backend", ()), + deployment(REGIONAL, "eu-backend", ("region:eu",)), + deployment(REGIONAL, "us-backend", ("region:us",)), + deployment(BATCHED, "eu-batch-backend", ("region:eu",), "hosted_vllm"), + deployment(BATCHED, "us-batch-backend", ("region:us",), "hosted_vllm"), + deployment(FAIL_OPEN_BATCHED, "eu-fail-open-backend", ("default", "region:eu"), "hosted_vllm", fail_open=True), + deployment(OUTAGE, "eu-down-backend", ("region:eu",)), + deployment(OUTAGE, "us-up-backend", ("region:us",)), + deployment(DOWN, "untagged-down-backend", ()), + deployment(USER_AGENT_ROUTED, "cli-backend", (), tag_regex=("^User-Agent: claude-cli/",)), + deployment(USER_AGENT_ROUTED, "browser-backend", ("default",)), + ] + config["router_settings"] = { + **config["router_settings"], + "enable_tag_filtering": True, + "tag_routing_prefix": prefix, + } + path: Final = directory / f"tag-routing-prefix-{prefix.rstrip(':') or 'none'}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def upstream() -> Iterator[Wire]: + with wire_server(_respond) as wire: + yield wire + + +@pytest.fixture(scope="module") +def prefixed(tmp_path_factory: pytest.TempPathFactory, upstream: Wire) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("tag-routing-prefix") + with ( + gateway_from_environment() as base, + owned_proxy(base, directory, {}, config=_config(directory, upstream, "route:")) as proxy, + ): + yield proxy + + +@pytest.fixture(scope="module") +def unprefixed(tmp_path_factory: pytest.TempPathFactory, upstream: Wire) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("tag-routing-no-prefix") + with ( + gateway_from_environment() as base, + owned_proxy(base, directory, {}, config=_config(directory, upstream, "")) as proxy, + ): + yield proxy + + +def _call( + endpoint: Endpoint, model: str, prompt: str, tags: Sequence[str], stream: bool +) -> tuple[str, Mapping[str, JsonValue], Mapping[str, str]]: + header: Final = {"x-litellm-tags": ",".join(tags)} + match endpoint: + case "chat": + messages: Final[JsonValue] = [{"role": "user", "content": prompt}] + return ( + "/v1/chat/completions", + {"model": model, "messages": messages, "metadata": {"tags": list(tags)}, "stream": stream}, + {}, + ) + case "messages": + return ( + "/v1/messages", + {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": prompt}], "stream": stream}, + header, + ) + case "responses": + return "/v1/responses", {"model": model, "input": prompt, "stream": stream}, header + + +def _send( + gateway: Gateway, + upstream: Wire, + endpoint: Endpoint, + model: str, + tags: Sequence[str], + *, + key: str | None = None, + stream: bool = False, +) -> tuple[httpx.Response, tuple[str, ...]]: + marker: Final = f"marker-{uuid.uuid4().hex}" + path, body, headers = _call(endpoint, model, f"route this {marker}", tags, stream) + response: Final = gateway.request("POST", path, body, key=key, headers=headers) + return response, _routed_backends(upstream, marker) + + +def _routed_backends(upstream: Wire, marker: str) -> tuple[str, ...]: + return tuple( + string_value(JSON_OBJECT.validate_json(request.body)["model"]) + for request in upstream.drain() + if request.method == "POST" and marker in request.body.decode() + ) + + +def _team_key(scenario: Scenario, tags: Sequence[str]) -> str: + return scenario.key(team_id=scenario.team(tags=list(tags))) + + +def _policy_key(scenario: Scenario, owner: Literal["key", "team", "project"], tags: Sequence[str]) -> str: + match owner: + case "key": + return scenario.key(metadata={"tags": list(tags)}) + case "team": + return _team_key(scenario, tags) + case "project": + team: Final = scenario.team() + return scenario.key(team_id=team, project_id=scenario.project(team, tags=list(tags))) + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"]) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_an_unprefixed_caller_tag_does_not_refuse_a_group_with_untagged_deployments( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint, stream: bool +) -> None: + response, backends = _send(prefixed, upstream, endpoint, UNTAGGED, ["user_id:234"], stream=stream) + + assert response.status_code == 200, response.text + assert backends == ("untagged-backend",) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_prefixed_caller_tag_routes_next_to_an_ignored_unprefixed_one( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint +) -> None: + response, backends = _send(prefixed, upstream, endpoint, REGIONAL, ["user_id:234", "route:region:us"]) + + assert response.status_code == 200, response.text + assert backends == ("us-backend",) + + +@pytest.mark.parametrize( + ("policy_tag", "backend"), [("®ion:eu", "eu-backend"), ("!region:eu", "us-backend")], ids=["require", "exclude"] +) +@pytest.mark.parametrize("owner", ["key", "team", "project"]) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_an_unprefixed_key_team_or_project_tag_still_constrains_routing( + prefixed: Gateway, + upstream: Wire, + endpoint: Endpoint, + owner: Literal["key", "team", "project"], + policy_tag: str, + backend: str, +) -> None: + with prefixed.scenario() as scenario: + key: Final = _policy_key(scenario, owner, [policy_tag]) + routed: Final = tuple(_send(prefixed, upstream, endpoint, REGIONAL, ["user_id:234"], key=key) for _ in range(8)) + + assert [response.status_code for response, _ in routed] == [200] * 8, [r.text for r, _ in routed] + assert [backends for _, backends in routed] == [(backend,)] * 8 + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_prefixed_caller_tag_cannot_route_around_a_team_requirement( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint +) -> None: + with prefixed.scenario() as scenario: + key: Final = _team_key(scenario, ["®ion:eu"]) + response, backends = _send(prefixed, upstream, endpoint, REGIONAL, ["route:region:us"], key=key) + + assert response.status_code == 401, response.text + assert REFUSED in response.text, response.text + assert backends == () + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_an_unprefixed_caller_constraint_is_ignored_once_a_prefix_is_set( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint +) -> None: + response, backends = _send(prefixed, upstream, endpoint, REGIONAL, ["®ion:nowhere"]) + + assert response.status_code == 200, response.text + assert len(backends) == 1 + + +@pytest.mark.parametrize( + "metadata", + [ + {"metadata": {"tags": ["®ion:nowhere"], "inherited_tags": ["®ion:nowhere"]}}, + {"litellm_metadata": {"tags": ["®ion:nowhere"], "inherited_tags": ["®ion:nowhere"]}}, + ], + ids=["metadata", "litellm_metadata"], +) +def test_a_caller_cannot_make_an_unprefixed_tag_route_by_sending_inherited_tags( + prefixed: Gateway, upstream: Wire, metadata: dict[str, JsonValue] +) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + messages: Final[JsonValue] = [{"role": "user", "content": f"route this {marker}"}] + response: Final = prefixed.request( + "POST", "/v1/chat/completions", {"model": REGIONAL, "messages": messages, **metadata} + ) + + assert response.status_code == 200, response.text + assert len(_routed_backends(upstream, marker)) == 1 + + +@pytest.mark.parametrize( + ("tags", "status", "backends"), + [ + ([], 200, ("untagged-backend",)), + (["user_id:234", "user_id:234"], 200, ("untagged-backend",)), + ([f"user_id:{'x' * 5000}"], 200, ("untagged-backend",)), + (["route:"], 401, ()), + (["route:user_id:234"], 401, ()), + ], + ids=["empty", "duplicate", "5kb", "bare-prefix", "prefixed-unknown-tag"], +) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_odd_caller_tag_values_on_an_untagged_group( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint, tags: list[str], status: int, backends: tuple[str, ...] +) -> None: + response, routed = _send(prefixed, upstream, endpoint, UNTAGGED, tags) + + assert response.status_code == status, response.text + assert routed == backends + + +@pytest.mark.parametrize("tags", ["user_id:234", 234, [234]], ids=["string", "int", "int-list"]) +def test_malformed_body_tags_never_reach_an_upstream_error(prefixed: Gateway, upstream: Wire, tags: JsonValue) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + messages: Final[JsonValue] = [{"role": "user", "content": f"route this {marker}"}] + response: Final = prefixed.request( + "POST", "/v1/chat/completions", {"model": UNTAGGED, "messages": messages, "metadata": {"tags": tags}} + ) + + assert (response.status_code, _routed_backends(upstream, marker)) == (200, ("untagged-backend",)), response.text + + +def test_a_wrong_key_is_refused_before_routing(prefixed: Gateway, upstream: Wire) -> None: + response, backends = _send(prefixed, upstream, "chat", REGIONAL, ["route:region:us"], key="sk-not-a-real-key") + + assert response.status_code == 401, response.text + assert backends == () + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_team_requirement_holds_while_its_only_deployment_is_down( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint +) -> None: + with prefixed.scenario() as scenario: + key: Final = _team_key(scenario, ["®ion:eu"]) + response, backends = _send(prefixed, upstream, endpoint, OUTAGE, ["user_id:234"], key=key) + + assert response.status_code >= 500, response.text + assert set(backends) == {"eu-down-backend"} + + +def test_a_fallback_group_ignores_the_unprefixed_spend_tag_too(prefixed: Gateway, upstream: Wire) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + messages: Final[JsonValue] = [{"role": "user", "content": f"route this {marker}"}] + response: Final = prefixed.request( + "POST", + "/v1/chat/completions", + {"model": DOWN, "messages": messages, "metadata": {"tags": ["user_id:234"]}, "fallbacks": [UNTAGGED]}, + ) + backends: Final = _routed_backends(upstream, marker) + + assert response.status_code == 200, response.text + assert (set(backends), backends[-1]) == ({"untagged-down-backend", "untagged-backend"}, "untagged-backend") + + +@pytest.mark.parametrize( + ("user_agent", "backend"), [("claude-cli/2.0", "cli-backend"), ("Mozilla/5.0", "browser-backend")] +) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_user_agent_routing_is_unchanged_next_to_an_ignored_spend_tag( + prefixed: Gateway, upstream: Wire, endpoint: Endpoint, user_agent: str, backend: str +) -> None: + def send() -> tuple[int, tuple[str, ...]]: + marker: Final = f"marker-{uuid.uuid4().hex}" + path, body, headers = _call(endpoint, USER_AGENT_ROUTED, f"route this {marker}", ["user_id:234"], False) + response: Final = prefixed.request("POST", path, body, headers={**headers, "User-Agent": user_agent}) + return response.status_code, _routed_backends(upstream, marker) + + assert [send() for _ in range(4)] == [(200, (backend,))] * 4 + + +def test_a_mixed_concurrent_burst_routes_every_request_by_its_own_tags(prefixed: Gateway, upstream: Wire) -> None: + with prefixed.scenario() as scenario: + require_eu: Final = _team_key(scenario, ["®ion:eu"]) + exclude_eu: Final = scenario.key(metadata={"tags": ["!region:eu"]}) + plans: Final = tuple( + (endpoint, stream, model, tags, key, backend, f"marker-{uuid.uuid4().hex}") + for round_index in range(3) + for endpoint in ENDPOINTS + for stream in (False, True) + for model, tags, key, backend in ( + (UNTAGGED, ["user_id:234"], None, "untagged-backend"), + (REGIONAL, ["user_id:234"], require_eu, "eu-backend"), + (REGIONAL, ["user_id:234"], exclude_eu, "us-backend"), + (REGIONAL, ["user_id:234", "route:region:us", f"round:{round_index}"], None, "us-backend"), + ) + ) + + def send(plan: tuple[Endpoint, bool, str, list[str], str | None, str, str]) -> int: + endpoint, stream, model, tags, key, _backend, marker = plan + path, body, headers = _call(endpoint, model, f"route this {marker}", tags, stream) + return prefixed.request("POST", path, body, key=key, headers=headers).status_code + + with ThreadPoolExecutor(max_workers=24) as pool: + statuses: Final = tuple(pool.map(send, plans)) + captured: Final = tuple( + (request.body.decode(), string_value(JSON_OBJECT.validate_json(request.body)["model"])) + for request in upstream.drain() + if request.method == "POST" + ) + + assert statuses == (200,) * len(plans) + assert [tuple(backend for text, backend in captured if plan[-1] in text) for plan in plans] == [ + (plan[5],) for plan in plans + ] + + +@pytest.mark.parametrize( + ("tags", "backend"), + [(["region:us"], "us-backend"), (["!region:eu"], "us-backend"), (["®ion:eu"], "eu-backend")], + ids=["positive", "exclude", "require"], +) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_without_a_prefix_unprefixed_caller_tags_still_route( + unprefixed: Gateway, upstream: Wire, endpoint: Endpoint, tags: list[str], backend: str +) -> None: + routed: Final = tuple(_send(unprefixed, upstream, endpoint, REGIONAL, tags) for _ in range(4)) + + assert [response.status_code for response, _ in routed] == [200] * 4, [r.text for r, _ in routed] + assert [backends for _, backends in routed] == [(backend,)] * 4 + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_without_a_prefix_an_unknown_caller_tag_still_refuses_an_untagged_group( + unprefixed: Gateway, upstream: Wire, endpoint: Endpoint +) -> None: + response, backends = _send(unprefixed, upstream, endpoint, UNTAGGED, ["user_id:234"]) + + assert response.status_code == 401, response.text + assert REFUSED in response.text, response.text + assert backends == () + + +def _batch_rows(model: str, marker: str, count: int) -> bytes: + rows: Final = ( + { + "custom_id": f"row-{index}", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": f"route this {marker} {index}"}]}, + } + for index in range(count) + ) + return b"".join(json.dumps(row).encode() + b"\n" for row in rows) + + +def _run_batch(gateway: Gateway, key: str, model: str, marker: str, count: int) -> Mapping[str, JsonValue]: + uploaded: Final = gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model}, + {"file": ("rows.jsonl", _batch_rows(model, marker, count), "application/jsonl")}, + key=key, + ) + assert uploaded.status_code == 200, uploaded.text + created: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": string_value(JSON_OBJECT.validate_json(uploaded.content)["id"]), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + key=key, + ) + assert created.status_code == 200, created.text + batch_id: Final = string_value(JSON_OBJECT.validate_json(created.content)["id"]) + return eventually( + lambda: JSON_OBJECT.validate_json(gateway.request("GET", f"/v1/batches/{batch_id}", key=key).content), + lambda batch: batch.get("status") in ("completed", "failed", "expired", "cancelled"), + seconds=60, + ) + + +@pytest.mark.parametrize("owner", ["key", "team", "project"]) +def test_a_proxy_run_batch_keeps_applying_an_unprefixed_key_team_or_project_tag( + prefixed: Gateway, upstream: Wire, owner: Literal["key", "team", "project"] +) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + with prefixed.scenario() as scenario: + finished: Final = _run_batch(prefixed, _policy_key(scenario, owner, ["®ion:eu"]), BATCHED, marker, 8) + + assert finished["status"] == "completed", finished + assert _routed_backends(upstream, marker) == ("eu-batch-backend",) * 8 + + +def test_a_proxy_run_batch_cannot_fall_open_past_a_team_exclusion(unprefixed: Gateway, upstream: Wire) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + with unprefixed.scenario() as scenario: + key: Final = _team_key(scenario, ["!region:eu"]) + finished: Final = _run_batch(unprefixed, key, FAIL_OPEN_BATCHED, marker, 2) + batch_backends: Final = _routed_backends(upstream, marker) + response, chat_backends = _send(unprefixed, upstream, "chat", FAIL_OPEN_BATCHED, [], key=key) + + assert (finished["status"], finished["request_counts"]) == ( + "completed", + {"completed": 0, "failed": 2, "total": 2}, + ), finished + assert batch_backends == () + assert response.status_code == 401, response.text + assert chat_backends == () diff --git a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py index 4189a8dba91..f5c656c485c 100644 --- a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -702,6 +702,21 @@ async def test_create_dispatches_each_row_with_the_batch_model_and_the_key_metad assert metadata["user_api_key_user_email"] == "user@example.com" +async def test_create_dispatches_each_row_with_the_key_team_and_project_tags_as_inherited_tags() -> None: + user = UserAPIKeyAuth( + api_key="sk-batch-key", + team_id="team-1", + metadata={"tags": ["key-tag", "shared"]}, + team_metadata={"tags": ["®ion:eu", "shared"]}, + project_metadata={"tags": ["project-tag"]}, + ) + harness = make_runner(user=user) + await harness.create_and_finish() + + inherited = [call.kwargs["metadata"]["inherited_tags"] for call in harness.router.acompletion.await_args_list] + assert inherited == [("key-tag", "shared", "®ion:eu", "project-tag")] * 2 + + async def test_create_dispatches_cli_session_rows_under_the_per_user_alias_not_the_login_token() -> None: session = UserAPIKeyAuth( api_key="cli-session-Qm7xJ2kP9sLw4vT1nR8yAa", diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 57c0177f9b4..f0e85208933 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -9,6 +9,8 @@ import pytest import litellm from litellm._logging import verbose_logger +from litellm.router_strategy.tag_based_routing import get_deployments_for_tag +from litellm.types.router import RouterErrors async def _routed_model_ids( @@ -3171,3 +3173,96 @@ async def test_chat_request_carrying_litellm_metadata_still_routes_on_proxy_merg ) assert deployment["model_info"]["id"] == "team-a-deployment" + + +def _tag_router(model_list: list[dict[str, object]], tag_routing_prefix: str = "route:") -> litellm.Router: + return litellm.Router(model_list=model_list, enable_tag_filtering=True, tag_routing_prefix=tag_routing_prefix) + + +def _deployment(model_id: str, tags: list[str]) -> dict[str, object]: + return { + "model_name": "chat", + "litellm_params": { + "model": "gpt-4o", + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + **({"tags": tags} if tags else {}), + }, + "model_info": {"id": model_id}, + } + + +async def _routed_to(router: litellm.Router, metadata: dict[str, object]) -> str: + response: Final = await router.acompletion( + model="chat", messages=[{"role": "user", "content": "hi"}], metadata=metadata, mock_response="hi" + ) + return response._hidden_params["model_id"] + + +async def _pool(router: litellm.Router, metadata: dict[str, object]) -> frozenset[str]: + deployments: Final = await get_deployments_for_tag( + llm_router_instance=router, + model="chat", + healthy_deployments=router.get_model_list(model_name="chat"), + request_kwargs={"metadata": metadata}, + ) + return frozenset(deployment["model_info"]["id"] for deployment in deployments) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("tags", [["user_id:234"], ["®ion:nowhere"]]) +async def test_tag_routing_prefix_ignores_unprefixed_caller_tags_on_an_untagged_group(tags): + router = _tag_router([_deployment("untagged", [])]) + + assert await _routed_to(router, {"tags": tags}) == "untagged" + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("tags", [["region:eu"], ["!region:eu"], ["®ion:eu"], ["user_id:234"]]) +async def test_tag_routing_prefix_ignores_unprefixed_caller_tags_on_a_tagged_group(tags): + router = _tag_router([_deployment("eu-1", ["region:eu"]), _deployment("us-1", ["region:us"])]) + + assert await _pool(router, {"tags": tags}) == {"eu-1", "us-1"} + + +@pytest.mark.asyncio() +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + ({"tags": ["user_id:234", "route:region:us"]}, "us-1"), + ({"tags": ["route:region:eu", "!region:eu"]}, "eu-1"), + ({"tags": ["®ion:eu", "user_id:234"], "inherited_tags": ["®ion:eu"]}, "eu-1"), + ({"tags": ["!region:eu", "user_id:234"], "inherited_tags": ["!region:eu"]}, "us-1"), + ({"tags": ["user_id:234", "region:us"], "inherited_tags": ["region:us"]}, "us-1"), + ], + ids=["prefixed-caller-tag", "unprefixed-twin-ignored", "policy-require", "policy-exclude", "policy-positive"], +) +async def test_tag_routing_prefix_routes_on_prefixed_and_policy_tags(metadata, expected): + router = _tag_router([_deployment("eu-1", ["region:eu"]), _deployment("us-1", ["region:us"])]) + + assert await _pool(router, metadata) == {expected} + + +@pytest.mark.asyncio() +async def test_tag_routing_prefix_caller_cannot_route_around_a_policy_requirement(): + router = _tag_router([_deployment("eu-1", ["region:eu"]), _deployment("us-1", ["region:us"])]) + + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await _routed_to(router, {"tags": ["®ion:eu", "route:region:us"], "inherited_tags": ["®ion:eu"]}) + + +@pytest.mark.asyncio() +async def test_without_tag_routing_prefix_an_unknown_caller_tag_still_refuses_an_untagged_group(): + router = _tag_router([_deployment("untagged", [])], tag_routing_prefix="") + + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await _routed_to(router, {"tags": ["user_id:234"]}) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize(("tags", "expected"), [(["region:eu"], {"eu-1"}), (["!region:eu"], {"us-1"})]) +async def test_without_tag_routing_prefix_unprefixed_caller_tags_still_route(tags, expected): + router = _tag_router( + [_deployment("eu-1", ["region:eu"]), _deployment("us-1", ["region:us"])], tag_routing_prefix="" + ) + + assert await _pool(router, {"tags": tags}) == expected From d698106ed7ce85a493383d745b9f95953ba78867 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:00:57 +0000 Subject: [PATCH 078/124] fix(router)!: schedule default_priority requests once and reject a malformed priority (#47498) * fix(router): schedule default_priority requests once and reject a non-integer priority A router configured with default_priority failed every request that carried no priority with a 500 naming a missing 'priority' argument, and an explicit priority re-entered acompletion and hit the same error. acompletion now resolves the priority once (the request's integer, else the default, with 0 kept as a real priority) and schedules through _schedule_factory directly, while schedule_acompletion delegates to acompletion so the two paths share one queue pass. A non-integer priority answers 400 naming the param, or is dropped under drop_params, instead of leaking to the provider or crashing the scheduler. * fix(router): honor router-level drop_params and schedule prompt-management models under a priority * fix(router): drop a non-integer priority when the model's deployments set drop_params * fix(router): resolve the drop_params deployment scope for team routes and deployment ids * fix(router): read the request and router drop_params flags without a loop-assigned Final * refactor(router): type the prompt-management wrapper's return and mock the provider in its test * perf(router): consult drop_params for the priority only when it is not an integer * test(integration): assert the new priority contract for non-integer and zero priorities * docs(router): describe default_priority as the priority every request without one takes * fix(router): parse default_priority once and apply it to text completions and the queue route A digit string default is taken and any other non-integer default is ignored with a warning instead of failing every request. atext_completion and /queue/chat/completions resolve the priority the way acompletion does: a request without one takes the default, an integer queues once, and a non-integer answers 400 or takes the default under drop_params. Integration cells cover every default-priority surface on an owned two-instance rig. * fix(router): type the priority dispatch so the lint budgets hold * fix(router): read priority like the scheduler did, require a priority on the queue route, and keep litellm's drop_params order * fix(router): drop a priority named in additional_drop_params and report the queued priority on streamed queue responses * fix(router): serve deployment-id requests without waiting behind the queue and ignore a boolean default_priority A request whose model is a deployment id now counts that deployment as healthy in the scheduler, since routing by id already ignores cooldown. Before, the scheduler found no healthy deployment for an id, so the request waited behind every entry ahead of it. A boolean default_priority (true, or YAML off) is ignored at startup with a warning, like any other value that is not a whole number. The scheduler integration tests now share worker state for the defaulted pair, cool a group through /v1/responses so no stale in-memory queue is left behind, and use "high" instead of "1" for the non-integer cases. The concurrent burst test no longer asserts that the queue drains, matching the explicit-priority burst test, since the scheduler's read-modify-write queue can race across workers. * test(router): keep the queue drain check as a BUG-skipped integration test * fix(router): queue default_priority chat requests once instead of failing them * test(router): open with the default_priority regression tests alone * fix(router): honor deployment drop_params for a malformed priority on a request routed by provider model name --- litellm/litellm_core_utils/core_helpers.py | 2 +- litellm/proxy/proxy_server.py | 8 +- litellm/router.py | 153 +++-- litellm/router_utils/request_priority.py | 88 +++ .../router_code_coverage.py | 4 + .../test_priority_scheduler_queue_cleanup.py | 644 ++++++++++++++++-- .../proxy/proxy_server/test_routes_queue.py | 105 ++- .../proxy_server/test_streaming_helpers.py | 1 + .../router_utils/test_request_priority.py | 112 +++ tests/unit/test_router/test_router.py | 353 ++++++++++ 10 files changed, 1374 insertions(+), 96 deletions(-) create mode 100644 litellm/router_utils/request_priority.py create mode 100644 tests/unit/router_utils/test_request_priority.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 1e5bc7ecba4..b7c6f805233 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -520,7 +520,7 @@ def reconstruct_model_name( # Helper functions used for OTEL logging def get_parent_otel_span_from_kwargs( - kwargs: dict[str, object] | None = None, + kwargs: Mapping[str, object] | None = None, ) -> Span | None: try: if kwargs is None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1840586a08e..ee8f20be243 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18282,6 +18282,7 @@ async def async_queue_request( raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) router: Final = llm_router + queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data) if "stream" in data and data["stream"] is True: # use generate_responses to stream responses async def produce_queue_stream() -> StreamingResponse: @@ -18293,6 +18294,7 @@ async def async_queue_request( request=request, ), media_type="text/event-stream", + headers={"x-litellm-priority": str(queued_priority)}, ) async def audit_late_failure(exc: Exception) -> HTTPException | None: @@ -18307,7 +18309,7 @@ async def async_queue_request( ) response: Final = await router.schedule_acompletion(**data) - fastapi_response.headers.update({"x-litellm-priority": str(data["priority"])}) + fastapi_response.headers.update({"x-litellm-priority": str(queued_priority)}) return response except Exception as e: await proxy_logging_obj.post_call_failure_hook( @@ -18322,6 +18324,10 @@ async def async_queue_request( ) elif isinstance(e, ProxyException): raise e + elif isinstance(e, litellm.BadRequestError): + raise ProxyException( + message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code + ) raise ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/router.py b/litellm/router.py index e3c1e56b7e8..ba3b4d53725 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -327,6 +327,14 @@ from litellm.router_utils.reasoning_effort_capability import ( intersect_supported_reasoning_efforts, resolve_supported_reasoning_efforts, ) +from litellm.router_utils.request_priority import ( + InvalidPriority, + parse_default_priority, + priority_error, + request_drops_params, + resolve_request_priority, + unqueueable_priority_message, +) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( find_deployment_metadata, get_counted_usage_tokens, @@ -1219,7 +1227,7 @@ class Router: caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None. client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600. polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms. - default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None. + default_priority: (Optional[int]): the priority given to every chat or text completion request that sets no 'priority' of its own; '.acompletion()' and '.atext_completion()' queue it through the scheduler. Default is None. num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2. timeout (Optional[float]): Timeout for requests. Defaults to None. default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}. @@ -1359,7 +1367,15 @@ class Router: ### SCHEDULER ### self.scheduler = Scheduler(polling_interval=polling_interval, redis_cache=redis_cache) - self.default_priority = default_priority + parsed_default_priority: Final = parse_default_priority(default_priority) + if isinstance(parsed_default_priority, InvalidPriority): + verbose_router_logger.warning( + "router_settings.default_priority is ignored, no request is queued by default: %s", + parsed_default_priority.message, + ) + self.default_priority = ( + None if isinstance(parsed_default_priority, InvalidPriority) else parsed_default_priority + ) self.default_deployment = ( None # use this to track the users default deployment, when they want to use model = * ) @@ -3188,22 +3204,27 @@ class Router: kwargs["original_function"] = self._acompletion self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) - request_priority: Final = kwargs.get("priority") or self.default_priority start_time: Final = time.time() - _is_prompt_management_model: Final = self._is_prompt_management_model(model) - - if _is_prompt_management_model: - return await self._prompt_management_factory( - model=model, - messages=messages, - kwargs=kwargs, - ) - controls: Final = per_request_fallback_controls(kwargs) - kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop - if request_priority is not None and isinstance(request_priority, int): - response = await self.schedule_acompletion(**kwargs) + is_prompt_management_model: Final = self._is_prompt_management_model(model) + if not is_prompt_management_model: + controls: Final = per_request_fallback_controls(kwargs) + kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop + request_priority, request_kwargs = self.split_request_priority(model=model, kwargs=kwargs) + original_function: Final[Callable[..., Awaitable[object]]] = ( + self._prompt_management_acompletion(model=model, messages=messages) + if is_prompt_management_model + else self.async_function_with_fallbacks + ) + if request_priority is None: + response = await original_function(**request_kwargs) else: - response = await self.async_function_with_fallbacks(**kwargs) + response = await self._schedule_factory( + model=model, + priority=request_priority, + original_function=original_function, + args=(), + kwargs=request_kwargs, + ) end_time: Final = time.time() _duration: Final = end_time - start_time asyncio.create_task( @@ -5047,13 +5068,18 @@ class Router: @overload async def schedule_acompletion( - self, model: str, messages: list[AllMessageValues], priority: int, stream: Literal[False] = False, **kwargs + self, + model: str, + messages: list[AllMessageValues], + priority: int | None = None, + stream: Literal[False] = False, + **kwargs, ) -> ModelResponse: ... @overload async def schedule_acompletion( - self, model: str, messages: list[AllMessageValues], priority: int, stream: Literal[True], **kwargs + self, model: str, messages: list[AllMessageValues], priority: int | None, stream: Literal[True], **kwargs ) -> CustomStreamWrapper: ... @@ -5063,36 +5089,36 @@ class Router: self, model: str, messages: list[AllMessageValues], - priority: int, + priority: int | None = None, stream=False, **kwargs, ): - await self._wait_for_scheduler_turn( - model=model, - priority=priority, - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), - request_kwargs=cast(Mapping[str, object], kwargs), # cast-ok: kwargs keys are param names + queued_priority, _ = self.split_request_priority(model=model, kwargs={**kwargs, "priority": priority}) + if queued_priority is None: + raise priority_error(model=model, message=unqueueable_priority_message(priority)) + return await self.acompletion(model=model, messages=messages, stream=stream, priority=queued_priority, **kwargs) + + def split_request_priority( + self, model: str, kwargs: Mapping[str, object] + ) -> tuple[int | None, Mapping[str, object]]: + request_priority: Final = resolve_request_priority( + requested=kwargs.get("priority"), + default_priority=self.default_priority, + drops_params=lambda: request_drops_params( + kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) + ), ) - try: - _response: Final = await self.acompletion(model=model, messages=messages, stream=stream, **kwargs) - response_hidden_params: Final = get_hidden_params(_response) - if response_hidden_params is not None: - additional_headers: Final = cast( # cast-ok: router headers are stored as a mutable mapping - dict[str, object], response_hidden_params.setdefault("additional_headers", {}) - ) - additional_headers.update({"x-litellm-request-prioritization-used": True}) - return _response - except Exception as e: - setattr(e, "priority", priority) - raise e + if isinstance(request_priority, InvalidPriority): + raise priority_error(model=model, message=request_priority.message) + return request_priority, {key: value for key, value in kwargs.items() if key != "priority"} async def _schedule_factory( self, model: str, priority: int, - original_function: Callable, + original_function: Callable[..., Awaitable[object]], args: tuple[object, ...], - kwargs: dict[str, object], + kwargs: Mapping[str, object], ): await self._wait_for_scheduler_turn( model=model, @@ -5116,7 +5142,11 @@ class Router: async def _wait_for_scheduler_turn( self, model: str, priority: int, parent_otel_span: Span | None, request_kwargs: Mapping[str, object] ) -> None: + pinned: Final = self._pinned_deployment(model) + async def healthy_deployments() -> Sequence[object]: + if pinned is not None: + return (pinned,) deployments, _ = await self._async_get_healthy_deployments( model=model, parent_otel_span=parent_otel_span, @@ -5142,6 +5172,30 @@ class Router: split_litellm_model: Final = litellm_model.split("/")[0] return split_litellm_model in litellm._known_custom_logger_compatible_callbacks + def _pinned_deployment(self, model: str) -> Deployment | None: + return self.get_deployment(model_id=model) if model not in self.model_names else None + + def _request_deployment_params(self, model: str, kwargs: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + deployment: Final = self._pinned_deployment(model) + if deployment is not None: + return (deployment.litellm_params.model_dump(exclude_none=True),) + candidates: Final = ( + self._get_deployment_by_litellm_model(model=model) + if kwargs.get("specific_deployment") is True + else self.get_model_list(model_name=model, team_id=get_request_team_id(kwargs)) or () + ) + return tuple(candidate["litellm_params"] for candidate in candidates) + + def _prompt_management_acompletion( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: _prompt_management_factory takes the caller's list + ) -> Callable[..., Awaitable[object]]: + async def acompletion(**kwargs: object) -> object: # kwargs-ok: called with the request's own keyword payload + return await self._prompt_management_factory(model=model, messages=messages, kwargs=dict(kwargs)) + + return acompletion + async def _prompt_management_factory( self, model: str, @@ -5646,23 +5700,22 @@ class Router: is_async: bool | None = False, **kwargs, ): - if kwargs.get("priority", None) is not None: - return await self._schedule_factory( - model=model, - priority=kwargs.pop("priority"), - original_function=self.atext_completion, - args=(model, prompt), - kwargs=kwargs, - ) try: kwargs["model"] = model kwargs["prompt"] = prompt kwargs["original_function"] = self._atext_completion self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) - response: Final = await self.async_function_with_fallbacks(**kwargs) - - return response + request_priority, request_kwargs = self.split_request_priority(model=model, kwargs=kwargs) + if request_priority is None: + return await self.async_function_with_fallbacks(**request_kwargs) + return await self._schedule_factory( + model=model, + priority=request_priority, + original_function=self.async_function_with_fallbacks, + args=(), + kwargs=request_kwargs, + ) except Exception as e: asyncio.create_task( send_llm_exception_alert( @@ -14122,7 +14175,7 @@ class Router: _get_model_from_alias = get_model_from_alias - def _get_deployment_by_litellm_model(self, model: str) -> list: + def _get_deployment_by_litellm_model(self, model: str) -> list[DeploymentTypedDict]: """ Get the deployment by litellm model. """ diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py new file mode 100644 index 00000000000..14608cc1bcf --- /dev/null +++ b/litellm/router_utils/request_priority.py @@ -0,0 +1,88 @@ +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params + + +@dataclass(frozen=True, slots=True) +class InvalidPriority: + value: object + + @property + def message(self) -> str: + return f"priority must be an integer, got {type(self.value).__name__} {self.value!r}" + + +def priority_error(model: str, message: str) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=message, + model=model, + llm_provider="", + body={"message": message, "type": "invalid_request_error", "param": "priority", "code": "400"}, + ) + + +_PRIORITY: Final = TypeAdapter(int) + + +def _as_priority(value: object) -> int | InvalidPriority: + try: + return _PRIORITY.validate_python(value) + except ValidationError: + return InvalidPriority(value=value) + + +def resolve_request_priority( + requested: object, default_priority: int | None, drops_params: Callable[[], bool] +) -> int | None | InvalidPriority: + if requested is None: + return default_priority + priority: Final = _as_priority(requested) + if isinstance(priority, InvalidPriority) and drops_params(): + return default_priority + return priority + + +def parse_default_priority(value: object) -> int | None | InvalidPriority: + if value is None: + return None + if isinstance(value, bool): + return InvalidPriority(value=value) + return _as_priority(value) + + +def unqueueable_priority_message(requested: object) -> str: + priority: Final = None if requested is None else _as_priority(requested) + if isinstance(priority, InvalidPriority): + return f"{priority.message}, and there is no router_settings.default_priority to queue it at instead" + return "priority is required to queue a request, send one or set router_settings.default_priority" + + +def _explicit_drop_params(scope: Mapping[str, object]) -> bool | None: + return normalize_drop_params(scope.get("drop_params")) + + +def _drops_priority_by_name(scope: Mapping[str, object]) -> bool: + names: Final = scope.get("additional_drop_params") + return isinstance(names, list) and "priority" in names + + +def request_drops_params( + kwargs: Mapping[str, object], + router_defaults: Mapping[str, object], + deployment_params: Iterable[Mapping[str, object]], +) -> bool: + deployments: Final = tuple(deployment_params) + if litellm.drop_params is True or any(_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)): + return True + request_flag: Final = _explicit_drop_params(kwargs) + if request_flag is not None: + return request_flag + router_flag: Final = _explicit_drop_params(router_defaults) + if router_flag is not None: + return router_flag + return any(_explicit_drop_params(params) is True for params in deployments) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 3f0ad07a876..7809a23e955 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -114,6 +114,10 @@ ignored_function_names = [ "_aanthropic_messages_yield_recovered", # Tested through every mid-stream retry and fallback test in test_router.py "_anthropic_messages_policy_retries", # Tested through the retry budget precedence test in test_router.py "_get_wildcard_deployments", # Tested through the get_model_list_of_routed_group wildcard test in test_router.py + "_prompt_management_acompletion", # Tested through the scheduled prompt-management model test in test_router.py + "_request_deployment_params", # Tested through the team-route and deployment-id drop_params priority tests in test_router.py + "split_request_priority", # Tested through the acompletion, atext_completion and queue route priority tests + "_pinned_deployment", # Tested through the deployment-id scheduler test in test_router.py "_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py "_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py diff --git a/tests/integration/routing/test_priority_scheduler_queue_cleanup.py b/tests/integration/routing/test_priority_scheduler_queue_cleanup.py index 993da52fc67..9fc21843b04 100644 --- a/tests/integration/routing/test_priority_scheduler_queue_cleanup.py +++ b/tests/integration/routing/test_priority_scheduler_queue_cleanup.py @@ -11,6 +11,7 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from contextlib import ExitStack from dataclasses import dataclass +from itertools import product from pathlib import Path from queue import SimpleQueue from typing import Final, Literal, TypeVar @@ -24,7 +25,7 @@ import yaml from integration._support.client import Gateway, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.openai_wire import answering_model_discovery, chat_reply, openai_error, responses_reply -from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.process import OwnedState, graceful_stop_seconds, owned_proxy_process from integration._support.redis_process import OwnedRedis, owned_redis from integration._support.timing import assert_cut_short from integration._support.wire import Reply, Request, Wire, wire_server @@ -61,6 +62,18 @@ INMEM_GROUP: Final = "sched-inmem" OUTAGE_GROUP: Final = "sched-outage" KILL_GROUP: Final = "sched-kill" GHOST_ENTRY: Final[list[JsonValue]] = [1, "ghost"] +DEFAULT_PRIORITY: Final = 3 +DEFAULT_GROUPS: Final = tuple(f"sched-default-{row}" for row in ("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9")) +PROMPT_GROUP: Final = "sched-default-prompt" +PROMPT_ID: Final = "sched_default" +PROMPT_PREFIX: Final = "Scheduled by default for" +DROP_GROUP: Final = "sched-default-drop" +TEAM_GROUP: Final = "sched-default-team" +ZERO_GROUPS: Final = ("sched-zero-plain", "sched-zero-dropped") +DIGIT_GROUPS: Final = ("sched-digit-plain", "sched-digit-dropped") +IGNORED_GROUP: Final = "sched-ignored" +PRIORITIZED: Final = "x-litellm-request-prioritization-used" +IGNORED_WARNING: Final = "router_settings.default_priority is ignored" JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) JSON_LIST: Final = TypeAdapter(list[JsonValue]) @@ -160,15 +173,24 @@ def assert_never(received: Sequence[str], markers: Sequence[str]) -> None: assert not set(markers) & set(received), (markers, received) -def deployment_entry(group: str, wire: Wire) -> dict[str, JsonValue]: +def deployment_entry( + group: str, + wire: Wire, + *, + model_name: str | None = None, + model: str | None = None, + model_info: Mapping[str, JsonValue] | None = None, + **litellm_params: JsonValue, +) -> dict[str, JsonValue]: return { - "model_name": group, + "model_name": model_name or group, "litellm_params": { - "model": f"openai/{upstream_name(group)}", + "model": model or f"openai/{upstream_name(group)}", "api_base": wire.url + "/v1", "api_key": "synthetic-openai-key", + **litellm_params, }, - "model_info": {"id": f"{group}-deployment"}, + "model_info": {"id": f"{group}-deployment", **(model_info or {})}, } @@ -179,12 +201,16 @@ def owned_config( settings: Mapping[str, JsonValue], *, cancel_on_disconnect: bool = False, + extra_models: Sequence[Mapping[str, JsonValue]] = (), + litellm_settings: Mapping[str, JsonValue] | None = None, ) -> Path: base: Final = JSON_OBJECT.validate_python( yaml.safe_load((Path(__file__).parents[1] / "proxy_config.yaml").read_text()) ) general_settings: Final = base.get("general_settings", {}) assert isinstance(general_settings, dict) + base_litellm_settings: Final = base.get("litellm_settings", {}) + assert isinstance(base_litellm_settings, dict) general: Final[dict[str, JsonValue]] = { **general_settings, **({"cancel_on_disconnect": True} if cancel_on_disconnect else {}), @@ -192,8 +218,9 @@ def owned_config( config: Final[dict[str, JsonValue]] = { **base, "general_settings": general, + "litellm_settings": {**base_litellm_settings, **(litellm_settings or {})}, "router_settings": dict(settings), - "model_list": [deployment_entry(group, wire) for group in groups], + "model_list": [*(deployment_entry(group, wire) for group in groups), *(dict(entry) for entry in extra_models)], } path: Final = directory / f"scheduler-{uuid.uuid4().hex[:8]}.yaml" path.write_text(yaml.safe_dump(config)) @@ -209,6 +236,10 @@ def redis_settings(cache: OwnedRedis) -> dict[str, JsonValue]: return {"num_retries": 0, "redis_host": cache.host, "redis_port": cache.port} +def priority_field(priority: int | None) -> dict[str, JsonValue]: + return {} if priority is None else {"priority": priority} + + def chat_body(group: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]: return {"model": group, "messages": [{"role": "user", "content": marker}], **extra} @@ -279,26 +310,21 @@ def stream_identities(text: str) -> frozenset[str]: return frozenset(str(frame["id"]) for frame in frames) -def call(gateway: Gateway, endpoint: Endpoint, group: str, marker: str, priority: int = 1) -> Answer: +def call(gateway: Gateway, endpoint: Endpoint, group: str, marker: str, priority: int | None = 1) -> Answer: + field: Final = priority_field(priority) match endpoint: case "chat": - return post(gateway, "/v1/chat/completions", chat_body(group, marker, priority=priority)) + return post(gateway, "/v1/chat/completions", chat_body(group, marker, **field)) case "chat-stream": - return stream_text( - gateway, "/v1/chat/completions", chat_body(group, marker, priority=priority, stream=True) - ) + return stream_text(gateway, "/v1/chat/completions", chat_body(group, marker, stream=True, **field)) case "completions": - return post(gateway, "/v1/completions", completion_body(group, marker, priority=priority)) + return post(gateway, "/v1/completions", completion_body(group, marker, **field)) case "completions-stream": - return stream_text( - gateway, "/v1/completions", completion_body(group, marker, priority=priority, stream=True) - ) + return stream_text(gateway, "/v1/completions", completion_body(group, marker, stream=True, **field)) case "queue": - return post(gateway, "/queue/chat/completions", chat_body(group, marker, priority=priority)) + return post(gateway, "/queue/chat/completions", chat_body(group, marker, **field)) case "queue-stream": - return stream_text( - gateway, "/queue/chat/completions", chat_body(group, marker, priority=priority, stream=True) - ) + return stream_text(gateway, "/queue/chat/completions", chat_body(group, marker, stream=True, **field)) def served_identity(status: int, text: str, marker: str) -> str: @@ -363,15 +389,19 @@ def waiting_entries(cache: OwnedRedis, group: str, priority: int) -> list[JsonVa ) -def cooled(gateway: Gateway, upstream: Upstream, group: str) -> None: +def cooled_with(upstream: Upstream, group: str, send: Callable[[], Answer]) -> None: upstream.refusing[upstream_name(group)].set() - trip: Final = post(gateway, "/v1/chat/completions", chat_body(group, new_marker())) + trip: Final = send() assert trip.status == 401, (trip.status, trip.text) - eventually( - lambda: post(gateway, "/v1/chat/completions", chat_body(group, new_marker())), - lambda answer: answer.status == 429 and NO_DEPLOYMENTS in answer.text, - seconds=10, - ) + eventually(send, lambda answer: answer.status == 429 and NO_DEPLOYMENTS in answer.text, seconds=10) + + +def cooled(gateway: Gateway, upstream: Upstream, group: str) -> None: + cooled_with(upstream, group, lambda: post(gateway, "/v1/chat/completions", chat_body(group, new_marker()))) + + +def cooled_outside_the_scheduler(gateway: Gateway, upstream: Upstream, group: str) -> None: + cooled_with(upstream, group, lambda: post(gateway, "/v1/responses", {"model": group, "input": new_marker()})) def assert_refused_at_once(answer: Answer) -> None: @@ -488,16 +518,42 @@ def test_other_prioritized_endpoints_are_served_once_and_billed( spend_row_lands(identity) -@pytest.mark.parametrize("priority", ("1", [1], "", "p" * 5120, 0), ids=("string", "list", "empty", "5kb", "zero")) -def test_non_integer_priority_bypasses_the_scheduler_and_is_forwarded( +def upstream_body_for(wire: Wire, marker: str) -> dict[str, JsonValue]: + (upstream_request,) = tuple(request for request in wire.drain() if marker.encode() in request.body) + return JSON_OBJECT.validate_json(upstream_request.body) + + +@pytest.mark.parametrize("priority", ("high", [1], "", "p" * 5120, 1.5), ids=("word", "list", "empty", "5kb", "float")) +def test_non_integer_priority_is_refused_before_the_upstream( gateway: Gateway, rig_model: tuple[str, Wire], priority: JsonValue ) -> None: model, wire = rig_model marker: Final = new_marker() answer: Final = post(gateway, "/v1/chat/completions", chat_body(model, marker, priority=priority)) + assert answer.status == 400, (answer.status, answer.text) + error: Final = JSON_OBJECT.validate_json(answer.text)["error"] + assert isinstance(error, dict), answer.text + assert error["param"] == "priority", answer.text + assert "priority must be an integer" in str(error["message"]), answer.text + assert_never(received_markers(wire), (marker,)) + + +def test_non_integer_priority_is_dropped_under_drop_params(gateway: Gateway, rig_model: tuple[str, Wire]) -> None: + model, wire = rig_model + marker: Final = new_marker() + answer: Final = post(gateway, "/v1/chat/completions", chat_body(model, marker, priority="high", drop_params=True)) assert_served(answer, marker) - (upstream_request,) = tuple(request for request in wire.drain() if marker.encode() in request.body) - assert JSON_OBJECT.validate_json(upstream_request.body).get("priority") == priority + assert answer.headers.get("x-litellm-request-prioritization-used") is None, answer.headers + assert "priority" not in upstream_body_for(wire, marker) + + +def test_zero_priority_is_scheduled_and_not_forwarded(gateway: Gateway, rig_model: tuple[str, Wire]) -> None: + model, wire = rig_model + marker: Final = new_marker() + answer: Final = post(gateway, "/v1/chat/completions", chat_body(model, marker, priority=0)) + assert_served(answer, marker) + assert answer.headers.get("x-litellm-request-prioritization-used") == "True", answer.headers + assert "priority" not in upstream_body_for(wire, marker) def test_unauthenticated_prioritized_request_never_reaches_the_upstream( @@ -747,7 +803,9 @@ def test_disconnected_waiter_is_removed_from_the_queue(pair: Pair) -> None: Outcome = Answer | httpx.TransportError -def burst(gateway: Gateway, group: str, count: int) -> tuple[tuple[Endpoint, str, Outcome], ...]: +def burst( + gateway: Gateway, group: str, count: int, priority: int | None = 1 +) -> tuple[tuple[Endpoint, str, Outcome], ...]: plan: Final[tuple[tuple[Endpoint, str], ...]] = tuple( (ENDPOINTS[index % len(ENDPOINTS)], new_marker()) for index in range(count) ) @@ -756,7 +814,7 @@ def burst(gateway: Gateway, group: str, count: int) -> tuple[tuple[Endpoint, str endpoint, marker = item with httpx.Client(base_url=gateway.client.base_url, timeout=60, trust_env=False) as client: try: - return call(Gateway(client, gateway.key, gateway.upstream_url), endpoint, group, marker) + return call(Gateway(client, gateway.key, gateway.upstream_url), endpoint, group, marker, priority) except httpx.TransportError as error: return error @@ -772,10 +830,14 @@ def assert_all_served(outcomes: Sequence[tuple[Endpoint, str, Outcome]]) -> tupl return tuple(marker for _, marker, _ in outcomes) -def served_eventually(gateway: Gateway, group: str, marker: str) -> None: +def default_for(priority: int | None) -> dict[str, JsonValue]: + return {} if priority is not None else {"default_priority": DEFAULT_PRIORITY} + + +def served_eventually(gateway: Gateway, group: str, marker: str, priority: int | None = 1) -> None: def send() -> Outcome: try: - return post(gateway, "/v1/chat/completions", chat_body(group, marker, priority=1)) + return post(gateway, "/v1/chat/completions", chat_body(group, marker, **priority_field(priority))) except httpx.TransportError as error: return error @@ -784,22 +846,25 @@ def served_eventually(gateway: Gateway, group: str, marker: str) -> None: assert_served(answer, marker) +@pytest.mark.parametrize("priority", (1, None), ids=("explicit", "default")) @pytest.mark.timeout(OWNED_CELL_TIMEOUT) -def test_prioritized_requests_survive_a_redis_outage(gateway: Gateway, tmp_path: Path) -> None: +def test_prioritized_requests_survive_a_redis_outage(gateway: Gateway, tmp_path: Path, priority: int | None) -> None: upstream: Final = upstream_for((OUTAGE_GROUP,)) with owned_redis(tmp_path) as cache, wire_server(answering_model_discovery(upstream.respond)) as wire: - config: Final = owned_config(tmp_path, wire, (OUTAGE_GROUP,), redis_settings(cache)) + config: Final = owned_config( + tmp_path, wire, (OUTAGE_GROUP,), {**redis_settings(cache), **default_for(priority)} + ) overrides: Final = { "REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port), "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "1", } with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: - before: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12)) + before: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12, priority)) cache.stop() - during: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12)) + during: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12, priority)) cache.start() - after: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12)) + after: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12, priority)) assert_once(received_markers(wire), (*before, *during, *after)) eventually(lambda: queue_entries(cache, OUTAGE_GROUP), lambda entries: entries in (None, []), seconds=10) @@ -813,13 +878,14 @@ def established_upstream_connections(pid: int, wire: Wire) -> int: ) +@pytest.mark.parametrize("priority", (1, None), ids=("explicit", "default")) @pytest.mark.timeout(OWNED_CELL_TIMEOUT) def test_sibling_worker_keeps_serving_prioritized_requests_after_a_worker_is_killed( - gateway: Gateway, tmp_path: Path + gateway: Gateway, tmp_path: Path, priority: int | None ) -> None: upstream: Final = upstream_for((KILL_GROUP,), hold=(KILL_GROUP,)) with owned_redis(tmp_path) as cache, wire_server(answering_model_discovery(upstream.respond)) as wire: - config: Final = owned_config(tmp_path, wire, (KILL_GROUP,), redis_settings(cache)) + config: Final = owned_config(tmp_path, wire, (KILL_GROUP,), {**redis_settings(cache), **default_for(priority)}) overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: workers: Final = eventually( @@ -828,7 +894,7 @@ def test_sibling_worker_keeps_serving_prioritized_requests_after_a_worker_is_kil seconds=graceful_stop_seconds(), ) with ThreadPoolExecutor(max_workers=1) as pool: - pending: Final = pool.submit(burst, owned.gateway, KILL_GROUP, 20) + pending: Final = pool.submit(burst, owned.gateway, KILL_GROUP, 20, priority) eventually(upstream.held.qsize, lambda size: size == 20, seconds=60) held_by: Final = {pid: established_upstream_connections(pid, wire) for pid in workers} assert sum(held_by.values()) == 20, held_by @@ -844,10 +910,502 @@ def test_sibling_worker_keeps_serving_prioritized_requests_after_a_worker_is_kil assert len(failed) == held_by[victim_pid], (held_by, len(failed)) assert_all_served(answered) follow_up: Final = new_marker() - served_eventually(owned.gateway, KILL_GROUP, follow_up) + served_eventually(owned.gateway, KILL_GROUP, follow_up, priority) eventually( lambda: len(STARTED_WORKER.findall(owned.log.read_text())), lambda count: count == 3, seconds=graceful_stop_seconds(), ) assert_once(received_markers(wire), (*(marker for _, marker, _ in outcomes), follow_up)) + + +@dataclass(frozen=True, slots=True) +class Defaulted: + pair: Pair + team_key: str + + def as_team(self) -> Gateway: + return Gateway(self.pair.first.client, self.team_key, self.pair.first.upstream_url) + + +def prompt_directory(directory: Path) -> Path: + prompts: Final = directory / "prompts" + prompts.mkdir() + (prompts / f"{PROMPT_ID}.prompt").write_text( + "---\n" + f"model: openai/{upstream_name(PROMPT_GROUP)}\n" + "input:\n" + " schema:\n" + " user_message: string\n" + "---\n" + "\n" + f"{PROMPT_PREFIX} {{{{user_message}}}}\n" + ) + return prompts + + +@pytest.fixture(scope="module") +def defaulted(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Defaulted]: + directory: Final = tmp_path_factory.mktemp("scheduler-default") + upstream: Final = upstream_for((*DEFAULT_GROUPS, PROMPT_GROUP, DROP_GROUP, TEAM_GROUP)) + with ExitStack() as stack: + gateway: Final = stack.enter_context(gateway_from_environment()) + scenario: Final = stack.enter_context(gateway.scenario()) + team: Final = scenario.team(models=[]) + team_key: Final = scenario.key(team_id=team) + cache: Final = stack.enter_context(owned_redis(directory)) + wire: Final = stack.enter_context(wire_server(answering_model_discovery(upstream.respond))) + extra_models: Final = ( + deployment_entry( + PROMPT_GROUP, wire, model=f"dotprompt/openai/{upstream_name(PROMPT_GROUP)}", prompt_id=PROMPT_ID + ), + deployment_entry(DROP_GROUP, wire, drop_params=True), + deployment_entry( + TEAM_GROUP, + wire, + model_name=f"{TEAM_GROUP}-internal", + model_info={"team_id": team, "team_public_model_name": TEAM_GROUP}, + drop_params=True, + ), + ) + settings: Final[dict[str, JsonValue]] = { + **cooldown_settings(cache), + "default_priority": DEFAULT_PRIORITY, + "fallbacks": [{DEFAULT_GROUPS[7]: [DEFAULT_GROUPS[8]]}], + } + config: Final = owned_config( + directory, + wire, + DEFAULT_GROUPS, + settings, + cancel_on_disconnect=True, + extra_models=extra_models, + litellm_settings={"global_prompt_directory": str(prompt_directory(directory))}, + ) + overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} + shared: Final = OwnedState.SHARED_WITH_WORKER + first: Final = stack.enter_context( + owned_proxy_process(gateway, directory, overrides, config=config, workers=2, state=shared) + ) + second: Final = stack.enter_context( + owned_proxy_process(gateway, directory, overrides, config=config, workers=2, state=shared) + ) + yield Defaulted(Pair(first.gateway, second.gateway, cache, wire, upstream), team_key) + + +def assert_prioritized(answer: Answer) -> None: + assert answer.headers.get(PRIORITIZED) == "True", answer.headers + + +def assert_unprioritized(answer: Answer) -> None: + assert answer.headers.get(PRIORITIZED) is None, answer.headers + + +def assert_refused_as_non_integer(answer: Answer) -> None: + assert answer.status == 400, (answer.status, answer.text) + error: Final = JSON_OBJECT.validate_json(answer.text)["error"] + assert isinstance(error, dict), answer.text + assert error["param"] == "priority", answer.text + assert "priority must be an integer" in str(error["message"]), answer.text + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_raw_chat_without_priority_is_scheduled_at_the_default_and_billed(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[0] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", chat_body(group, marker)) + identity: Final = assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + spend_row_lands(identity) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_raw_chat_stream_without_priority_is_served_once(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[0] + marker: Final = new_marker() + answer: Final = stream_text(defaulted.pair.second, "/v1/chat/completions", chat_body(group, marker, stream=True)) + assert_served(answer, marker) + assert_prioritized(answer) + assert_once(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_sdk_chat_without_priority_is_scheduled_at_the_default_and_billed(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[0] + marker: Final = new_marker() + client: Final = sdk(defaulted.pair.first) + raw: Final = sdk_settled( + lambda: client.chat.completions.with_raw_response.create( + model=group, messages=[{"role": "user", "content": marker}] + ) + ) + assert served_identity(raw.status_code, raw.text, marker) == f"chatcmpl-{marker}" + assert raw.headers.get(PRIORITIZED) == "True", dict(raw.headers) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + spend_row_lands(f"chatcmpl-{marker}") + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_sdk_chat_stream_without_priority_is_served_once(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[0] + marker: Final = new_marker() + client: Final = sdk(defaulted.pair.second) + chunks: Final = sdk_settled( + lambda: tuple( + client.chat.completions.create(model=group, messages=[{"role": "user", "content": marker}], stream=True) + ) + ) + assert {chunk.id for chunk in chunks} == {f"chatcmpl-{marker}"}, chunks + content: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) + assert content == f"served {marker}", chunks + assert_once(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_async_sdk_chat_without_priority_is_scheduled_at_the_default(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[0] + marker: Final = new_marker() + gateway: Final = defaulted.pair.first + + async def send() -> tuple[int, str, str | None]: + async with openai.AsyncOpenAI( + base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=30 + ) as client: + raw: Final = await client.chat.completions.with_raw_response.create( + model=group, messages=[{"role": "user", "content": marker}] + ) + return raw.status_code, raw.text, raw.headers.get(PRIORITIZED) + + status, text, prioritized = sdk_settled(lambda: asyncio.run(send())) + assert served_identity(status, text, marker) == f"chatcmpl-{marker}" + assert prioritized == "True" + assert_once(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.parametrize("priority", (1, 0, -1, 2**40), ids=("one", "zero", "negative", "huge")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_explicit_integer_priority_under_a_default_is_scheduled_once(defaulted: Defaulted, priority: int) -> None: + group: Final = DEFAULT_GROUPS[1] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", chat_body(group, marker, priority=priority)) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_null_priority_under_a_default_takes_the_default(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[1] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.second, "/v1/chat/completions", chat_body(group, marker, priority=None)) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.parametrize( + "priority", + ("high", [1], 1.5, {"level": 3}, "", "p" * 5120), + ids=("word", "list", "float", "object", "empty", "5kb"), +) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_non_integer_priority_under_a_default_is_refused_before_the_upstream( + defaulted: Defaulted, priority: JsonValue +) -> None: + group: Final = DEFAULT_GROUPS[1] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", chat_body(group, marker, priority=priority)) + assert_refused_as_non_integer(answer) + assert_never(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_request_drop_params_replaces_a_non_integer_priority_with_the_default(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[1] + marker: Final = new_marker() + body: Final = chat_body(group, marker, priority="high", drop_params=True) + answer: Final = post(defaulted.pair.second, "/v1/chat/completions", body) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.parametrize("model", (DROP_GROUP, f"{DROP_GROUP}-deployment"), ids=("group", "deployment-id")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_deployment_drop_params_replaces_a_non_integer_priority_with_the_default( + defaulted: Defaulted, model: str +) -> None: + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", chat_body(model, marker, priority="high")) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_team_deployment_drop_params_replaces_a_non_integer_priority_with_the_default(defaulted: Defaulted) -> None: + marker: Final = new_marker() + answer: Final = post(defaulted.as_team(), "/v1/chat/completions", chat_body(TEAM_GROUP, marker, priority="high")) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +def assert_waits_at( + gateway: Gateway, + cache: OwnedRedis, + upstream: Upstream, + group: str, + priority: int, + body: Mapping[str, JsonValue], +) -> None: + cooled_outside_the_scheduler(gateway, upstream, group) + write_queue(cache, group, [[priority - 1, "ghost"]]) + with ThreadPoolExecutor(max_workers=1) as pool: + waiter: Final = pool.submit(post, gateway, "/v1/chat/completions", body) + entries: Final = waiting_entries(cache, group, priority) + write_queue(cache, group, [entry for entry in entries if entry_priority(entry) == priority]) + assert_refused_before_the_router_timeout(waiter.result(timeout=30)) + assert_drained(cache, group) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_requests_without_priority_wait_at_the_default_priority(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[2] + pair: Final = defaulted.pair + assert_waits_at(pair.second, pair.cache, pair.upstream, group, DEFAULT_PRIORITY, chat_body(group, new_marker())) + + +@pytest.mark.timeout(OWNED_CELL_TIMEOUT) +def test_zero_default_priority_queues_requests_without_one_and_dropped_ones(gateway: Gateway, tmp_path: Path) -> None: + upstream: Final = upstream_for(ZERO_GROUPS) + with owned_redis(tmp_path) as cache, wire_server(answering_model_discovery(upstream.respond)) as wire: + settings: Final[dict[str, JsonValue]] = { + **cooldown_settings(cache), + "default_priority": 0, + "default_litellm_params": {"drop_params": True}, + } + config: Final = owned_config(tmp_path, wire, ZERO_GROUPS, settings) + overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} + with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: + plain, dropped = ZERO_GROUPS + assert_waits_at(owned.gateway, cache, upstream, plain, 0, chat_body(plain, new_marker())) + assert_waits_at( + owned.gateway, cache, upstream, dropped, 0, chat_body(dropped, new_marker(), priority="high") + ) + + +@pytest.mark.timeout(OWNED_CELL_TIMEOUT) +def test_digit_string_default_priority_queues_requests_without_one_and_dropped_ones( + gateway: Gateway, tmp_path: Path +) -> None: + upstream: Final = upstream_for(DIGIT_GROUPS) + with owned_redis(tmp_path) as cache, wire_server(answering_model_discovery(upstream.respond)) as wire: + settings: Final[dict[str, JsonValue]] = {**cooldown_settings(cache), "default_priority": "2"} + config: Final = owned_config(tmp_path, wire, DIGIT_GROUPS, settings, litellm_settings={"drop_params": True}) + overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} + with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: + plain, dropped = DIGIT_GROUPS + assert_waits_at(owned.gateway, cache, upstream, plain, 2, chat_body(plain, new_marker())) + assert_waits_at( + owned.gateway, cache, upstream, dropped, 2, chat_body(dropped, new_marker(), priority="high") + ) + + +@pytest.mark.parametrize("configured", ("high", True), ids=("word", "bool")) +@pytest.mark.timeout(OWNED_CELL_TIMEOUT) +def test_an_invalid_default_priority_is_ignored_with_a_warning( + gateway: Gateway, tmp_path: Path, configured: JsonValue +) -> None: + upstream: Final = upstream_for((IGNORED_GROUP,)) + with wire_server(answering_model_discovery(upstream.respond)) as wire: + config: Final = owned_config( + tmp_path, wire, (IGNORED_GROUP,), {"num_retries": 0, "default_priority": configured} + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: + marker: Final = new_marker() + answer: Final = post(owned.gateway, "/v1/chat/completions", chat_body(IGNORED_GROUP, marker)) + assert_served(answer, marker) + assert_unprioritized(answer) + assert IGNORED_WARNING in owned.log.read_text() + assert_once(received_markers(wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_requests_without_priority_fall_back_to_the_fallback_group(defaulted: Defaulted) -> None: + primary, fallback = DEFAULT_GROUPS[7], DEFAULT_GROUPS[8] + defaulted.pair.upstream.refusing[upstream_name(primary)].set() + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", chat_body(primary, marker)) + assert_served(answer, marker) + assert_prioritized(answer) + attempts: Final = tuple( + str(JSON_OBJECT.validate_json(request.body)["model"]) + for request in defaulted.pair.wire.drain() + if marker.encode() in request.body + ) + assert attempts == (upstream_name(primary), upstream_name(fallback)), attempts + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_cooled_group_refuses_requests_without_priority_at_once(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[3] + cooled(defaulted.pair.second, defaulted.pair.upstream, group) + assert_refused_at_once(post(defaulted.pair.second, "/v1/chat/completions", chat_body(group, new_marker()))) + assert_drained(defaulted.pair.cache, group) + + +@pytest.mark.parametrize("priority", (None, 1), ids=("default", "explicit")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_prompt_management_model_is_scheduled(defaulted: Defaulted, priority: int | None) -> None: + marker: Final = new_marker() + body: Final = chat_body(PROMPT_GROUP, marker, prompt_variables={"user_message": marker}, **priority_field(priority)) + answer: Final = post(defaulted.pair.first, "/v1/chat/completions", body) + assert_served(answer, marker) + assert_prioritized(answer) + upstream_body: Final = upstream_body_for(defaulted.pair.wire, marker) + assert f"{PROMPT_PREFIX} {marker}" in json.dumps(upstream_body), upstream_body + assert "priority" not in upstream_body + + +@pytest.mark.parametrize("endpoint", ("completions", "completions-stream")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_text_completions_without_priority_are_scheduled_at_the_default( + defaulted: Defaulted, endpoint: Endpoint +) -> None: + group: Final = DEFAULT_GROUPS[4] + marker: Final = new_marker() + answer: Final = call(defaulted.pair.second, endpoint, group, marker, priority=None) + identity: Final = assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + spend_row_lands(identity) + + +@pytest.mark.parametrize("priority", ("high", [1]), ids=("word", "list")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_text_completion_with_a_non_integer_priority_is_refused_before_the_upstream( + defaulted: Defaulted, priority: JsonValue +) -> None: + group: Final = DEFAULT_GROUPS[4] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/v1/completions", completion_body(group, marker, priority=priority)) + assert_refused_as_non_integer(answer) + assert_never(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_text_completion_drop_params_replaces_a_non_integer_priority_with_the_default(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[4] + marker: Final = new_marker() + body: Final = completion_body(group, marker, priority="high", drop_params=True) + answer: Final = post(defaulted.pair.second, "/v1/completions", body) + assert_served(answer, marker) + assert_prioritized(answer) + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.parametrize("endpoint", ("queue", "queue-stream")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_queue_route_without_priority_takes_the_default(defaulted: Defaulted, endpoint: Endpoint) -> None: + group: Final = DEFAULT_GROUPS[5] + marker: Final = new_marker() + answer: Final = call(defaulted.pair.first, endpoint, group, marker, priority=None) + assert_served(answer, marker) + if endpoint == "queue": + assert answer.headers.get("x-litellm-priority") == str(DEFAULT_PRIORITY), answer.headers + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.parametrize("priority", (1, "1", True, 1.0), ids=("integer", "digit-string", "bool", "whole-float")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_queue_route_with_an_explicit_priority_under_a_default_reports_it( + defaulted: Defaulted, priority: JsonValue +) -> None: + group: Final = DEFAULT_GROUPS[5] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.second, "/queue/chat/completions", chat_body(group, marker, priority=priority)) + assert_served(answer, marker) + assert answer.headers.get("x-litellm-priority") == "1", answer.headers + assert "priority" not in upstream_body_for(defaulted.pair.wire, marker) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_queue_route_refuses_a_word_priority(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[5] + marker: Final = new_marker() + answer: Final = post(defaulted.pair.first, "/queue/chat/completions", chat_body(group, marker, priority="high")) + assert_refused_as_non_integer(answer) + assert_never(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.parametrize("path", ("/v1/messages", "/v1/responses"), ids=("messages", "responses")) +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_unscheduled_routes_without_priority_are_served_once_under_a_default(defaulted: Defaulted, path: str) -> None: + group: Final = DEFAULT_GROUPS[6] + marker: Final = new_marker() + body: Final[dict[str, JsonValue]] = ( + {"model": group, "max_tokens": 32, "messages": [{"role": "user", "content": marker}]} + if path == "/v1/messages" + else {"model": group, "input": marker} + ) + answer: Final = post(defaulted.pair.second, path, body) + assert answer.status == 200, (answer.status, answer.text) + assert markers_in(answer.text) == {marker}, answer.text + assert_once(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_unauthenticated_request_without_priority_never_reaches_the_upstream(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[6] + marker: Final = new_marker() + response: Final = defaulted.pair.first.client.post("/v1/chat/completions", json=chat_body(group, marker)) + assert response.status_code == 401, response.text + assert_never(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_cached_replay_under_a_default_reaches_the_upstream_once(defaulted: Defaulted) -> None: + group: Final = DEFAULT_GROUPS[6] + marker: Final = new_marker() + body: Final = chat_body(group, marker) + first: Final = post(defaulted.pair.first, "/v1/chat/completions", body) + assert_served(first, marker) + assert_prioritized(first) + replay: Final = post(defaulted.pair.second, "/v1/chat/completions", body) + assert_served(replay, marker) + assert_prioritized(replay) + assert_once(received_markers(defaulted.pair.wire), (marker,)) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_concurrent_burst_without_priority_across_instances_and_endpoints_is_served(defaulted: Defaulted) -> None: + serve_burst_without_priority(defaulted.pair, DEFAULT_GROUPS[0]) + + +@pytest.mark.timeout(PAIR_CELL_TIMEOUT) +def test_concurrent_burst_without_priority_across_instances_leaves_no_queue_entry(defaulted: Defaulted) -> None: + pytest.skip("BUG: concurrent queue writes across workers can leave a served request's entry in the queue") + group: Final = DEFAULT_GROUPS[0] + serve_burst_without_priority(defaulted.pair, group) + assert_drained(defaulted.pair.cache, group) + + +def serve_burst_without_priority(pair: Pair, group: str) -> None: + primer: Final = new_marker() + assert_served(post(pair.first, "/v1/chat/completions", chat_body(group, primer)), primer) + instances: Final[tuple[Instance, ...]] = ("first", "second") + plan: Final = tuple((instance, endpoint, new_marker()) for instance, endpoint in product(instances, ENDPOINTS * 2)) + + def one(item: tuple[Instance, Endpoint, str]) -> Answer: + instance, endpoint, marker = item + return call(pair.at(instance), endpoint, group, marker, priority=None) + + with ThreadPoolExecutor(max_workers=len(plan)) as pool: + answers: Final = tuple(pool.map(one, plan)) + for (_, _, marker), answer in zip(plan, answers, strict=True): + assert_served(answer, marker) + closing: Final = new_marker() + assert_served(post(pair.second, "/v1/chat/completions", chat_body(group, closing)), closing) + assert_once(received_markers(pair.wire), (primer, *(marker for _, _, marker in plan), closing)) diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index 27cc6300711..07a61c8587b 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -6,10 +6,12 @@ Pins (PR2): from __future__ import annotations -from unittest.mock import AsyncMock, MagicMock +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm import Router from litellm.proxy import proxy_server from .conftest import normalize # type: ignore[import-not-found] @@ -34,6 +36,7 @@ HAPPY_RESPONSE = { @pytest.fixture def patched_queue(monkeypatch): router = MagicMock() + router.split_request_priority = MagicMock(return_value=(0, {})) router.schedule_acompletion = AsyncMock(return_value=dict(HAPPY_RESPONSE)) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr( @@ -89,3 +92,103 @@ def test_queue_chat_completions_no_router_error(client, auth_as, queue_no_router response = client.post("/queue/chat/completions", json=payload) assert response.status_code == 500 assert len(response.content) > 0 + + +def _queue_router( + monkeypatch: pytest.MonkeyPatch, default_priority: int | None, mock_response: str = "queued" +) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4", "api_key": "sk-queue-test", "mock_response": mock_response}, + } + ], + default_priority=default_priority, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock())) + return router + + +@pytest.mark.parametrize( + ("default_priority", "priority_fields", "expected_priority"), + [ + (3, {"priority": "high", "drop_params": True}, 3), + (3, {"priority": "1"}, 1), + (None, {"priority": 1.0}, 1), + (3, {"priority": 1}, 1), + (3, {}, 3), + ], + ids=("dropped-word-takes-default", "digit-string", "integral-float", "explicit", "absent-takes-default"), +) +def test_queue_chat_completions_reports_the_priority_the_router_queued_at( + client, + auth_as, + monkeypatch: pytest.MonkeyPatch, + default_priority: int | None, + priority_fields: dict[str, object], + expected_priority: int, +) -> None: + router: Final = _queue_router(monkeypatch, default_priority) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post("/queue/chat/completions", json=payload) + assert response.status_code == 200, response.text + assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority] + assert response.headers.get("x-litellm-priority") == str(expected_priority) + + +@pytest.mark.parametrize( + ("default_priority", "priority_fields", "message"), + [ + (None, {}, "priority is required"), + (3, {"priority": "high"}, "priority must be an integer"), + (None, {"priority": "high", "drop_params": True}, "priority must be an integer, got str 'high'"), + ], + ids=("no-priority-and-no-default", "malformed", "dropped-word-and-no-default"), +) +def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_request( + client, + auth_as, + monkeypatch: pytest.MonkeyPatch, + default_priority: int | None, + priority_fields: dict[str, object], + message: str, +) -> None: + router: Final = _queue_router(monkeypatch, default_priority) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post("/queue/chat/completions", json=payload) + assert response.status_code == 400, response.text + error: Final = response.json()["error"] + assert (error["type"], error["param"]) == ("invalid_request_error", "priority") + assert message in error["message"] + assert add_request.await_count == 0 + + +def test_streamed_queue_responses_report_the_priority_the_router_queued_at( + client, auth_as, monkeypatch: pytest.MonkeyPatch +) -> None: + _queue_router(monkeypatch, default_priority=3) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": "1", "stream": True} + with auth_as(): + response: Final = client.post("/queue/chat/completions", json=payload) + assert response.status_code == 200, response.text + assert response.headers.get("x-litellm-priority") == "1" + + +def test_a_provider_bad_request_on_the_queue_route_keeps_its_own_message( + client, auth_as, monkeypatch: pytest.MonkeyPatch +) -> None: + _queue_router(monkeypatch, default_priority=3, mock_response="litellm.ContextWindowExceededError") + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]} + with auth_as(): + response: Final = client.post("/queue/chat/completions", json=payload) + assert response.status_code == 400, response.text + error: Final = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert "context window" in error["message"].lower() + assert "Authentication Error" not in error["message"] diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 227a099889d..9a3516d20d6 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -2322,6 +2322,7 @@ async def _queue_streaming(monkeypatch, interval, delay=0.3, fails_with=None): router = MagicMock() router.get_model_list.return_value = [] + router.split_request_priority.return_value = (0, {}) async def _schedule_after_the_scheduler_queue_drains(**kwargs): await asyncio.sleep(delay) diff --git a/tests/unit/router_utils/test_request_priority.py b/tests/unit/router_utils/test_request_priority.py new file mode 100644 index 00000000000..71ab441d199 --- /dev/null +++ b/tests/unit/router_utils/test_request_priority.py @@ -0,0 +1,112 @@ +from typing import Final + +import pytest + +import litellm +from litellm.router_utils.request_priority import ( + InvalidPriority, + parse_default_priority, + request_drops_params, + resolve_request_priority, + unqueueable_priority_message, +) + + +@pytest.mark.parametrize( + ("requested", "default_priority", "expected"), + [ + (None, None, None), + (None, 3, 3), + (2, 3, 2), + (0, 3, 0), + (0, None, 0), + (255, None, 255), + ("1", 3, 1), + (" 2 ", None, 2), + ("-1", 3, -1), + (1.0, 3, 1), + (True, None, 1), + ], +) +def test_an_absent_or_null_priority_takes_the_default_and_an_integer_like_value_wins( + requested: object, default_priority: int | None, expected: int | None +) -> None: + assert resolve_request_priority(requested, default_priority, drops_params=lambda: False) == expected + assert resolve_request_priority(requested, default_priority, drops_params=lambda: True) == expected + + +@pytest.mark.parametrize("requested", ["high", "", [1], 1.5, {"level": 1}]) +def test_a_non_integer_priority_is_rejected_unless_params_are_dropped(requested: object) -> None: + rejected: Final = resolve_request_priority(requested, 3, drops_params=lambda: False) + assert isinstance(rejected, InvalidPriority) + assert rejected.value == requested + assert rejected.message.startswith("priority must be an integer, got ") + assert resolve_request_priority(requested, 3, drops_params=lambda: True) == 3 + assert resolve_request_priority(requested, None, drops_params=lambda: True) is None + + +def test_the_drop_params_decision_is_consulted_only_for_a_non_integer_priority() -> None: + def _never() -> bool: + raise AssertionError("drop_params was consulted") + + assert resolve_request_priority(None, 3, drops_params=_never) == 3 + assert resolve_request_priority(2, 3, drops_params=_never) == 2 + assert resolve_request_priority(0, None, drops_params=_never) == 0 + assert resolve_request_priority("1", 3, drops_params=_never) == 1 + + +def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_then_the_deployments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert request_drops_params({}, {}, ()) is False + assert request_drops_params({"drop_params": True}, {}, ()) is True + assert request_drops_params({"drop_params": "true"}, {}, ()) is True + assert request_drops_params({}, {"drop_params": True}, ()) is True + assert request_drops_params({"drop_params": False}, {"drop_params": True}, ()) is False + assert request_drops_params({}, {}, ({"drop_params": True},)) is True + assert request_drops_params({}, {}, ({"drop_params": True}, {"model": "openai/gpt-5.4-nano"})) is True + assert request_drops_params({}, {}, ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})) is False + assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is False + monkeypatch.setattr(litellm, "drop_params", True) + assert request_drops_params({}, {}, ()) is True + assert request_drops_params({}, {}, ({"model": "openai/gpt-5.4-nano"},)) is True + assert request_drops_params({"drop_params": False}, {}, ()) is True + assert request_drops_params({}, {"drop_params": False}, ()) is True + assert request_drops_params({}, {}, ({"drop_params": False},)) is True + assert request_drops_params({"drop_params": "not-a-flag"}, {}, ()) is True + + +def test_a_priority_named_in_additional_drop_params_at_any_level_is_dropped(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + by_name: Final = {"additional_drop_params": ["priority"]} + assert request_drops_params(by_name, {}, ()) is True + assert request_drops_params({**by_name, "drop_params": False}, {}, ()) is True + assert request_drops_params({}, by_name, ()) is True + assert request_drops_params({}, {"drop_params": False}, (by_name,)) is True + assert request_drops_params({"additional_drop_params": ["seed"]}, {}, ()) is False + assert request_drops_params({}, {}, (params for params in ({"drop_params": True},))) is True + + +def test_a_request_it_cannot_queue_is_told_why() -> None: + assert "priority is required" in unqueueable_priority_message(None) + assert unqueueable_priority_message("high").startswith("priority must be an integer, got str 'high'") + + +@pytest.mark.parametrize( + ("configured", "expected"), + [(None, None), (3, 3), (0, 0), ("3", 3), (" 12 ", 12), ("0", 0), ("-1", -1), (2.0, 2)], + ids=["unset", "int", "zero", "digit-string", "padded-digit-string", "zero-string", "negative-string", "whole-float"], +) +def test_the_default_priority_is_taken_as_an_integer_or_an_integer_like_value( + configured: object, expected: int | None +) -> None: + assert parse_default_priority(configured) == expected + + +@pytest.mark.parametrize("configured", ["high", "", "1.5", 1.5, [3], {"level": 3}, True, False]) +def test_any_other_default_priority_is_invalid(configured: object) -> None: + parsed: Final = parse_default_priority(configured) + assert isinstance(parsed, InvalidPriority) + assert parsed.value == configured + assert parsed.message.startswith("priority must be an integer, got ") diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 90674dcb3f9..cc25d73b83a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -26055,6 +26055,359 @@ class TestRouterIndexManagement: ) +_PRIORITY_CHAT_COMPLETION: Final = { + "id": "chatcmpl-priority", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.4-nano", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "pong"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_PRIORITY_MESSAGES: Final = [{"role": "user", "content": "Reply with the single word pong"}] + + +def _priority_router(default_priority: int | None) -> Router: + return Router( + model_list=[ + { + "model_name": "qa-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire"}, + } + ], + default_priority=default_priority, + ) + + +def _priority_provider(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_PRIORITY_CHAT_COMPLETION) + ) + + +@pytest.mark.parametrize( + ("default_priority", "request_kwargs", "queued_priority"), + [ + (3, {}, 3), + (3, {"priority": None}, 3), + (3, {"priority": 2}, 2), + (None, {"priority": 2}, 2), + (None, {"priority": 0}, 0), + (None, {"priority": "1"}, 1), + (None, {"priority": True}, 1), + (3, {"priority": 1.0}, 1), + (3, {"priority": "high", "drop_params": True}, 3), + ], +) +async def test_acompletion_queues_the_resolved_priority_exactly_once( + default_priority: int | None, request_kwargs: dict[str, object], queued_priority: int +) -> None: + router: Final = _priority_router(default_priority) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.acompletion( + model="qa-chat", messages=_PRIORITY_MESSAGES, mock_response="pong", **request_kwargs + ) + assert add_request.await_count == 1 + assert add_request.await_args.kwargs["request"].priority == queued_priority + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +async def test_acompletion_without_any_priority_never_enters_the_scheduler() -> None: + router: Final = _priority_router(None) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES, mock_response="pong") + assert add_request.await_count == 0 + assert "x-litellm-request-prioritization-used" not in response._hidden_params.get("additional_headers", {}) + + +@pytest.mark.parametrize( + ("default_priority", "request_kwargs"), [(3, {}), (None, {"priority": 3})], ids=["default", "explicit"] +) +async def test_a_request_pinned_to_a_deployment_id_is_served_while_an_earlier_one_waits_in_the_queue( + default_priority: int | None, request_kwargs: dict[str, object] +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "qa-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire"}, + "model_info": {"id": "qa-chat-deployment"}, + } + ], + default_priority=default_priority, + ) + await router.scheduler.add_request(FlowItem(priority=0, request_id="ahead", model_name="qa-chat-deployment")) + response: Final = await asyncio.wait_for( + router.acompletion( + model="qa-chat-deployment", messages=_PRIORITY_MESSAGES, mock_response="pong", **request_kwargs + ), + timeout=5, + ) + assert response.choices[0].message.content == "pong" + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize("priority", ["high", [1], 1.5]) +async def test_acompletion_rejects_a_non_integer_priority_before_the_provider_sees_it( + priority: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(3) + with pytest.raises(litellm.BadRequestError, match="priority must be an integer") as raised: + await router.acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES, priority=priority) + assert raised.value.param == "priority" + assert not provider.called + + +async def test_acompletion_drops_a_non_integer_priority_from_the_provider_call_under_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(None) + await router.acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES, priority="high", drop_params=True) + assert "priority" not in json.loads(provider.calls.last.request.content) + + +@pytest.mark.parametrize("priority", [0, "1"]) +async def test_acompletion_with_an_integer_priority_sends_no_priority_to_the_provider( + priority: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(None) + await router.acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES, priority=priority) + assert "priority" not in json.loads(provider.calls.last.request.content) + + +async def test_schedule_acompletion_under_a_default_priority_queues_the_explicit_priority_once() -> None: + router: Final = _priority_router(3) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.schedule_acompletion( + model="qa-chat", messages=_PRIORITY_MESSAGES, priority=1, mock_response="pong" + ) + assert add_request.await_count == 1 + assert add_request.await_args.kwargs["request"].priority == 1 + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize( + ("deployment_params", "router_defaults"), + [ + ({}, {"drop_params": True}), + ({"drop_params": True}, {}), + ({"drop_params": "true"}, {}), + ({"additional_drop_params": ["priority"]}, {}), + ({}, {"additional_drop_params": ["priority"]}), + ], +) +async def test_acompletion_drops_a_non_integer_priority_under_the_router_or_deployment_drop_params( + deployment_params: dict[str, object], router_defaults: dict[str, object] +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "qa-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire", **deployment_params}, + } + ], + default_priority=3, + default_litellm_params=router_defaults, + ) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.acompletion( + model="qa-chat", messages=_PRIORITY_MESSAGES, mock_response="pong", priority="high" + ) + assert add_request.await_args.kwargs["request"].priority == 3 + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize( + ("deployment", "requested_model", "routing_fields"), + [ + ( + { + "model_name": "model_name_team-a_team-a-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire", "drop_params": True}, + "model_info": {"id": "team-a-chat", "team_id": "team-a", "team_public_model_name": "team-chat"}, + }, + "team-chat", + {"metadata": {"user_api_key_team_id": "team-a"}}, + ), + ( + { + "model_name": "qa-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire", "drop_params": True}, + "model_info": {"id": "qa-chat-deployment"}, + }, + "qa-chat-deployment", + {}, + ), + ( + { + "model_name": "qa-chat", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-priority-wire", "drop_params": True}, + }, + "openai/gpt-5.4-nano", + {"specific_deployment": True}, + ), + ], + ids=["team-route", "deployment-id", "provider-model-name"], +) +async def test_deployment_drop_params_drop_a_non_integer_priority_on_every_deployment_route( + deployment: dict[str, object], + requested_model: str, + routing_fields: dict[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + router: Final = Router(model_list=[deployment], default_priority=3) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.acompletion( + model=requested_model, + messages=_PRIORITY_MESSAGES, + mock_response="pong", + priority="high", + **routing_fields, + ) + assert add_request.await_args.kwargs["request"].priority == 3 + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +async def test_schedule_acompletion_queues_a_prompt_management_model(monkeypatch: pytest.MonkeyPatch) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "cached-chat", + "litellm_params": { + "model": "anthropic_cache_control_hook/openai/gpt-5.4-nano", + "api_key": "sk-priority-wire", + "prompt_id": "cache-points", + }, + } + ] + ) + provider: Final = AsyncMock(return_value=litellm.ModelResponse()) + monkeypatch.setattr(litellm, "acompletion", provider) + logging_obj: Final = LiteLLMLogging( + model="cached-chat", + messages=_PRIORITY_MESSAGES, + stream=False, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="prompt-management-priority", + function_id="f", + ) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.schedule_acompletion( + model="cached-chat", messages=_PRIORITY_MESSAGES, priority=1, litellm_logging_obj=logging_obj + ) + assert add_request.await_count == 1 + assert add_request.await_args.kwargs["request"].priority == 1 + assert provider.await_count == 1 + assert "priority" not in provider.await_args.kwargs + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize( + ("configured", "expected"), + [(3, 3), ("3", 3), ("-1", -1), (0, 0), (None, None)], + ids=["int", "digit-string", "negative-string", "zero", "unset"], +) +def test_the_router_parses_default_priority_once_at_init(configured: object, expected: int | None) -> None: + assert _priority_router(configured).default_priority == expected + + +@pytest.mark.parametrize("configured", ["high", 1.5, [3], False], ids=["word", "float", "list", "yaml-off"]) +def test_an_invalid_default_priority_is_ignored_with_a_warning( + configured: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + router: Final = _priority_router(configured) + assert router.default_priority is None + assert any("default_priority is ignored" in record.getMessage() for record in caplog.records), caplog.records + + +async def test_an_invalid_default_priority_leaves_every_request_unscheduled() -> None: + router: Final = _priority_router("high") + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES, mock_response="pong") + assert add_request.await_count == 0 + assert "x-litellm-request-prioritization-used" not in response._hidden_params.get("additional_headers", {}) + + +async def test_schedule_acompletion_without_a_priority_or_a_default_is_rejected_before_the_provider( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(None) + with pytest.raises(litellm.BadRequestError, match="priority is required") as raised: + await router.schedule_acompletion(model="qa-chat", messages=_PRIORITY_MESSAGES) + assert raised.value.param == "priority" + assert not provider.called + + +async def test_schedule_acompletion_without_a_priority_queues_the_default_once() -> None: + router: Final = _priority_router(3) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.schedule_acompletion( + model="qa-chat", messages=_PRIORITY_MESSAGES, mock_response="pong" + ) + assert add_request.await_count == 1 + assert add_request.await_args.kwargs["request"].priority == 3 + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + + +@pytest.mark.parametrize( + ("default_priority", "request_kwargs", "queued_priority"), + [ + (3, {}, 3), + (3, {"priority": None}, 3), + (3, {"priority": 2}, 2), + (None, {"priority": 0}, 0), + (None, {"priority": "1"}, 1), + (3, {"priority": "high", "drop_params": True}, 3), + ], +) +async def test_atext_completion_queues_the_resolved_priority_exactly_once( + default_priority: int | None, + request_kwargs: dict[str, object], + queued_priority: int, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(default_priority) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.atext_completion(model="qa-chat", prompt="pong?", **request_kwargs) + assert add_request.await_count == 1 + assert add_request.await_args.kwargs["request"].priority == queued_priority + assert response._hidden_params["additional_headers"]["x-litellm-request-prioritization-used"] is True + assert "priority" not in json.loads(provider.calls.last.request.content) + + +async def test_atext_completion_without_any_priority_never_enters_the_scheduler( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(None) + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + response: Final = await router.atext_completion(model="qa-chat", prompt="pong?") + assert add_request.await_count == 0 + assert "x-litellm-request-prioritization-used" not in response._hidden_params.get("additional_headers", {}) + assert provider.called + + +@pytest.mark.parametrize("priority", ["high", [1], 1.5]) +async def test_atext_completion_rejects_a_non_integer_priority_before_the_provider_sees_it( + priority: object, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + provider: Final = _priority_provider(respx_mock, monkeypatch) + router: Final = _priority_router(3) + with pytest.raises(litellm.BadRequestError, match="priority must be an integer") as raised: + await router.atext_completion(model="qa-chat", prompt="pong?", priority=priority) + assert raised.value.param == "priority" + assert not provider.called + + def test_router_multi_org_list() -> None: """ Pass list of orgs in 1 model definition, From c9d081b72a72ecf34c6a9bf7f1ac841402f33cb2 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:27:39 -0700 Subject: [PATCH 079/124] chore(cost-map): add fireworks 2026-11-07 deprecation dates from the prices api (#47518) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++------ model_prices_and_context_window.json | 19 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cc284c0ce4c..8b3771033a5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27672,6 +27672,7 @@ "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "fireworks_ai", @@ -27681,7 +27682,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -28072,6 +28073,7 @@ "fireworks_ai/gpt-oss-120b": { "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "fireworks_ai", @@ -28081,7 +28083,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63808,6 +63810,7 @@ "fireworks_ai/nemotron-lightning-3p5-30b-a3b": { "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_priority": 1.25e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 5e-08, "input_cost_per_token_priority": 6.25e-08, "litellm_provider": "fireworks_ai", @@ -63817,7 +63820,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63827,6 +63830,7 @@ "fireworks_ai/nemotron-3-ultra-nvfp4": { "cache_read_input_token_cost": 1.2e-07, "cache_read_input_token_cost_priority": 1.5e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 6e-07, "input_cost_per_token_priority": 7.5e-07, "litellm_provider": "fireworks_ai", @@ -63836,7 +63840,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63877,6 +63881,7 @@ "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_priority": 1.25e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 5e-08, "input_cost_per_token_priority": 6.25e-08, "litellm_provider": "fireworks_ai", @@ -63886,7 +63891,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63896,6 +63901,7 @@ "fireworks_ai/accounts/fireworks/models/nemotron-3-ultra-nvfp4": { "cache_read_input_token_cost": 1.2e-07, "cache_read_input_token_cost_priority": 1.5e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 6e-07, "input_cost_per_token_priority": 7.5e-07, "litellm_provider": "fireworks_ai", @@ -63905,7 +63911,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67049,6 +67055,7 @@ "fireworks_ai/accounts/fireworks/models/inkling": { "cache_read_input_token_cost": 1.7e-07, "cache_read_input_token_cost_priority": 1.7e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1e-06, "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cc284c0ce4c..8b3771033a5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27672,6 +27672,7 @@ "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "fireworks_ai", @@ -27681,7 +27682,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -28072,6 +28073,7 @@ "fireworks_ai/gpt-oss-120b": { "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 1.8e-07, "litellm_provider": "fireworks_ai", @@ -28081,7 +28083,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63808,6 +63810,7 @@ "fireworks_ai/nemotron-lightning-3p5-30b-a3b": { "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_priority": 1.25e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 5e-08, "input_cost_per_token_priority": 6.25e-08, "litellm_provider": "fireworks_ai", @@ -63817,7 +63820,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63827,6 +63830,7 @@ "fireworks_ai/nemotron-3-ultra-nvfp4": { "cache_read_input_token_cost": 1.2e-07, "cache_read_input_token_cost_priority": 1.5e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 6e-07, "input_cost_per_token_priority": 7.5e-07, "litellm_provider": "fireworks_ai", @@ -63836,7 +63840,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63877,6 +63881,7 @@ "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_priority": 1.25e-08, + "deprecation_date": "2026-11-07", "input_cost_per_token": 5e-08, "input_cost_per_token_priority": 6.25e-08, "litellm_provider": "fireworks_ai", @@ -63886,7 +63891,7 @@ "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, - "source": "https://docs.fireworks.ai/serverless/pricing", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -63896,6 +63901,7 @@ "fireworks_ai/accounts/fireworks/models/nemotron-3-ultra-nvfp4": { "cache_read_input_token_cost": 1.2e-07, "cache_read_input_token_cost_priority": 1.5e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 6e-07, "input_cost_per_token_priority": 7.5e-07, "litellm_provider": "fireworks_ai", @@ -63905,7 +63911,7 @@ "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -67049,6 +67055,7 @@ "fireworks_ai/accounts/fireworks/models/inkling": { "cache_read_input_token_cost": 1.7e-07, "cache_read_input_token_cost_priority": 1.7e-07, + "deprecation_date": "2026-11-07", "input_cost_per_token": 1e-06, "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", From e0d903b971fe36d57d12f5185b153c44b063afff Mon Sep 17 00:00:00 2001 From: Avani Prajapati <143805019+Avani-prajapati@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:06:19 +0530 Subject: [PATCH 080/124] fix(proxy)!: drop router_settings_override before routing and apply it on every branch (#47257) * fix(proxy): drop router_settings_override before routing and apply it on every branch route_request only popped router_settings_override inside the branch that runs after the api_key/api_base, comma-separated batch and user_config branches. Requests taking one of those earlier branches forwarded the field to the provider, which Anthropic rejects with a 400, and silently lost the key/team num_retries, fallbacks and timeout. The override is now popped and merged into the request before any branch is chosen, keeping the existing precedence where request values win and None values are skipped Fixes #43582 * fix(proxy): apply key/team fallbacks when the request carries api_key or api_base The previous commit skipped fallbacks, context_window_fallbacks and content_policy_fallbacks from router_settings_override whenever the request had api_key or api_base. That broke the scenario from #43582, where a trusted pre-call hook sets api_key and the key/team router_settings still need to apply. All per-request router settings are merged again on every branch, with None values skipped and request values still winning The tests now check that fallbacks run on the api_key and api_base paths and that the provider never receives router_settings_override * fix(proxy): answer 400 for a non-object router_settings_override and treat null as absent * fix(proxy): drop a malformed router_settings_override under drop_params * fix(proxy): route key and team router settings like any request, honor user_config and global drop_params * fix(proxy): keep Router-only override settings out of direct provider calls Calls that route_request sends straight to litellm (evals, endpoints that need no model, pass_through_all_models, the video fallback and the no-router paths) now leave out context_window_fallbacks, content_policy_fallbacks and model_group_retry_policy merged from key or team router settings. Only the Router reads them, and providers reject them as unknown parameters. * test(proxy): drop a comment from the router settings override test * fix(proxy): keep key router settings on the model read-through retry route_request now merges router_settings_override once, before the first attempt. The retry after a model read-through used to pop the merged enable_tag_filtering again and recompute an empty Router-only set, so a model created on another instance was served without the key's tag filtering * test(proxy): check the read-through retry keeps key tag filtering through a real router The regression test now routes through a real Router with two tagged deployments from the database instead of replacing Router.acompletion. Before the fix the retry picks the lower-order untagged-for-request deployment, after it the tagged one * fix(proxy): keep key Router-format fallbacks out of direct provider calls Key and team fallbacks use the Router's model-group mapping. litellm.acompletion reads each mapping as a fallback config, so a direct call retried the same model with the mapping merged into the provider body. The override's fallbacks now stay out of direct calls like the other Router-only settings, while num_retries and timeout still pass --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/route_llm_request.py | 124 ++++-- tests/unit/proxy/test_route_llm_request.py | 460 ++++++++++++++++++++- 2 files changed, 541 insertions(+), 43 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 8bab7215b17..e16e14c8d54 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -2,13 +2,14 @@ import asyncio from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal import httpx from fastapi import HTTPException, status -from pydantic import TypeAdapter, ValidationError +from pydantic import BeforeValidator, TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.router_utils.common_utils import is_proxy_admin_request @@ -38,6 +39,21 @@ else: LitellmRouter = Any +_PER_REQUEST_ROUTER_SETTINGS: Final = ( + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "timeout", + "model_group_retry_policy", + "routing_strategy", + "enable_tag_filtering", +) +_ROUTER_ONLY_SETTINGS: Final = frozenset( + ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", "model_group_retry_policy") +) + + def _route_user_config_request(data: dict, route_type: str): """Route a request using the user-provided router config.""" router_config: Final = data.pop("user_config") @@ -204,6 +220,18 @@ REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = Map ) JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_OPTIONAL_JSON_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( + Mapping[str, object] | None +) + + +def _none_unless_json_object(value: object) -> object: + return value if isinstance(value, Mapping) else None + + +_JSON_OBJECT_OR_DROPPED_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( + Annotated[Mapping[str, object] | None, BeforeValidator(_none_unless_json_object)] +) class ProxyMissingRequiredParamError(ProxyException): @@ -596,6 +624,43 @@ RouteType = Literal[ ] +def _merge_router_settings_override( + data: dict[str, object], # mutable-ok: route_request mutates the shared request body in place +) -> frozenset[str]: + data.pop("enable_tag_filtering", None) + drop_params: Final = litellm.drop_params is True or normalize_drop_params(data.get("drop_params")) is True + router_settings_override_adapter: Final = ( + _JSON_OBJECT_OR_DROPPED_ADAPTER if drop_params else _OPTIONAL_JSON_OBJECT_ADAPTER + ) + try: + router_settings_override: Final = router_settings_override_adapter.validate_python( + data.pop("router_settings_override", None) + ) + except ValidationError: + raise ProxyException( + message="router_settings_override must be a JSON object.", + type="invalid_request_error", + param="router_settings_override", + code=status.HTTP_400_BAD_REQUEST, + ) from None + if "user_config" in data and data["user_config"] is None: + data.pop("user_config") + user_config: Final = _JSON_OBJECT_OR_DROPPED_ADAPTER.validate_python(data.get("user_config")) + router_settings_from_override: Final = MappingProxyType( + { + key: router_settings_override[key] + for key in _PER_REQUEST_ROUTER_SETTINGS + if router_settings_override.get(key) is not None + and key not in data + and (user_config is None or key not in user_config) + } + if router_settings_override is not None + else {} + ) + data.update(router_settings_from_override) + return _ROUTER_ONLY_SETTINGS.intersection(router_settings_from_override) + + async def route_request( data: dict, llm_router: LitellmRouter | None, @@ -606,12 +671,14 @@ async def route_request( """ Common helper to route the request """ + router_only_settings_from_override: Final = _merge_router_settings_override(data) # pyright: ignore[reportUnknownArgumentType] # route_request takes an untyped dict body try: return await _route_request_single_attempt( data=data, llm_router=llm_router, user_model=user_model, route_type=route_type, + router_only_settings_from_override=router_only_settings_from_override, user_api_key_dict=user_api_key_dict, ) except (ProxyModelNotFoundError, ProxyMissingParamWithoutLoadedModelError) as e: @@ -634,6 +701,7 @@ async def route_request( llm_router=proxy_server.llm_router, user_model=user_model, route_type=route_type, + router_only_settings_from_override=router_only_settings_from_override, user_api_key_dict=user_api_key_dict, ) @@ -643,6 +711,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr llm_router: LitellmRouter | None, user_model: str | None, route_type: RouteType, + router_only_settings_from_override: frozenset[str], user_api_key_dict: UserAPIKeyAuth | None = None, ): raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router) @@ -651,9 +720,14 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) - data.pop("enable_tag_filtering", None) - if "user_config" in data and data["user_config"] is None: - data.pop("user_config") # pyright: ignore[reportUnknownMemberType] # route_request takes an untyped dict body + def direct_litellm_kwargs() -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in JSON_OBJECT_ADAPTER.validate_python(data).items() + if key not in router_only_settings_from_override + } + ) team_id: Final = get_team_id_from_data(data) router_model_names: Final = llm_router.model_names if llm_router is not None else [] @@ -668,7 +742,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr if llm_router is not None: return getattr(llm_router, f"{route_type}")(**data) else: - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) elif ( route_type == "acompletion" @@ -687,34 +761,6 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr elif "user_config" in data: return _route_user_config_request(data, route_type) - elif "router_settings_override" in data: - # Apply per-request router settings overrides from key/team config - # Instead of creating a new Router (expensive), merge settings into kwargs - # The Router already supports per-request overrides for these settings - override_settings: Final = data.pop("router_settings_override") - - # Settings that the Router accepts as per-request kwargs - # These override the global router settings for this specific request - per_request_settings: Final = [ - "fallbacks", - "context_window_fallbacks", - "content_policy_fallbacks", - "num_retries", - "timeout", - "model_group_retry_policy", - "routing_strategy", - "enable_tag_filtering", - ] - - for key in per_request_settings: - if override_settings.get(key) is not None and key not in data: - data[key] = override_settings[key] - - # Use main router with overridden kwargs - if llm_router is not None: - return getattr(llm_router, f"{route_type}")(**data) - else: - return getattr(litellm, f"{route_type}")(**data) elif llm_router is not None: _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) # Evals API: always route to litellm directly (not through router) @@ -755,7 +801,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr # If we can't get deployment creds, continue without them pass - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) # Skip model-based routing for container operations if route_type in [ "acreate_container", @@ -806,7 +852,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "aingest", ] and (data.get("model") is None or data.get("model") == ""): # These endpoints don't need a model, use custom_llm_provider directly - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) if "model" not in data: raise ProxyMissingRequiredParamError( @@ -845,7 +891,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr # Check wildcards before checking deployment_names # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match if llm_router.router_general_settings.pass_through_all_models: - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: return getattr(llm_router, f"{route_type}")(**data) elif data["model"] in llm_router.deployment_names: @@ -897,7 +943,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr return getattr(llm_router, f"{route_type}")(**data) except Exception: # If router fails (e.g., model not found in router), fall back to direct call - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) elif _is_a2a_agent_model(data.get("model", "")): from litellm.proxy.agent_endpoints.a2a_routing import ( route_a2a_agent_request, @@ -909,7 +955,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr # Fall through to raise exception below if result is None elif user_model is not None or route_type == "allm_passthrough_route": - return getattr(litellm, f"{route_type}")(**data) + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) # if no route found then it's a bad request route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 4e4e2f23950..8928b79bcd4 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1,4 +1,6 @@ +import contextlib import httpx +import json import litellm import pytest import respx @@ -10,7 +12,8 @@ from unittest.mock import MagicMock from fastapi import HTTPException -from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request +from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER, ProxyModelNotFoundError, route_request +from litellm.types.router import RouterGeneralSettings @pytest.mark.parametrize( @@ -390,7 +393,6 @@ async def test_route_request_with_router_settings_override(): "timeout": 30, "model_group_retry_policy": {"gpt-3.5-turbo": {"RateLimitErrorRetries": 3}}, "routing_strategy": "least-busy", - # This setting should be ignored (not in per_request_settings list) "model_group_alias": {"alias": "real_model"}, }, } @@ -437,12 +439,12 @@ async def test_route_request_with_router_settings_override_no_router(): litellm.acompletion = mock_completion try: - response = await route_request(data, None, None, "acompletion") + response = await route_request(data, None, "gpt-3.5-turbo", "acompletion") assert response == "success" # Verify litellm.acompletion was called with merged settings call_kwargs = mock_completion.call_args[1] - assert call_kwargs["fallbacks"] == [{"gpt-3.5-turbo": ["gpt-4"]}] + assert "fallbacks" not in call_kwargs assert call_kwargs["num_retries"] == 3 finally: litellm.acompletion = original_acompletion @@ -1039,6 +1041,408 @@ async def test_route_request_override_enable_tag_filtering_beats_body_value(): assert call_kwargs["enable_tag_filtering"] is True +_PROXY_DEPLOYMENT_BASE: Final = "https://proxy-deployment.example/v1" +_HELLO_MESSAGES: Final = ({"role": "user", "content": "Hello"},) + + +def _chat_completion_response(model: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _proxy_router_with_two_model_groups() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": f"openai/{model}", + "api_key": "sk-proxy", + "api_base": _PROXY_DEPLOYMENT_BASE, + }, + } + for model in ("gpt-3.5-turbo", "gpt-4") + ], + num_retries=0, + ) + + +@pytest.mark.parametrize( + "extra_body, upstream_base", + [ + pytest.param({"api_key": "sk-from-pre-call-hook"}, _PROXY_DEPLOYMENT_BASE, id="clientside-api-key"), + pytest.param( + {"api_base": "https://clientside.example/v1"}, "https://clientside.example/v1", id="clientside-api-base" + ), + pytest.param({"model": "gpt-3.5-turbo,gpt-4"}, _PROXY_DEPLOYMENT_BASE, id="comma-separated-batch"), + pytest.param({"drop_params": True}, _PROXY_DEPLOYMENT_BASE, id="drop-params"), + pytest.param( + { + "user_config": { + "model_list": [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "openai/gpt-3.5-turbo", + "api_key": "sk-user-config", + "api_base": "https://user-config.example/v1", + }, + } + ], + } + }, + "https://user-config.example/v1", + id="user-config", + ), + ], +) +@pytest.mark.asyncio +async def test_route_request_never_forwards_router_settings_override( + monkeypatch: pytest.MonkeyPatch, extra_body: dict[str, object], upstream_base: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": {"num_retries": 2, "timeout": None}, + **extra_body, + } + + with respx.mock(assert_all_called=False) as respx_mock: + primary: Final = respx_mock.post(f"{upstream_base}/chat/completions", json__model="gpt-3.5-turbo").mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "boom"}}), + _chat_completion_response("gpt-3.5-turbo"), + ] + ) + respx_mock.post(f"{upstream_base}/chat/completions", json__model="gpt-4").mock( + return_value=_chat_completion_response("gpt-4") + ) + response: Final = await (await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion")) + + primary_response: Final = response[0] if isinstance(response, list) else response + sent_bodies: Final = [json.loads(call.request.content) for call in primary.calls] + assert not [body for body in sent_bodies if "router_settings_override" in body], sent_bodies + assert primary_response._hidden_params["additional_headers"]["x-litellm-max-retries"] == 2, ( + primary_response._hidden_params + ) + + +@pytest.mark.parametrize("extra_body", [{}, {"api_key": "sk-from-pre-call-hook"}], ids=["router", "clientside-api-key"]) +@pytest.mark.asyncio +async def test_route_request_treats_null_router_settings_override_as_absent( + monkeypatch: pytest.MonkeyPatch, extra_body: dict[str, str] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": None, + **extra_body, + } + + with respx.mock(assert_all_called=False) as respx_mock: + upstream: Final = respx_mock.post(f"{_PROXY_DEPLOYMENT_BASE}/chat/completions").mock( + return_value=_chat_completion_response("gpt-3.5-turbo") + ) + response: Final = await (await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion")) + + assert response.choices[0].message.content == "hi", response + assert "router_settings_override" not in json.loads(upstream.calls.last.request.content) + + +@pytest.mark.parametrize( + "global_drop_params, extra_body", + [ + pytest.param(False, {}, id="drop-params-off"), + pytest.param(False, {"drop_params": False}, id="request-drop-params-false"), + ], +) +@pytest.mark.parametrize("override", ["fallbacks", ["num_retries"], 3]) +@pytest.mark.asyncio +async def test_route_request_rejects_non_object_router_settings_override( + monkeypatch: pytest.MonkeyPatch, global_drop_params: bool, extra_body: dict[str, bool | str], override: object +) -> None: + from litellm.proxy._types import ProxyException + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": override, + **extra_body, + } + + with respx.mock(assert_all_called=False) as respx_mock: + upstream: Final = respx_mock.post(f"{_PROXY_DEPLOYMENT_BASE}/chat/completions").mock( + return_value=_chat_completion_response("gpt-3.5-turbo") + ) + with pytest.raises(ProxyException) as exc_info: + await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion") + + assert exc_info.value.code == "400" + assert exc_info.value.param == "router_settings_override" + assert upstream.call_count == 0 + + +@pytest.mark.parametrize( + "global_drop_params, extra_body", + [ + pytest.param(False, {"drop_params": True}, id="request-drop-params"), + pytest.param(True, {}, id="global-drop-params"), + pytest.param(True, {"drop_params": False}, id="global-drop-params-with-request-false"), + pytest.param(False, {"drop_params": "true"}, id="request-drop-params-string"), + ], +) +@pytest.mark.parametrize("override", ["fallbacks", ["num_retries"], 3]) +@pytest.mark.asyncio +async def test_route_request_drops_non_object_router_settings_override_under_drop_params( + monkeypatch: pytest.MonkeyPatch, global_drop_params: bool, extra_body: dict[str, bool | str], override: object +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": override, + **extra_body, + } + + with respx.mock(assert_all_called=False) as respx_mock: + upstream: Final = respx_mock.post(f"{_PROXY_DEPLOYMENT_BASE}/chat/completions").mock( + return_value=_chat_completion_response("gpt-3.5-turbo") + ) + response: Final = await (await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion")) + + assert response.choices[0].message.content == "hi", response + assert upstream.call_count == 1 + assert "router_settings_override" not in json.loads(upstream.calls.last.request.content) + + +@pytest.mark.parametrize( + "user_config_settings, expected_upstream_calls", + [ + pytest.param({}, 2, id="key-setting-fills-unset-user-config-setting"), + pytest.param({"num_retries": 0}, 1, id="user-config-setting-beats-key-setting"), + ], +) +@pytest.mark.asyncio +async def test_route_request_user_config_router_settings_beat_key_settings( + monkeypatch: pytest.MonkeyPatch, user_config_settings: dict[str, int], expected_upstream_calls: int +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + user_config_base: Final = "https://user-config.example/v1" + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": {"num_retries": 1}, + "user_config": { + **user_config_settings, + "model_list": [ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "openai/gpt-3.5-turbo", + "api_key": "sk-user-config", + "api_base": user_config_base, + }, + } + ], + }, + } + + with respx.mock(assert_all_called=False) as respx_mock: + upstream: Final = respx_mock.post(f"{user_config_base}/chat/completions").mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "boom"}}), + _chat_completion_response("gpt-3.5-turbo"), + ] + ) + with contextlib.suppress(litellm.InternalServerError): + await (await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion")) + + assert upstream.call_count == expected_upstream_calls + + +@pytest.mark.parametrize( + "extra_body", + [ + pytest.param({}, id="no-key-settings"), + pytest.param({"router_settings_override": {"num_retries": 2}}, id="key-settings"), + ], +) +@pytest.mark.asyncio +async def test_route_request_refuses_blocked_model_whether_or_not_key_has_router_settings( + extra_body: dict[str, dict[str, int]], +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "sk-proxy"}, + "model_info": {"blocked": True}, + } + ], + ) + data: Final = {"model": "gpt-3.5-turbo", "messages": list(_HELLO_MESSAGES), **extra_body} + + with pytest.raises(litellm.PermissionDeniedError): + await route_request(data, router, None, "acompletion") + + +@pytest.mark.parametrize( + "credential", + [ + pytest.param({"api_key": "sk-from-pre-call-hook"}, id="clientside-api-key"), + pytest.param({"api_base": _PROXY_DEPLOYMENT_BASE}, id="clientside-api-base"), + ], +) +@pytest.mark.parametrize( + "fallback_field, upstream_error", + [ + pytest.param("fallbacks", httpx.Response(500, json={"error": {"message": "boom"}}), id="fallbacks"), + pytest.param( + "context_window_fallbacks", + httpx.Response( + 400, + json={ + "error": { + "message": "This model's maximum context length is 16385 tokens", + "code": "context_length_exceeded", + } + }, + ), + id="context-window-fallbacks", + ), + pytest.param( + "content_policy_fallbacks", + httpx.Response( + 400, + json={ + "error": { + "message": "Your request was rejected by the content filter", + "type": "invalid_request_error", + "code": "content_policy_violation", + } + }, + ), + id="content-policy-fallbacks", + ), + ], +) +@pytest.mark.asyncio +async def test_route_request_applies_override_fallbacks_when_request_carries_credentials( + monkeypatch: pytest.MonkeyPatch, + credential: dict[str, str], + fallback_field: str, + upstream_error: httpx.Response, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + data: Final = { + "model": "gpt-3.5-turbo", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": {fallback_field: [{"gpt-3.5-turbo": ["gpt-4"]}]}, + **credential, + } + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.post(f"{_PROXY_DEPLOYMENT_BASE}/chat/completions", json__model="gpt-3.5-turbo").mock( + return_value=upstream_error + ) + fallback: Final = respx_mock.post(f"{_PROXY_DEPLOYMENT_BASE}/chat/completions", json__model="gpt-4").mock( + return_value=_chat_completion_response("gpt-4") + ) + response: Final = await (await route_request(data, _proxy_router_with_two_model_groups(), None, "acompletion")) + + assert fallback.call_count == 1, [str(call.request.url) for call in respx_mock.calls] + assert "router_settings_override" not in json.loads(fallback.calls.last.request.content) + assert response.choices[0].message.content == "hi", response + + +@pytest.mark.parametrize( + "router_only_setting", + [ + pytest.param({"model_group_retry_policy": {"gpt-4": {"TimeoutErrorRetries": 1}}}, id="model-group-retry-policy"), + pytest.param({"context_window_fallbacks": [{"gpt-4": ["gpt-3.5-turbo"]}]}, id="context-window-fallbacks"), + pytest.param({"content_policy_fallbacks": [{"gpt-4": ["gpt-3.5-turbo"]}]}, id="content-policy-fallbacks"), + ], +) +@pytest.mark.asyncio +async def test_route_request_keeps_router_only_override_settings_out_of_pass_through_provider_body( + monkeypatch: pytest.MonkeyPatch, router_only_setting: dict[str, object] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "sk-pass-through") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4", "api_key": "sk-proxy", "api_base": _PROXY_DEPLOYMENT_BASE}, + } + ], + router_general_settings=RouterGeneralSettings(pass_through_all_models=True), + ) + data: Final = { + "model": "openai/gpt-pass-through", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": router_only_setting, + } + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=_chat_completion_response("gpt-pass-through") + ) + await (await route_request(data, router, None, "acompletion")) + + assert set(router_only_setting).isdisjoint(JSON_OBJECT_ADAPTER.validate_json(upstream.calls.last.request.content)) + + +@pytest.mark.asyncio +async def test_route_request_keeps_key_fallbacks_out_of_pass_through_provider_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "sk-pass-through") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4", "api_key": "sk-proxy", "api_base": _PROXY_DEPLOYMENT_BASE}, + } + ], + router_general_settings=RouterGeneralSettings(pass_through_all_models=True), + ) + data: Final = { + "model": "openai/gpt-pass-through", + "messages": list(_HELLO_MESSAGES), + "router_settings_override": {"fallbacks": [{"gpt-4": ["gpt-4"]}], "num_retries": 0}, + } + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(500, json={"error": {"message": "upstream down"}}) + ) + with pytest.raises(litellm.InternalServerError): + await (await route_request(data, router, None, "acompletion")) + + provider_bodies: Final = tuple(JSON_OBJECT_ADAPTER.validate_json(call.request.content) for call in upstream.calls) + assert provider_bodies and all("gpt-4" not in body for body in provider_bodies), provider_bodies + + @pytest.mark.parametrize( "route_type, param, route", [ @@ -1524,6 +1928,54 @@ async def test_route_request_read_through_recovers_model_created_on_sibling_repl assert table.find_many_wheres[0] == {"model_name": model_name} +def _tagged_db_model_row(model_name: str, tag: str, order: int) -> SimpleNamespace: + return SimpleNamespace( + model_id=f"{model_name}-{tag}-id", + model_name=model_name, + litellm_params={ + "model": "openai/gpt-4o", + "api_key": "fake", + "mock_response": f"{tag}-deployment", + "tags": [tag], + "order": order, + }, + model_info={}, + blocked=False, + ) + + +@pytest.mark.asyncio +async def test_route_request_read_through_retry_keeps_key_tag_filtering(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm.proxy.proxy_server as proxy_server + + model_name: Final = "sibling-replica-model-for-key-with-tag-filtering" + router: Final = litellm.Router( + model_list=[{"model_name": "some-other-model", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}] + ) + fake_prisma, _ = _fake_prisma_client_with_models( + [_tagged_db_model_row(model_name, "free", order=1), _tagged_db_model_row(model_name, "paid", order=2)] + ) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + + response: Final = await ( + await route_request( + { + "model": model_name, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["paid"]}, + "router_settings_override": {"enable_tag_filtering": True}, + }, + router, + None, + "acompletion", + ) + ) + + assert response.choices[0].message.content == "paid-deployment" + + @pytest.mark.asyncio async def test_route_request_unknown_model_raises_and_hits_db_once_within_ttl(monkeypatch): import litellm From 270d108aceb9850b6e924406c13a4bd6dbc7cd94 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:38:40 +0000 Subject: [PATCH 081/124] fix(anthropic): json_schema envelope edge cases on the Anthropic json tool path (#47478) * test(anthropic): cover json_schema envelope edge cases on the Anthropic json tool path * fix(anthropic): unwrap required-only keys, walk past tool-named envelopes, keep non-ASCII text, flush cut streams, check Vertex output_format * fix(anthropic): send held json tool text on cut Bedrock Invoke streams and drop a non-object output_format under drop_params A Bedrock Invoke stream that ends before the json tool block closes now sends the held text and closing brace, as the direct Anthropic stream already did. A non-object output_format on the Anthropic messages route and its providers is dropped under drop_params and otherwise forwarded, so the provider answers its own 400. Vertex Claude names the proxy setting in its 400 and reads drop_params through the shared helper. Adds integration cases for the json_schema envelope and the cut stream. * fix(anthropic): drop a non-object output_format on chat, unwrap a whole envelope on a cut stream, escape lone surrogates Chat completions to Anthropic, Azure AI, Bedrock Invoke and Vertex now drop a non-object output_format under drop_params, matching /v1/messages. A stream cut after a complete held envelope is unwrapped like a normal end. A json_schema answer holding a lone surrogate keeps its \u escape so the response can still be encoded * fix(anthropic): join a split answer when the stream ends in its last block A cut during the second json tool block now runs the held block through the same flush as a block stop, so it joins the streamed keys or the earlier split blocks instead of sending two objects back to back. The Vertex output_format fold drops its unreachable drop_params branch, since both parent configs drop a non-object value first. * test(vertex): expect the LiteLLM 400 for a non-object output_format on /v1/messages --- litellm/llms/anthropic/chat/handler.py | 34 ++++- litellm/llms/anthropic/chat/transformation.py | 15 +- .../pass_through/messages/transformation.py | 9 ++ litellm/llms/base_llm/base_utils.py | 50 +++--- litellm/llms/bedrock/chat/invoke_handler.py | 12 ++ .../anthropic/output_params_utils.py | 22 ++- ...vertex_anthropic_structured_output_wire.py | 29 ++-- .../response_format/__init__.py | 0 ...t_completions_response_format_anthropic.py | 123 +++++++++++++++ ...st_chat_completions_streaming_anthropic.py | 144 +++++++++++++++++- .../chat/test_anthropic_chat_handler.py | 110 +++++++++++++ .../test_anthropic_chat_transformation.py | 75 +++++++++ ...t_anthropic_messages_structured_outputs.py | 50 ++++++ ...azure_anthropic_messages_transformation.py | 45 ++++++ .../llms/bedrock/chat/test_invoke_handler.py | 81 ++++++++++ ...artner_models_anthropic_messages_config.py | 39 +++++ ...partner_models_anthropic_transformation.py | 38 +++++ 17 files changed, 829 insertions(+), 47 deletions(-) create mode 100644 tests/integration/translation/chat_completions/response_format/__init__.py create mode 100644 tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 718536120b6..f0143f64be8 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -28,6 +28,7 @@ from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, names_a_response_format_property, partial_response_format_answer, + response_format_answer_json, response_format_envelope_contents, split_response_format_answer, ) @@ -104,17 +105,17 @@ def _key_after_a_string_verdict(string_value_start: str, schema: ResponseFormatS next_key: Final = _STRING_VALUE_THEN_KEY.match(string_value_start) if next_key is None: return "undecided" - return "pass_through" if next_key.group(1) in schema.property_names else "hold" + return "pass_through" if next_key.group(1) in schema.declared_names else "hold" def _is_a_label(text: str, schema: ResponseFormatSchema, tool_name: str) -> bool: - return text in schema.property_names or text == tool_name + return text in schema.declared_names or text == tool_name def _label_verdict(label: str, next_key: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict: if _is_a_label(label, schema, tool_name): return "hold" - return "pass_through" if next_key in schema.property_names else "undecided" + return "pass_through" if next_key in schema.declared_names else "undecided" def _leading_labels_verdict(string_value_start: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict: @@ -159,9 +160,9 @@ def _prefix_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: return "hold" if first_value is None and _NAME_VALUE_PREFIX.match(buffered_json): return "undecided" - if first_key.group(1) == "values" and "values" not in schema.property_names: + if first_key.group(1) == "values" and "values" not in schema.declared_names: return "hold" - if first_key.group(1) in schema.property_names or not schema.property_names: + if first_key.group(1) in schema.declared_names or not schema.declared_names: return "pass_through" return _first_value_verdict(buffered_json[first_key.end() :], schema, tool_name) @@ -172,13 +173,13 @@ def _held_answer(held: str, schema: ResponseFormatSchema) -> str: except (json.JSONDecodeError, RecursionError): return held contents: Final = response_format_envelope_contents(payload, schema, RESPONSE_FORMAT_TOOL_NAME) - return held if contents is None else json.dumps(contents) + return held if contents is None else response_format_answer_json(contents) def _split_answer_text(blocks: tuple[str, ...], schema: ResponseFormatSchema) -> str: answer: Final = split_response_format_answer(blocks, schema) if answer is not None: - return json.dumps(answer) + return response_format_answer_json(answer) return "".join(_held_answer(block, schema) for block in blocks) @@ -1283,7 +1284,7 @@ class ModelResponseIterator: return None self._streamed_answer_keys = streamed | frozenset(fields) separator: Final = ", " if streamed else "" - return f"{separator}{json.dumps(fields)[1:-1]}" if fields else "" + return f"{separator}{response_format_answer_json(fields)[1:-1]}" if fields else "" def _flush_response_format_buffer(self) -> str: schema: Final = self.response_format_schema @@ -1314,6 +1315,17 @@ class ModelResponseIterator: return "" return _split_answer_text(blocks, self.response_format_schema) + def cut_stream_chunk(self) -> ModelResponseStream | None: + unsent: Final = self._flush_response_format_buffer() if self._response_format_buffer.getvalue() else "" + text: Final = unsent + self._release_answer_close() + self._drain_split_answer_blocks() + if not text: + return None + return ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=text))], + id=self.response_id, + model=self.served_model, + ) + def _handle_message_delta( self, chunk: dict ) -> tuple[str, Usage | None, dict[str, Any] | None, Mapping[str, object]]: @@ -1428,6 +1440,9 @@ class ModelResponseIterator: result = self._handle_accumulated_json_chunk(data_str="", is_final=True) if result is not None: return result + cut_stream_chunk = self.cut_stream_chunk() + if cut_stream_chunk is not None: + return cut_stream_chunk raise StopIteration except ValueError as e: raise RuntimeError(f"Error receiving chunk from stream: {e}") @@ -1474,6 +1489,9 @@ class ModelResponseIterator: result = self._handle_accumulated_json_chunk(data_str="", is_final=True) if result is not None: return result + cut_stream_chunk = self.cut_stream_chunk() + if cut_stream_chunk is not None: + return cut_stream_chunk raise StopAsyncIteration except ValueError as e: raise RuntimeError(f"Error receiving chunk from stream: {e}") diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9624dbb582b..867dd935a03 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -23,7 +23,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, is_object_schema, @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, + response_format_answer_json, response_format_schema, split_response_format_answer, type_to_response_format_param, @@ -163,6 +164,7 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128 _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) +_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2436,6 +2438,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=litellm_params.get("drop_params") is True, output_key="top_k", ) + output_format: Final = request_params.get("output_format") + if ( + output_format is not None + and not isinstance(output_format, Mapping) + and drops_unsupported_params(_LITELLM_PARAMS_ADAPTER.validate_python(litellm_params).get("drop_params")) + ): + request_params.pop("output_format") data: Final = { "model": model, @@ -3166,7 +3175,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) if split_answer is not None: - return litellm.Message(content=json.dumps(split_answer)) + return litellm.Message(content=response_format_answer_json(split_answer)) try: payload: Final = _DECODED_JSON.validate_python(json.loads(json_mode_content_str)) except (json.JSONDecodeError, RecursionError): @@ -3174,7 +3183,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return litellm.Message(content=json_mode_content_str) if response_format_schema is not None: return litellm.Message( - content=json.dumps( + content=response_format_answer_json( unwrap_response_format_envelope(payload, response_format_schema, RESPONSE_FORMAT_TOOL_NAME) ) ) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 0e8c1b49e8e..a71cd72b768 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -7,6 +7,7 @@ import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards @@ -627,6 +628,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ): anthropic_messages_optional_request_params.pop("safeguards") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + output_format: Final[object] = anthropic_messages_optional_request_params.get("output_format") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the request params arrive as a bare dict + if ( + output_format is not None + and not isinstance(output_format, Mapping) + and drops_unsupported_params(litellm_params.drop_params) + ): + anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + self._translate_reasoning_effort_to_anthropic( model=model, optional_params=anthropic_messages_optional_request_params, diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 5ed73e13188..640930a780d 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -5,6 +5,7 @@ Utility functions for base LLM classes. import copy import itertools import json +import re from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass @@ -172,7 +173,7 @@ _convert_tool_response_to_message = convert_tool_response_to_message @dataclass(frozen=True, slots=True) class ResponseFormatSchema: - property_names: frozenset[str] + declared_names: frozenset[str] required_names: frozenset[str] = frozenset() allows_undeclared_keys: bool = False @@ -216,9 +217,10 @@ def response_format_schema(value: object) -> ResponseFormatSchema | None: properties: Final = as_json_object(document.get("properties", {})) if properties is None: return None + required_names: Final = _names(document.get("required", ())) return ResponseFormatSchema( - property_names=frozenset(properties), - required_names=_names(document.get("required", ())), + declared_names=frozenset(properties) | required_names, + required_names=required_names, allows_undeclared_keys=_allows_undeclared_keys(document), ) @@ -244,8 +246,9 @@ def _json_object_or_encoded_object(value: JsonValue) -> Mapping[str, JsonValue] def _label_keys(payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema) -> frozenset[str]: if not schema.required_names: return frozenset() - declared: Final = schema.property_names | schema.required_names - return frozenset(key for key, value in payload_object.items() if isinstance(value, str) and key not in declared) + return frozenset( + key for key, value in payload_object.items() if isinstance(value, str) and key not in schema.declared_names + ) def _envelope_layer( @@ -256,7 +259,7 @@ def _envelope_layer( ) label_keys: Final = name_keys or _label_keys(payload_object, schema) wrapper_keys: Final = tuple(key for key in payload_object if key not in label_keys) - if len(wrapper_keys) != 1 or wrapper_keys[0] in schema.property_names: + if len(wrapper_keys) != 1 or wrapper_keys[0] in schema.declared_names: return None wrapped: Final = payload_object[wrapper_keys[0]] inner: Final = _json_object_or_encoded_object(wrapped) if name_keys else as_json_object(wrapped) @@ -275,15 +278,16 @@ def _envelope_layers( def _schema_object_inside_envelope( payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str ) -> Mapping[str, JsonValue] | None: - layers: Final = _envelope_layers(payload_object, schema, tool_name) - candidates: Final = layers if schema.property_names else itertools.islice(layers, 1) - return next((layer.inner for layer in candidates if _holds_the_schema_object(layer, schema)), None) + walk: Final = _envelope_layers(payload_object, schema, tool_name) + layers: Final = tuple(walk if schema.declared_names else itertools.islice(walk, 1)) + schema_object: Final = next((layer.inner for layer in layers if _shares_a_schema_key(layer.inner, schema)), None) + if schema_object is not None: + return schema_object + return next((layer.inner for layer in layers if layer.names_the_tool), None) -def _holds_the_schema_object(layer: _EnvelopeLayer, schema: ResponseFormatSchema) -> bool: - if layer.names_the_tool: - return True - return not schema.accepts_a_lone_undeclared_key and not schema.property_names.isdisjoint(layer.inner) +def _shares_a_schema_key(candidate: Mapping[str, JsonValue], schema: ResponseFormatSchema) -> bool: + return not schema.accepts_a_lone_undeclared_key and not schema.declared_names.isdisjoint(candidate) def response_format_envelope_contents(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object | None: @@ -293,7 +297,15 @@ def response_format_envelope_contents(payload: object, schema: ResponseFormatSch schema_object: Final = _schema_object_inside_envelope(payload_object, schema, tool_name) if schema_object is not None: return schema_object - return None if "values" in schema.property_names else payload_object.get("values") + return None if "values" in schema.declared_names else payload_object.get("values") + + +_LONE_SURROGATE: Final = re.compile("[\ud800-\udfff]") + + +def response_format_answer_json(value: object) -> str: + text: Final = json.dumps(value, ensure_ascii=False) + return json.dumps(value) if _LONE_SURROGATE.search(text) else text def unwrap_response_format_envelope(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object: @@ -305,7 +317,7 @@ def _field_names( blocks: tuple[Mapping[str, JsonValue], ...], key: str, schema: ResponseFormatSchema ) -> tuple[str, ...] | None: values: Final = (block[key] for block in blocks) - names: Final = tuple(value for value in values if isinstance(value, str) and value in schema.property_names) + names: Final = tuple(value for value in values if isinstance(value, str) and value in schema.declared_names) return names if len(frozenset(names)) == len(blocks) else None @@ -324,7 +336,7 @@ def _split_answer_fields( blocks: tuple[Mapping[str, JsonValue], ...], schema: ResponseFormatSchema ) -> Mapping[str, JsonValue] | None: keys: Final = frozenset(blocks[0]) - if not keys.isdisjoint(schema.property_names) or any(frozenset(block) != keys for block in blocks): + if not keys.isdisjoint(schema.declared_names) or any(frozenset(block) != keys for block in blocks): return None named: Final = tuple((key, names) for key in keys if (names := _field_names(blocks, key, schema)) is not None) if len(named) != 1: @@ -340,7 +352,7 @@ def _merged_partial_answers( blocks: tuple[Mapping[str, JsonValue], ...], schema: ResponseFormatSchema ) -> Mapping[str, JsonValue] | None: keys: Final = frozenset(itertools.chain.from_iterable(blocks)) - if not keys <= schema.property_names or len(keys) != sum(len(block) for block in blocks): + if not keys <= schema.declared_names or len(keys) != sum(len(block) for block in blocks): return None return dict(itertools.chain.from_iterable(block.items() for block in blocks)) @@ -358,14 +370,14 @@ def split_response_format_answer( def partial_response_format_answer(encoded_block: str, schema: ResponseFormatSchema) -> Mapping[str, JsonValue] | None: block: Final = _json_object_or_encoded_object(encoded_block) - return block if block is not None and frozenset(block) <= schema.property_names else None + return block if block is not None and frozenset(block) <= schema.declared_names else None def names_a_response_format_property(encoded_block: str, schema: ResponseFormatSchema) -> bool: block: Final = _json_object_or_encoded_object(encoded_block) if block is None: return False - return any(isinstance(value, str) and value in schema.property_names for value in block.values()) + return any(isinstance(value, str) and value in schema.declared_names for value in block.values()) def _dict_to_response_format_helper(response_format: dict, ref_template: str | None = None) -> dict: diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 7553dacd35d..3d4821fcaa2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -857,6 +857,9 @@ class AWSEventStreamDecoder: undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers) if undecoded_stream_error is not None: raise undecoded_stream_error + stream_end_chunk: Final = self._stream_end_chunk() + if stream_end_chunk is not None: + yield stream_end_chunk async def aiter_bytes( self, iterator: AsyncIterator[bytes], *, response_headers: Mapping[str, str] | None = None @@ -878,6 +881,12 @@ class AWSEventStreamDecoder: undecoded_stream_error: Final = tally.undecoded_stream_error(response_headers) if undecoded_stream_error is not None: raise undecoded_stream_error + stream_end_chunk: Final = self._stream_end_chunk() + if stream_end_chunk is not None: + yield stream_end_chunk + + def _stream_end_chunk(self) -> ModelResponseStream | None: + return None def _response_stream_shape(self) -> "Shape": response_stream_shape: Final = get_bedrock_response_stream_shape() @@ -956,6 +965,9 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder): def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream: return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data) + def _stream_end_chunk(self) -> ModelResponseStream | None: + return self.anthropic_model_response_iterator.cut_stream_chunk() + class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder): def __init__( diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index d5950d49d2b..db27427a5be 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -45,11 +45,22 @@ def _as_output_config(value: object) -> Mapping[str, object] | None: def _fold_output_format_into_output_config( - output_config: Mapping[str, object] | None, output_format: object + output_config: Mapping[str, object] | None, output_format: object, model: str ) -> Mapping[str, object] | None: if output_format is None: return output_config - return {**(output_config or {}), "format": output_format} + output_format_object: Final = _as_output_config(output_format) + if output_format_object is not None: + return {**(output_config or {}), "format": output_format_object} + from litellm.exceptions import UnsupportedParamsError + + raise UnsupportedParamsError( + message=( + f"{model} on vertex_ai needs `output_format` to be a JSON object, got {type(output_format).__name__}. " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + status_code=400, + ) def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: @@ -67,13 +78,16 @@ def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: body has no empty dict. * ``output_format`` is moved into ``output_config.format``, because Vertex AI Claude rejects the top-level field as deprecated. When both - are set, ``output_format`` wins, as it does in the Anthropic adapter. + are set, a dict ``output_format`` wins, as it does in the Anthropic + adapter. A non-dict ``output_format`` raises a 400 + ``UnsupportedParamsError``, since the parent config already dropped it + under ``drop_params``. * Non-dict values for ``output_config`` are dropped to avoid sending malformed payloads downstream. """ request: Final[dict[str, object]] = data # mutable-ok: the caller's request body is sanitized in place output_config: Final = _fold_output_format_into_output_config( - _as_output_config(request.get("output_config")), request.pop("output_format", None) + _as_output_config(request.get("output_config")), request.pop("output_format", None), model ) if output_config is None: request.pop("output_config", None) diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index cb295815ae2..2918f364c56 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -1819,25 +1819,32 @@ def test_vertex_chat_without_response_format_sends_no_tool_and_no_output_config( assert not {"tools", "output_config", "output_format"} & sent.body.keys(), sent.body -@pytest.mark.parametrize( - "output_format", - [5, [], {}, "x" * 5000], - ids=["int", "list", "empty", "5kb"], -) -def test_vertex_messages_malformed_output_format_reaches_the_caller_as_the_peer_400( +def test_vertex_messages_empty_output_format_reaches_the_caller_as_the_peer_400(gateway: Gateway) -> None: + marker, marker_hex = _marker("native") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url, _NATIVE_MODEL) + response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, marker, output_format={})) + assert response.status_code == 400, response.text + assert _BAD_FORMAT in response.text, response.text + sent: Final = _one_delivery(wire, marker_hex) + assert "output_format" not in sent.body, sent.body + assert object_value(sent.body["output_config"])["format"] == {}, sent.body + assert gateway.request("GET", "/health/liveliness").status_code == 200 + + +@pytest.mark.parametrize("output_format", [5, [], "x" * 5000], ids=["int", "list", "5kb"]) +def test_vertex_messages_non_object_output_format_is_refused_without_reaching_the_peer( gateway: Gateway, output_format: JsonValue ) -> None: - marker, marker_hex = _marker("native") + marker, _ = _marker("native") with wire_server(_peer) as wire, gateway.scenario() as scenario: model: Final = _vertex_deployment(gateway, scenario, wire.url, _NATIVE_MODEL) response: Final = gateway.request( "POST", "/v1/messages", _messages_body(model, marker, output_format=output_format) ) assert response.status_code == 400, response.text - assert _BAD_FORMAT in response.text, response.text - sent: Final = _one_delivery(wire, marker_hex) - assert "output_format" not in sent.body, sent.body - assert object_value(sent.body["output_config"])["format"] == output_format, sent.body + assert "needs `output_format` to be a JSON object" in response.text, response.text + assert _delivered(wire) == () assert gateway.request("GET", "/health/liveliness").status_code == 200 diff --git a/tests/integration/translation/chat_completions/response_format/__init__.py b/tests/integration/translation/chat_completions/response_format/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py new file mode 100644 index 00000000000..07e4358ff97 --- /dev/null +++ b/tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py @@ -0,0 +1,123 @@ +from dataclasses import replace +from typing import Final +from unittest.mock import ANY + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.chat_completions.bases.anthropic import CLAUDE_HAIKU_5_5_TEST_CASE +from integration.translation.runner import assert_translation + +PLACE_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": True, +} +PLACE_PROMPT: Final = "Which city is the capital of France? Answer as JSON with city and country." +PLACE_TOOL_USE: Final = { + "type": "tool_use", + "id": "toolu_01Sr1iptyZnsy1eNxabMZejF", + "name": "json_tool_call", + "input": {"city": "Paris", "country": "France"}, + "caller": {"type": "direct"}, +} + +CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace( + CLAUDE_HAIKU_5_5_TEST_CASE, + scenario="json_schema", + litellm_request={ + "model": "anthropic/claude-haiku-5-5", + "max_tokens": 128, + "messages": [{"role": "user", "content": PLACE_PROMPT}], + "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, + "cache": {"no-cache": True}, + }, + expected_provider_request={ + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], + "max_tokens": 128, + "tool_choice": {"name": "json_tool_call", "type": "tool"}, + "tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}], + }, + mock_provider_response={ + "model": "claude-haiku-5-5", + "id": "msg_011CgCoP2UR3vcM5ZWmEBd8D", + "type": "message", + "role": "assistant", + "content": [PLACE_TOOL_USE], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 528, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 55, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + expected_litellm_response={ + "id": ANY, + "created": ANY, + "object": "chat.completion", + "model": "anthropic/claude-haiku-5-5", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"content": '{"city": "Paris", "country": "France"}', "role": "assistant"}, + } + ], + "usage": { + "completion_tokens": 55, + "prompt_tokens": 528, + "total_tokens": 583, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 55}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 528, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "service_tier": "standard", + "speed": "standard", + }, + }, +) + +CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace( + CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, + scenario="json_schema_tool_named_envelope", + mock_provider_response={ + **CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE.mock_provider_response, + "content": [ + { + **PLACE_TOOL_USE, + "input": {"name": "json_tool_call", "arguments": {"input": {"city": "Paris", "country": "France"}}}, + } + ], + }, +) + + +@pytest.mark.parametrize( + "case", + [CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE], + ids=lambda case: case.id, +) +def test_chat_completions_response_format_anthropic( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py index ccba51cff23..b2c0d39f9aa 100644 --- a/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py +++ b/tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py @@ -5,8 +5,12 @@ import pytest from integration._support.client import Gateway from integration._support.provider import SharedProvider from integration.translation.case import SSEFrame, StreamingTranslationTestCase -from integration.translation.chat_completions.bases.anthropic import CLAUDE_SONNET_5_5_TEST_CASE +from integration.translation.chat_completions.bases.anthropic import ( + CLAUDE_HAIKU_5_5_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, +) from integration.translation.runner import assert_streaming_translation +from pydantic import JsonValue CLAUDE_SONNET_5_5_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( scenario="streaming", @@ -128,7 +132,143 @@ CLAUDE_SONNET_5_5_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( ) -@pytest.mark.parametrize("case", [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE], ids=lambda case: case.id) +PLACE_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": True, +} +PLACE_PROMPT: Final = "Which city is the capital of France? Answer as JSON with city and country." + + +def _place_delta(partial_json: str) -> SSEFrame: + return SSEFrame( + event="content_block_delta", + data={ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": partial_json}, + }, + ) + + +def _place_chunk(delta: dict[str, JsonValue], *, finish_reason: str | None = None) -> SSEFrame: + finish: Final = {} if finish_reason is None else {"finish_reason": finish_reason} + return SSEFrame( + data={ + "id": ANY, + "object": "chat.completion.chunk", + "created": ANY, + "model": "anthropic/claude-haiku-5-5", + "choices": [{"index": 0, "delta": delta, **finish}], + } + ) + + +CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="json_schema_cut_stream", + litellm_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.litellm_endpoint, + litellm_request={ + "model": "anthropic/claude-haiku-5-5", + "max_tokens": 128, + "messages": [{"role": "user", "content": PLACE_PROMPT}], + "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}}, + "cache": {"no-cache": True}, + "stream": True, + }, + expected_provider_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_endpoint, + expected_provider_headers=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_headers, + expected_provider_request={ + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}], + "max_tokens": 128, + "tool_choice": {"name": "json_tool_call", "type": "tool"}, + "tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}], + "stream": True, + }, + mock_provider_frames=( + SSEFrame( + event="message_start", + data={ + "type": "message_start", + "message": { + "model": "claude-haiku-5-5", + "id": "msg_011CgCoP5RkexwE6N7PRZEca", + "type": "message", + "role": "assistant", + "content": [], + "container": None, + "stop_reason": None, + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 528, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 16, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + }, + ), + SSEFrame( + event="content_block_start", + data={ + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_01B4QSMQtft8TA3Rn6XFziWe", + "name": "json_tool_call", + "input": {}, + "caller": {"type": "direct"}, + }, + }, + ), + SSEFrame(event="ping", data={"type": "ping"}), + _place_delta(""), + _place_delta('{"city"'), + _place_delta(': "Pari'), + _place_delta('s"'), + _place_delta(', "coun'), + _place_delta('try": "'), + _place_delta('France"}'), + ), + expected_litellm_frames=( + _place_chunk({"role": "assistant", "content": '{"city": "Pari'}), + _place_chunk({"content": 's"'}), + _place_chunk({"content": ', "coun'}), + _place_chunk({"content": 'try": "'}), + _place_chunk({"content": 'France"'}), + _place_chunk({"content": "}"}), + _place_chunk({}, finish_reason="stop"), + SSEFrame(data="[DONE]"), + ), + expected_litellm_response={ + "id": ANY, + "object": "chat.completion", + "created": ANY, + "model": "anthropic/claude-haiku-5-5", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": '{"city": "Paris", "country": "France"}'}, + "finish_reason": "stop", + } + ], + }, +) + + +@pytest.mark.parametrize( + "case", + [CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE], + ids=lambda case: case.id, +) def test_chat_completions_streaming_anthropic( case: StreamingTranslationTestCase, gateway: Gateway, provider: SharedProvider ) -> None: diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index a2d9f321122..1f4a5aaa256 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -3108,6 +3108,92 @@ def test_streamed_json_mode_answer_with_a_foreign_name_matches_the_non_streamed_ assert unwrap_response_format_envelope(answer, schema, RESPONSE_FORMAT_TOOL_NAME) == answer +def test_streamed_json_mode_tool_name_envelope_around_an_input_wrapper_is_unwrapped_to_the_answer(): + envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "arguments": {"input": _STREAMED_RISK_OBJECT}}) + + contents = _streamed_json_mode_contents((envelope[:9], envelope[9:40], envelope[40:])) + + assert json.loads("".join(contents)) == _STREAMED_RISK_OBJECT + + +def test_streamed_json_mode_unwrapped_answer_keeps_non_ascii_text(): + answer = {"tier": 3, "summary": "Hôtel Lutetia, 東京"} + envelope = json.dumps({"input": answer}, ensure_ascii=False) + + contents = _streamed_json_mode_contents((envelope[:9], envelope[9:30], envelope[30:])) + + assert "Hôtel Lutetia, 東京" in "".join(contents) + assert json.loads("".join(contents)) == answer + + +def test_streamed_json_mode_unwrapped_answer_with_a_lone_surrogate_stays_utf8_encodable(): + answer = {"tier": 3, "summary": "smile \ud83d end"} + envelope = json.dumps({"input": answer}) + + contents = _streamed_json_mode_contents((envelope[:9], envelope[9:30], envelope[30:])) + + assert json.loads("".join(contents).encode("utf-8")) == answer + + +_CUT_STREAM_FRAGMENTS: Final = ( + pytest.param( + json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_RISK_OBJECT})[:-1], + id="held_envelope", + ), + pytest.param(json.dumps(_STREAMED_RISK_OBJECT), id="streamed_answer"), +) + + +def _cut_stream_lines(sent: str) -> tuple[bytes, ...]: + events: Final = _json_tool_events((sent[:9], sent[9:30], sent[30:])) + return tuple(f"data: {json.dumps(event)}\n".encode() for event in events[:-2]) + + +def _cut_stream_iterator(sync_stream: bool, streaming_response: object) -> ModelResponseIterator: + return ModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=True, + response_format_schema=response_format_schema(_STREAMED_RISK_SCHEMA), + ) + + +@pytest.mark.parametrize("sent", _CUT_STREAM_FRAGMENTS) +def test_streamed_json_mode_text_is_delivered_when_the_stream_ends_before_the_block_stops(sent: str): + iterator = _cut_stream_iterator(True, iter(_cut_stream_lines(sent))) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in iterator) == sent + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sent", _CUT_STREAM_FRAGMENTS) +async def test_async_streamed_json_mode_text_is_delivered_when_the_stream_ends_before_the_block_stops(sent: str): + async def lines(): + for line in _cut_stream_lines(sent): + yield line + + iterator = _cut_stream_iterator(False, lines()) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in iterator]) == sent + + +def test_streamed_json_mode_complete_envelope_is_unwrapped_when_the_stream_ends_before_the_block_stops(): + envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_RISK_OBJECT}) + + iterator = _cut_stream_iterator(True, iter(_cut_stream_lines(envelope))) + + assert json.loads("".join(chunk.choices[0].delta.content or "" for chunk in iterator)) == _STREAMED_RISK_OBJECT + + +def test_streamed_json_mode_answer_keyed_by_a_required_only_name_is_kept_whole(): + required_only_schema = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["payload"]} + encoded = json.dumps({"payload": {"id": "abc", "rows": 3}}) + + contents = _streamed_json_mode_contents((encoded[:12], encoded[12:24], encoded[24:]), required_only_schema) + + assert "".join(contents) == encoded + + def _json_tool_block_events(index: int, encoded: str) -> tuple[dict[str, object], ...]: return ( { @@ -3202,6 +3288,30 @@ def test_streamed_json_mode_answer_split_into_one_partial_object_per_field_arriv assert contents[3] == blocks[0][:-1] +@pytest.mark.parametrize( + "blocks", + [ + pytest.param((_STREAMED_TIER_FIELD, _STREAMED_SUMMARY_FIELD), id="one_block_per_field"), + pytest.param( + tuple(json.dumps({name: value}) for name, value in _STREAMED_RISK_OBJECT.items()), + id="one_partial_object_per_field", + ), + ], +) +def test_streamed_json_mode_split_answer_arrives_as_one_object_when_the_stream_ends_in_its_last_block( + blocks: tuple[str, ...], +): + message_start, *_ = _json_tool_events(()) + events = ( + message_start, + *chain.from_iterable(_json_tool_block_events(index, encoded) for index, encoded in enumerate(blocks)), + )[:-1] + + iterator = _cut_stream_iterator(True, iter(tuple(f"data: {json.dumps(event)}\n".encode() for event in events))) + + assert json.loads("".join(chunk.choices[0].delta.content or "" for chunk in iterator)) == _STREAMED_RISK_OBJECT + + def test_streamed_json_mode_empty_object_then_field_blocks_arrives_as_one_object(): blocks = ("{}", *(json.dumps({name: value}) for name, value in _STREAMED_RISK_OBJECT.items())) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index ef8447712d6..32f18b952ac 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2251,6 +2251,38 @@ def test_effort_with_other_features(): assert "thinking" in result +_OBJECT_OUTPUT_FORMAT: Final = {"type": "json_schema", "schema": {"type": "object", "properties": {}}} + + +@pytest.mark.parametrize( + ("output_format", "request_drop_params", "global_drop_params", "forwarded"), + [ + pytest.param("json", True, False, False, id="string_dropped_under_request_drop_params"), + pytest.param("json", None, True, False, id="string_dropped_under_global_drop_params"), + pytest.param("json", None, False, True, id="string_forwarded_without_drop_params"), + pytest.param(_OBJECT_OUTPUT_FORMAT, True, False, True, id="object_kept_under_drop_params"), + ], +) +def test_anthropic_chat_drops_only_a_non_object_output_format_and_only_under_drop_params( + output_format: object, + request_drop_params: bool | None, + global_drop_params: bool, + forwarded: bool, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + + result = AnthropicConfig().transform_request( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"output_format": output_format}, + litellm_params={"drop_params": request_drop_params}, + headers={}, + ) + + assert result.get("output_format") == (output_format if forwarded else None) + + def test_anthropic_drop_params_strips_output_config_for_pre_4_5_models(): """``drop_params=True`` strips unsupported ``output_config`` for pre-4.5 models.""" config = AnthropicConfig() @@ -7882,6 +7914,24 @@ def _json_tool_call(arguments: object) -> list[ChatCompletionToolCallChunk]: {"summary": "likes tea"}, id="tool_name_envelope_on_a_schema_allowing_extra_keys", ), + pytest.param( + {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["payload"]}, + {"payload": {"id": "abc", "rows": 3}}, + {"payload": {"id": "abc", "rows": 3}}, + id="key_listed_only_in_required", + ), + pytest.param( + _RISK_SCHEMA, + {"name": RESPONSE_FORMAT_TOOL_NAME, "arguments": {"input": _RISK_OBJECT}}, + _RISK_OBJECT, + id="tool_name_envelope_around_an_input_wrapper", + ), + pytest.param( + _PER_USER_SCHEMA, + {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _PER_USER_ANSWER}, + _PER_USER_ANSWER, + id="tool_name_envelope_around_an_answer_with_one_allowed_key", + ), ], ) def test_convert_tool_response_to_message_unwraps_only_an_unambiguous_schema_envelope( @@ -7896,6 +7946,31 @@ def test_convert_tool_response_to_message_unwraps_only_an_unambiguous_schema_env assert json.loads(message.content) == expected_content +def test_convert_tool_response_to_message_keeps_non_ascii_text_in_an_unwrapped_answer(): + answer = {"tier": 3, "summary": "Hôtel Lutetia, 東京"} + + message = AnthropicConfig.convert_tool_response_to_message( + tool_calls=_json_tool_call({"input": answer}), + response_format_schema=_schema(_RISK_SCHEMA), + ) + + assert message is not None + assert "Hôtel Lutetia, 東京" in message.content + assert json.loads(message.content) == answer + + +def test_convert_tool_response_to_message_answer_with_a_lone_surrogate_stays_utf8_encodable(): + answer = {"tier": 3, "summary": "smile \ud83d end"} + + message = AnthropicConfig.convert_tool_response_to_message( + tool_calls=_json_tool_call({"input": answer}), + response_format_schema=_schema(_RISK_SCHEMA), + ) + + assert message is not None + assert json.loads(message.content.encode("utf-8")) == answer + + _TOO_DEEP_TO_DECODE: Final = "[" * 100_000 + "]" * 100_000 diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py index 670e3de5110..db3ff58d80a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py @@ -2,10 +2,16 @@ Tests for structured outputs support in Anthropic /v1/messages endpoint. """ +from collections.abc import Mapping +from typing import Final + import pytest + +import litellm from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) +from litellm.types.router import GenericLiteLLMParams def test_output_format_supported_and_transforms_correctly(): @@ -109,3 +115,47 @@ def test_output_format_works_with_bedrock_and_azure(): headers={}, ) assert "output_format" in azure_result + + +_OBJECT_OUTPUT_FORMAT: Final = {"type": "json_schema", "schema": {"type": "object", "properties": {}}} + + +def _request_with_output_format( + output_format: object, litellm_params: GenericLiteLLMParams +) -> Mapping[str, object]: + return AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-haiku-5-5", + messages=[{"role": "user", "content": "What is 6 times 7?"}], + anthropic_messages_optional_request_params={"max_tokens": 1024, "output_format": output_format}, + litellm_params=litellm_params, + headers={}, + ) + + +@pytest.mark.parametrize("output_format", ["json", ["json_schema"], 42]) +def test_malformed_output_format_is_dropped_under_drop_params(output_format: object) -> None: + result: Final = _request_with_output_format(output_format, GenericLiteLLMParams(drop_params=True)) + + assert "output_format" not in result + + +def test_malformed_output_format_is_dropped_under_the_global_drop_params_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + + result: Final = _request_with_output_format("json", GenericLiteLLMParams()) + + assert "output_format" not in result + + +def test_object_output_format_is_kept_under_drop_params() -> None: + result: Final = _request_with_output_format(_OBJECT_OUTPUT_FORMAT, GenericLiteLLMParams(drop_params=True)) + + assert result["output_format"] == _OBJECT_OUTPUT_FORMAT + + +def test_malformed_output_format_is_forwarded_for_the_provider_to_reject_without_drop_params() -> None: + result: Final = _request_with_output_format("json", GenericLiteLLMParams()) + + assert result["output_format"] == "json" diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 176dcfc86c9..38483b8a1fa 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -7,6 +7,7 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) +from collections.abc import Mapping from typing import Final from unittest.mock import patch @@ -551,3 +552,47 @@ async def test_outgoing_azure_request_sends_safeguards_with_the_dangerous_tool_u sent: Final = received[0] assert "dangerous-tool-use-2026-09-03" in sent.headers["anthropic-beta"].split(","), dict(sent.headers) assert json.loads(sent.content)["safeguards"] == list(AUTO_MODE_SAFEGUARDS) + + +async def _azure_request_body_for_output_format(output_format: object, drop_params: bool) -> Mapping[str, object]: + received: Final[list[httpx.Request]] = [] + + def foundry_answers(request: httpx.Request) -> httpx.Response: + received.append(request) + return _foundry_answer(request) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers)) + try: + await litellm.anthropic.messages.acreate( + max_tokens=256, + messages=[{"role": "user", "content": "What is 6 times 7?"}], + model="azure_ai/claude-sonnet-4-6", + api_key="test-api-key", + api_base="https://test-resource.services.ai.azure.com", + output_format=output_format, + drop_params=drop_params, + client=upstream, + ) + finally: + await upstream.close() + + assert len(received) == 1, received + return json.loads(received[0].content) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_outgoing_azure_request_drops_a_malformed_output_format_under_drop_params() -> None: + body: Final = await _azure_request_body_for_output_format("json", drop_params=True) + + assert "output_format" not in body + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_outgoing_azure_request_keeps_an_object_output_format_under_drop_params() -> None: + output_format: Final = {"type": "json_schema", "schema": {"type": "object", "properties": {}}} + + body: Final = await _azure_request_body_for_output_format(output_format, drop_params=True) + + assert body["output_format"] == output_format diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 069438e25c0..f3ceac1d93c 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1880,3 +1880,84 @@ async def test_converse_stream_thinking_merges_into_content_with_think_tags() -> "Hi I am Anthropic, I am a helpful assistant" ) assert [getattr(chunk.choices[0].delta, "reasoning_content", None) for chunk in chunks] == [None] * len(chunks) + + +_PLACE_JSON_MODE_REQUEST: Final = json.dumps( + { + "tools": [ + { + "name": "json_tool_call", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": True, + }, + } + ] + } +) + +_CUT_JSON_TOOL_STREAM: Final = b"".join( + _bedrock_event_stream_frame(event) + for event in ( + { + "type": "message_start", + "message": { + "id": "msg_cut_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 528, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "toolu_cut_1", "name": "json_tool_call", "input": {}}, + }, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"city"'}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ': "Paris", '}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '"country": "France"}'}, + }, + ) +) + + +def _cut_json_tool_stream_decoder() -> AWSEventStreamDecoder: + return _stream_decoder( + "anthropic", + model="anthropic.claude-haiku-5-5", + json_mode=True, + sync_stream=True, + response_format_schema=response_format_schema_from_request(_PLACE_JSON_MODE_REQUEST, json_mode=True), + ) + + +def _streamed_content(chunks: Sequence[object]) -> str: + return "".join( + chunk.choices[0].delta.content or "" + for chunk in chunks + if isinstance(chunk, ModelResponseStream) and chunk.choices + ) + + +def test_anthropic_invoke_stream_cut_before_the_json_tool_block_ends_sends_the_held_close() -> None: + chunks: Final = list(_cut_json_tool_stream_decoder().iter_bytes(iter([_CUT_JSON_TOOL_STREAM]))) + + assert json.loads(_streamed_content(chunks)) == {"city": "Paris", "country": "France"} + + +@pytest.mark.asyncio +async def test_async_anthropic_invoke_stream_cut_before_the_json_tool_block_ends_sends_the_held_close() -> None: + async def _chunks() -> AsyncIterator[bytes]: + yield _CUT_JSON_TOOL_STREAM + + chunks: Final = [chunk async for chunk in _cut_json_tool_stream_decoder().aiter_bytes(_chunks())] + + assert json.loads(_streamed_content(chunks)) == {"city": "Paris", "country": "France"} diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 113ddcfe2a9..ae9b9684f2d 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -1,6 +1,7 @@ import copy import json import os +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch @@ -537,6 +538,44 @@ def test_messages_request_moves_output_format_into_output_config(): assert result["output_config"] == {"effort": "high", "format": output_format} +_CALLER_OUTPUT_CONFIG: Final = { + "effort": "high", + "format": {"type": "json_schema", "schema": {"type": "object", "properties": {"answer": {"type": "integer"}}}}, +} + + +def _messages_request_with_output_format(output_format: object, drop_params: bool) -> Mapping[str, object]: + return VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "What is 6 times 7?"}], + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "output_format": output_format, + "output_config": copy.deepcopy(_CALLER_OUTPUT_CONFIG), + }, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + +@pytest.mark.parametrize("output_format", ["json", ["json_schema"], 42]) +def test_messages_request_drops_a_malformed_output_format_under_drop_params(output_format: object): + result = _messages_request_with_output_format(output_format, drop_params=True) + + assert "output_format" not in result + assert result["output_config"] == _CALLER_OUTPUT_CONFIG + + +@pytest.mark.parametrize("output_format", ["json", ["json_schema"], 42]) +def test_messages_request_rejects_a_malformed_output_format_without_drop_params(output_format: object): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError) as error: + _messages_request_with_output_format(output_format, drop_params=False) + + assert error.value.status_code == 400 + + def test_provider_config_manager_reuses_vertex_anthropic_messages_config_instance(): """ Regression test: repeated provider config lookups for the same Vertex Claude model diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ace24c8a3f4..21bda6f0345 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -918,6 +918,44 @@ def test_sanitize_folds_output_format_into_output_config(model: str, data: dict, assert request["output_config"] == expected_output_config +def _vertex_chat_request_with_output_format( + output_format: object, litellm_params: Mapping[str, object] +) -> Mapping[str, object]: + return VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "What is 6 times 7?"}], + optional_params={"max_tokens": 64, "output_format": output_format}, + litellm_params=dict(litellm_params), + headers={}, + ) + + +def test_vertex_chat_request_drops_a_malformed_output_format_under_per_request_drop_params(): + request: Final = _vertex_chat_request_with_output_format("json", {"drop_params": True}) + + assert "output_format" not in request + assert "output_config" not in request + + +def test_vertex_chat_request_drops_a_malformed_output_format_under_the_global_drop_params_flag( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "drop_params", True) + + request: Final = _vertex_chat_request_with_output_format("json", {}) + + assert "output_format" not in request + assert "output_config" not in request + + +def test_vertex_chat_request_rejects_a_malformed_output_format_without_drop_params_naming_the_proxy_setting(): + with pytest.raises(litellm.UnsupportedParamsError) as error: + _vertex_chat_request_with_output_format("json", {}) + + assert error.value.status_code == 400 + assert "litellm_settings:\n drop_params: true" in str(error.value) + + def test_vertex_ai_native_response_format_reaches_vertex_as_output_config_format(local_model_cost_map): config = VertexAIAnthropicConfig() schema: Final = {"type": "object", "properties": {"result": {"type": "string"}}, "required": ["result"]} From 66cd1a651647639f298af0fc9554ecc9d9cd212b Mon Sep 17 00:00:00 2001 From: Zuraiz Anjum Date: Tue, 20 Oct 2026 13:50:29 +0500 Subject: [PATCH 082/124] fix(streaming): restore the token estimate for streams that send no usage (#44230) * fix(streaming): restore the token estimate for streams that send no usage A stream without stream_options got a made-up Usage(0, 0) written into its final chunk's hidden params. Since #42323 the chunk builder treats that as a provider-reported zero, so these streams were logged and billed at 0 tokens and $0 instead of the tokenizer estimate. Only attach the summed usage when some chunk actually carried usage. * test(streaming): use a fixed start time and check the estimated cost * fix(streaming): keep reported usage on the synthesized finish chunk and test it * test(streaming): cover explicit zero usage reported by the provider * fix(streaming): set the token estimate on a copy of the final chunk and ignore usage with no counts * fix(streaming): keep final chunk edits in the logged response when the provider sends no usage * fix(streaming): mark the unreported usage placeholder so every chunk rebuild falls back to the estimate * test(streaming): wait on the success callback event instead of sleep-polling --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 3 + .../litellm_core_utils/streaming_handler.py | 142 +++++------ litellm/types/utils.py | 4 + .../test_streaming_handler.py | 235 ++++++++++++++++++ .../proxy/test_common_request_processing.py | 60 ++++- 5 files changed, 372 insertions(+), 72 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index b42210e2793..39f675099d7 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -46,6 +46,7 @@ from litellm.types.utils import ( StreamingChoices, TextChoices, TextCompletionResponse, + UnreportedUsage, Usage, ) from litellm.utils import print_verbose, token_counter @@ -876,6 +877,8 @@ class ChunkProcessor: if isinstance(usage_chunk, dict): return Usage(**usage_chunk) + if isinstance(usage_chunk, UnreportedUsage): + return None if usage_chunk is None or isinstance(usage_chunk, Usage): return usage_chunk return Usage(**usage_chunk.model_dump()) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index a627b26b9af..6ec426d6da5 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -24,7 +24,12 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.asyncify import asyncify -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, stored_attribute +from litellm.litellm_core_utils.hidden_params import ( + HIDDEN_PARAMS_ATTR, + get_hidden_params, + set_hidden_param, + stored_attribute, +) from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) @@ -43,6 +48,8 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, StreamingChoices, + TextCompletionResponse, + UnreportedUsage, Usage, ) from litellm.types.utils import GenericStreamingChunk as GChunk @@ -62,6 +69,7 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call" _SYNC_ITER_EXHAUSTED: Final = object() _GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__) +_CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None _USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.OPENROUTER.value}) @@ -335,7 +343,8 @@ class CustomStreamWrapper: self.sent_stream_usage = False self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False self.tool_call = False - self.chunks: list = [] # keep track of the returned chunks - used for calculating the input/output tokens for stream options + # keep track of the returned chunks - used for calculating the input/output tokens for stream options + self.chunks: list[ModelResponseStream] = [] self._repeated_messages_count = 1 self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: int | None = None @@ -544,7 +553,7 @@ class CustomStreamWrapper: last_content: Final = self.chunks[-1].choices[0].delta.content if ( - last_content is None or not isinstance(last_content, str) or len(last_content) <= 2 + not isinstance(last_content, str) or len(last_content) <= 2 ): # ignore empty content - https://github.com/BerriAI/litellm/issues/5158#issuecomment-2287156946 self._repeated_messages_count = 1 return @@ -1927,6 +1936,51 @@ class CustomStreamWrapper: response.hidden_params["additional_headers"] = {} response.hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = _cost + def _builder_failure_response(self, error: Exception) -> ModelResponseStream | None: + verbose_logger.warning( + "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.", + str(error), + ) + try: + return self.model_response_creator(chunk={"usage": calculate_total_usage(chunks=self.chunks)}) + except Exception: + return None + + def _build_complete_streaming_response(self) -> _CompleteStreamingResponse: + try: + return litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, + ) + except Exception as e: + return self._builder_failure_response(e) + + async def _async_build_complete_streaming_response(self) -> _CompleteStreamingResponse: + try: + return await asyncify(litellm.stream_chunk_builder)( + chunks=self.chunks, + messages=self.messages, + logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, + ) + except Exception as e: + return self._builder_failure_response(e) + + def _set_final_chunk_usage(self, chunk: ModelResponseStream) -> None: + reported_usage: Final = _reported_total_usage(chunks=self.chunks) + set_hidden_param( + chunk, + "usage", + UnreportedUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + if reported_usage is None + else reported_usage, + ) + self._last_returned_hidden_params = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR) + ) + def __next__(self) -> "ModelResponseStream": if self._closed: raise StopIteration @@ -2007,12 +2061,7 @@ class CustomStreamWrapper: continue # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: - usage = calculate_total_usage(chunks=self.chunks) - response_hidden_params_for_final_chunk = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) - ) - response_hidden_params_for_final_chunk["usage"] = usage - self._last_returned_hidden_params = response_hidden_params_for_final_chunk + self._set_final_chunk_usage(response) # Add MCP metadata to final chunk if present response = self._add_mcp_metadata_to_final_chunk(response) # RETURN RESULT @@ -2020,30 +2069,7 @@ class CustomStreamWrapper: except StopIteration: if self.sent_last_chunk is True: - try: - complete_streaming_response = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages, - logging_obj=self.logging_obj, - count_prompt_tokens=self.count_prompt_tokens, - ) - except Exception as e: - # stream_chunk_builder can re-raise (as APIError) on large agentic - # streams. The raise originates inside this except-StopIteration block, - # so the sibling `except Exception` below does not catch it; it would - # escape __next__ and drop the request from SpendLogs. Recover - # best-effort usage from the raw chunks so cost is still tracked - verbose_logger.warning( - "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.", - str(e), - ) - try: - complete_streaming_response = self.model_response_creator( - chunk={"usage": calculate_total_usage(chunks=self.chunks)} - ) - except Exception: - complete_streaming_response = None - + complete_streaming_response: Final = self._build_complete_streaming_response() response = self.model_response_creator() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) @@ -2107,9 +2133,8 @@ class CustomStreamWrapper: # last content chunk (stripped there), but never add the synthetic "stop" used when it sent none. if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: self.chunks.append(processed_chunk) - if self.stream_options is None: # add usage as hidden param - usage = calculate_total_usage(chunks=self.chunks) - processed_chunk.hidden_params["usage"] = usage + if self.stream_options is None: + self._set_final_chunk_usage(processed_chunk) ## LOGGING executor.submit( self.run_success_logging_and_cache_storage, @@ -2224,12 +2249,7 @@ class CustomStreamWrapper: # add usage as hidden param if self.sent_last_chunk is True and self.stream_options is None: - usage = calculate_total_usage(chunks=self.chunks) - processed_chunk_hidden_params = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) - ) - processed_chunk_hidden_params["usage"] = usage - self._last_returned_hidden_params = processed_chunk_hidden_params + self._set_final_chunk_usage(processed_chunk) # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: @@ -2285,29 +2305,7 @@ class CustomStreamWrapper: async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream": if self.sent_last_chunk is True: - # log the final chunk with accurate streaming values - try: - complete_streaming_response = await asyncify(litellm.stream_chunk_builder)( - chunks=self.chunks, - messages=self.messages, - logging_obj=self.logging_obj, - count_prompt_tokens=self.count_prompt_tokens, - ) - except Exception as e: - # see sync __next__: a raise from stream_chunk_builder inside this - # except handler escapes __anext__ and drops the request from SpendLogs. - # Recover best-effort usage from the raw chunks so cost is still tracked - verbose_logger.warning( - "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.", - str(e), - ) - try: - complete_streaming_response = self.model_response_creator( - chunk={"usage": calculate_total_usage(chunks=self.chunks)} - ) - except Exception: - complete_streaming_response = None - + complete_streaming_response: Final = await self._async_build_complete_streaming_response() response: Final = self.model_response_creator() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) @@ -2384,11 +2382,7 @@ class CustomStreamWrapper: if self.received_finish_reason is not None or self.intermittent_finish_reason is not None: self.chunks.append(processed_chunk) if self.stream_options is None: - usage: Final = calculate_total_usage(chunks=self.chunks) - processed_chunk_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) - ) - processed_chunk_hidden_params["usage"] = usage + self._set_final_chunk_usage(processed_chunk) # see sync __next__'s sibling branch: deliberately do NOT restore # here - this chunk is still this call's own data, and restoring # before returning it would corrupt the caller's own log @@ -2659,7 +2653,13 @@ def _coerce_token_details( return details_type(**(raw if isinstance(raw, dict) else raw.model_dump())) -def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: +def _reported_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) -> Usage | None: + if not any("usage" in chunk and chunk["usage"] is not None for chunk in chunks): + return None + return calculate_total_usage(chunks=chunks) + + +def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( attach_cache_creation_token_details, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 576c4f093d3..2721a0c1cc1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2062,6 +2062,10 @@ class Usage(SafeAttributeModel, CompletionUsage): setattr(self, key, value) +class UnreportedUsage(Usage): + pass + + class StreamingChoices(OpenAIObject): def __init__( self, diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 691442e80d0..68aabc3c01e 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1,4 +1,5 @@ import asyncio +import datetime import json import sys import threading @@ -18,6 +19,7 @@ import litellm from litellm import verbose_logger from litellm._logging import session_id_var, trace_id_var from litellm.exceptions import MidStreamFallbackError +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import ( @@ -26,6 +28,7 @@ from litellm.litellm_core_utils.streaming_handler import ( _ProviderChunkParsed, ) from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response from litellm.types.utils import ( CompletionTokensDetailsWrapper, Delta, @@ -2820,6 +2823,238 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): ), f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("ends_with_finish_chunk", [True, False]) +@pytest.mark.asyncio +async def test_stream_without_provider_usage_falls_back_to_the_token_estimate( + sync_mode: bool, ends_with_finish_chunk: bool +) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + content_chunk: Final = ModelResponseStream( + id="chatcmpl-no-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + ) + finish_chunk: Final = ModelResponseStream( + id="chatcmpl-no-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[content_chunk, finish_chunk] if ends_with_finish_chunk else [content_chunk] + ), + model=model, + custom_llm_provider="openai", + logging_obj=Logging( + model=model, + messages=messages, + stream=True, + call_type="completion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="no-usage-call", + function_id="no-usage-fn", + ), + stream_options=None, + ) + + yielded: Final = ( + [(chunk, chunk._hidden_params.get("usage")) for chunk in wrapper] + if sync_mode + else [(chunk, chunk._hidden_params.get("usage")) async for chunk in wrapper] + ) + collected: Final = [chunk for chunk, _ in yielded] + + expected: Final = ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + assembled: Final = litellm.stream_chunk_builder(chunks=collected, messages=messages) + assert assembled is not None + assert (assembled.usage.prompt_tokens, assembled.usage.completion_tokens) == expected + last_chunk, usage_when_yielded = yielded[-1] + assert usage_when_yielded is not None + usage_after_drain: Final = last_chunk._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == expected + + rates: Final = litellm.model_cost[model] + spend: Final = expected[0] * rates["input_cost_per_token"] + expected[1] * rates["output_cost_per_token"] + assert litellm.completion_cost(completion_response=assembled) == pytest.approx(spend) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_with_usage_after_the_finish_chunk_bills_only_the_reported_usage(sync_mode: bool) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + reported: Final = (6, 20) + chunks: Final = [ + ModelResponseStream(id="cmpl-late-usage", created=1741037890, model=model, choices=choices, usage=usage) + for choices, usage in ( + ([StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], None), + ([StreamingChoices(index=0, delta=Delta(content=""), finish_reason="length")], None), + ([], Usage(prompt_tokens=reported[0], completion_tokens=reported[1], total_tokens=sum(reported))), + ) + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model=model, + custom_llm_provider="openai", + logging_obj=Logging( + model=model, + messages=messages, + stream=True, + call_type="completion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="late-usage-call", + function_id="late-usage-fn", + ), + stream_options=None, + ) + + collected: Final = [chunk for chunk in wrapper] if sync_mode else [chunk async for chunk in wrapper] + + usage_after_drain: Final = collected[-1]._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == reported + assembled: Final = litellm.stream_chunk_builder(chunks=collected, messages=messages) + assert assembled is not None + rates: Final = litellm.model_cost[model] + spend: Final = reported[0] * rates["input_cost_per_token"] + reported[1] * rates["output_cost_per_token"] + assert litellm.completion_cost(completion_response=assembled) == pytest.approx(spend) + + +class _LoggedResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.responses: list[object] = [] + self.logged: Final = threading.Event() + + def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.responses.append(response_obj) + self.logged.set() + + async def async_log_success_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.responses.append(response_obj) + self.logged.set() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_without_provider_usage_logs_the_estimate_with_the_final_chunk_mcp_metadata( + sync_mode: bool, +) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + tool_calls: Final = [{"name": "lookup_fox", "arguments": "{}"}] + recorder: Final = _LoggedResponseRecorder() + logging_obj: Final = Logging( + model=model, + messages=messages, + stream=True, + call_type="completion" if sync_mode else "acompletion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="mcp-no-usage-call", + function_id="mcp-no-usage-fn", + dynamic_success_callbacks=[recorder], + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": "mcp-no-usage-call"} + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-mcp", created=1741037890, model=model, choices=choices) + for choices in ( + [StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + [StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + _add_mcp_metadata_to_response(wrapper, openai_tools=None, tool_calls=tool_calls) + + _ = [chunk for chunk in wrapper] if sync_mode else [chunk async for chunk in wrapper] + + assert await asyncio.to_thread(recorder.logged.wait, 5) + assert len(recorder.responses) == 1 + logged: Final = recorder.responses[0] + assert isinstance(logged, ModelResponse) + assert logged.choices[0].message.provider_specific_fields == {"mcp_tool_calls": tool_calls} + assert (logged.usage.prompt_tokens, logged.usage.completion_tokens) == ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("ends_with_finish_chunk", [True, False]) +@pytest.mark.parametrize("reported_tokens", [(30, 12), (0, 0)]) +@pytest.mark.asyncio +async def test_stream_with_provider_usage_hands_that_usage_to_the_last_chunk( + logging_obj: Logging, sync_mode: bool, ends_with_finish_chunk: bool, reported_tokens: tuple[int, int] +) -> None: + model: Final = "gpt-4o" + reported: Final = Usage( + prompt_tokens=reported_tokens[0], + completion_tokens=reported_tokens[1], + total_tokens=sum(reported_tokens), + ) + content_chunk: Final = ModelResponseStream( + id="chatcmpl-with-usage", + created=1741037890, + model=model, + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content="Hello there"), + finish_reason=None if ends_with_finish_chunk else "stop", + ) + ], + **({} if ends_with_finish_chunk else {"usage": reported}), + ) + finish_chunk: Final = ModelResponseStream( + id="chatcmpl-with-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + usage=reported, + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[content_chunk, finish_chunk] if ends_with_finish_chunk else [content_chunk] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + + yielded: Final = ( + [(chunk, chunk._hidden_params.get("usage")) for chunk in wrapper] + if sync_mode + else [(chunk, chunk._hidden_params.get("usage")) async for chunk in wrapper] + ) + + last_chunk, usage_when_yielded = yielded[-1] + assert usage_when_yielded is not None + assert (usage_when_yielded.prompt_tokens, usage_when_yielded.completion_tokens) == reported_tokens + usage_after_drain: Final = last_chunk._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == reported_tokens + + @pytest.mark.asyncio async def test_custom_stream_wrapper_aclose(): """Test that aclose() delegates to the underlying completion_stream's aclose()""" diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 679e1bfd85a..f837dc3fb5f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.bug_report import ( build_bug_report, ) from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic -from litellm.types.utils import CachingDetails +from litellm.types.utils import CachingDetails, Delta, ModelResponseStream, StreamingChoices from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -76,6 +76,8 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( ) from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.utils import ModelResponseListIterator from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment @@ -8480,9 +8482,11 @@ class _RecordingSuccessLogger(CustomLogger): def __init__(self): super().__init__() self.success_events = [] + self.logged: Final = asyncio.Event() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): self.success_events.append({"kwargs": kwargs, "response_obj": response_obj}) + self.logged.set() class TestStreamingClientDisconnectBilling: @@ -8593,6 +8597,60 @@ class TestStreamingClientDisconnectBilling: assert getattr(partial_response, "service_tier") == "priority" assert partial_response.usage.total_tokens > 0 + @pytest.mark.asyncio + async def test_disconnect_after_finish_chunk_bills_the_estimate_when_the_provider_sent_no_usage(self): + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = LiteLLMLoggingObj( + model=model, + messages=messages, + stream=True, + call_type="acompletion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="no-usage-disconnect-call", + function_id="no-usage-disconnect-fn", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": "no-usage-disconnect-call"} + ) + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-no-usage", created=1741037890, model=model, choices=choices) + for choices in ( + [StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + [StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + stream_iter: Final = response.__aiter__() + first_chunk: Final = await stream_iter.__anext__() + finish_chunk: Final = await stream_iter.__anext__() + assert first_chunk.choices[0].finish_reason is None + assert finish_chunk.choices[0].finish_reason == "stop" + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": logging_obj}, + response, + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + usage: Final = recorder.success_events[0]["response_obj"].usage + assert (usage.prompt_tokens, usage.completion_tokens) == ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + @pytest.mark.asyncio async def test_completed_stream_does_not_double_bill_on_late_disconnect(self): recorder = _RecordingSuccessLogger() From 0ca29c7d8485bf1d2bc14c7ed88f7905e6bed890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Antunes?= <107940399+fabioAntGit@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:01:40 +0100 Subject: [PATCH 083/124] feat(harness): add pi harness for litellm.agent() (#45467) * feat(harness): add pi harness for litellm.agent() Adds Harness.PI, which drives the pi coding agent CLI (`pi --mode json`) once per turn against the per-session model endpoint. The endpoint is declared as a `litellm` openai-completions provider in the models.json of a LiteLLM-owned PI_CODING_AGENT_DIR, so the user's own pi settings, auth, extensions and MCP servers are never read, and the token travels only through an env var referenced as `$LITELLM_HARNESS_TOKEN` pi exits 0 after a failed provider call, so a turn fails when the last assistant message ends with stopReason error/aborted. Permission modes map to a `--tools` allowlist, disable_tools to `--exclude-tools`, and sessions resume with `--session` from a persisted session dir. build_instructions and turn_prompt move from the OpenCode config into the shared harness utils so both configs use them Unit tests run on JSONL recorded from pi 1.1.0 * test(harness): align pi harness tests with the other CLI harness suites Mirror the OpenCode suite's layout, fakes and helpers, and add the cases every CLI harness covers: turn before start, early close kills the process, persist failure, endpoint model fallback, long JSONL lines, skills without SKILL.md and a missing endpoint. Add an endpoint_requests.jsonl fixture recording what pi sends to the session endpoint, and re-record the turn fixtures in one run so their ids agree * fix(harness): parametrize harness e2e only over harnesses with a model group harness_params() iterated every Harness member, so Harness.TOOL_LOOP, which has no entry in BINARIES or DEFAULT_MODEL_GROUPS, raised KeyError and the whole suite failed to collect * feat(harness): accept pi config and MCP servers through PiOptions.config PiOptions gains `config`, like the other harness options. `mcpServers` is written to the private agent dir's mcp.json and every other key to its settings.json. Keys that would reroute model calls, bypass permissions=, or load code as the host user (defaultProvider, defaultModel, defaultTools, defaultProjectTrust, extensions, packages, ...) raise OptionsMismatch A plain-name `--tools` allowlist drops MCP tools in pi 1.1.0, so when servers are configured the allowlist also gets `mcp__*` plus the tool each exposure is reached through (`codemode` by default, `tool_search` for `deferred`). disable_tools still excludes individual MCP tools * refactor(harness): give harness options immutable mapping defaults `field(default_factory=dict)` left every config/env default typed as `dict[Unknown, Unknown]`, which basedpyright reports as reportUnknownVariableType under strict mode: 9 errors across the five options classes, two of them added by PiOptions. A shared module-level `MappingProxyType({})` default is accepted by frozen dataclasses, matches the no-mutation rule the rest of the harness follows, and drops the file to zero errors so the rule shrinks against the merge base instead of growing. * fix(harness): reject managed env keys and a missing model in the pi harness PiOptions.env could name PI_CODING_AGENT_DIR or LITELLM_HARNESS_TOKEN and the value was dropped in silence, because transform_session_setup merges the managed keys last. The ordering stays as defense in depth, but validate_environment now rejects those keys up front the way the Claude Code config does, so a bad option is an error rather than a setting that appears to apply and does not. transform_turn_request also stringified the model it resolved, so a call with neither model= nor an endpoint model would have put the literal "None" on pi's --model. Both call sites now go through session_model(), which raises the same ValueError transform_session_setup already raised. * test(harness): escape the regex dot in the pi skill manifest match * fix(harness): keep harness options importable on Python 3.11 Python 3.11 rejects an unhashable dataclass default, and MappingProxyType only became hashable in 3.12, so the shared MappingProxyType({}) defaults made importing litellm.harness.options raise ValueError and broke every harness on 3.11. The defaults now come from typed default factories that still return read-only mappings Co-Authored-By: Claude Opus 5.5 * test(harness): address review feedback on the pi harness tests Drop the endpoint-request fixture test, which only read a recorded file and could not fail when PiHarnessConfig changed, along with its now unused fixture. Remove the section-divider comments and a comment that restated MANAGED_ENV_KEYS, and type the test helpers and fakes Co-Authored-By: Claude Opus 5.5 * test(harness): cover the last uncovered pi harness lines Assert that a tool_execution_end event without a result yields an empty ToolResult output, and that a PiOptions.config value JSON cannot encode raises TypeError. The Python 3.11 default check now asserts on the list of unhashable defaults instead of relying on hash() raising, which the test-quality gate flagged as an assertion-free test * test(harness): annotate harness option tests and stop rebinding helper params Mark the option test locals Final and check read-only defaults through the MutableMapping interface instead of writing into them. In the pi suite, type the parametrized test arguments, and have started() and parse_all() bind new names instead of reassigning their sandbox and state parameters * test(harness): test harness option defaults by constructing the options Replace the field-introspection check with one that builds each options class twice and asserts every mapping default is empty, read-only and not shared between instances. Sharing one MappingProxyType across instances is what broke class creation on Python 3.11, and that sharing is observable on every Python version, so the test fails on the old code without inspecting dataclass declarations * test(harness): stop mutating test helper inputs and assert whole values make_ctx took model and endpoint out of **kwargs with pop, which mutated its own parameter; they are explicit parameters now, with a frozen default endpoint. assistant_end builds its message in one expression instead of adding errorMessage afterwards. Two docstrings that explained an assertion became that assertion's message, and the options test compares whole values so a failure names the offending fields * ci(unit): run the pi harness unit tests in the llms-h-to-z shard main now lists every tests/unit/llms provider directory per shard, so the new tests/unit/llms/pi directory was claimed by no shard and assert_ci_coverage failed on both the shard and the test-file checks * test(harness): mark pi test locals Final and freeze the test output model Every single-assignment local and module constant in the pi suite is now Final. The name-mapping loop became one comprehension compared as a whole list, test_start_persists_sessions_dir takes the sandbox by index instead of binding _ twice, and the Answer output model is frozen like every other pydantic model * fix(harness): send PiOptions.thinking to the gateway as reasoning_effort pi clamps --thinking to "off" for a model that does not declare reasoning, and the generated models.json only declared {"id": model}, so every thinking level was dropped and the gateway never saw reasoning_effort. xhigh and max are also only offered when thinkingLevelMap names them. The model now declares reasoning plus that map whenever a level other than "off" is requested, and nothing when thinking is unset, so pi does not send its default effort on every call The pi tests also drop **kwargs from make_ctx and started in favour of explicit parameters, and annotate their helpers and the fake sandbox with read-only types; the fake records calls by rebinding tuples instead of appending to lists * docs(harness): list pi among the harnesses in the module and agent() docstrings * fix(harness): keep pi calls provider-agnostic and harness option defaults copyable Declare supportsStore false so pi stops sending store, take contextWindow and maxTokens from the model map, treat a length stop or a silent exit as an error, and give every options class plain dict defaults again so deepcopy, pickle and asdict work * fix(harness): look up pi gateway groups as litellm_proxy and ask pi for cache_control marks A gateway session's model is the bare group name, so the model map lookup now passes custom_llm_provider=litellm_proxy, the way the SDK prices it. That keeps an unmapped group from printing get_llm_provider's provider list banner into the caller's stdout. pi only adds Anthropic cache_control marks for OpenRouter Anthropic models, so every Claude call through the harness missed the prompt cache. Models the map says support prompt caching now declare cacheControlFormat anthropic, which LiteLLM translates for each provider. * fix(harness): cap pi's maxTokens at its own default and fork sessions saved in another directory pi sends maxTokens as max_completion_tokens on every call, and the gateway reserves that many tokens against the key's TPM limit, so passing the model's full output cap (128000 for claude-haiku-5-5) made keys with lower limits answer 429. maxTokens is now the lower of the model's cap and pi's own 16384 default. agent_resume() into another sandbox directory stopped at pi's interactive fork prompt. The native session id now carries the directory pi saved the session in, and a resume from a different directory passes --fork, so the conversation continues in the new sandbox. * fix(harness): leave pi room for the thinking budget and price cached prompt tokens pi's maxTokens now adds LiteLLM's thinking budget for the requested level to pi's 16384 default, within the model's output cap, so Anthropic budget thinking at max no longer answers 400. Harness turns now pass cache read and write tokens to cost_per_token, so cached prompts are priced at their cache rates on every harness. * test(harness): cover cached prompt cost on the gateway and non-stream routes * test(harness): mark the cost test constants Final * test(harness): type the cached cost test helpers without Any * fix(harness): price OpenAI cache write tokens in harness cost --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/ci-coverage-allowlist.yml | 2 +- .github/workflows/test-unit.yml | 1 + README.md | 6 +- litellm/__init__.py | 1 + litellm/harness/__init__.py | 4 +- litellm/harness/endpoint.py | 119 ++- litellm/harness/options.py | 29 +- litellm/harness/sync.py | 2 +- litellm/harness/types.py | 1 + litellm/llms/base_llm/harness/utils.py | 16 + .../llms/opencode/harness/transformation.py | 18 +- litellm/llms/pi/__init__.py | 0 litellm/llms/pi/harness/__init__.py | 0 litellm/llms/pi/harness/transformation.py | 569 ++++++++++ litellm/utils.py | 4 + .../check_provider_folders_documented.py | 1 + tests/harness_e2e/conftest.py | 6 +- tests/unit/harness/test_endpoint.py | 190 +++- tests/unit/harness/test_init.py | 1 + tests/unit/harness/test_options.py | 24 + tests/unit/harness/test_types.py | 1 + tests/unit/llms/pi/__init__.py | 0 tests/unit/llms/pi/harness/__init__.py | 0 .../unit/llms/pi/harness/fixtures/__init__.py | 0 .../llms/pi/harness/fixtures/api_error.jsonl | 12 + .../llms/pi/harness/fixtures/mcp_tool.jsonl | 27 + .../fixtures/readonly_denied_bash.jsonl | 29 + .../harness/fixtures/structured_output.jsonl | 19 + .../harness/fixtures/turn1_write_read.jsonl | 42 + .../pi/harness/fixtures/turn2_resume.jsonl | 15 + .../llms/pi/harness/test_transformation.py | 991 ++++++++++++++++++ 31 files changed, 2056 insertions(+), 74 deletions(-) create mode 100644 litellm/llms/pi/__init__.py create mode 100644 litellm/llms/pi/harness/__init__.py create mode 100644 litellm/llms/pi/harness/transformation.py create mode 100644 tests/unit/harness/test_options.py create mode 100644 tests/unit/llms/pi/__init__.py create mode 100644 tests/unit/llms/pi/harness/__init__.py create mode 100644 tests/unit/llms/pi/harness/fixtures/__init__.py create mode 100644 tests/unit/llms/pi/harness/fixtures/api_error.jsonl create mode 100644 tests/unit/llms/pi/harness/fixtures/mcp_tool.jsonl create mode 100644 tests/unit/llms/pi/harness/fixtures/readonly_denied_bash.jsonl create mode 100644 tests/unit/llms/pi/harness/fixtures/structured_output.jsonl create mode 100644 tests/unit/llms/pi/harness/fixtures/turn1_write_read.jsonl create mode 100644 tests/unit/llms/pi/harness/fixtures/turn2_resume.jsonl create mode 100644 tests/unit/llms/pi/harness/test_transformation.py diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index db2f652b877..5dffa78776d 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -5,7 +5,7 @@ description: >- test_paths: - reason: >- - litellm.agent() end-to-end suite. It drives the real claude, codex and opencode CLIs and + litellm.agent() end-to-end suite. It drives the real claude, codex, opencode and pi CLIs and deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus LITELLM_PROXY_API_BASE / LITELLM_PROXY_API_KEY, and skips without them. Run manually before changing litellm/harness; the mocked coverage runs in tests/unit/harness and diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 1514c58d022..eefda48b682 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -322,6 +322,7 @@ jobs: tests/unit/llms/pass_through tests/unit/llms/perplexity tests/unit/llms/pg_vector + tests/unit/llms/pi tests/unit/llms/publicai tests/unit/llms/ragflow tests/unit/llms/recraft diff --git a/README.md b/README.md index 77f7e3baa4d..5fd84b529dc 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse

-Agents - Run Claude Code, Codex, OpenCode or Deep Agents on any model (Python SDK) +Agents - Run Claude Code, Codex, OpenCode, pi or Deep Agents on any model (Python SDK) ### Python SDK - Agents @@ -296,7 +296,7 @@ import litellm from litellm import Harness, sandbox result = litellm.agent( - Harness.CLAUDE_CODE, # or Harness.CODEX, Harness.OPENCODE, Harness.DEEPAGENTS + Harness.CLAUDE_CODE, # or Harness.CODEX, Harness.OPENCODE, Harness.PI, Harness.DEEPAGENTS "Find why tests/test_router.py is flaky and fix it.", sandbox=sandbox.local("./repo"), model="litellm_proxy/claude-sonnet-4-5", # a model group on your AI Gateway @@ -305,7 +305,7 @@ result = litellm.agent( print(result.text, result.cost, [f.path for f in result.files]) ``` -Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call the agent makes goes through your AI Gateway, tagged `harness,claude_code`. Drop the `litellm_proxy/` prefix to call a provider directly. Install `starlette uvicorn` plus the agent's CLI (`claude`, `codex` or `opencode`), or `deepagents langchain-litellm` for Deep Agents. +Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call the agent makes goes through your AI Gateway, tagged `harness,claude_code`. Drop the `litellm_proxy/` prefix to call a provider directly. Install `starlette uvicorn` plus the agent's CLI (`claude`, `codex`, `opencode` or `pi`), or `deepagents langchain-litellm` for Deep Agents. [**Docs: Agent Harnesses**](https://docs.litellm.ai/docs/harness) diff --git a/litellm/__init__.py b/litellm/__init__.py index 1f9b6c5264c..0fe3d091c2a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2385,6 +2385,7 @@ _AGENT_EXPORTS: Final = frozenset( "ClaudeCodeOptions", "CodexOptions", "OpenCodeOptions", + "PiOptions", "DeepAgentsOptions", "ToolLoopOptions", } diff --git a/litellm/harness/__init__.py b/litellm/harness/__init__.py index 213591c5949..d6693b0e681 100644 --- a/litellm/harness/__init__.py +++ b/litellm/harness/__init__.py @@ -1,4 +1,4 @@ -"""Agent harnesses: run Claude Code, Codex, OpenCode, Deep Agents or Tool Loop on any LiteLLM model. +"""Agent harnesses: run Claude Code, Codex, OpenCode, pi, Deep Agents or Tool Loop on any LiteLLM model. The entrypoints live on the top-level package: @@ -30,6 +30,7 @@ from litellm.harness.options import ( CodexOptions, DeepAgentsOptions, OpenCodeOptions, + PiOptions, ToolLoopOptions, ) from litellm.harness.runtime import ( @@ -78,6 +79,7 @@ __all__ = ( "OpenCodeOptions", "OptionsMismatch", "OutputInvalid", + "PiOptions", "Reasoning", "Result", "SandboxError", diff --git a/litellm/harness/endpoint.py b/litellm/harness/endpoint.py index 1f500fafba2..0abf7c11280 100644 --- a/litellm/harness/endpoint.py +++ b/litellm/harness/endpoint.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol import httpx import openai -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm.constants import ( @@ -172,16 +172,57 @@ def usage_from_mapping(usage: object) -> tuple[int, int]: return _as_int(input_tokens), _as_int(output_tokens) -def usage_from_body(body: object) -> tuple[int, int]: - """Usage from a non-streaming JSON response body.""" - if not isinstance(body, Mapping): - return 0, 0 - if isinstance(body.get("usage"), Mapping): - return usage_from_mapping(body["usage"]) - response = body.get("response") - if isinstance(response, Mapping): - return usage_from_mapping(response.get("usage")) - return 0, 0 +@dataclass(frozen=True, slots=True) +class PromptCache: + """Prompt tokens read from or written to the provider's cache. Anthropic's usage counts them outside input_tokens, + while OpenAI's prompt_tokens and the Responses API's input_tokens already include them.""" + + read_tokens: int = 0 + creation_tokens: int = 0 + outside_input: bool = False + + def prompt_tokens(self, input_tokens: int) -> int: + if self.outside_input: + return input_tokens + self.read_tokens + self.creation_tokens + return input_tokens + + +NO_PROMPT_CACHE: Final = PromptCache() + + +def json_object(value: object) -> Mapping[str, object]: + try: + return _JSON_OBJECT_ADAPTER.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def prompt_cache_from_mapping(usage: object) -> PromptCache: + """Cache token counts from a usage dict using OpenAI, Anthropic, or Responses field names.""" + fields: Final = json_object(usage) + details: Final = json_object(fields.get("prompt_tokens_details", fields.get("input_tokens_details"))) + return PromptCache( + read_tokens=_as_int(fields.get("cache_read_input_tokens") or details.get("cached_tokens")), + creation_tokens=_as_int( + fields.get("cache_creation_input_tokens") + or details.get("cache_write_tokens") + or details.get("cache_creation_tokens") + ), + outside_input="prompt_tokens" not in fields and "input_tokens_details" not in fields, + ) + + +def usage_mapping_of_body(body: object) -> Mapping[str, object]: + fields: Final = json_object(body) + usage: Final = json_object(fields.get("usage")) + return usage if usage else json_object(json_object(fields.get("response")).get("usage")) + + +def usage_mapping_of_json(raw: bytes) -> Mapping[str, object]: + try: + return usage_mapping_of_body(_JSON_VALUE_ADAPTER.validate_python(json.loads(raw))) + except ValueError: + return MappingProxyType({}) class SSEUsageParser: @@ -190,6 +231,7 @@ class SSEUsageParser: def __init__(self) -> None: self.input_tokens = 0 self.output_tokens = 0 + self.prompt_cache = NO_PROMPT_CACHE self._buffer = b"" def feed(self, chunk: bytes) -> None: @@ -226,21 +268,23 @@ class SSEUsageParser: elif event_type == "response.completed": self._absorb_response_completed(event) elif isinstance(event.get("usage"), Mapping): - self._set(*usage_from_mapping(event["usage"])) + self._absorb_usage(event["usage"]) def _absorb_message_start(self, event: Mapping[str, object]) -> None: - message = event.get("message") - if isinstance(message, Mapping): - self._set(*usage_from_mapping(message.get("usage"))) + self._absorb_usage(json_object(event.get("message")).get("usage")) def _absorb_message_delta(self, event: Mapping[str, object]) -> None: # message_delta output_tokens is cumulative for the whole message. - self._set(*usage_from_mapping(event.get("usage"))) + self._absorb_usage(event.get("usage")) def _absorb_response_completed(self, event: Mapping[str, object]) -> None: - response = event.get("response") - if isinstance(response, Mapping): - self._set(*usage_from_mapping(response.get("usage"))) + self._absorb_usage(json_object(event.get("response")).get("usage")) + + def _absorb_usage(self, usage: object) -> None: + self._set(*usage_from_mapping(usage)) + prompt_cache: Final = prompt_cache_from_mapping(usage) + if prompt_cache.read_tokens or prompt_cache.creation_tokens: + self.prompt_cache = prompt_cache def _set(self, input_tokens: int, output_tokens: int) -> None: if input_tokens: @@ -249,13 +293,19 @@ class SSEUsageParser: self.output_tokens = output_tokens -def compute_cost(model: str | None, input_tokens: int, output_tokens: int) -> float: - """Cost from LiteLLM's price map. Never raises; unknown models cost 0.0.""" +def compute_cost( + model: str | None, input_tokens: int, output_tokens: int, prompt_cache: PromptCache = NO_PROMPT_CACHE +) -> float: + """Cost from LiteLLM's price map, cached prompt tokens at their cache prices. Never raises; unknown models cost 0.0.""" if not model or not (input_tokens or output_tokens): return 0.0 try: prompt_cost, completion_cost = litellm.cost_per_token( - model=model, prompt_tokens=input_tokens, completion_tokens=output_tokens + model=model, + prompt_tokens=prompt_cache.prompt_tokens(input_tokens), + completion_tokens=output_tokens, + cache_read_input_tokens=prompt_cache.read_tokens, + cache_creation_input_tokens=prompt_cache.creation_tokens, ) return float(prompt_cost) + float(completion_cost) except Exception: # accounting must never break a call; the price-map lookup raises bare Exception @@ -565,9 +615,10 @@ class ModelEndpoint: input_tokens: int, output_tokens: int, cost: float | None, + prompt_cache: PromptCache, ) -> None: if cost is None: - cost = compute_cost(model, input_tokens, output_tokens) + cost = compute_cost(model, input_tokens, output_tokens, prompt_cache) self.usage.add(input_tokens, output_tokens, cost) async def _forward(self, request: Request, route: str, body: Mapping[str, object]) -> Response: @@ -617,14 +668,12 @@ class ModelEndpoint: ) -> None: if is_sse: parser.close() - tokens = (parser.input_tokens, parser.output_tokens) - else: - try: - body: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(collected)) - tokens = usage_from_body(body) - except ValueError: - tokens = (0, 0) - self._record(model, tokens[0], tokens[1], header_cost(upstream.headers)) + self._record( + model, parser.input_tokens, parser.output_tokens, header_cost(upstream.headers), parser.prompt_cache + ) + return + usage: Final = usage_mapping_of_json(collected) + self._record(model, *usage_from_mapping(usage), header_cost(upstream.headers), prompt_cache_from_mapping(usage)) def _sdk_kwargs( self, body: Mapping[str, object] @@ -667,9 +716,9 @@ class ModelEndpoint: return self._responses.StreamingResponse( self._sdk_stream(route, response, model), media_type=SSE_MEDIA_TYPE ) - data = to_jsonable(response) - input_tokens, output_tokens = usage_from_body(data) - self._record(model, input_tokens, output_tokens, hidden_cost(response)) + data: Final = to_jsonable(response) + usage: Final = usage_mapping_of_body(data) + self._record(model, *usage_from_mapping(usage), hidden_cost(response), prompt_cache_from_mapping(usage)) return self._json(data) async def _sdk_stream(self, route: str, iterator: AsyncIterable[object], model: str | None) -> AsyncIterator[bytes]: @@ -688,4 +737,4 @@ class ModelEndpoint: yield f"event: error\ndata: {json.dumps(error_body(e, message))}\n\n".encode() finally: parser.close() - self._record(model, parser.input_tokens, parser.output_tokens, None) + self._record(model, parser.input_tokens, parser.output_tokens, None, parser.prompt_cache) diff --git a/litellm/harness/options.py b/litellm/harness/options.py index 29536a3e093..27a5b031e04 100644 --- a/litellm/harness/options.py +++ b/litellm/harness/options.py @@ -4,28 +4,37 @@ from __future__ import annotations from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from typing import Literal +from typing import Literal, TypeAlias + +PiThinkingLevel: TypeAlias = Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] @dataclass(frozen=True) class ClaudeCodeOptions: - config: Mapping[str, object] = field(default_factory=dict) - env: Mapping[str, str] = field(default_factory=dict) + config: Mapping[str, object] = field(default_factory=dict[str, object]) + env: Mapping[str, str] = field(default_factory=dict[str, str]) @dataclass(frozen=True) class CodexOptions: reasoning_effort: Literal["low", "medium", "high", "xhigh"] | None = None web_search: bool = False - config: Mapping[str, object] = field(default_factory=dict) - env: Mapping[str, str] = field(default_factory=dict) + config: Mapping[str, object] = field(default_factory=dict[str, object]) + env: Mapping[str, str] = field(default_factory=dict[str, str]) @dataclass(frozen=True) class OpenCodeOptions: agent: str = "build" - config: Mapping[str, object] = field(default_factory=dict) - env: Mapping[str, str] = field(default_factory=dict) + config: Mapping[str, object] = field(default_factory=dict[str, object]) + env: Mapping[str, str] = field(default_factory=dict[str, str]) + + +@dataclass(frozen=True) +class PiOptions: + thinking: PiThinkingLevel | None = None + config: Mapping[str, object] = field(default_factory=dict[str, object]) + env: Mapping[str, str] = field(default_factory=dict[str, str]) @dataclass(frozen=True) @@ -36,7 +45,9 @@ class DeepAgentsOptions: @dataclass(frozen=True) class ToolLoopOptions: - completion_kwargs: Mapping[str, object] = field(default_factory=dict) + completion_kwargs: Mapping[str, object] = field(default_factory=dict[str, object]) -HarnessOptions = ClaudeCodeOptions | CodexOptions | OpenCodeOptions | DeepAgentsOptions | ToolLoopOptions +HarnessOptions: TypeAlias = ( + ClaudeCodeOptions | CodexOptions | OpenCodeOptions | PiOptions | DeepAgentsOptions | ToolLoopOptions +) diff --git a/litellm/harness/sync.py b/litellm/harness/sync.py index 18336a3d06e..70ef3b75095 100644 --- a/litellm/harness/sync.py +++ b/litellm/harness/sync.py @@ -411,7 +411,7 @@ def agent( options: HarnessOptions | None = None, install: bool = False, ) -> Result | EventStream: - """Run an agent harness (Claude Code, Codex, OpenCode, Deep Agents, Tool Loop) on one prompt. + """Run an agent harness (Claude Code, Codex, OpenCode, pi, Deep Agents, Tool Loop) on one prompt. Returns a Result. With stream=True it returns an iterator of events instead. Prefix the model with `litellm_proxy/` to route every model call through your diff --git a/litellm/harness/types.py b/litellm/harness/types.py index 6b0118f4349..d853a33c740 100644 --- a/litellm/harness/types.py +++ b/litellm/harness/types.py @@ -26,6 +26,7 @@ class Harness(Enum): CLAUDE_CODE = "claude_code" CODEX = "codex" OPENCODE = "opencode" + PI = "pi" DEEPAGENTS = "deepagents" TOOL_LOOP = "tool_loop" diff --git a/litellm/llms/base_llm/harness/utils.py b/litellm/llms/base_llm/harness/utils.py index 8dcf784c90f..7080eb01ac1 100644 --- a/litellm/llms/base_llm/harness/utils.py +++ b/litellm/llms/base_llm/harness/utils.py @@ -70,6 +70,22 @@ def structured_output_instruction(schema: Mapping[str, object]) -> str: ) +def build_instructions(ctx: SessionContext) -> str | None: + """System instructions plus the structured-output instruction, for runtimes that take one prompt file.""" + schema_part: Final = ( + structured_output_instruction(ctx.output.model_json_schema()) if ctx.output is not None else None + ) + sections: Final = tuple(section for section in (ctx.instructions, schema_part) if section) + return "\n\n".join(sections) if sections else None + + +def turn_prompt(ctx: SessionContext, prompt: str) -> str: + """Repeat the schema instruction in the user turn; system instructions alone are too weak.""" + if ctx.output is None: + return prompt + return f"{prompt}\n\n{structured_output_instruction(ctx.output.model_json_schema())}" + + def strict_json_schema(schema: JSONValue, depth: int = 0) -> JSONValue: """Make a JSON schema acceptable to OpenAI strict structured outputs. diff --git a/litellm/llms/opencode/harness/transformation.py b/litellm/llms/opencode/harness/transformation.py index 787bc99a2d7..144687233e8 100644 --- a/litellm/llms/opencode/harness/transformation.py +++ b/litellm/llms/opencode/harness/transformation.py @@ -40,11 +40,12 @@ from litellm.llms.base_llm.harness.transformation import ( event_list, ) from litellm.llms.base_llm.harness.utils import ( + build_instructions, last_json_object, native_tool_names, normalize_tool_name, stderr_tail_text, - structured_output_instruction, + turn_prompt, ) if TYPE_CHECKING: @@ -274,21 +275,6 @@ def build_opencode_config( return {**user, **managed, **dict(present)} # mutable-ok: opencode config JSON -def build_instructions(ctx: SessionContext) -> str | None: - schema_part: Final = ( - structured_output_instruction(ctx.output.model_json_schema()) if ctx.output is not None else None - ) - sections: Final = tuple(section for section in (ctx.instructions, schema_part) if section) - return "\n\n".join(sections) if sections else None - - -def turn_prompt(ctx: SessionContext, prompt: str) -> str: - """Repeat the schema instruction in the user turn; system instructions alone are too weak.""" - if ctx.output is None: - return prompt - return f"{prompt}\n\n{structured_output_instruction(ctx.output.model_json_schema())}" - - class OpenCodeHarnessConfig(BaseCLIHarnessConfig): harness = Harness.OPENCODE options_type = OpenCodeOptions diff --git a/litellm/llms/pi/__init__.py b/litellm/llms/pi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/pi/harness/__init__.py b/litellm/llms/pi/harness/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/pi/harness/transformation.py b/litellm/llms/pi/harness/transformation.py new file mode 100644 index 00000000000..9f660b544ae --- /dev/null +++ b/litellm/llms/pi/harness/transformation.py @@ -0,0 +1,569 @@ +""" +pi harness config: `pi --mode json` (JSONL session events), once per turn. + +Every model call goes to one custom provider (`litellm`, api=openai-completions) declared in +the models.json of a LiteLLM-owned PI_CODING_AGENT_DIR, so the user's own pi settings, auth, +extensions, MCP servers and skills are never read. PiOptions.config is pi's own config: +`mcpServers` goes to that dir's mcp.json and every other key to its settings.json, minus the +keys LiteLLM manages. The provider's apiKey is the `$`-reference +pi resolves from LITELLM_HARNESS_TOKEN, so the token is never in argv or on disk. pi exits 0 +after a failed provider call, so failure is read from the last assistant message instead. +Verified against @earendil-works/pi-coding-agent 1.1.0 and 1.3.0. +""" + +from __future__ import annotations + +import itertools +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm.constants import ( + ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, + DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, +) +from litellm.exceptions import ModelNotMappedError +from litellm.harness.errors import CapabilityUnsupported, HarnessError, OptionsMismatch +from litellm.harness.options import PiOptions, PiThinkingLevel +from litellm.harness.types import ( + Capabilities, + Event, + Harness, + PermissionMode, + Reasoning, + Text, + ToolCall, + ToolResult, +) +from litellm.llms.base_llm.harness.transformation import ( + BaseCLIHarnessConfig, + HarnessSessionSetup, + HarnessTurnError, + HarnessTurnRequest, + HarnessTurnResponse, + event_list, +) +from litellm.llms.base_llm.harness.utils import ( + build_instructions, + last_json_object, + native_tool_names, + normalize_tool_name, + stderr_tail_text, + turn_prompt, +) +from litellm.types.utils import ModelInfo +from litellm.utils import get_model_info + +if TYPE_CHECKING: + from litellm.harness.context import SessionContext + +PI_BINARY: Final = "pi" +PI_PROVIDER_ID: Final = "litellm" +PI_TOKEN_ENV: Final = "LITELLM_HARNESS_TOKEN" +PI_CONFIG_DIR_ENV: Final = "PI_CODING_AGENT_DIR" +AGENT_DIRNAME: Final = "agent" +SESSIONS_DIRNAME: Final = "sessions" +SKILLS_DIRNAME: Final = "skills" +INSTRUCTIONS_FILENAME: Final = "instructions.md" +MODELS_FILENAME: Final = f"{AGENT_DIRNAME}/models.json" +SETTINGS_FILENAME: Final = f"{AGENT_DIRNAME}/settings.json" +MCP_FILENAME: Final = f"{AGENT_DIRNAME}/mcp.json" +MCP_CONFIG_KEY: Final = "mcpServers" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +MCP_TOOL_PATTERN: Final = "mcp__*" +EXPOSURE_TOOLS: Final = (("codemode", "codemode"), ("codemode-deferred", "codemode"), ("deferred", "tool_search")) + +# settings.json keys that would reroute model calls, bypass permissions=, or load code as the host user. +MANAGED_CONFIG_KEYS: Final = frozenset( + { + "defaultProvider", + "defaultModel", + "enabledModels", + "defaultThinkingLevel", + "defaultTools", + "defaultProjectTrust", + "sessionDir", + "httpProxy", + "packages", + "extensions", + "skills", + } +) + +MANAGED_ENV_KEYS: Final = frozenset({PI_CONFIG_DIR_ENV, PI_TOKEN_ENV}) + +PI_ISOLATION_ENV: Final[Mapping[str, str]] = MappingProxyType( + { + "PI_OFFLINE": "1", + "PI_SKIP_VERSION_CHECK": "1", + "PI_TELEMETRY": "0", + } +) + +PERMISSION_TOOLS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + "read-only": ("read", "grep", "find", "ls"), + "edit": ("read", "edit", "write", "grep", "find", "ls"), + "full": ("read", "bash", "edit", "write", "grep", "find", "ls"), + } +) + +NORMALIZED_TO_NATIVE: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + "read": ("read",), + "write": ("write",), + "edit": ("edit",), + "bash": ("bash", "powershell"), + "glob": ("find",), + "grep": ("grep",), + "ls": ("ls",), + } +) + +NATIVE_TO_NORMALIZED: Final[Mapping[str, str]] = MappingProxyType( + { + "read": "read", + "write": "write", + "edit": "edit", + "bash": "bash", + "powershell": "bash", + "find": "glob", + "grep": "grep", + "ls": "ls", + } +) + +PI_THINKING_LEVEL_MAP: Final[Mapping[str, str]] = MappingProxyType({"xhigh": "xhigh", "max": "max"}) + +# pi adds `store: false` to every call otherwise, which LiteLLM rejects for providers that have no `store`. +PI_PROVIDER_COMPAT: Final[Mapping[str, object]] = MappingProxyType({"supportsStore": False}) + +PI_CACHE_CONTROL_COMPAT: Final[Mapping[str, str]] = MappingProxyType({"cacheControlFormat": "anthropic"}) + +PI_DEFAULT_MAX_TOKENS: Final = 16384 + +LITELLM_THINKING_BUDGETS: Final[Mapping[str, int]] = MappingProxyType( + { + "minimal": max(DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS), + "low": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, + "medium": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + "high": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + "xhigh": DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, + "max": DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET, + } +) + +SESSION_CWD_SEPARATOR: Final = "@" + +PI_BUILTIN_TOOLS: Final = frozenset({*NATIVE_TO_NORMALIZED, "codemode", "tool_search"}) +FAILED_STOP_REASONS: Final[Mapping[str, str]] = MappingProxyType( + { + "error": "the model call failed", + "aborted": "aborted", + "length": "the reply hit the model's output token limit", + } +) + + +class _Frozen(BaseModel): + model_config = ConfigDict(frozen=True) + + +class _McpServer(_Frozen): + exposure: str = "codemode" + tool_exposure: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({}), alias="toolExposure") + + def exposures(self) -> frozenset[str]: + return frozenset({self.exposure, *self.tool_exposure.values()}) + + +_MCP_SERVERS: Final = TypeAdapter(Mapping[str, _McpServer]) + + +class _ContentBlock(_Frozen): + type: str + text: str = "" + + +class _Role(_Frozen): + role: str = "" + + +class _AssistantMessage(_Frozen): + content: tuple[_ContentBlock, ...] = () + stop_reason: str | None = Field(default=None, alias="stopReason") + error_message: str | None = Field(default=None, alias="errorMessage") + + +class _ToolOutput(_Frozen): + content: tuple[_ContentBlock, ...] = () + + +class _ToolStart(_Frozen): + tool_call_id: str = Field(alias="toolCallId") + tool_name: str = Field(alias="toolName") + args: Mapping[str, object] = Field(default_factory=lambda: MappingProxyType({})) + + +class _ToolEnd(_Frozen): + tool_call_id: str = Field(alias="toolCallId") + result: _ToolOutput | str | None = None + is_error: bool = Field(default=False, alias="isError") + + +class _MessageUpdate(_Frozen): + type: str + delta: str = "" + + +@dataclass +class PiStreamState: + """What the parser has learned from one `pi --mode json` run.""" + + session_id: str | None = None + session_cwd: str | None = None + final_text: str = "" + stop_reason: str | None = None + error: str | None = None + + def record_session(self, session_id: object, cwd: object) -> None: + if isinstance(session_id, str) and self.session_id is None: + self.session_id = session_id + self.session_cwd = cwd if isinstance(cwd, str) else None + + def record_message_end(self, message: object) -> None: + if _Role.model_validate(message).role != "assistant": + return + assistant: Final = _AssistantMessage.model_validate(message) + self.final_text = _text_of(assistant.content) + self.stop_reason = assistant.stop_reason + self.error = assistant.error_message + + def mark_aborted(self) -> None: + self.stop_reason = "aborted" + + +def _text_of(blocks: Sequence[_ContentBlock]) -> str: + return "".join(block.text for block in blocks if block.type == "text") + + +def _tool_output_text(result: _ToolOutput | str | None) -> str: + if result is None: + return "" + if isinstance(result, str): + return result + return _text_of(result.content) + + +def _message_update_events(update: object) -> Sequence[Event]: + try: + event: Final = _MessageUpdate.model_validate(update) + except ValidationError: + return event_list() + if not event.delta: + return event_list() + if event.type == "text_delta": + return event_list(Text(delta=event.delta)) + if event.type == "thinking_delta": + return event_list(Reasoning(delta=event.delta)) + return event_list() + + +def _tool_start_events(line: Mapping[str, object]) -> Sequence[Event]: + start: Final = _ToolStart.model_validate(line) + return event_list( + ToolCall( + id=start.tool_call_id, + name=normalize_tool_name(start.tool_name, NATIVE_TO_NORMALIZED), + native_name=start.tool_name, + input=start.args, + builtin=start.tool_name in PI_BUILTIN_TOOLS, + ) + ) + + +def _tool_end_events(line: Mapping[str, object]) -> Sequence[Event]: + end: Final = _ToolEnd.model_validate(line) + return event_list(ToolResult(id=end.tool_call_id, output=_tool_output_text(end.result), is_error=end.is_error)) + + +def tool_args( + permissions: PermissionMode, disable_tools: Sequence[str], mcp_tools: Sequence[str] = () +) -> tuple[str, ...]: + """`--tools` allowlist for a permission mode and MCP servers, plus `--exclude-tools` for disable_tools.""" + if permissions not in PERMISSION_TOOLS: + raise CapabilityUnsupported( + f"Harness.PI does not support permissions={permissions!r} (supported: {sorted(PERMISSION_TOOLS)})" + ) + denied: Final = native_tool_names(disable_tools, NORMALIZED_TO_NATIVE) + exclude: Final = ("--exclude-tools", ",".join(denied)) if denied else () + return ("--tools", ",".join((*PERMISSION_TOOLS[permissions], *mcp_tools)), *exclude) + + +def session_args(native_session_id: str | None, workdir: str) -> tuple[str, ...]: + """`--session` for a session pi saved in this workdir, `--fork` for one saved in another, since pi otherwise + stops at an interactive "Fork this session into current directory?" prompt the harness cannot answer.""" + if native_session_id is None: + return () + session_id, _, saved_cwd = native_session_id.partition(SESSION_CWD_SEPARATOR) + return ("--fork" if saved_cwd and saved_cwd != workdir else "--session", session_id) + + +def mcp_tool_entries(config: Mapping[str, object]) -> tuple[str, ...]: + """`--tools` entries that keep configured MCP servers reachable. + + A plain-name `--tools` list drops MCP tools in pi 1.1.0, so `mcp__*` is listed whenever servers + are configured, plus the tool each exposure is reached through: `codemode` (pi's default) or + `tool_search` for `deferred`. + """ + servers: Final = _MCP_SERVERS.validate_python(config.get(MCP_CONFIG_KEY) or {}) + if not servers: + return () + exposures: Final = frozenset(itertools.chain.from_iterable(server.exposures() for server in servers.values())) + reach: Final = tuple(tool for exposure, tool in EXPOSURE_TOOLS if exposure in exposures) + return (MCP_TOOL_PATTERN, *reach) + + +def session_model(ctx: SessionContext) -> str: + """The model pi is pointed at, from agent(model=) or the session endpoint.""" + model: Final = ctx.model or (ctx.endpoint.model if ctx.endpoint else None) + if not model: + raise ValueError("Harness.PI needs model= (a gateway model group or litellm model)") + return model + + +def mapped_model_info(model: str, through_gateway: bool) -> ModelInfo | None: + """LiteLLM's model map entry. A gateway model group is looked up as litellm_proxy, the way the SDK prices it, + which also keeps an unmapped group from printing get_llm_provider's provider-list banner on stdout.""" + try: + return get_model_info(model, custom_llm_provider="litellm_proxy" if through_gateway else None) + except ModelNotMappedError: + return None + + +def max_output_tokens(info: ModelInfo | None, thinking: PiThinkingLevel | None) -> int | None: + """pi's own 16384 default plus the thinking budget LiteLLM sends for the level, within the model's output cap. + pi sends maxTokens as max_completion_tokens on every call and the gateway reserves that many tokens against the + key's TPM limit, so the model's full output cap would starve keys with lower limits, while Anthropic budget + thinking needs max_tokens above the budget, which is how pi sizes its own Anthropic calls.""" + budget: Final = LITELLM_THINKING_BUDGETS.get(thinking, 0) if thinking else 0 + max_output: Final = info.get("max_output_tokens") if info is not None else None + if isinstance(max_output, int): + return min(max_output, PI_DEFAULT_MAX_TOKENS + budget) + return PI_DEFAULT_MAX_TOKENS + budget if budget else None + + +def model_limits(info: ModelInfo | None, thinking: PiThinkingLevel | None) -> Mapping[str, int]: + """contextWindow from LiteLLM's model map, since pi assumes 128000 otherwise, and maxTokens per max_output_tokens.""" + limits: Final = ( + ("contextWindow", info.get("max_input_tokens") if info is not None else None), + ("maxTokens", max_output_tokens(info, thinking)), + ) + return MappingProxyType({key: value for key, value in limits if isinstance(value, int)}) + + +def model_compat(info: ModelInfo | None) -> Mapping[str, object]: + """Anthropic cache_control marks, which LiteLLM translates per provider, for models that support prompt caching.""" + caches: Final = info is not None and info.get("supports_prompt_caching") is True + return MappingProxyType({"compat": PI_CACHE_CONTROL_COMPAT} if caches else {}) + + +def build_models_json(model: str, base_url: str, thinking: PiThinkingLevel | None, info: ModelInfo | None) -> str: + # pi clamps --thinking to "off" unless the model declares reasoning, and offers xhigh/max only when + # thinkingLevelMap names them. Declaring reasoning without a requested level would make pi send its + # default reasoning_effort on every call, so it is declared only when a level is asked for + reasoning: Final = ( + MappingProxyType({"reasoning": True, "thinkingLevelMap": PI_THINKING_LEVEL_MAP}) + if thinking is not None and thinking != "off" + else MappingProxyType({}) + ) + provider: Final = MappingProxyType( + { + "baseUrl": base_url, + "api": "openai-completions", + "apiKey": f"${PI_TOKEN_ENV}", + "compat": PI_PROVIDER_COMPAT, + "models": ( + MappingProxyType({"id": model, **model_limits(info, thinking), **model_compat(info), **reasoning}), + ), + } + ) + return json.dumps({"providers": {PI_PROVIDER_ID: provider}}, default=_plain_json) + + +def validate_user_config(config: Mapping[str, object]) -> None: + """Reject PiOptions.config keys LiteLLM manages, and a malformed mcpServers.""" + managed: Final = sorted(MANAGED_CONFIG_KEYS.intersection(config)) + if managed: + raise OptionsMismatch( + f"PiOptions.config may not set {', '.join(managed)}; LiteLLM manages it. Use the matching " + "agent() argument (model=, permissions=, disable_tools=, skills=) or PiOptions.thinking instead" + ) + try: + _MCP_SERVERS.validate_python(config.get(MCP_CONFIG_KEY) or {}) + except ValidationError as e: + raise OptionsMismatch( + f"PiOptions.config[{MCP_CONFIG_KEY!r}] must map each server name to its config: {e}" + ) from e + + +def _plain_json(value: object) -> object: + if isinstance(value, Mapping): + return _JSON_OBJECT.validate_python(value) + raise TypeError(f"{type(value).__name__} in PiOptions.config is not JSON serializable") + + +def config_files(config: Mapping[str, object]) -> tuple[tuple[str, bytes], ...]: + """settings.json for every key but mcpServers, and mcp.json for mcpServers; each only when non-empty.""" + settings: Final = MappingProxyType({key: value for key, value in config.items() if key != MCP_CONFIG_KEY}) + servers: Final = config.get(MCP_CONFIG_KEY) + settings_file: Final = ((SETTINGS_FILENAME, json.dumps(settings, default=_plain_json)),) if settings else () + mcp_file: Final = ((MCP_FILENAME, json.dumps({MCP_CONFIG_KEY: servers}, default=_plain_json)),) if servers else () + return tuple((path, text.encode("utf-8")) for path, text in (*settings_file, *mcp_file)) + + +class PiHarnessConfig(BaseCLIHarnessConfig[PiOptions, PiStreamState]): + harness = Harness.PI + options_type = PiOptions + capabilities = Capabilities( + structured_output=True, + tool_approval=False, + tool_filtering=True, + history=False, + custom_tools=False, + skills=True, + resume=True, + permission_modes=frozenset(PERMISSION_TOOLS), + ) + + def get_binary(self) -> str: + return PI_BINARY + + def get_install_hint(self) -> str: + return "npm install -g @earendil-works/pi-coding-agent" + + def validate_environment(self, ctx: SessionContext) -> None: + options: Final = self.get_options(ctx) + clashing: Final = sorted(MANAGED_ENV_KEYS.intersection(options.env)) + if clashing: + raise OptionsMismatch(f"PiOptions.env may not set {', '.join(clashing)}; LiteLLM manages it") + validate_user_config(options.config) + tool_args(ctx.permissions, ctx.disable_tools, mcp_tool_entries(options.config)) + + def transform_session_setup(self, ctx: SessionContext, private_dir: str) -> HarnessSessionSetup: + if ctx.endpoint is None: + raise HarnessError("pi needs the session model endpoint") + model: Final = session_model(ctx) + options: Final = self.get_options(ctx) + base_url: Final = ctx.sandbox.host_url(ctx.endpoint.port).rstrip("/") + "/v1" + info: Final = mapped_model_info(model, through_gateway=ctx.gateway is not None) + models_file: Final = ( + MODELS_FILENAME, + build_models_json(model, base_url, options.thinking, info).encode("utf-8"), + ) + instructions: Final = build_instructions(ctx) + instructions_file: Final = ( + ((INSTRUCTIONS_FILENAME, instructions.encode("utf-8")),) if instructions is not None else () + ) + return HarnessSessionSetup( + files=MappingProxyType(dict((models_file, *config_files(options.config), *instructions_file))), + persisted_dirs=((SESSIONS_DIRNAME, "pi/sessions"),), + skills_dir=SKILLS_DIRNAME, + env=MappingProxyType( + { + **PI_ISOLATION_ENV, + **options.env, + PI_CONFIG_DIR_ENV: f"{private_dir}/{AGENT_DIRNAME}", + PI_TOKEN_ENV: ctx.endpoint.token, + } + ), + ) + + def transform_turn_request( + self, + ctx: SessionContext, + setup: HarnessSessionSetup, + private_dir: str, + prompt: str, + native_session_id: str | None, + ) -> HarnessTurnRequest: + options: Final = self.get_options(ctx) + # --no-approve: a repo's .pi/extensions would otherwise run as the host user at startup. + # --no-skills: ~/.agents/skills is discovered outside PI_CODING_AGENT_DIR. + argv: Final = ( + PI_BINARY, + "--mode", + "json", + "--no-approve", + "--no-skills", + "--provider", + PI_PROVIDER_ID, + "--model", + session_model(ctx), + "--session-dir", + f"{private_dir}/{SESSIONS_DIRNAME}", + *session_args(native_session_id, ctx.sandbox.workdir), + *tool_args(ctx.permissions, ctx.disable_tools, mcp_tool_entries(options.config)), + *(("--thinking", options.thinking) if options.thinking else ()), + *(("--skill", f"{private_dir}/{SKILLS_DIRNAME}") if ctx.skills else ()), + *( + ("--append-system-prompt", f"{private_dir}/{INSTRUCTIONS_FILENAME}") + if INSTRUCTIONS_FILENAME in setup.files + else () + ), + ) + # The prompt goes on stdin; pi prepends piped stdin to the (empty) first message. + return HarnessTurnRequest(argv=argv, env=setup.env, stdin=turn_prompt(ctx, prompt), cwd=ctx.sandbox.workdir) + + def create_stream_state(self) -> PiStreamState: + return PiStreamState() + + def transform_stream_line(self, line: Mapping[str, object], state: PiStreamState) -> Sequence[Event]: + """Usage on message_update is ignored on purpose: the session endpoint accounts it.""" + event_type: Final = line.get("type") + if event_type == "session": + state.record_session(line.get("id"), line.get("cwd")) + return event_list() + if event_type == "message_update": + return _message_update_events(line.get("assistantMessageEvent")) + if event_type == "message_end": + state.record_message_end(line.get("message")) + return event_list() + if event_type == "tool_execution_start": + return _tool_start_events(line) + if event_type == "tool_execution_end": + return _tool_end_events(line) + if event_type == "agent_settled" and line.get("aborted") is True: + state.mark_aborted() + return event_list() + + def get_native_session_id(self, state: PiStreamState) -> str | None: + if state.session_id is None or state.session_cwd is None: + return state.session_id + return f"{state.session_id}{SESSION_CWD_SEPARATOR}{state.session_cwd}" + + def transform_turn_response( + self, + ctx: SessionContext, + state: PiStreamState, + exit_code: int, + stderr_tail: Sequence[str], + ) -> HarnessTurnResponse: + failure: Final = FAILED_STOP_REASONS.get(state.stop_reason or "") + if failure is not None: + raise HarnessTurnError(f"pi turn failed: {state.error or failure}") + if exit_code != 0: + raise HarnessTurnError(f"pi exited with code {exit_code}: {stderr_tail_text(stderr_tail) or 'no output'}") + if state.stop_reason is None: + raise HarnessTurnError(f"pi exited without a reply: {stderr_tail_text(stderr_tail) or 'no output'}") + output_json: Final = last_json_object(state.final_text) if ctx.output is not None else None + return HarnessTurnResponse(final_text=state.final_text, output_json=output_json) diff --git a/litellm/utils.py b/litellm/utils.py index 27c9667f032..14ee35e571e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10597,6 +10597,10 @@ class ProviderConfigManager: ) return OpenCodeHarnessConfig() + if harness == _Harness.PI: + from litellm.llms.pi.harness.transformation import PiHarnessConfig + + return PiHarnessConfig() if harness == _Harness.DEEPAGENTS: from litellm.llms.deepagents.harness.transformation import ( DeepAgentsHarnessConfig, diff --git a/tests/code_coverage_tests/check_provider_folders_documented.py b/tests/code_coverage_tests/check_provider_folders_documented.py index 6f58b393cd1..b4e60246774 100644 --- a/tests/code_coverage_tests/check_provider_folders_documented.py +++ b/tests/code_coverage_tests/check_provider_folders_documented.py @@ -32,6 +32,7 @@ EXCLUDED_FOLDERS = { "claude_code", "codex", "opencode", + "pi", "deepagents", "tool_loop", } diff --git a/tests/harness_e2e/conftest.py b/tests/harness_e2e/conftest.py index ba679d6d5bf..9521519d18a 100644 --- a/tests/harness_e2e/conftest.py +++ b/tests/harness_e2e/conftest.py @@ -1,6 +1,6 @@ """Fixtures for the litellm.agent() end-to-end tests. -These run the real harness runtimes (claude, codex, opencode, deepagents) against a real +These run the real harness runtimes (claude, codex, opencode, pi, deepagents) against a real LiteLLM AI Gateway, routed with the `litellm_proxy/` model prefix. They skip unless LITELLM_PROXY_API_BASE and LITELLM_PROXY_API_KEY are set. Model groups can be overridden per harness with HARNESS_E2E_MODEL_. @@ -23,6 +23,7 @@ DEFAULT_MODEL_GROUPS = { Harness.CLAUDE_CODE: "claude-haiku-4-5-20251001", Harness.CODEX: "bedrock_mantle/openai.gpt-5.4", Harness.OPENCODE: "claude-haiku-4-5-20251001", + Harness.PI: "claude-haiku-4-5-20251001", Harness.DEEPAGENTS: "claude-haiku-4-5-20251001", } @@ -30,6 +31,7 @@ BINARIES = { Harness.CLAUDE_CODE: "claude", Harness.CODEX: "codex", Harness.OPENCODE: "opencode", + Harness.PI: "pi", } requires_gateway = pytest.mark.skipif( @@ -62,7 +64,7 @@ def harness_params() -> list: not harness_available(h), reason=f"{h.value} runtime not installed" ), ) - for h in Harness + for h in DEFAULT_MODEL_GROUPS ] diff --git a/tests/unit/harness/test_endpoint.py b/tests/unit/harness/test_endpoint.py index 8b56debdc29..19bd6d0450e 100644 --- a/tests/unit/harness/test_endpoint.py +++ b/tests/unit/harness/test_endpoint.py @@ -1,7 +1,7 @@ import json import sys from collections.abc import AsyncIterator -from typing import Any +from typing import Any, Final import httpx import pytest @@ -13,13 +13,17 @@ from litellm.harness.endpoint import ( SSEUsageParser, UsageTracker, compute_cost, - usage_from_body, + usage_from_mapping, + usage_mapping_of_body, ) from litellm.harness.errors import HarnessInstallFailed from litellm.harness.context import GatewayTarget from litellm.harness.types import Harness, Usage from litellm.types.utils import ModelResponse, ModelResponseStream +REAL_COST_PER_TOKEN: Final = litellm.cost_per_token +CACHED_MODEL: Final = "claude-haiku-5-5" + GATEWAY = GatewayTarget(api_base="https://gw.example.com", api_key="sk-gateway-secret") ANTHROPIC_SSE = ( @@ -82,7 +86,7 @@ def no_real_cost(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( litellm, "cost_per_token", - lambda model, prompt_tokens, completion_tokens: ( + lambda model, prompt_tokens, completion_tokens, **cache_tokens: ( prompt_tokens * 0.001, completion_tokens * 0.002, ), @@ -294,6 +298,180 @@ async def test_sdk_messages_stream_handles_dicts_and_bytes( assert (ep.usage.input_tokens, ep.usage.output_tokens) == (4, 9) +async def chunks_of(chunks: list[dict[str, object]]) -> AsyncIterator[dict[str, object]]: + for chunk in chunks: + yield chunk + + +@pytest.mark.parametrize( + ("path", "sdk", "chunks"), + [ + ( + "/v1/messages", + (litellm.anthropic.messages, "acreate"), + [ + { + "type": "message_start", + "message": { + "usage": { + "input_tokens": 100, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 100, + "output_tokens": 1, + } + }, + }, + {"type": "message_delta", "usage": {"output_tokens": 10}}, + ], + ), + ( + "/chat/completions", + (litellm, "acompletion"), + [ + { + "choices": [], + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 10, + "prompt_tokens_details": {"cached_tokens": 800, "cache_creation_tokens": 100}, + }, + } + ], + ), + ( + "/v1/responses", + (litellm, "aresponses"), + [ + { + "type": "response.completed", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 10, + "input_tokens_details": {"cached_tokens": 800, "cache_creation_tokens": 100}, + } + }, + } + ], + ), + ( + "/chat/completions", + (litellm, "acompletion"), + [ + { + "choices": [], + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 10, + "prompt_tokens_details": {"cached_tokens": 800, "cache_write_tokens": 100}, + }, + } + ], + ), + ( + "/v1/responses", + (litellm, "aresponses"), + [ + { + "type": "response.completed", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 10, + "input_tokens_details": {"cached_tokens": 800, "cache_write_tokens": 100}, + } + }, + } + ], + ), + ], + ids=["messages", "chat", "responses", "chat cache_write_tokens", "responses cache_write_tokens"], +) +async def test_sdk_stream_cost_prices_cached_prompt_tokens_like_litellm( + monkeypatch: pytest.MonkeyPatch, path: str, sdk: tuple[object, str], chunks: list[dict[str, object]] +) -> None: + async def fake_sdk(**kwargs: object) -> AsyncIterator[dict[str, object]]: + return chunks_of(chunks) + + monkeypatch.setattr(litellm, "cost_per_token", REAL_COST_PER_TOKEN) + monkeypatch.setattr(*sdk, fake_sdk) + async with ModelEndpoint(Harness.PI, CACHED_MODEL, None) as ep: + async with httpx.AsyncClient(base_url=ep.url) as client: + await client.post(path, json={"stream": True}, headers=auth(ep)) + cached_price: Final = sum( + REAL_COST_PER_TOKEN( + model=CACHED_MODEL, + prompt_tokens=1000, + completion_tokens=10, + usage_object=litellm.Usage( + prompt_tokens=1000, + completion_tokens=10, + total_tokens=1010, + prompt_tokens_details={"cached_tokens": 800, "cache_creation_tokens": 100}, + ), + ) + ) + assert cached_price < sum(REAL_COST_PER_TOKEN(model=CACHED_MODEL, prompt_tokens=1000, completion_tokens=10)) + assert ep.usage.cost == pytest.approx(cached_price), ( + "SDK mode priced cached prompt tokens at the full input price, overstating the cost of every cached turn" + ) + + +ANTHROPIC_CACHED_USAGE: Final = { + "input_tokens": 100, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 100, + "output_tokens": 10, +} + + +def anthropic_cached_sse() -> bytes: + start: Final = {"type": "message_start", "message": {"usage": {**ANTHROPIC_CACHED_USAGE, "output_tokens": 1}}} + delta: Final = {"type": "message_delta", "usage": {"output_tokens": 10}} + return ( + f"event: message_start\ndata: {json.dumps(start)}\n\n" + f"event: message_delta\ndata: {json.dumps(delta)}\n\n" + ).encode() + + +async def anthropic_cached_message(**kwargs: object) -> dict[str, object]: + return {"type": "message", "content": [], "usage": ANTHROPIC_CACHED_USAGE} + + +@pytest.mark.parametrize("route", ["gateway stream", "gateway json", "sdk json"]) +async def test_claude_code_cost_counts_anthropic_cached_prompt_tokens( + monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + monkeypatch.setattr(litellm, "cost_per_token", REAL_COST_PER_TOKEN) + monkeypatch.setattr(litellm.anthropic.messages, "acreate", anthropic_cached_message) + upstream: Final = ( + sse_response(anthropic_cached_sse()) + if route == "gateway stream" + else httpx.Response(200, json={"usage": ANTHROPIC_CACHED_USAGE}) + ) + endpoint: Final = ( + ModelEndpoint(Harness.CLAUDE_CODE, CACHED_MODEL, None) + if route == "sdk json" + else gateway_endpoint(Recorder(upstream), model=CACHED_MODEL) + ) + async with endpoint as ep: + async with httpx.AsyncClient(base_url=ep.url) as client: + await client.post("/v1/messages", json={"stream": route == "gateway stream"}, headers=auth(ep)) + cached_price: Final = sum( + REAL_COST_PER_TOKEN( + model=CACHED_MODEL, + prompt_tokens=1000, + completion_tokens=10, + cache_read_input_tokens=800, + cache_creation_input_tokens=100, + ) + ) + assert cached_price > sum(REAL_COST_PER_TOKEN(model=CACHED_MODEL, prompt_tokens=100, completion_tokens=10)) + assert ep.usage.cost == pytest.approx(cached_price), ( + "Anthropic counts cached prompt tokens outside input_tokens, so leaving them out understates the turn's cost" + ) + + async def test_sdk_error_is_sanitized(monkeypatch: pytest.MonkeyPatch) -> None: async def failing(**kwargs: Any) -> Any: raise litellm.RateLimitError( @@ -330,12 +508,12 @@ def test_load_server_deps_maps_import_error(monkeypatch: pytest.MonkeyPatch) -> def test_usage_helpers() -> None: - assert usage_from_body({"usage": {"prompt_tokens": 1, "completion_tokens": 2}}) == ( + assert usage_from_mapping(usage_mapping_of_body({"usage": {"prompt_tokens": 1, "completion_tokens": 2}})) == ( 1, 2, ) - assert usage_from_body({"response": {"usage": {"input_tokens": 3}}}) == (3, 0) - assert usage_from_body("nope") == (0, 0) + assert usage_from_mapping(usage_mapping_of_body({"response": {"usage": {"input_tokens": 3}}})) == (3, 0) + assert usage_from_mapping(usage_mapping_of_body("nope")) == (0, 0) parser = SSEUsageParser() for i in range(0, len(ANTHROPIC_SSE), 7): # split across arbitrary chunk borders diff --git a/tests/unit/harness/test_init.py b/tests/unit/harness/test_init.py index 75ec3b60c74..f70d565fdec 100644 --- a/tests/unit/harness/test_init.py +++ b/tests/unit/harness/test_init.py @@ -34,6 +34,7 @@ PUBLIC_NAMES = [ "ClaudeCodeOptions", "CodexOptions", "OpenCodeOptions", + "PiOptions", "DeepAgentsOptions", "ToolLoopOptions", "HarnessError", diff --git a/tests/unit/harness/test_options.py b/tests/unit/harness/test_options.py new file mode 100644 index 00000000000..b2890de40d8 --- /dev/null +++ b/tests/unit/harness/test_options.py @@ -0,0 +1,24 @@ +import copy +import dataclasses +from collections.abc import Mapping +from typing import Final, get_args + +import pytest + +from litellm.harness.options import HarnessOptions + + +@pytest.mark.parametrize("options_type", get_args(HarnessOptions), ids=lambda t: t.__name__) +def test_default_options_survive_deepcopy_and_asdict(options_type: type) -> None: + options: Final = options_type() + assert copy.deepcopy(options) == options + assert dataclasses.asdict(options) == vars(options) + + +@pytest.mark.parametrize("options_type", get_args(HarnessOptions), ids=lambda t: t.__name__) +def test_each_options_instance_gets_its_own_empty_mappings(options_type: type) -> None: + first: Final = vars(options_type()) + second: Final = vars(options_type()) + mapping_names: Final = tuple(name for name, value in first.items() if isinstance(value, Mapping)) + assert {name: first[name] for name in mapping_names} == dict.fromkeys(mapping_names, {}) + assert [name for name in mapping_names if first[name] is second[name]] == [], "defaults shared between instances" diff --git a/tests/unit/harness/test_types.py b/tests/unit/harness/test_types.py index e59c4e983fb..f13469dfe5a 100644 --- a/tests/unit/harness/test_types.py +++ b/tests/unit/harness/test_types.py @@ -30,6 +30,7 @@ def test_harness_is_plain_enum(): ("codex", "Harness.CODEX"), ("claude-code", "Harness.CLAUDE_CODE"), ("OPENCODE", "Harness.OPENCODE"), + ("pi", "Harness.PI"), ], ) def test_require_harness_hint(given, hint): diff --git a/tests/unit/llms/pi/__init__.py b/tests/unit/llms/pi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/pi/harness/__init__.py b/tests/unit/llms/pi/harness/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/pi/harness/fixtures/__init__.py b/tests/unit/llms/pi/harness/fixtures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/pi/harness/fixtures/api_error.jsonl b/tests/unit/llms/pi/harness/fixtures/api_error.jsonl new file mode 100644 index 00000000000..e8720dd9ca3 --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/api_error.jsonl @@ -0,0 +1,12 @@ +{"type":"session","version":3,"id":"01a11cb0-c576-7785-9c03-73fd7035291c","timestamp":"2026-10-08T18:05:09.367Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482709539,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_end","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482709539,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482709537}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482709537}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"error","timestamp":1791482709583,"errorMessage":"400: {\"message\":\"litellm.BadRequestError: You passed in model=no-such-model-xyz. There are no healthy deployments for this model\",\"type\":\"invalid_request_error\",\"code\":\"400\"}","durationMs":32,"thinkingLevel":"off"}} +{"type":"message_end","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"error","timestamp":1791482709583,"errorMessage":"400: {\"message\":\"litellm.BadRequestError: You passed in model=no-such-model-xyz. There are no healthy deployments for this model\",\"type\":\"invalid_request_error\",\"code\":\"400\"}","durationMs":32,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"error","timestamp":1791482709583,"errorMessage":"400: {\"message\":\"litellm.BadRequestError: You passed in model=no-such-model-xyz. There are no healthy deployments for this model\",\"type\":\"invalid_request_error\",\"code\":\"400\"}","durationMs":32,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482709539,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]},{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482709537},{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"error","timestamp":1791482709583,"errorMessage":"400: {\"message\":\"litellm.BadRequestError: You passed in model=no-such-model-xyz. There are no healthy deployments for this model\",\"type\":\"invalid_request_error\",\"code\":\"400\"}","durationMs":32,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/fixtures/mcp_tool.jsonl b/tests/unit/llms/pi/harness/fixtures/mcp_tool.jsonl new file mode 100644 index 00000000000..bc46f5f2aac --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/mcp_tool.jsonl @@ -0,0 +1,27 @@ +{"type":"session","version":3,"id":"01a11ced-fa14-70ec-ab43-6949fb5c4695","timestamp":"2026-10-08T19:12:00.533Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791486720735,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}},{"name":"mcp__moyai__echo","description":"Echo the given text back","parameters":{"type":"object","properties":{"text":{"type":"string"}},"required":["text"]}}]}} +{"type":"message_end","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791486720735,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}},{"name":"mcp__moyai__echo","description":"Echo the given text back","parameters":{"type":"object","properties":{"text":{"type":"string"}},"required":["text"]}}]}} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791486720733}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791486720733}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791486720755}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":0,"id":"call_m1","toolName":"mcp__moyai__echo"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":"{\"text\":\"hello\"}"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":0,"toolCall":{"type":"toolCall","id":"call_m1","name":"mcp__moyai__echo","arguments":{"text":"hello"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_m1","name":"mcp__moyai__echo","arguments":{"text":"hello"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791486720755,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":34,"thinkingLevel":"off"}} +{"type":"tool_execution_start","toolCallId":"call_m1","toolName":"mcp__moyai__echo","args":{"text":"hello"}} +{"type":"tool_execution_end","toolCallId":"call_m1","toolName":"mcp__moyai__echo","result":{"content":[{"type":"text","text":"moyai says: hello"}],"details":{"server":"moyai","tool":"echo"},"structuredContent":{"content":[{"type":"text","text":"moyai says: hello"}]}},"isError":false,"durationMs":2} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"call_m1","toolName":"mcp__moyai__echo","content":[{"type":"text","text":"moyai says: hello"}],"details":{"server":"moyai","tool":"echo"},"isError":false,"durationMs":2,"timestamp":1791486720794}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"call_m1","toolName":"mcp__moyai__echo","content":[{"type":"text","text":"moyai says: hello"}],"details":{"server":"moyai","tool":"echo"},"isError":false,"durationMs":2,"timestamp":1791486720794}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_m1","name":"mcp__moyai__echo","arguments":{"text":"hello"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791486720755,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":34,"thinkingLevel":"off"},"toolResults":[{"role":"toolResult","toolCallId":"call_m1","toolName":"mcp__moyai__echo","content":[{"type":"text","text":"moyai says: hello"}],"details":{"server":"moyai","tool":"echo"},"isError":false,"durationMs":2,"timestamp":1791486720794}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791486720798}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"The MCP tool"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" replied."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"The MCP tool replied."}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"The MCP tool replied."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791486720798,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":5,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"The MCP tool replied."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791486720798,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":5,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791486720735,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}},{"name":"mcp__moyai__echo","description":"Echo the given text back","parameters":{"type":"object","properties":{"text":{"type":"string"}},"required":["text"]}}]},{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791486720733},{"role":"assistant","content":[{"type":"toolCall","id":"call_m1","name":"mcp__moyai__echo","arguments":{"text":"hello"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791486720755,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":34,"thinkingLevel":"off"},{"role":"toolResult","toolCallId":"call_m1","toolName":"mcp__moyai__echo","content":[{"type":"text","text":"moyai says: hello"}],"details":{"server":"moyai","tool":"echo"},"isError":false,"durationMs":2,"timestamp":1791486720794},{"role":"assistant","content":[{"type":"text","text":"The MCP tool replied."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791486720798,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":5,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/fixtures/readonly_denied_bash.jsonl b/tests/unit/llms/pi/harness/fixtures/readonly_denied_bash.jsonl new file mode 100644 index 00000000000..f5645a6fb63 --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/readonly_denied_bash.jsonl @@ -0,0 +1,29 @@ +{"type":"session","version":3,"id":"01a11cb0-a988-7781-93cc-8f9b9f61d05e","timestamp":"2026-10-08T18:05:02.216Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482702706,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_end","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482702706,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482702704}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482702704}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482702749}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":0,"id":"call_b1","toolName":"bash"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":"{\"command\":\"echo x > blocked.txt\"}"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":0,"toolCall":{"type":"toolCall","id":"call_b1","name":"bash","arguments":{"command":"echo x > blocked.txt"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_b1","name":"bash","arguments":{"command":"echo x > blocked.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482702749,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":89,"thinkingLevel":"off"}} +{"type":"tool_execution_start","toolCallId":"call_b1","toolName":"bash","args":{"command":"echo x > blocked.txt"}} +{"type":"tool_execution_end","toolCallId":"call_b1","toolName":"bash","result":{"content":[{"type":"text","text":"Tool bash not found"}],"details":{}},"isError":true} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"call_b1","toolName":"bash","content":[{"type":"text","text":"Tool bash not found"}],"details":{},"isError":true,"timestamp":1791482702841}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"call_b1","toolName":"bash","content":[{"type":"text","text":"Tool bash not found"}],"details":{},"isError":true,"timestamp":1791482702841}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_b1","name":"bash","arguments":{"command":"echo x > blocked.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482702749,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":89,"thinkingLevel":"off"},"toolResults":[{"role":"toolResult","toolCallId":"call_b1","toolName":"bash","content":[{"type":"text","text":"Tool bash not found"}],"details":{},"isError":true,"timestamp":1791482702841}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482702847}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"FAILED: I ca"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"nnot write f"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"iles in this"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" mode."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"FAILED: I cannot write files in this mode."}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"FAILED: I cannot write files in this mode."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482702847,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":9,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"FAILED: I cannot write files in this mode."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482702847,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":9,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482702706,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]},{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482702704},{"role":"assistant","content":[{"type":"toolCall","id":"call_b1","name":"bash","arguments":{"command":"echo x > blocked.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482702749,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":89,"thinkingLevel":"off"},{"role":"toolResult","toolCallId":"call_b1","toolName":"bash","content":[{"type":"text","text":"Tool bash not found"}],"details":{},"isError":true,"timestamp":1791482702841},{"role":"assistant","content":[{"type":"text","text":"FAILED: I cannot write files in this mode."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482702847,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":9,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/fixtures/structured_output.jsonl b/tests/unit/llms/pi/harness/fixtures/structured_output.jsonl new file mode 100644 index 00000000000..009074e7f29 --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/structured_output.jsonl @@ -0,0 +1,19 @@ +{"type":"session","version":3,"id":"01a11cb0-bd27-7561-9a95-f21aaec081b9","timestamp":"2026-10-08T18:05:07.240Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","addendum":"\nBe terse.\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}\n","cwd":"\n/work\n"},"timestamp":1791482707686,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_end","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","addendum":"\nBe terse.\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}\n","cwd":"\n/work\n"},"timestamp":1791482707686,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}"}],"timestamp":1791482707684}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}"}],"timestamp":1791482707684}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482707753}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"Here you go:"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" {\"city\": \"P"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"aris\", \"coun"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"try\": \"Franc"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"e\"}"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"Here you go: {\"city\": \"Paris\", \"country\": \"France\"}"}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"Here you go: {\"city\": \"Paris\", \"country\": \"France\"}"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482707753,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":131,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"Here you go: {\"city\": \"Paris\", \"country\": \"France\"}"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482707753,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":131,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","addendum":"\nBe terse.\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}\n","cwd":"\n/work\n"},"timestamp":1791482707686,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]},{"role":"user","content":[{"type":"text","text":"do the task\n\nWhen you have finished, your final message must be a single JSON object that matches this JSON schema, with no other text before or after it:\n{\"properties\": {\"city\": {\"title\": \"City\", \"type\": \"string\"}, \"country\": {\"title\": \"Country\", \"type\": \"string\"}}, \"required\": [\"city\", \"country\"], \"title\": \"Answer\", \"type\": \"object\"}"}],"timestamp":1791482707684},{"role":"assistant","content":[{"type":"text","text":"Here you go: {\"city\": \"Paris\", \"country\": \"France\"}"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482707753,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":131,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/fixtures/turn1_write_read.jsonl b/tests/unit/llms/pi/harness/fixtures/turn1_write_read.jsonl new file mode 100644 index 00000000000..d508b33070d --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/turn1_write_read.jsonl @@ -0,0 +1,42 @@ +{"type":"session","version":3,"id":"01a11cb0-87cf-752e-a3e3-365c7a169871","timestamp":"2026-10-08T18:04:53.583Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482693985,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_end","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482693985,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]}} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482693983}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482693983}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482694116}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_delta","contentIndex":0,"delta":"I should write the file first."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":1,"id":"call_w1","toolName":"write"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":1,"delta":"{\"path\":\"hello.txt\",\"content\":\"hi\"}"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_end","contentIndex":0,"content":"I should write the file first."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":1,"toolCall":{"type":"toolCall","id":"call_w1","name":"write","arguments":{"path":"hello.txt","content":"hi"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"thinking","thinking":"I should write the file first.","thinkingSignature":"reasoning_content"},{"type":"toolCall","id":"call_w1","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694116,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":146,"thinkingLevel":"off"}} +{"type":"tool_execution_start","toolCallId":"call_w1","toolName":"write","args":{"path":"hello.txt","content":"hi"}} +{"type":"tool_execution_end","toolCallId":"call_w1","toolName":"write","result":{"content":[{"type":"text","text":"Successfully wrote to hello.txt"}]},"isError":false,"durationMs":103} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"call_w1","toolName":"write","content":[{"type":"text","text":"Successfully wrote to hello.txt"}],"isError":false,"durationMs":103,"timestamp":1791482694371}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"call_w1","toolName":"write","content":[{"type":"text","text":"Successfully wrote to hello.txt"}],"isError":false,"durationMs":103,"timestamp":1791482694371}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"thinking","thinking":"I should write the file first.","thinkingSignature":"reasoning_content"},{"type":"toolCall","id":"call_w1","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694116,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":146,"thinkingLevel":"off"},"toolResults":[{"role":"toolResult","toolCallId":"call_w1","toolName":"write","content":[{"type":"text","text":"Successfully wrote to hello.txt"}],"isError":false,"durationMs":103,"timestamp":1791482694371}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482694375}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":0,"id":"call_r1","toolName":"read"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":"{\"path\":\"hello.txt\"}"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":0,"toolCall":{"type":"toolCall","id":"call_r1","name":"read","arguments":{"path":"hello.txt"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_r1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694375,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":5,"thinkingLevel":"off"}} +{"type":"tool_execution_start","toolCallId":"call_r1","toolName":"read","args":{"path":"hello.txt"}} +{"type":"tool_execution_end","toolCallId":"call_r1","toolName":"read","result":{"content":[{"type":"text","text":"hi"}],"structuredContent":"hi"},"isError":false,"durationMs":11} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"call_r1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"durationMs":11,"timestamp":1791482694394}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"call_r1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"durationMs":11,"timestamp":1791482694394}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_r1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694375,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":5,"thinkingLevel":"off"},"toolResults":[{"role":"toolResult","toolCallId":"call_r1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"durationMs":11,"timestamp":1791482694394}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482694397}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"Done! hello."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"txt contains"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":": hi"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"Done! hello.txt contains: hi"}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"Done! hello.txt contains: hi"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482694397,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":4,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"Done! hello.txt contains: hi"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482694397,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":4,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.","tools":"\n- read: Read file contents\n- bash: Execute bash commands (ls, grep, find, etc.)\n- edit: Make precise file edits with exact text replacement, including multiple disjoint edits in one call\n- write: Create or overwrite files\n- grep: Search file contents for patterns (respects .gitignore)\n- find: Find files by glob pattern (respects .gitignore)\n- ls: List directory contents\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n","rules":"\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Be concise in your responses\n- Show file paths clearly when working with files\n","docs":"\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /usr/lib/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /usr/lib/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /usr/lib/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n","cwd":"\n/work\n"},"timestamp":1791482693985,"toolsAdded":[{"name":"read","description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","parameters":{"type":"object","required":["path"],"properties":{"path":{"type":"string","description":"Path to the file to read (relative or absolute)"},"offset":{"type":"number","description":"Line number to start reading from (1-indexed)"},"limit":{"type":"number","description":"Maximum number of lines to read"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"bash","description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","parameters":{"type":"object","required":["command"],"properties":{"command":{"type":"string","description":"Shell command to execute"},"timeout":{"type":"number","description":"Timeout in seconds (optional, no default timeout)"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"edit","description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","parameters":{"type":"object","required":["path","edits"],"properties":{"path":{"type":"string","description":"Path to the file to edit (relative or absolute)"},"edits":{"type":"array","items":{"type":"object","required":["oldText","newText"],"properties":{"oldText":{"type":"string","description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call."},"newText":{"type":"string","description":"Replacement text for this targeted edit."}}},"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead."}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"write","description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","parameters":{"type":"object","required":["path","content"],"properties":{"path":{"type":"string","description":"Path to the file to write (relative or absolute)"},"content":{"type":"string","description":"Content to write to the file"}}},"constrainedSampling":{"type":"json_schema","strict":"prefer"}},{"name":"grep","description":"Search file contents for a pattern. Returns matching lines with file paths and line numbers. Respects .gitignore. Output is truncated to 100 matches or 50KB (whichever is hit first). Long lines are truncated to 500 chars.","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Search pattern (regex or literal string)"},"path":{"type":"string","description":"Directory or file to search (default: current directory)"},"glob":{"type":"string","description":"Filter files by glob pattern, e.g. '*.ts' or '**/*.spec.ts'"},"ignoreCase":{"type":"boolean","description":"Case-insensitive search (default: false)"},"literal":{"type":"boolean","description":"Treat pattern as literal string instead of regex (default: false)"},"context":{"type":"number","description":"Number of lines to show before and after each match (default: 0)"},"limit":{"type":"number","description":"Maximum number of matches to return (default: 100)"}}}},{"name":"find","description":"Search for files by glob pattern. Returns matching file paths relative to the search directory. Respects .gitignore. Output is truncated to 1000 results or 50KB (whichever is hit first).","parameters":{"type":"object","required":["pattern"],"properties":{"pattern":{"type":"string","description":"Glob pattern to match files, e.g. '*.ts', '**/*.json', or 'src/**/*.spec.ts'"},"path":{"type":"string","description":"Directory to search in (default: current directory)"},"limit":{"type":"number","description":"Maximum number of results (default: 1000)"}}}},{"name":"ls","description":"List directory contents. Returns entries sorted alphabetically, with '/' suffix for directories. Includes dotfiles. Output is truncated to 500 entries or 50KB (whichever is hit first).","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Directory to list (default: current directory)"},"limit":{"type":"number","description":"Maximum number of entries to return (default: 500)"}}}}]},{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482693983},{"role":"assistant","content":[{"type":"thinking","thinking":"I should write the file first.","thinkingSignature":"reasoning_content"},{"type":"toolCall","id":"call_w1","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694116,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":146,"thinkingLevel":"off"},{"role":"toolResult","toolCallId":"call_w1","toolName":"write","content":[{"type":"text","text":"Successfully wrote to hello.txt"}],"isError":false,"durationMs":103,"timestamp":1791482694371},{"role":"assistant","content":[{"type":"toolCall","id":"call_r1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1791482694375,"responseId":"chatcmpl-1","rawStopReason":"tool_calls","durationMs":5,"thinkingLevel":"off"},{"role":"toolResult","toolCallId":"call_r1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"durationMs":11,"timestamp":1791482694394},{"role":"assistant","content":[{"type":"text","text":"Done! hello.txt contains: hi"}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482694397,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":4,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/fixtures/turn2_resume.jsonl b/tests/unit/llms/pi/harness/fixtures/turn2_resume.jsonl new file mode 100644 index 00000000000..8d50c16e564 --- /dev/null +++ b/tests/unit/llms/pi/harness/fixtures/turn2_resume.jsonl @@ -0,0 +1,15 @@ +{"type":"session","version":3,"id":"01a11cb0-87cf-752e-a3e3-365c7a169871","timestamp":"2026-10-08T18:04:53.583Z","cwd":"/work"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482698176}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482698176}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1791482698355}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"You asked me"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" to create h"}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"ello.txt."}} +{"type":"message_update","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"You asked me to create hello.txt."}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"You asked me to create hello.txt."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482698355,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":169,"thinkingLevel":"off"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"You asked me to create hello.txt."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482698355,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":169,"thinkingLevel":"off"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"user","content":[{"type":"text","text":"do the task"}],"timestamp":1791482698176},{"role":"assistant","content":[{"type":"text","text":"You asked me to create hello.txt."}],"api":"openai-completions","provider":"litellm","model":"claude-haiku-4-5-20251001","usage":{"input":120,"output":9,"cacheRead":0,"cacheWrite":0,"reasoning":0,"totalTokens":129,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1791482698355,"responseId":"chatcmpl-1","rawStopReason":"stop","durationMs":169,"thinkingLevel":"off"}],"willRetry":false} +{"type":"agent_settled","aborted":false} diff --git a/tests/unit/llms/pi/harness/test_transformation.py b/tests/unit/llms/pi/harness/test_transformation.py new file mode 100644 index 00000000000..615a255bb7e --- /dev/null +++ b/tests/unit/llms/pi/harness/test_transformation.py @@ -0,0 +1,991 @@ +"""Unit tests for the pi harness config. No network, no real CLI. + +Fixtures under fixtures/ are sanitized `pi --mode json` output recorded from +@earendil-works/pi-coding-agent 1.1.0 against a scripted OpenAI-compatible endpoint. +""" + +import asyncio +import itertools +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +from pydantic import BaseModel, ConfigDict + +from litellm.harness.context import GatewayTarget, SessionContext +from litellm.harness.errors import ( + CapabilityUnsupported, + HarnessError, + HarnessInstallFailed, + OptionsMismatch, +) +from litellm.harness.handlers.cli_handler import PERSIST_DIR_SCRIPT, CLIHarnessHandler +from litellm.harness.options import HarnessOptions, OpenCodeOptions, PiOptions, PiThinkingLevel +from litellm.harness.sandbox.base import CompletedRun +from litellm.harness.types import Event, Harness, PermissionMode, Reasoning, Text, ToolCall, ToolResult +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.base_llm.harness.transformation import ( + HarnessSessionSetup, + HarnessTurnError, +) +from litellm.llms.pi.harness.transformation import ( + INSTRUCTIONS_FILENAME, + MANAGED_CONFIG_KEYS, + MANAGED_ENV_KEYS, + MCP_FILENAME, + MODELS_FILENAME, + PI_CONFIG_DIR_ENV, + PI_DEFAULT_MAX_TOKENS, + PI_ISOLATION_ENV, + PI_TOKEN_ENV, + SETTINGS_FILENAME, + PiHarnessConfig, + PiStreamState, + build_models_json, + config_files, + mcp_tool_entries, + tool_args, + validate_user_config, +) +from litellm.types.utils import ModelInfo +from litellm.utils import ProviderConfigManager, get_model_info + +FIXTURES: Final = Path(__file__).parent / "fixtures" +TOKEN: Final = "tok-secret-123" +SESSION: Final = "01a11cb0-87cf-752e-a3e3-365c7a169871" +PRIVATE: Final = "/tmp/pi-1" +MODEL: Final = "claude-haiku-4-5-20251001" +FULL_TOOLS: Final = "read,bash,edit,write,grep,find,ls" +MOYAI: Final = {"command": "/path/to/mcp-server", "args": [], "env": {}, "exposure": "direct"} +CONFIG: Final = PiHarnessConfig() + + +def load_fixture(name: str) -> Sequence[Mapping[str, object]]: + return tuple(json.loads(line) for line in (FIXTURES / name).read_text().splitlines() if line) + + +def parse(obj: Mapping[str, object], state: PiStreamState) -> Sequence[Event]: + return CONFIG.transform_stream_line(obj, state) + + +def parse_all(name: str, state: PiStreamState | None = None) -> tuple[Sequence[Event], PiStreamState]: + run_state: Final = state or CONFIG.create_stream_state() + events: Final = tuple(itertools.chain.from_iterable(parse(obj, run_state) for obj in load_fixture(name))) + return events, run_state + + +def assistant_end(text: str, stop_reason: str, error: str | None = None) -> Mapping[str, object]: + error_field: Final = {"errorMessage": error} if error else {} + return { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": text}], + "stopReason": stop_reason, + **error_field, + }, + } + + +class FakeStdin: + def __init__(self) -> None: + self.data = b"" + self.closed = False + + def write(self, data: bytes) -> None: + self.data += data + + async def drain(self) -> None: + return None + + def close(self) -> None: + self.closed = True + + +class FakeProcess: + def __init__(self, stdout: bytes, stderr: bytes = b"", exit_code: int = 0) -> None: + self.stdin = FakeStdin() + self.stdout = asyncio.StreamReader() + self.stdout.feed_data(stdout) + self.stdout.feed_eof() + self.stderr = asyncio.StreamReader() + self.stderr.feed_data(stderr) + self.stderr.feed_eof() + self._exit_code = exit_code + self.killed = False + + async def wait(self) -> int: + return self._exit_code + + async def kill(self) -> None: + self.killed = True + + +@dataclass(frozen=True, slots=True) +class Exec: + cmd: Sequence[str] + env: Mapping[str, str] + cwd: str | None + + +@dataclass +class FakeSandbox: + workdir: str = "/work" + has_binary: bool = True + persist_ok: bool = True + outputs: tuple[FakeProcess, ...] = () + files: Mapping[str, bytes] = field(default_factory=lambda: MappingProxyType({})) + execs: tuple[Exec, ...] = () + runs: tuple[Sequence[str], ...] = () + tempdirs: int = 0 + + async def exec( + self, cmd: Sequence[str], *, env: Mapping[str, str] | None = None, cwd: str | None = None + ) -> FakeProcess: + self.execs = (*self.execs, Exec(cmd=cmd, env=MappingProxyType(dict(env or {})), cwd=cwd)) + return self.outputs[len(self.execs) - 1] + + async def run( + self, + cmd: Sequence[str], + *, + env: Mapping[str, str] | None = None, + cwd: str | None = None, + timeout: float | None = None, + ) -> CompletedRun: + self.runs = (*self.runs, cmd) + if self.persist_ok: + return CompletedRun("", "", 0) + return CompletedRun("", "read-only fs", 1) + + async def read(self, path: str) -> bytes: + return self.files[path] + + async def write(self, path: str, data: bytes) -> None: + self.files = MappingProxyType({**self.files, path: data}) + + def host_url(self, port: int) -> str: + return f"http://host.docker.internal:{port}" + + async def which(self, binary: str) -> str | None: + return f"/usr/bin/{binary}" if self.has_binary else None + + async def tempdir(self) -> str: + self.tempdirs += 1 + return f"/tmp/pi-{self.tempdirs}" + + async def snapshot(self) -> Mapping[str, str]: + return MappingProxyType({}) + + async def close(self) -> None: + return None + + +@dataclass(frozen=True) +class FakeEndpoint: + port: int = 4555 + token: str = TOKEN + model: str | None = None + + +DEFAULT_ENDPOINT: Final = FakeEndpoint() + + +class Answer(BaseModel): + model_config = ConfigDict(frozen=True) + + city: str + country: str + + +def make_ctx( + sandbox: FakeSandbox | None = None, + *, + model: str | None = MODEL, + gateway: GatewayTarget | None = None, + endpoint: FakeEndpoint | None = DEFAULT_ENDPOINT, + options: HarnessOptions | None = None, + permissions: PermissionMode = "full", + instructions: str | None = None, + output: type[BaseModel] | None = None, + skills: Sequence[str] = (), + disable_tools: Sequence[str] = (), +) -> SessionContext: + return SessionContext( + harness=Harness.PI, + sandbox=sandbox or FakeSandbox(), + session_id="s1", + model=model, + gateway=gateway, + endpoint=endpoint, + options=options, + permissions=permissions, + instructions=instructions, + output=output, + skills=skills, + disable_tools=disable_tools, + ) + + +def setup_for(ctx: SessionContext) -> HarnessSessionSetup: + return CONFIG.transform_session_setup(ctx, PRIVATE) + + +def setup_models(setup: HarnessSessionSetup) -> Mapping[str, object]: + return json.loads(setup.files[MODELS_FILENAME]) + + +def fixture_proc(name: str, exit_code: int = 0) -> FakeProcess: + return FakeProcess((FIXTURES / name).read_bytes(), exit_code=exit_code) + + +async def collect(handler: CLIHarnessHandler, ctx: SessionContext, prompt: str) -> Sequence[Event]: + return [e async for e in handler.turn(ctx, prompt)] + + +async def started(ctx: SessionContext) -> CLIHarnessHandler: + handler: Final = CLIHarnessHandler(PiHarnessConfig()) + await handler.start(ctx) + return handler + + +def flag(argv: Sequence[str], name: str) -> str: + args: Final = list(argv) + return args[args.index(name) + 1] + + +def written_models(sandbox: FakeSandbox, private: str = PRIVATE) -> Mapping[str, object]: + return json.loads(sandbox.files[f"{private}/{MODELS_FILENAME}"]) + + +def test_parse_write_read_turn(): + events, state = parse_all("turn1_write_read.jsonl") + assert state.session_id == SESSION + assert [type(e) for e in events] == [ + Reasoning, + ToolCall, + ToolResult, + ToolCall, + ToolResult, + Text, + Text, + Text, + ] + assert events[0] == Reasoning(delta="I should write the file first.") + write, write_result = events[1], events[2] + assert write.name == "write" and write.native_name == "write" + assert write.builtin is True + assert write.input == {"path": "hello.txt", "content": "hi"} + assert write_result.id == write.id == "call_w1" + assert write_result.output == "Successfully wrote to hello.txt" + assert write_result.is_error is False + read, read_result = events[3], events[4] + assert read.name == "read" and read_result.output == "hi" + assert "".join(e.delta for e in events[5:]) == state.final_text + assert state.final_text == "Done! hello.txt contains: hi" + assert state.stop_reason == "stop" and state.error is None + + +def test_parse_continued_session(): + events, state = parse_all("turn2_resume.jsonl") + assert state.session_id == SESSION + assert all(isinstance(e, Text) for e in events) + assert state.final_text == "You asked me to create hello.txt." + + +def test_parse_denied_tool_is_error_result(): + events, state = parse_all("readonly_denied_bash.jsonl") + call: Final = next(e for e in events if isinstance(e, ToolCall)) + result: Final = next(e for e in events if isinstance(e, ToolResult)) + assert call.native_name == "bash" and call.input == {"command": "echo x > blocked.txt"} + assert result == ToolResult(id="call_b1", output="Tool bash not found", is_error=True) + assert state.final_text.startswith("FAILED") + + +def test_parse_mcp_tool_turn(): + events, state = parse_all("mcp_tool.jsonl") + call: Final = next(e for e in events if isinstance(e, ToolCall)) + result: Final = next(e for e in events if isinstance(e, ToolResult)) + assert call == ToolCall( + id="call_m1", + name="mcp__moyai__echo", + native_name="mcp__moyai__echo", + input={"text": "hello"}, + builtin=False, + ) + assert result == ToolResult(id="call_m1", output="moyai says: hello", is_error=False) + assert state.final_text == "The MCP tool replied." + + +def test_parse_api_error_records_error(): + events, state = parse_all("api_error.jsonl") + assert events == () + assert state.stop_reason == "error" + assert "no healthy deployments" in state.error + + +def test_parse_reasoning_and_tool_error_and_name_mapping(): + state: Final = PiStreamState() + reasoning: Final = parse( + { + "type": "message_update", + "assistantMessageEvent": {"type": "thinking_delta", "delta": "thinking hard"}, + }, + state, + ) + assert reasoning == [Reasoning(delta="thinking hard")] + failed: Final = parse( + { + "type": "tool_execution_end", + "toolCallId": "c1", + "toolName": "bash", + "result": {"content": [{"type": "text", "text": "boom"}]}, + "isError": True, + }, + state, + ) + assert failed == [ToolResult(id="c1", output="boom", is_error=True)] + calls: Final = [ + parse({"type": "tool_execution_start", "toolCallId": "x", "toolName": native, "args": {}}, state)[0] + for native in ("find", "powershell", "grep", "ls", "edit") + ] + assert [(call.name, call.builtin) for call in calls] == [ + ("glob", True), + ("bash", True), + ("grep", True), + ("ls", True), + ("edit", True), + ] + mcp: Final = parse( + {"type": "tool_execution_start", "toolCallId": "m", "toolName": "mcp__gh__search", "args": {"q": 1}}, + state, + ) + assert mcp[0].builtin is False and mcp[0].input == {"q": 1} + plain: Final = parse( + {"type": "tool_execution_end", "toolCallId": "m", "toolName": "mcp__gh__search", "result": "raw"}, + state, + ) + assert plain == [ToolResult(id="m", output="raw", is_error=False)] + no_result: Final = parse({"type": "tool_execution_end", "toolCallId": "e", "toolName": "bash"}, state) + assert no_result == [ToolResult(id="e", output="", is_error=False)] + + +def test_final_text_is_last_assistant_message(): + state: Final = PiStreamState() + parse(assistant_end("working", "toolUse"), state) + parse(assistant_end("answer", "stop"), state) + parse( + {"type": "message_end", "message": {"role": "user", "content": [{"type": "text", "text": "q"}]}}, + state, + ) + parse({"type": "message_end", "message": {"role": "system", "content": ""}}, state) + assert state.final_text == "answer" + parse( + { + "type": "message_end", + "message": { + "role": "assistant", + "stopReason": "stop", + "content": [ + {"type": "thinking", "thinking": "hmm"}, + {"type": "text", "text": "a"}, + {"type": "toolCall", "id": "x", "name": "read", "arguments": {}}, + {"type": "text", "text": "b"}, + ], + }, + }, + state, + ) + assert state.final_text == "ab" + + +def test_message_update_ignores_non_delta_events(): + state: Final = PiStreamState() + for update in [ + {"type": "text_start", "contentIndex": 0}, + {"type": "text_end", "content": "full"}, + {"type": "toolcall_delta", "delta": "{"}, + {"type": "text_delta", "delta": ""}, + ]: + assert parse({"type": "message_update", "assistantMessageEvent": update}, state) == [] + assert parse({"type": "message_update"}, state) == [] + + +def test_retried_error_then_success_is_not_an_error(): + state: Final = PiStreamState() + parse(assistant_end("", "error", "529 overloaded"), state) + parse(assistant_end("recovered", "stop"), state) + assert state.stop_reason == "stop" and state.error is None + assert CONFIG.transform_turn_response(make_ctx(), state, 0, []).final_text == "recovered" + + +def test_aborted_run_is_an_error(): + state: Final = PiStreamState() + parse(assistant_end("partial", "stop"), state) + parse({"type": "agent_settled", "aborted": False}, state) + assert state.stop_reason == "stop" + parse({"type": "agent_settled", "aborted": True}, state) + assert state.stop_reason == "aborted" + + +def test_permission_mapping(): + assert tool_args("full", ()) == ("--tools", FULL_TOOLS) + assert tool_args("read-only", ()) == ("--tools", "read,grep,find,ls") + assert tool_args("edit", ()) == ("--tools", "read,edit,write,grep,find,ls") + with pytest.raises(CapabilityUnsupported): + tool_args("ask", ()) + + +def test_disable_tools_map_to_native_excludes(): + args: Final = tool_args("full", ["bash", "glob", "write", "mcp__x"]) + assert flag(args, "--tools") == FULL_TOOLS + assert flag(args, "--exclude-tools") == "bash,powershell,find,write,mcp__x" + + +def test_mcp_servers_keep_tools_reachable(): + assert mcp_tool_entries({}) == () + assert mcp_tool_entries({"mcpServers": {}}) == () + assert mcp_tool_entries({"mcpServers": {"moyai": MOYAI}}) == ("mcp__*",) + assert mcp_tool_entries({"mcpServers": {"s": {"command": "x"}}}) == ("mcp__*", "codemode") + assert mcp_tool_entries({"mcpServers": {"s": {"url": "u", "exposure": "deferred"}}}) == ( + "mcp__*", + "tool_search", + ) + assert mcp_tool_entries( + {"mcpServers": {"s": {"command": "x", "exposure": "hidden", "toolExposure": {"a": "direct", "b": "deferred"}}}} + ) == ("mcp__*", "tool_search") + args: Final = tool_args("read-only", ["mcp__moyai__drop"], mcp_tool_entries({"mcpServers": {"moyai": MOYAI}})) + assert flag(args, "--tools") == "read,grep,find,ls,mcp__*" + assert flag(args, "--exclude-tools") == "mcp__moyai__drop" + + +def test_config_files_split_settings_and_mcp(): + files: Final = dict(config_files({"mcpServers": {"moyai": MOYAI}, "compaction": {"enabled": False}})) + assert json.loads(files[MCP_FILENAME]) == {"mcpServers": {"moyai": MOYAI}} + assert json.loads(files[SETTINGS_FILENAME]) == {"compaction": {"enabled": False}} + assert dict(config_files({"mcpServers": {"moyai": MOYAI}})).keys() == {MCP_FILENAME} + assert dict(config_files({"theme": "dark"})).keys() == {SETTINGS_FILENAME} + assert config_files({}) == () + + +def test_config_files_reject_values_that_are_not_json(): + with pytest.raises(TypeError, match=r"set in PiOptions\.config is not JSON serializable"): + config_files({"theme": {"dark", "light"}}) + + +@pytest.mark.parametrize( + "config", + [ + *({key: "x"} for key in sorted(MANAGED_CONFIG_KEYS)), + {"mcpServers": "not-a-mapping"}, + {"mcpServers": {"moyai": "not-a-mapping"}}, + ], +) +def test_managed_keys_rejected(config: Mapping[str, object]) -> None: + with pytest.raises(OptionsMismatch): + validate_user_config(config) + + +def test_options_config_cannot_load_code_or_reroute_model(): + for key in ("extensions", "packages", "defaultProjectTrust", "defaultProvider", "defaultModel", "httpProxy"): + assert key in MANAGED_CONFIG_KEYS + with pytest.raises(OptionsMismatch, match="extensions"): + validate_user_config({"extensions": ["./evil.ts"]}) + + +def test_build_models_json(): + models: Final = json.loads(build_models_json("m1", "http://h:1/v1", None, None)) + assert models == { + "providers": { + "litellm": { + "baseUrl": "http://h:1/v1", + "api": "openai-completions", + "apiKey": f"${PI_TOKEN_ENV}", + "compat": {"supportsStore": False}, + "models": [{"id": "m1"}], + } + } + }, ( + "an unmapped model keeps pi's own limits and no cache_control marks, " + "and pi must not send `store`, which non-OpenAI providers reject" + ) + + +def test_mapped_model_declares_litellm_context_and_output_limits() -> None: + info: Final = get_model_info(MODEL) + models: Final = setup_models(setup_for(make_ctx())) + declared: Final = models["providers"]["litellm"]["models"][0] + assert (declared["contextWindow"], declared["maxTokens"]) == ( + info["max_input_tokens"], + min(info["max_output_tokens"], PI_DEFAULT_MAX_TOKENS), + ) + + +@pytest.mark.parametrize("max_output_tokens", [PI_DEFAULT_MAX_TOKENS // 4, PI_DEFAULT_MAX_TOKENS * 8]) +def test_pi_asks_for_no_more_output_tokens_than_the_model_or_its_own_default(max_output_tokens: int) -> None: + info: Final = ModelInfo(**{**get_model_info(MODEL), "max_output_tokens": max_output_tokens}) + declared: Final = json.loads(build_models_json(MODEL, "http://h:1/v1", None, info))["providers"]["litellm"] + assert declared["models"][0]["maxTokens"] == min(max_output_tokens, PI_DEFAULT_MAX_TOKENS), ( + "pi sends maxTokens as max_completion_tokens on every call, and the gateway reserves it against the key's " + "TPM limit, so a model's full output cap made keys under that limit answer 429" + ) + + +@pytest.mark.parametrize("info", [get_model_info(MODEL), None], ids=["mapped", "unmapped"]) +@pytest.mark.parametrize("thinking", ["minimal", "low", "medium", "high", "xhigh", "max"]) +def test_thinking_budget_leaves_pi_its_default_room_for_the_answer( + thinking: PiThinkingLevel, info: ModelInfo | None +) -> None: + declared: Final = json.loads(build_models_json(MODEL, "http://h:1/v1", thinking, info))["providers"]["litellm"] + sent: Final = AnthropicConfig().map_openai_params( + {"reasoning_effort": thinking, "max_completion_tokens": declared["models"][0]["maxTokens"]}, + {}, + MODEL, + drop_params=False, + ) + assert sent["max_tokens"] - sent["thinking"]["budget_tokens"] >= PI_DEFAULT_MAX_TOKENS, ( + "Anthropic answers 400 unless max_tokens is above the thinking budget LiteLLM sends for the level, " + "so a budget as large as pi's default left thinking=max with no request it could make" + ) + assert sent["max_tokens"] <= get_model_info(MODEL)["max_output_tokens"] + + +def test_model_that_supports_prompt_caching_asks_pi_for_cache_control_marks() -> None: + assert get_model_info(MODEL).get("supports_prompt_caching") is True + declared: Final = setup_models(setup_for(make_ctx()))["providers"]["litellm"]["models"][0] + assert declared["compat"] == {"cacheControlFormat": "anthropic"}, ( + "pi only marks cache_control for OpenRouter Anthropic models by itself, so every Claude call missed the cache" + ) + + +GATEWAY: Final = GatewayTarget(api_base="http://gateway:4000", api_key="sk-gateway") + + +@pytest.mark.parametrize( + ("group", "declared_limits"), + [ + (MODEL, {"contextWindow": get_model_info(MODEL)["max_input_tokens"]}), + ("team-default-model", {}), + ], +) +def test_gateway_group_is_looked_up_without_printing_to_stdout( + group: str, declared_limits: Mapping[str, int], capsys: pytest.CaptureFixture[str] +) -> None: + declared: Final = setup_models(setup_for(make_ctx(model=group, gateway=GATEWAY)))["providers"]["litellm"]["models"][ + 0 + ] + assert {key: declared[key] for key in declared_limits} == declared_limits + assert ("maxTokens" in declared) == bool(declared_limits) + assert capsys.readouterr().out == "", "an unmapped gateway group printed LiteLLM's provider-list banner on stdout" + + +MAPPED_MODEL: Final = { + "id": MODEL, + "contextWindow": get_model_info(MODEL)["max_input_tokens"], + "maxTokens": min(get_model_info(MODEL)["max_output_tokens"], PI_DEFAULT_MAX_TOKENS), + "compat": {"cacheControlFormat": "anthropic"}, +} +REASONING_MODEL: Final = {**MAPPED_MODEL, "reasoning": True, "thinkingLevelMap": {"xhigh": "xhigh", "max": "max"}} + + +@pytest.mark.parametrize( + ("thinking", "declared_model"), + [ + (None, MAPPED_MODEL), + ("off", MAPPED_MODEL), + ("minimal", REASONING_MODEL), + ("high", REASONING_MODEL), + ("xhigh", REASONING_MODEL), + ("max", REASONING_MODEL), + ], +) +def test_thinking_declares_a_reasoning_model_only_when_a_level_is_requested( + thinking: PiThinkingLevel | None, declared_model: Mapping[str, object] +) -> None: + models: Final = setup_models(setup_for(make_ctx(options=PiOptions(thinking=thinking)))) + declared: Final = {key: value for key, value in models["providers"]["litellm"]["models"][0].items() if key != "maxTokens"} + assert declared == {key: value for key, value in declared_model.items() if key != "maxTokens"}, ( + "pi drops --thinking to off unless the model declares reasoning, and drops xhigh/max to high " + "unless thinkingLevelMap names them; declaring reasoning with no level makes pi send its default effort" + ) + + +def test_config_metadata(): + assert CONFIG.get_binary() == "pi" + assert "@earendil-works/pi-coding-agent" in CONFIG.get_install_hint() + assert CONFIG.uses_model_endpoint is True + assert CONFIG.capabilities.permission_modes == {"read-only", "edit", "full"} + assert CONFIG.capabilities.resume is True + assert CONFIG.capabilities.tool_filtering is True + assert isinstance(ProviderConfigManager.get_provider_harness_config(Harness.PI), PiHarnessConfig) + + +def test_validate_environment_rejects_wrong_options_managed_config_and_ask_mode(): + CONFIG.validate_environment(make_ctx()) + CONFIG.validate_environment(make_ctx(options=PiOptions(config={"mcpServers": {"moyai": MOYAI}}))) + with pytest.raises(OptionsMismatch): + CONFIG.validate_environment(make_ctx(options=OpenCodeOptions())) + with pytest.raises(OptionsMismatch, match="defaultModel"): + CONFIG.validate_environment(make_ctx(options=PiOptions(config={"defaultModel": "openai/x"}))) + with pytest.raises(CapabilityUnsupported): + CONFIG.validate_environment(make_ctx(permissions="ask")) + + +@pytest.mark.parametrize("key", sorted(MANAGED_ENV_KEYS)) +def test_managed_env_keys_rejected_instead_of_silently_ignored(key: str) -> None: + assert key in {PI_CONFIG_DIR_ENV, PI_TOKEN_ENV} + with pytest.raises(OptionsMismatch, match=key): + CONFIG.validate_environment(make_ctx(options=PiOptions(env={key: "/tmp/evil"}))) + CONFIG.validate_environment(make_ctx(options=PiOptions(env={f"{key}_SUFFIX": "ok"}))) + + +def test_session_setup_token_only_in_env(): + setup: Final = setup_for(make_ctx()) + assert set(setup.files) == {MODELS_FILENAME} + assert all(TOKEN.encode() not in data for data in setup.files.values()) + assert setup.env[PI_TOKEN_ENV] == TOKEN + provider: Final = setup_models(setup)["providers"]["litellm"] + assert provider["baseUrl"] == "http://host.docker.internal:4555/v1" + assert provider["apiKey"] == f"${PI_TOKEN_ENV}" + + +def test_session_setup_env_and_persisted_sessions(): + options: Final = PiOptions(env={"FOO": "1", PI_TOKEN_ENV: "evil", PI_CONFIG_DIR_ENV: "/home/u/.pi/agent"}) + setup: Final = setup_for(make_ctx(options=options)) + assert list(setup.persisted_dirs) == [("sessions", "pi/sessions")] + assert setup.skills_dir == "skills" + env: Final = setup.env + assert env[PI_CONFIG_DIR_ENV] == f"{PRIVATE}/agent", "managed keys must win over PiOptions.env" + assert env[PI_TOKEN_ENV] == TOKEN, "managed keys must win over PiOptions.env" + for key, value in PI_ISOLATION_ENV.items(): + assert env[key] == value + assert env["FOO"] == "1" + + +def test_session_setup_writes_user_config_next_to_managed_models(): + options: Final = PiOptions(config={"mcpServers": {"moyai": MOYAI}, "compaction": {"enabled": False}}) + setup: Final = setup_for(make_ctx(options=options)) + assert set(setup.files) == {MODELS_FILENAME, MCP_FILENAME, SETTINGS_FILENAME} + assert json.loads(setup.files[MCP_FILENAME]) == {"mcpServers": {"moyai": MOYAI}} + assert setup_models(setup)["providers"]["litellm"]["apiKey"] == f"${PI_TOKEN_ENV}" + + +def test_session_setup_errors(): + with pytest.raises(HarnessError): + setup_for(make_ctx(endpoint=None)) + with pytest.raises(ValueError, match="needs model="): + setup_for(make_ctx(model=None, endpoint=FakeEndpoint(model=None))) + + +def test_turn_request_without_a_model_raises_instead_of_passing_none(): + ctx: Final = make_ctx(model=None, endpoint=FakeEndpoint(model=None)) + with pytest.raises(ValueError, match="needs model="): + CONFIG.transform_turn_request(ctx, HarnessSessionSetup(), PRIVATE, "hi", None) + + +def test_session_setup_instructions_and_skills(): + ctx: Final = make_ctx(instructions="Be terse.", output=Answer, skills=["/s/greeter"]) + setup: Final = setup_for(ctx) + written: Final = setup.files[INSTRUCTIONS_FILENAME].decode() + assert written.startswith("Be terse.") + assert '"city"' in written and "single JSON object" in written + argv: Final = CONFIG.transform_turn_request(ctx, setup, PRIVATE, "hi", None).argv + assert flag(argv, "--append-system-prompt") == f"{PRIVATE}/instructions.md" + assert flag(argv, "--skill") == f"{PRIVATE}/skills" + plain: Final = make_ctx() + plain_setup: Final = setup_for(plain) + plain_argv: Final = CONFIG.transform_turn_request(plain, plain_setup, PRIVATE, "hi", None).argv + assert INSTRUCTIONS_FILENAME not in plain_setup.files + assert "--append-system-prompt" not in plain_argv and "--skill" not in plain_argv + + +def test_turn_request_argv_and_session_continuation(): + ctx: Final = make_ctx(options=PiOptions(thinking="high")) + setup: Final = setup_for(ctx) + first: Final = CONFIG.transform_turn_request(ctx, setup, PRIVATE, "hello", None) + assert list(first.argv) == [ + "pi", + "--mode", + "json", + "--no-approve", + "--no-skills", + "--provider", + "litellm", + "--model", + MODEL, + "--session-dir", + f"{PRIVATE}/sessions", + "--tools", + FULL_TOOLS, + "--thinking", + "high", + ] + assert first.cwd == "/work" + assert first.stdin == "hello" + assert first.env == setup.env + second: Final = CONFIG.transform_turn_request(ctx, setup, PRIVATE, "again", SESSION) + assert flag(second.argv, "--session") == SESSION + assert "again" not in " ".join(second.argv) + + +def test_turn_prompt_repeats_schema_when_output_set(): + ctx: Final = make_ctx(output=Answer) + request: Final = CONFIG.transform_turn_request(ctx, setup_for(ctx), PRIVATE, "hi", None) + assert request.stdin.startswith("hi\n\n") and '"city"' in request.stdin + + +def test_turn_response_paths(): + _, state = parse_all("structured_output.jsonl") + ok: Final = CONFIG.transform_turn_response(make_ctx(output=Answer), state, 0, []) + assert json.loads(ok.output_json) == {"city": "Paris", "country": "France"} + plain: Final = CONFIG.transform_turn_response(make_ctx(), state, 0, []) + assert plain.output_json is None and plain.final_text == state.final_text + with pytest.raises(HarnessTurnError, match="boom"): + CONFIG.transform_turn_response(make_ctx(), PiStreamState(stop_reason="error", error="boom"), 0, []) + with pytest.raises(HarnessTurnError, match="aborted"): + CONFIG.transform_turn_response(make_ctx(), PiStreamState(stop_reason="aborted"), 0, []) + with pytest.raises(HarnessTurnError, match="code 3: no output"): + CONFIG.transform_turn_response(make_ctx(), PiStreamState(), 3, []) + + +def test_reply_cut_off_at_the_output_token_limit_is_an_error(): + state: Final = PiStreamState() + parse(assistant_end("partial", "length"), state) + with pytest.raises(HarnessTurnError, match="output token limit"): + CONFIG.transform_turn_response(make_ctx(), state, 0, []) + + +def test_exit_zero_without_a_reply_is_an_error(): + with pytest.raises(HarnessTurnError, match="without a reply: Session found in different project"): + CONFIG.transform_turn_response(make_ctx(), PiStreamState(), 0, ["Session found in different project: /a"]) + + +async def test_start_writes_token_only_in_env(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("turn1_write_read.jsonl"),)) + ctx: Final = make_ctx(sandbox) + handler: Final = await started(ctx) + assert written_models(sandbox)["providers"]["litellm"]["apiKey"] == f"${PI_TOKEN_ENV}" + assert all(TOKEN.encode() not in data for data in sandbox.files.values()) + await collect(handler, ctx, "create hello.txt containing hi then read it") + assert TOKEN not in json.dumps(list(sandbox.execs[0].cmd)) + assert sandbox.execs[0].env[PI_TOKEN_ENV] == TOKEN + + +async def test_start_persists_sessions_dir(): + sandbox: Final = FakeSandbox() + await started(make_ctx(sandbox)) + assert sandbox.runs == (["sh", "-c", PERSIST_DIR_SCRIPT, "sh", "/tmp/pi-1/sessions", "pi/sessions"],) + + +async def test_persist_failure_still_uses_private_sessions(): + sandbox: Final = FakeSandbox(persist_ok=False, outputs=(fixture_proc("turn1_write_read.jsonl"),)) + ctx: Final = make_ctx(sandbox) + await collect(await started(ctx), ctx, "x") + assert flag(sandbox.execs[0].cmd, "--session-dir") == "/tmp/pi-1/sessions" + + +async def test_turn_argv_env_and_session_continuation(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("turn1_write_read.jsonl"), fixture_proc("turn2_resume.jsonl"))) + ctx: Final = make_ctx(sandbox, options=PiOptions(env={"FOO": "1"})) + handler: Final = await started(ctx) + events: Final = await collect(handler, ctx, "create hello.txt containing hi then read it") + first: Final = sandbox.execs[0] + assert first.cmd == [ + "pi", + "--mode", + "json", + "--no-approve", + "--no-skills", + "--provider", + "litellm", + "--model", + MODEL, + "--session-dir", + "/tmp/pi-1/sessions", + "--tools", + FULL_TOOLS, + ] + assert first.cwd == "/work" + assert first.env["PI_CODING_AGENT_DIR"] == "/tmp/pi-1/agent" + assert first.env["PI_OFFLINE"] == "1" and first.env["PI_TELEMETRY"] == "0" + assert first.env["FOO"] == "1" + assert any(isinstance(e, ToolCall) for e in events) + assert ctx.final_text == "Done! hello.txt contains: hi" + + await collect(handler, ctx, "what file did you create?") + second: Final = sandbox.execs[1].cmd + assert flag(second, "--session") == SESSION + assert "what file" not in " ".join(second) + assert ctx.final_text == "You asked me to create hello.txt." + + +async def test_prompt_is_sent_on_stdin_not_argv(): + proc: Final = fixture_proc("turn1_write_read.jsonl") + sandbox: Final = FakeSandbox(outputs=(proc,)) + ctx: Final = make_ctx(sandbox) + await collect(await started(ctx), ctx, "secret prompt text") + assert proc.stdin.data == b"secret prompt text" and proc.stdin.closed + assert "secret prompt text" not in sandbox.execs[0].cmd + + +@pytest.mark.parametrize(("workdir", "resume_flag"), [("/work", "--session"), ("/elsewhere", "--fork")]) +def test_session_saved_in_another_workdir_is_forked_into_this_one(workdir: str, resume_flag: str) -> None: + _, state = parse_all("turn1_write_read.jsonl") + ctx: Final = make_ctx(FakeSandbox(workdir=workdir)) + argv: Final = CONFIG.transform_turn_request( + ctx, setup_for(ctx), PRIVATE, "again", CONFIG.get_native_session_id(state) + ).argv + assert flag(argv, resume_flag) == SESSION + assert ({"--session", "--fork"} - {resume_flag}).isdisjoint(argv), ( + "pi answers --session for a session saved in another directory with a fork prompt the harness never answers" + ) + + +async def test_resume_sets_session(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("turn2_resume.jsonl"),)) + ctx: Final = make_ctx(sandbox) + handler: Final = await started(ctx) + await handler.resume(ctx, "prev-session") + await collect(handler, ctx, "hi") + assert flag(sandbox.execs[0].cmd, "--session") == "prev-session" + + +async def test_read_only_and_disable_tools_argv(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("readonly_denied_bash.jsonl"),)) + ctx: Final = make_ctx(sandbox, permissions="read-only", disable_tools=("grep",)) + events: Final = await collect(await started(ctx), ctx, "x") + cmd: Final = sandbox.execs[0].cmd + assert flag(cmd, "--tools") == "read,grep,find,ls" + assert flag(cmd, "--exclude-tools") == "grep" + assert ToolResult(id="call_b1", output="Tool bash not found", is_error=True) in events + + +async def test_instructions_and_structured_output(): + proc: Final = fixture_proc("structured_output.jsonl") + sandbox: Final = FakeSandbox(outputs=(proc,)) + ctx: Final = make_ctx(sandbox, instructions="Be terse.", output=Answer) + handler: Final = await started(ctx) + written: Final = sandbox.files["/tmp/pi-1/instructions.md"].decode() + assert written.startswith("Be terse.") + assert '"city"' in written and "single JSON object" in written + await collect(handler, ctx, "x") + assert flag(sandbox.execs[0].cmd, "--append-system-prompt") == "/tmp/pi-1/instructions.md" + assert '"city"' in proc.stdin.data.decode() + assert json.loads(ctx.output_json) == {"city": "Paris", "country": "France"} + + +async def test_skills_copied_to_private_skills_path(tmp_path: Path) -> None: + skill: Final = tmp_path / "greeter" + (skill / "ref").mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: greeter\ndescription: d\n---\nbody") + (skill / "ref" / "notes.txt").write_text("n") + sandbox: Final = FakeSandbox(outputs=(fixture_proc("turn2_resume.jsonl"),)) + ctx: Final = make_ctx(sandbox, skills=(str(skill),)) + handler: Final = await started(ctx) + assert sandbox.files["/tmp/pi-1/skills/greeter/SKILL.md"].startswith(b"---") + assert sandbox.files["/tmp/pi-1/skills/greeter/ref/notes.txt"] == b"n" + await collect(handler, ctx, "x") + cmd: Final = sandbox.execs[0].cmd + assert flag(cmd, "--skill") == "/tmp/pi-1/skills" + assert "--no-skills" in cmd + + +async def test_skill_without_manifest_rejected(tmp_path: Path) -> None: + with pytest.raises(ValueError, match=r"SKILL\.md"): + await started(make_ctx(skills=(str(tmp_path),))) + + +async def test_missing_binary(): + with pytest.raises(HarnessInstallFailed, match="@earendil-works/pi-coding-agent"): + await started(make_ctx(FakeSandbox(has_binary=False))) + + +async def test_start_missing_endpoint_raises(): + with pytest.raises(HarnessError): + await started(make_ctx(endpoint=None)) + + +async def test_mcp_servers_written_and_reachable(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("mcp_tool.jsonl"),)) + ctx: Final = make_ctx(sandbox, options=PiOptions(config={"mcpServers": {"moyai": MOYAI}})) + handler: Final = await started(ctx) + assert json.loads(sandbox.files["/tmp/pi-1/agent/mcp.json"]) == {"mcpServers": {"moyai": MOYAI}} + events: Final = await collect(handler, ctx, "call the moyai echo tool") + assert flag(sandbox.execs[0].cmd, "--tools") == f"{FULL_TOOLS},mcp__*" + assert ToolResult(id="call_m1", output="moyai says: hello", is_error=False) in events + + +async def test_wrong_options_and_ask_mode_rejected(): + with pytest.raises(OptionsMismatch): + await started(make_ctx(options=OpenCodeOptions())) + with pytest.raises(OptionsMismatch): + await started(make_ctx(options=PiOptions(config={"sessionDir": "/elsewhere"}))) + with pytest.raises(CapabilityUnsupported): + await started(make_ctx(permissions="ask")) + + +async def test_turn_before_start_raises(): + handler: Final = CLIHarnessHandler(PiHarnessConfig()) + with pytest.raises(RuntimeError, match="before start"): + await collect(handler, make_ctx(), "x") + + +async def test_api_error_raises_even_on_exit_zero(): + ctx: Final = make_ctx(FakeSandbox(outputs=(fixture_proc("api_error.jsonl", exit_code=0),))) + handler: Final = await started(ctx) + with pytest.raises(HarnessTurnError, match="no healthy deployments"): + await collect(handler, ctx, "x") + + +async def test_nonzero_exit_raises_with_stderr_tail(): + failing: Final = FakeProcess(b"", stderr=b"line1\nfatal: bad flag\n", exit_code=2) + ctx: Final = make_ctx(FakeSandbox(outputs=(failing,))) + handler: Final = await started(ctx) + with pytest.raises(HarnessTurnError, match="code 2: line1\nfatal: bad flag"): + await collect(handler, ctx, "x") + + +async def test_early_close_kills_process_and_stop_is_idempotent(): + proc: Final = fixture_proc("turn1_write_read.jsonl") + ctx: Final = make_ctx(FakeSandbox(outputs=(proc,))) + handler: Final = await started(ctx) + gen: Final = handler.turn(ctx, "x") + await gen.__anext__() + await gen.aclose() + assert proc.killed + await handler.stop(ctx) + await handler.stop(ctx) + + +async def test_long_jsonl_line_is_parsed(): + text: Final = "x" * 200_000 + lines: Final = ( + {"type": "session", "version": 3, "id": "s"}, + {"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "delta": text}}, + assistant_end(text, "stop"), + ) + stdout: Final = "\n".join(json.dumps(dict(line)) for line in lines).encode() + ctx: Final = make_ctx(FakeSandbox(outputs=(FakeProcess(stdout),))) + events: Final = await collect(await started(ctx), ctx, "x") + assert events == [Text(delta=text)] + assert ctx.final_text == text + + +async def test_model_falls_back_to_endpoint_model(): + sandbox: Final = FakeSandbox(outputs=(fixture_proc("turn1_write_read.jsonl"),)) + ctx: Final = make_ctx(sandbox, model=None, endpoint=FakeEndpoint(model="gw-model")) + await collect(await started(ctx), ctx, "x") + assert flag(sandbox.execs[0].cmd, "--model") == "gw-model" + assert written_models(sandbox)["providers"]["litellm"]["models"] == [{"id": "gw-model"}] + + +def test_turn_request_never_trusts_project_files(): + ctx: Final = make_ctx() + argv: Final = list(CONFIG.transform_turn_request(ctx, setup_for(ctx), PRIVATE, "hi", None).argv) + assert argv[:4] == ["pi", "--mode", "json", "--no-approve"], ( + "without --no-approve a repo's .pi/extensions run as the host user at startup" + ) From dfac770e2b997384d1b41bef0a531dac31bd5b02 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:09:42 +0000 Subject: [PATCH 084/124] feat(proxy): map baggage values to budget tags (#47358) * feat(proxy): map baggage values to budget tags * fix(proxy): preserve mapped tags in the parsed request cache * fix(proxy): preserve mapper accounting and tag policy * fix(proxy): preserve baggage tags across native transports * fix(proxy): keep mapper rejection and passthrough logging safe * fix(proxy): scope baggage tag mappings to billed inference and keep them out of tag routing Mapped tags now apply only to billed LLM API routes, skipping token counting, model listing, MCP and A2A. They classify spend and budgets but never select deployments unless a key, team or client set the same tag. WebSocket inference returns 403 only while a required rule exists. A tag created with /tag/new resets its spend counter so its budget counts from creation. The General settings editor keeps a new mapping unsaved until every required field is filled in. * fix(proxy): keep client-tag rejection identical with baggage, map A2A calls, skip request previews * fix(proxy): keep mapped tags out of requester metadata, answer 503 on tag lookup errors, map custom pass-through routes, check the key before the WebSocket 403 * fix(proxy): keep caller-sent tags routable, keep mapped tags out of policy and guardrail matching, honor allow_requests_on_db_unavailable, reject control characters in baggage values * test(proxy): cover baggage tag mappings end to end Adds integration cells for request_tag_mappings across chat, messages, responses, A2A and custom pass-through spend, required and budgeted rules, the WebSocket refusal, unbilled routes, reject_clientside_metadata_tags, tag routing and policy isolation, a dropped tag read with and without allow_requests_on_db_unavailable, and the /tag/new spend counter reset * fix(proxy): keep base tag merges for requests that map nothing, accept JSON-string metadata tags, count litellm_metadata caller tags Requests with no mapped baggage now reuse the cached body exactly as before, so header and key tag merges still normalize metadata for routing. The client body copy is made only when a tag maps, and the base merges are mirrored onto it. A request that maps nothing no longer gets a 400 for non-object metadata. caller_tags now counts tags sent in both metadata and litellm_metadata. Optional mappings no longer reject JSON-string metadata tags under reject_clientside_metadata_tags. * fix(router): keep baggage-mapped tags from selecting tag-scoped pre-routing strategies Auto, complexity, adaptive and quality router selection, the consumed-tags stamp and compression policy lookup now read request tags without baggage-mapped tags, unless the caller sent the tag or the key, team or project carries it, matching deployment tag filtering. * feat(proxy): parse baggage headers into request tag mappings * fix(proxy): match plus-sign baggage keys and keep deployment tags in guardrail modes * fix(proxy): keep baggage-mapped tags behind key, team and deployment tags and serve non-object or invalid DB mapping settings like unset Mapped tags are added after the key, team, deployment and caller tags, so a tag-mode guardrail picks the same mode with or without baggage. A non-object metadata field maps like empty metadata, and an invalid request_tag_mappings value loaded from the DB is ignored with an ERROR log instead of failing every request * fix(router): merge deployment tags in linear time and keep malformed client tags as sent under mapped baggage Merge serving deployment tags into the request tags in one pass, so a large client tag list no longer blocks the event loop. Leave a non-list metadata.tags untouched when baggage maps a tag, so the request gets the same answer as without baggage. Add the C17 integration cell for the team tag's guardrail mode under a mapped deployment tag. * test(spend): cover string metadata.tags under mapped baggage * fix(proxy): check mapped tag budgets when metadata.tags is a non-list or holds non-strings Mapped baggage tags are applied again after the header and key tag merges, so a header or key tag that replaces a string metadata.tags no longer hides the mapped tag from the tag budget check. An empty or false metadata.tags is read like a missing one, and a tags list with non-string items is kept as sent with the mapped tags added after it, instead of failing auth with a 401 --- .../integrations/custom_guardrail.py | 16 +- litellm/constants.py | 1 + litellm/proxy/_types.py | 15 + litellm/proxy/auth/auth_checks.py | 102 ++- litellm/proxy/auth/route_checks.py | 20 + litellm/proxy/auth/user_api_key_auth.py | 54 +- litellm/proxy/common_utils/callback_utils.py | 2 + .../proxy/common_utils/http_parsing_utils.py | 10 + .../common_utils/request_tag_mappings.py | 67 ++ .../guardrails/auto_router_compression.py | 6 +- litellm/proxy/litellm_pre_call_utils.py | 219 ++++- .../tag_management_endpoints.py | 2 + .../llm_passthrough_endpoints.py | 19 +- .../pass_through_endpoints.py | 10 +- litellm/proxy/proxy_server.py | 16 + .../spend_tracking/budget_reservation.py | 25 +- litellm/router.py | 24 +- litellm/router_strategy/tag_based_routing.py | 89 +- litellm/types/proxy/request_tag_mappings.py | 27 + litellm/types/router.py | 12 + .../spend/test_baggage_tag_mappings.py | 847 ++++++++++++++++++ .../spend/test_tag_budget_enforcement.py | 50 +- .../integrations/test_custom_guardrail.py | 81 ++ ...st_auth_checks_object_access_and_lookup.py | 160 ++++ .../unit/proxy/auth/test_user_api_key_auth.py | 180 ++++ .../common_utils/test_request_tag_mappings.py | 473 ++++++++++ .../test_tag_management_endpoints.py | 60 ++ .../test_llm_passthrough_endpoints.py | 36 + .../test_pass_through_endpoints.py | 36 + .../unit/proxy/test_litellm_pre_call_utils.py | 146 +++ ...test_proxy_server_endpoints_and_startup.py | 31 + .../test_router_tag_routing.py | 106 +++ tests/unit/test_router/test_router.py | 20 + .../_components/RequestTagMappings.test.tsx | 145 +++ .../_components/RequestTagMappings.tsx | 189 ++++ .../general_settings.integration.test.tsx | 30 + .../_components/general_settings.tsx | 37 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 35 + 38 files changed, 3267 insertions(+), 131 deletions(-) create mode 100644 litellm/proxy/common_utils/request_tag_mappings.py create mode 100644 litellm/types/proxy/request_tag_mappings.py create mode 100644 tests/integration/spend/test_baggage_tag_mappings.py create mode 100644 tests/unit/proxy/common_utils/test_request_tag_mappings.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.tsx diff --git a/enterprise/litellm_enterprise/integrations/custom_guardrail.py b/enterprise/litellm_enterprise/integrations/custom_guardrail.py index cbac4cb5b15..58c24219b47 100644 --- a/enterprise/litellm_enterprise/integrations/custom_guardrail.py +++ b/enterprise/litellm_enterprise/integrations/custom_guardrail.py @@ -20,9 +20,15 @@ class EnterpriseCustomGuardrailHelper: the tag's value (the mode for that tag). - If no request tag matches, fall back to default mode(s). """ + from litellm.litellm_core_utils.core_helpers import ( + get_metadata_variable_name_from_kwargs, + ) from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) + from litellm.router_strategy.tag_based_routing import ( + without_baggage_mapped_tags, + ) from litellm.proxy._types import CommonProxyErrors from litellm.proxy.proxy_server import premium_user @@ -36,9 +42,13 @@ class EnterpriseCustomGuardrailHelper: proxy_server_request = data.get("proxy_server_request", {}) - request_tags = StandardLoggingPayloadSetup.get_request_tags( - litellm_params=data, - proxy_server_request=proxy_server_request, + request_tags = without_baggage_mapped_tags( + StandardLoggingPayloadSetup.get_request_tags( + litellm_params=data, + proxy_server_request=proxy_server_request, + ), + data.get(get_metadata_variable_name_from_kwargs(data)), + keep_serving_deployment_tags=True, ) # Check if any request tag matches a Mode tag key diff --git a/litellm/constants.py b/litellm/constants.py index c7673db6976..bc94760b187 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1645,6 +1645,7 @@ OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_to CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" +BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e4f77ac0005..523f478d771 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -86,6 +86,7 @@ from litellm.types.proxy.carried_budget_state import ( UserBudgetSnapshot as UserBudgetSnapshot, # noqa: PLC0414 # public re-export ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry +from litellm.types.proxy.request_tag_mappings import RequestTagMapping from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterConfig from litellm.types.router_weights import validate_router_settings_dict @@ -2457,6 +2458,20 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ + request_tag_mappings: Sequence[RequestTagMapping] | None = Field( + None, + description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.", + ) + + @field_validator("request_tag_mappings") + @classmethod + def validate_request_tag_mapping_duplicates( + cls, rules: Sequence[RequestTagMapping] | None + ) -> Sequence[RequestTagMapping] | None: + if rules is not None and len({(rule.key, rule.value) for rule in rules}) != len(rules): + raise ValueError("Only one mapping is allowed for each baggage key/value, including its any-value fallback") + return rules + completion_model: str | None = Field(None, description="proxy level default model for all chat completion calls") max_in_flight_requests_per_worker: int | None = Field( None, gt=0, description="maximum concurrent requests handled by each worker" diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 0f94d2952ec..5a09a0efbf6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -877,18 +877,27 @@ def _enforce_user_param_check(general_settings: dict, request: Request, request_ raise Exception(f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}") -def _reject_clientside_metadata_tags_check(general_settings: dict, request_body: dict, route: str) -> None: +def _reject_clientside_metadata_tags_check( + general_settings: dict, request_body: dict, route: str, request: Request | None = None +) -> None: if not general_settings.get("reject_clientside_metadata_tags", False): return - if ( - RouteChecks.is_llm_api_route(route=route) - and "metadata" in request_body - and isinstance(request_body["metadata"], dict) - and "tags" in request_body["metadata"] - ): + original_client_tags: Final = ( + getattr(request.state, "litellm_baggage_client_metadata_tags", None) if request is not None else None + ) + has_client_tags: Final = ( + original_client_tags + if isinstance(original_client_tags, bool) + else ( + "metadata" in request_body + and isinstance(request_body["metadata"], dict) + and "tags" in request_body["metadata"] + ) + ) + if RouteChecks.is_llm_api_route(route=route) and has_client_tags: raise ProxyException( - message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.", + message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via key or team metadata, or by configured request_tag_mappings.", type=ProxyErrorTypes.bad_request_error, param="metadata.tags", code=status.HTTP_400_BAD_REQUEST, @@ -1337,24 +1346,19 @@ async def common_checks( ) # Run before apply_key_tags_pre_auth injects key metadata.tags into request_body. - _reject_clientside_metadata_tags_check(general_settings, request_body, route) + _reject_clientside_metadata_tags_check(general_settings, request_body, route, request) # If this is a free model, skip all budget checks if not skip_all_budget_checks: # Key metadata.tags are injected into request_body here so the tag budget # check can read them; this mutation must run before the gathered checks. + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + if valid_token is not None: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - - LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route( - request_data=request_body, - route=route, - ) - - LiteLLMProxyRequestSetup.apply_key_tags_pre_auth( - request_data=request_body, - user_api_key_dict=valid_token, - ) + for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body): + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route) + LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token) + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body) async def _user_max_budget_check() -> None: # 4.1 personal budget @@ -2456,7 +2460,7 @@ async def _fetch_uncached_tags( uncached_tags: Sequence[str], prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache, -) -> tuple[tuple[str, LiteLLM_TagTable], ...]: +) -> tuple[tuple[str, LiteLLM_TagTable], ...] | Exception: """Rows for the tags a cache probe missed; names absent from the registry never reach the DB.""" if not uncached_tags: return () @@ -2484,13 +2488,46 @@ async def _fetch_uncached_tags( model_type=LiteLLM_TagTable, ttl=get_management_object_ttl(user_api_key_cache), ) - except Exception as e: # noqa: BLE001 # fail-safe: a tag fetch error must yield "no budget objects", never break auth - verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e) - return () + except Exception as e: # noqa: BLE001 # returned as a value: budget checks fail open, the baggage mapper fails closed + return e else: return fetched +async def _probe_and_fetch_tags( + tag_names: Sequence[str], + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> tuple[Mapping[str, LiteLLM_TagTable], Exception | None]: + probed: Final = [ + ( + tag_name, + await user_api_key_cache.async_get_cache(key=tag_cache_key(tag_name), model_type=LiteLLM_TagTable), + ) + for tag_name in tag_names + ] + fetched: Final = await _fetch_uncached_tags( + uncached_tags=tuple(tag_name for tag_name, tag_obj in probed if tag_obj is None), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + rows: Final = () if isinstance(fetched, Exception) else fetched + found: Final = {tag_name: tag_obj for tag_name, tag_obj in (*probed, *rows) if tag_obj is not None} + return found, fetched if isinstance(fetched, Exception) else None + + +@log_db_metrics +@with_service_target(AUTH_OBJECTS_TARGET) +async def read_tag_objects_batch( + tag_names: Sequence[str], + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> Mapping[str, LiteLLM_TagTable] | Exception: + """Like ``get_tag_objects_batch``, but a failed DB read comes back as the error instead of missing tags.""" + found, error = await _probe_and_fetch_tags(tag_names, prisma_client, user_api_key_cache) + return found if error is None else error + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def get_tag_objects_batch( @@ -2521,19 +2558,10 @@ async def get_tag_objects_batch( if prisma_client is None or not tag_names: return {} - probed: Final = [ - ( - tag_name, - await user_api_key_cache.async_get_cache(key=tag_cache_key(tag_name), model_type=LiteLLM_TagTable), - ) - for tag_name in tag_names - ] - fetched: Final = await _fetch_uncached_tags( - uncached_tags=tuple(tag_name for tag_name, tag_obj in probed if tag_obj is None), - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - ) - return {tag_name: tag_obj for tag_name, tag_obj in (*probed, *fetched) if tag_obj is not None} + found, error = await _probe_and_fetch_tags(tag_names, prisma_client, user_api_key_cache) + if error is not None: + verbose_proxy_logger.debug("Error batch fetching tags from database: %s", error) + return dict(found) @log_db_metrics diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index f5e8304c83c..720bfabb95a 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -91,6 +91,21 @@ _SELF_SCOPED_CREDENTIAL_ROUTES: Final = ( ) +_UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( + { + "/models", + "/v1/models", + "/utils/token_counter", + "/utils/transform_request", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + } +) +_TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "count-tokens"}) +_TOKEN_COUNTING_ACTION: Final = "countTokens" + + class RouteChecks: @staticmethod def should_call_route( @@ -412,6 +427,11 @@ class RouteChecks: detail=f"user not allowed to access this route. Route={route} is an admin only route", ) + @staticmethod + def is_unbilled_route(route: str) -> bool: + resource, _, action = route.rsplit("/", 1)[-1].partition(":") + return route in _UNBILLED_ROUTES or resource in _TOKEN_COUNTING_SEGMENTS or action == _TOKEN_COUNTING_ACTION + @staticmethod def is_llm_api_route(route: str) -> bool: """ diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a1abed202a2..27fa4731baf 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -18,6 +18,7 @@ from typing import Final, NamedTuple, Protocol, Union, cast import fastapi from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader +from pydantic import TypeAdapter from starlette.exceptions import WebSocketException import litellm @@ -268,6 +269,7 @@ _CLAUDE_MODEL_ROUTES: Final = frozenset( f"/{prefix}{endpoint}" for prefix in ("", "v1/") for endpoint in ("messages", "chat/completions", "responses") ) _CLAUDE_MODEL_NORMALIZED: Final = "litellm.claude_model_normalized" +_GENERAL_SETTINGS_ADAPTER: Final = TypeAdapter(dict[str, object]) async def _normalize_claude_model( @@ -1185,6 +1187,18 @@ async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAu return await user_api_key_auth_websocket_for_model(websocket, model=None) +def _has_required_baggage_tag_mapping() -> bool: + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.request_tag_mappings import REQUEST_TAG_MAPPINGS_ADAPTER + + raw_mappings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_server.general_settings).get( + "request_tag_mappings" + ) + return bool(raw_mappings) and any( + rule.required for rule in REQUEST_TAG_MAPPINGS_ADAPTER.validate_python(raw_mappings) + ) + + async def user_api_key_auth_websocket_for_model( websocket: WebSocket, model: str | None, *, close_on_auth_error: bool = True ) -> UserAPIKeyAuth: @@ -1240,7 +1254,11 @@ async def user_api_key_auth_websocket_for_model( # Call user_api_key_auth with the extracted API key # Note: You'll need to modify this to work with WebSocket context if needed try: - return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}") + authorized: Final = await user_api_key_auth(request=request, api_key=f"Bearer {api_key}") + if _has_required_baggage_tag_mapping(): + raise HTTPException( + 403, "Required baggage tag mappings need HTTP inference, WebSocket inference is disabled" + ) except Exception as e: if not close_on_auth_error: raise @@ -1249,6 +1267,8 @@ async def user_api_key_auth_websocket_for_model( log_model_access_denial(e) await websocket.close(code=status.WS_1008_POLICY_VIOLATION) raise HTTPException(status_code=403, detail=str(e)) + else: + return authorized async def user_api_key_auth_accepted_websocket_for_model(websocket: WebSocket, model: str) -> UserAPIKeyAuth: @@ -3759,20 +3779,17 @@ async def run_centralized_common_checks( # later seed in common_checks pushes key tags and the # _tag_max_budget_check read into `litellm_metadata`, hiding header # tags from per-tag budget enforcement on LITELLM_METADATA_ROUTES. - LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route( - request_data=request_data, - route=route, - ) - # Merge x-litellm-tags into request_data BEFORE common_checks runs. # _tag_max_budget_check inside common_checks only inspects request_data; # without this pre-merge, header-supplied tags bypass tag-budget # enforcement. - LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( - request=request, - request_data=request_data, - user_api_key_dict=user_api_key_auth_obj, - ) + for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_data): + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route) + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( + request=request, + request_data=tag_target, + user_api_key_dict=user_api_key_auth_obj, + ) bind_admission_counter_keys(user_api_key_auth_obj, end_user_id=end_user_id) try: @@ -4001,7 +4018,7 @@ def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> async def _authorize_authenticated_request( user_api_key_auth_obj: UserAPIKeyAuth, request: Request, - request_data: dict, + request_data: dict[str, object], route: str, api_key: str, *, @@ -4061,6 +4078,19 @@ async def _authorize_authenticated_request( billable=request_data.get("method") in (None, "message/send", "message/stream", "SendMessage", "SendStreamingMessage"), ) + if "request_tag_mappings" in general_settings and request.method == "POST": + from litellm.proxy.common_utils.request_tag_mappings import is_baggage_mapped_route + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + + if is_baggage_mapped_route(route): + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=authorized_data, route=route) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request=request, + request_data=authorized_data, + general_settings=_GENERAL_SETTINGS_ADAPTER.validate_python(general_settings), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) await run_centralized_common_checks( user_api_key_auth_obj=user_api_key_auth_obj, request=request, diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4fa7709e5c7..46434a997b7 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -12,6 +12,7 @@ import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH, @@ -543,6 +544,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + BAGGAGE_MAPPED_TAGS_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, "disable_global_guardrails", diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index ea33bc763d4..521a9a15bd0 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -372,6 +372,16 @@ def safe_set_request_parsed_body( _safe_set_request_parsed_body: Final = safe_set_request_parsed_body +def detach_request_parsed_body( + request: Request, request_data: Mapping[str, object] +) -> dict[str, object]: # mutable-ok: the client's body copy, which the base tag merges write into + client_body: Final = dict(request_data) + cached: Final[object] = request.scope.get("parsed_body") + if isinstance(cached, tuple) and cached[1] is request_data: + request.scope["parsed_body"] = (cached[0], client_body) # rebind-ok: cache holds the client copy + return client_body + + def rewrite_request_model( request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader request: Request | None, diff --git a/litellm/proxy/common_utils/request_tag_mappings.py b/litellm/proxy/common_utils/request_tag_mappings.py new file mode 100644 index 00000000000..dca269dca16 --- /dev/null +++ b/litellm/proxy/common_utils/request_tag_mappings.py @@ -0,0 +1,67 @@ +from collections.abc import Mapping, Sequence +from typing import Final +from urllib.parse import unquote_plus + +from fastapi import status +from opentelemetry import baggage +from opentelemetry.baggage.propagation import W3CBaggagePropagator +from opentelemetry.context import Context +from pydantic import TypeAdapter + +from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException +from litellm.proxy.auth.route_checks import RouteChecks +from litellm.types.proxy.request_tag_mappings import RequestTagMapping + +REQUEST_TAG_MAPPINGS_ADAPTER: Final = TypeAdapter(tuple[RequestTagMapping, ...]) + + +def is_baggage_mapped_route(route: str) -> bool: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + return ( + (RouteChecks.is_llm_api_route(route) or InitPassThroughEndpointHelpers.is_registered_pass_through_route(route)) + and not RouteChecks.is_unbilled_route(route) + and not RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.mcp_inference_routes.value) + ) + + +def mapping_rejection(message: str) -> ProxyException: + return ProxyException( + message=message, type=ProxyErrorTypes.bad_request_error, param=None, code=status.HTTP_400_BAD_REQUEST + ) + + +def resolve_baggage_tag_mappings(header: str, rules: Sequence[RequestTagMapping]) -> tuple[tuple[str, bool], ...]: + """Exact mappings win over the any-value fallback for the same baggage key.""" + members: Final = tuple(_member_without_properties(member) for member in header.split(",")) + context: Final = W3CBaggagePropagator().extract({"baggage": ",".join(members)}, context=Context()) + values: Final = baggage.get_all(context) + member_keys: Final = tuple(unquote_plus(member.partition("=")[0]).strip() for member in members) + resolved: Final = ( + _resolve_key(key, values, member_keys, rules) for key in dict.fromkeys(rule.key for rule in rules) + ) + return tuple(mapping for mapping in resolved if mapping is not None) + + +def _member_without_properties(member: str) -> str: + key, separator, value = member.partition(";")[0].partition("=") + return f"{key.strip()}{separator}{value.strip()}" + + +def _resolve_key( + key: str, values: Mapping[str, object], member_keys: Sequence[str], rules: Sequence[RequestTagMapping] +) -> tuple[str, bool] | None: + key_rules: Final = tuple(rule for rule in rules if rule.key == key) + key_spellings: Final = frozenset((key, unquote_plus(key))) + sent_keys: Final = tuple(member_key for member_key in member_keys if member_key in key_spellings) + value: Final = values.get(sent_keys[0]) if len(sent_keys) == 1 else None + selected: Final = next( + (rule for rule in key_rules if rule.value == value and rule.value is not None), None + ) or next((rule for rule in key_rules if rule.value is None), None) + usable: Final = isinstance(value, str) and value.isprintable() and bool(value) + if not usable or selected is None: + if any(rule.required for rule in key_rules): + raise mapping_rejection(f"Missing, invalid or unmapped baggage value for {key}") + return None + tag: Final = selected.tag if selected.tag is not None else f"{selected.tag_prefix}{value}" + return tag, selected.require_budgeted_tag diff --git a/litellm/proxy/guardrails/auto_router_compression.py b/litellm/proxy/guardrails/auto_router_compression.py index ecb182ab33c..1267fb15db8 100644 --- a/litellm/proxy/guardrails/auto_router_compression.py +++ b/litellm/proxy/guardrails/auto_router_compression.py @@ -156,15 +156,13 @@ async def arm_pre_call( if not isinstance(model_alias, str) or not model_alias: return - from litellm.router_strategy.tag_based_routing import ( - get_tags_from_request_kwargs, - ) + from litellm.router_strategy.tag_based_routing import get_routing_tags_from_request_kwargs policy: Final = policy_for_model( llm_router=llm_router, model_alias=model_alias, request_kwargs=data, - request_tags=get_tags_from_request_kwargs(data), + request_tags=get_routing_tags_from_request_kwargs(data), ) if policy is None: return diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 67a1d640103..c0222dbcd0b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -5,7 +5,8 @@ import json import re import time from collections import OrderedDict -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Iterable, Mapping, MutableMapping, Sequence +from dataclasses import dataclass from datetime import datetime from types import MappingProxyType from typing import ( @@ -17,7 +18,7 @@ from typing import ( cast, # noqa: TID251 # narrows untyped deployment and credential payloads ) -from fastapi import HTTPException, Request +from fastapi import HTTPException, Request, status from pydantic import TypeAdapter from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers @@ -27,6 +28,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm._uuid import uuid from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, CALLER_SESSION_ID_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, @@ -80,6 +82,7 @@ from litellm.proxy.common_utils.callback_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + detach_request_parsed_body, safe_get_request_headers, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata @@ -90,6 +93,7 @@ _SPECIAL_HEADERS_CACHE: Final = frozenset(str(v.value).lower() for v in SpecialH _REDACTED_HEADER_VALUE: Final = "***REDACTED***" _DECODED_JSON: Final = TypeAdapter(object) +_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names() | frozenset( {"cookie", "proxy-authorization"} ) @@ -209,8 +213,19 @@ def _drop_client_caller_session(request_body: Mapping[str, object]) -> None: metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None) +_PROXY_SET_METADATA_KEYS: Final = frozenset({CALLER_SESSION_ID_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY}) +_METADATA_TAGS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + def _requester_metadata_snapshot(metadata: Mapping[str, object]) -> Mapping[str, object]: - return copy.deepcopy({key: value for key, value in metadata.items() if key != CALLER_SESSION_ID_METADATA_KEY}) + requester: Final = {key: value for key, value in metadata.items() if key not in _PROXY_SET_METADATA_KEYS} + stamp: Final = metadata.get(BAGGAGE_MAPPED_TAGS_METADATA_KEY) + tags: Final = requester.get("tags") + if not isinstance(stamp, BaggageMappedTagsStamp) or not isinstance(tags, list): + return copy.deepcopy(requester) + sent_tags: Final = [tag for tag in _METADATA_TAGS_ADAPTER.validate_python(tags) if tag not in stamp.tags] + without_tags: Final = {key: value for key, value in requester.items() if key != "tags"} + return copy.deepcopy({**without_tags, "tags": sent_tags} if sent_tags else without_tags) def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None: @@ -254,7 +269,7 @@ from litellm.router import Router from litellm.secret_managers.main import get_secret_bool from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS -from litellm.types.router import holds_api_key_secret_reference +from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_reference from litellm.types.services import ServiceTypes from litellm.types.utils import ( API_KEY_SECRET_REFERENCE_REJECTION, @@ -285,8 +300,11 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as OtelSpan from litellm.integrations.otel.model.destination import OtelDestination + from litellm.models.tag import LiteLLM_TagTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.policy_engine.attachment_registry import AttachmentRegistry from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig + from litellm.proxy.utils import PrismaClient from litellm.types.proxy.policy_engine import Policy, PolicyMatchContext ProxyConfig = _ProxyConfig @@ -537,6 +555,39 @@ def _normalized_metadata_object(field: str, value: object) -> Mapping[str, objec raise _invalid_metadata_type_error(field=field, value=value) +def _serves_through_db_outage() -> bool: + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler + + return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() + + +def _tag_budgets_unavailable(message: str) -> ProxyException: + return ProxyException( + message=message, type=ProxyErrorTypes.no_db_connection, param=None, code=status.HTTP_503_SERVICE_UNAVAILABLE + ) + + +async def _unbudgeted_mapped_tag( + budgeted_tags: tuple[str, ...], prisma_client: "PrismaClient | None", user_api_key_cache: "UserApiKeyCache" +) -> str | ProxyException | None: + from litellm.proxy.auth.auth_checks import read_tag_objects_batch + + if prisma_client is None: + return _tag_budgets_unavailable("Baggage tag mappings require a database to check budgeted tags") + tag_objects: Final = await read_tag_objects_batch( + tag_names=budgeted_tags, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache + ) + if isinstance(tag_objects, Exception): + return _tag_budgets_unavailable("Baggage tag mappings could not read tag budgets from the database") + return next((tag for tag in budgeted_tags if not _has_tag_budget(tag_objects.get(tag))), None) + + +def _has_tag_budget(tag: "LiteLLM_TagTable | None") -> bool: + if tag is None or tag.litellm_budget_table is None: + return False + return tag.litellm_budget_table.max_budget is not None + + def _normalized_metadata_slot( request_data: MutableMapping[str, object], metadata_variable_name: str ) -> dict[str, object]: @@ -554,6 +605,43 @@ def _normalized_metadata_slot( return normalized +def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException: + if value is None: + return {} + try: + return dict(_normalized_metadata_object(field, value)) + except ProxyException as invalid_type: + return invalid_type + + +@dataclass(frozen=True, slots=True) +class _BaggageClientBody: + body: dict[str, object] # mutable-ok: the client's cached body, which the base tag merges write into + + +def _detach_auth_metadata_from_client_body( + request: Request, + request_data: MutableMapping[str, object], # mutable-ok: auth's own copy takes the normalized metadata objects +) -> None: + client_body: Final = detach_request_parsed_body(request, request_data) + request.state.litellm_baggage_client_body = _BaggageClientBody(client_body) # rebind-ok: mirror tag merges + slots: Final = tuple( + (field, _metadata_object_or_error(field, request_data[field])) + for field in ("metadata", "litellm_metadata") + if field in request_data + ) + for field, normalized in slots: + request_data[field] = {} if isinstance(normalized, ProxyException) else normalized # rebind-ok: auth's copy + + +def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> tuple[str, ...]: + tag_lists: Final = (metadata.get("tags") for metadata in metadata_objects) + tags: Final = itertools.chain.from_iterable( + _OBJECT_LIST_ADAPTER.validate_python(tag_list) for tag_list in tag_lists if isinstance(tag_list, list) + ) + return tuple(tag for tag in tags if isinstance(tag, str)) + + def _strip_untrusted_request_header_controls( headers: Any, *, @@ -1132,6 +1220,8 @@ _get_dynamic_logging_metadata: Final = get_dynamic_logging_metadata _TENANT_OTEL_PARAMS: Final = TypeAdapter(StandardCallbackDynamicParams) +_STRING_TAGS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_CLIENT_TAGS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDynamicParams: @@ -2045,7 +2135,7 @@ class LiteLLMProxyRequestSetup: @staticmethod def pre_seed_litellm_metadata_for_route( - request_data: dict, + request_data: MutableMapping[str, object], # mutable-ok: seed shared request metadata before admission route: str, ) -> None: """Pre-seed ``litellm_metadata`` for routes that track tags there. @@ -2064,6 +2154,87 @@ class LiteLLMProxyRequestSetup: if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES): request_data.setdefault("litellm_metadata", {}) + @staticmethod + def pre_auth_tag_merge_targets( + request: Request | None, + request_data: dict[str, object], # mutable-ok: auth's request body, which the tag merges write into + ) -> tuple[dict[str, object], ...]: # mutable-ok: every body the base tag merges write into + client_body: Final[object] = getattr(getattr(request, "state", None), "litellm_baggage_client_body", None) + if not isinstance(client_body, _BaggageClientBody) or client_body.body is request_data: + return (request_data,) + return (request_data, client_body.body) + + @staticmethod + async def apply_baggage_tag_mappings_pre_auth( + request: Request, + request_data: MutableMapping[str, object], # mutable-ok: enrich the shared request before budget admission + general_settings: Mapping[str, object], + prisma_client: "PrismaClient | None", + user_api_key_cache: "UserApiKeyCache", + ) -> None: + """Resolve configured baggage tags before the existing tag-budget gate.""" + from litellm.proxy.common_utils.request_tag_mappings import ( + REQUEST_TAG_MAPPINGS_ADAPTER, + mapping_rejection, + resolve_baggage_tag_mappings, + ) + + raw_rules: Final = general_settings.get("request_tag_mappings") + if not isinstance(raw_rules, list) or not raw_rules: + return + rules: Final = REQUEST_TAG_MAPPINGS_ADAPTER.validate_python(raw_rules) + header: Final = ",".join(request.headers.getlist("baggage")) + try: + mapped: Final = resolve_baggage_tag_mappings(header, rules) + except ProxyException: + _detach_auth_metadata_from_client_body(request, request_data) + raise + if not mapped: + return + client_metadata: Final[object] = request_data.get("metadata") + _detach_auth_metadata_from_client_body(request, request_data) + budgeted_tags: Final = tuple(tag for tag, require_budget in mapped if require_budget) + unbudgeted_tag: Final = ( + await _unbudgeted_mapped_tag(budgeted_tags, prisma_client, user_api_key_cache) if budgeted_tags else None + ) + if isinstance(unbudgeted_tag, ProxyException) and not _serves_through_db_outage(): + raise unbudgeted_tag + if isinstance(unbudgeted_tag, str): + raise mapping_rejection(f"Mapped tag {unbudgeted_tag} requires an existing tag budget") + client_has_tags: Final = (isinstance(client_metadata, dict) and "tags" in client_metadata) or ( + get_metadata_variable_name_from_kwargs(request_data) == "metadata" + and any(member.strip() for member in request.headers.get("x-litellm-tags", "").split(",")) + ) + mapped_tags: Final = tuple(tag for tag, _ in mapped) + request.state.litellm_baggage_client_metadata_tags = client_has_tags # rebind-ok: request input flag + request.state.litellm_baggage_mapped_tags = mapped_tags # rebind-ok: per-request immutable mapped tags + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_data) + + @staticmethod + def apply_authenticated_baggage_tags( + request: Request, + request_data: MutableMapping[str, object], # mutable-ok: restore authenticated tags in downstream metadata + metadata_key: str | None = None, + ) -> None: + request_state: Final[object] = getattr(request, "state", None) + raw_tags: Final[object] = getattr(request_state, "litellm_baggage_mapped_tags", None) + if not isinstance(raw_tags, tuple) or not raw_tags: + return + tags: Final = _STRING_TAGS_ADAPTER.validate_python(raw_tags) + slot: Final = metadata_key if metadata_key is not None else get_metadata_variable_name_from_kwargs(request_data) + metadata: Final = _normalized_metadata_slot(request_data, slot) + existing: Final = metadata.get("tags") + if existing and not isinstance(existing, list): + return + existing_tags: Final = _CLIENT_TAGS_ADAPTER.validate_python(existing) if isinstance(existing, list) else () + prior_stamp: Final = metadata.get(BAGGAGE_MAPPED_TAGS_METADATA_KEY) + prior_mapped: Final = prior_stamp.tags if isinstance(prior_stamp, BaggageMappedTagsStamp) else () + sent_tags: Final = tuple(tag for tag in existing_tags if tag not in prior_mapped) + metadata["tags"] = [*existing_tags, *(tag for tag in tags if tag not in existing_tags)] + metadata[BAGGAGE_MAPPED_TAGS_METADATA_KEY] = BaggageMappedTagsStamp( + tags=tuple(tag for tag in tags if tag not in sent_tags) + ) + @staticmethod def apply_key_tags_pre_auth( request_data: dict, @@ -2221,12 +2392,16 @@ async def add_litellm_data_to_request( data.pop(_internal_key, None) _reject_url_valued_destinations(data) _raw_metadata_by_field: Final = { - _metadata_field: data.pop(_metadata_field) + _metadata_field: _DECODED_JSON.validate_python(data.pop(_metadata_field)) for _metadata_field in ("metadata", "litellm_metadata") if data.get(_metadata_field) is not None } - for _metadata_field, _raw_metadata in _raw_metadata_by_field.items(): - data[_metadata_field] = _normalized_metadata_object(_metadata_field, _raw_metadata) + _normalized_metadata_by_field: Final = { + _metadata_field: _normalized_metadata_object(_metadata_field, _raw_metadata) + for _metadata_field, _raw_metadata in _raw_metadata_by_field.items() + } + data.update(_normalized_metadata_by_field) + _client_sent_metadata_tags: Final = _metadata_string_tags(_normalized_metadata_by_field.values()) # Strip spoofable auth metadata from user-supplied metadata dict _user_metadata = data.get("metadata") if isinstance(_user_metadata, dict): @@ -2654,15 +2829,14 @@ async def add_litellm_data_to_request( tags_to_add=_caller_body_tags, ) - # caller_tags: exactly what this request itself supplied (x-litellm-tags header, - # body "tags", or body "metadata.tags" on litellm_metadata routes), never - # anything from key/team/project metadata. Read directly from the header and - # body values here, the same way inherited_tags above is read directly from - # key/team/project metadata -- neither is derived by inspecting the shared - # "tags" list, which a pre-auth pass (apply_client_tag_policy_pre_auth) may - # have already merged caller header tags into before this function runs. + # caller_tags: the tags the request's headers and body carry (x-litellm-tags, body "tags", and body + # "metadata.tags" / "litellm_metadata.tags" read before the mapped tags are added below), + # never derived from the shared "tags" list. data[_metadata_variable_name]["caller_tags"] = tuple( # rebind-ok: matches file idiom - dict.fromkeys((*(tags or ()), *(_caller_body_tags or ()))) + dict.fromkeys((*(tags or ()), *_client_sent_metadata_tags)) + ) + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags( + request=request, request_data=data, metadata_key=_metadata_variable_name ) # Team Callbacks controls @@ -3958,6 +4132,16 @@ def _apply_resolved_guardrails_to_metadata( verbose_proxy_logger.debug("Policy engine: added guardrails to request metadata: %s", combined) +def _policy_match_tags(request_data: Mapping[str, object], metadata_variable_name: str) -> list[str] | None: + from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body + from litellm.router_strategy.tag_based_routing import without_baggage_mapped_tags + + tags: Final = without_baggage_mapped_tags( + get_tags_from_request_body(request_data), request_data.get(metadata_variable_name) + ) + return list(tags) if tags else None + + async def add_guardrails_from_policy_engine( data: dict, metadata_variable_name: str, @@ -3982,7 +4166,6 @@ async def add_guardrails_from_policy_engine( user_api_key_dict: The user's API key authentication info """ from litellm._logging import verbose_proxy_logger - from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.types.proxy.policy_engine import PolicyMatchContext @@ -4000,7 +4183,7 @@ async def add_guardrails_from_policy_engine( return # Extract tags and build context - all_tags: Final = get_tags_from_request_body(data) or None + all_tags: Final = _policy_match_tags(data, metadata_variable_name) _team_alias: Final = user_api_key_dict.team_alias _key_alias: Final = user_api_key_dict.key_alias context: Final = PolicyMatchContext( diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 9a1cf701142..c12a90ed6a2 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -275,6 +275,7 @@ async def new_tag( """ from litellm.proxy._types import CommonProxyErrors from litellm.proxy.proxy_server import ( + invalidate_spend_counter, litellm_proxy_admin_name, llm_router, prisma_client, @@ -316,6 +317,7 @@ async def new_tag( ) await _evict_tag_cache_keys((tag_cache_key(tag.name), tag_registry_cache_key())) + await invalidate_spend_counter(counter_key=f"spend:tag:{tag.name}") # Update models with new tag if tag.models: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f2605a8ec6f..d9d3b6f8c92 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -222,7 +222,12 @@ async def _json_request_body(request: Request) -> Mapping[str, object]: return await request.json() -def get_passthrough_router_request_metadata(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]: +_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def get_passthrough_router_request_metadata( + user_api_key_dict: UserAPIKeyAuth, request: Request | None = None +) -> Mapping[str, object]: """ Build the request metadata carrying key-level spend attribution and the pre-call budget reservation for a router-model passthrough request. @@ -243,13 +248,17 @@ def get_passthrough_router_request_metadata(user_api_key_dict: UserAPIKeyAuth) - """ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - request_data: Final = {"litellm_metadata": {}} + request_data: Final[dict[str, object]] = { # mutable-ok: legacy enrichment writes this wrapper + "litellm_metadata": {} + } LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=request_data, user_api_key_dict=user_api_key_dict, _metadata_variable_name="litellm_metadata", ) - return request_data["litellm_metadata"] + if request is not None: + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_data) + return _METADATA_ADAPTER.validate_python(request_data["litellm_metadata"]) async def llm_passthrough_factory_proxy_route( @@ -549,7 +558,7 @@ async def vllm_proxy_route( params=None, headers=None, cookies=None, - litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict, request), ), ) @@ -2229,7 +2238,7 @@ async def _relay_router_model( params=None, headers=None, cookies=None, - litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict, request), ) except httpx.HTTPStatusError as upstream_error: return await _relay_upstream_response(upstream_error.response) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 311dab058a4..fe7ac30e434 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2021,6 +2021,9 @@ def _apply_key_team_project_controls( ) +_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object]) + + def _update_metadata_with_tags_in_header(request: Request, metadata: dict) -> dict: """ If tags are in the request headers, add them to the metadata @@ -2046,7 +2049,12 @@ def _update_metadata_with_tags_in_header(request: Request, metadata: dict) -> di tags_to_add=tags_to_add, ) - return metadata + restored_metadata: Final = _METADATA_ADAPTER.validate_python(metadata) + request_data: Final[dict[str, object]] = { # mutable-ok: legacy enrichment writes this wrapper + "metadata": restored_metadata + } + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_data) + return restored_metadata class _PassThroughRequestEnvelope(TypedDict, total=False): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ee8f20be243..c5128ddab87 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7104,6 +7104,7 @@ class ProxyConfig: "vector_store_deny_by_default", "search_tool_deny_by_default", "vertex_ai_pass_through_drop_anthropic_betas", + "request_tag_mappings", ) if name in typed_general_settings } @@ -8348,6 +8349,7 @@ class ProxyConfig: previous_cleanup_schedule: tuple[object, ...], ) -> None: effects: Final = ( + self._apply_request_tag_mappings_setting, self._apply_alerting_settings, self._apply_pass_through_settings, self._apply_boolean_settings, @@ -8359,6 +8361,19 @@ class ProxyConfig: for effect in effects: await effect(db_values) + async def _apply_request_tag_mappings_setting(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + if "request_tag_mappings" not in db_values: + return + mappings: Final = self.settings.get("request_tag_mappings") + try: + ConfigGeneralSettings.model_validate(MappingProxyType({"request_tag_mappings": mappings})) + except ValidationError as error: + self.settings.pop("request_tag_mappings", None) + verbose_proxy_logger.error( + "Ignoring invalid general_settings.request_tag_mappings from the DB, so no baggage maps to tags: %s", + error, + ) + async def _apply_alerting_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: alerting: Final = self.settings.get("alerting") if "alerting" in db_values and isinstance(alerting, list): @@ -19640,6 +19655,7 @@ _PLUGIN_KEY_REDACTED: Final = "***" _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingProxyType( { + "request_tag_mappings": "RequestTagMappings", "max_parallel_requests": "Integer", "global_max_parallel_requests": "Integer", "max_in_flight_requests_per_worker": "Integer", diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 4c9df95cd8d..803df9f83ca 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -228,29 +228,6 @@ def _raise_counter_budget_exceeded( ) -_UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( - { - "/models", - "/v1/models", - "/utils/token_counter", - "/responses/input_tokens", - "/v1/responses/input_tokens", - "/openai/v1/responses/input_tokens", - } -) -_TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "count-tokens"}) -_TOKEN_COUNTING_ACTION: Final = "countTokens" - - -def _is_token_counting_route(route: str) -> bool: - resource, _, action = route.rsplit("/", 1)[-1].partition(":") - return resource in _TOKEN_COUNTING_SEGMENTS or action == _TOKEN_COUNTING_ACTION - - -def _is_unbilled_route(route: str) -> bool: - return route in _UNBILLED_ROUTES or _is_token_counting_route(route) - - async def reserve_budget_for_request( request_body: dict, route: str, @@ -269,7 +246,7 @@ async def reserve_budget_for_request( ) -> dict | None: if valid_token is None or not RouteChecks.is_llm_api_route(route=route): return None - if _is_unbilled_route(route): + if RouteChecks.is_unbilled_route(route): return None if get_model_from_request(request_body, route, llm_router=llm_router) is None: return None diff --git a/litellm/router.py b/litellm/router.py index ba3b4d53725..aadbdaa8520 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -62,6 +62,7 @@ from litellm.caching.caching import ( ) from litellm.caching.redis_cache import log_redis_failure from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, @@ -162,9 +163,12 @@ from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2, PrefetchedUsage from litellm.router_strategy.simple_shuffle import simple_shuffle from litellm.router_strategy.tag_based_routing import ( + baggage_stamp_for_serving_deployment, get_deployments_for_tag, + get_routing_tags_from_request_kwargs, get_tags_from_request_kwargs, is_valid_deployment_tag, + tags_with_serving_deployment_tags, ) from litellm.router_utils.access_windows import access_windows_config_error, filter_reserved_deployments from litellm.router_utils.add_retry_fallback_headers import ( @@ -4681,13 +4685,13 @@ class Router: ## DEPLOYMENT-LEVEL TAGS deployment_tags: Final = deployment.get("litellm_params", {}).get("tags") - if deployment_tags: - existing_tags = kwargs[metadata_variable_name].get("tags") or [] - merged_tags: Final = list(existing_tags) - for tag in deployment_tags: - if tag not in merged_tags: - merged_tags.append(tag) - kwargs[metadata_variable_name]["tags"] = merged_tags + merged_tags: Final = tags_with_serving_deployment_tags(kwargs[metadata_variable_name], deployment_tags) + if merged_tags is not None: + kwargs[metadata_variable_name]["tags"] = list(merged_tags) + + baggage_stamp: Final = baggage_stamp_for_serving_deployment(kwargs[metadata_variable_name], deployment_tags) + if baggage_stamp is not None: + kwargs[metadata_variable_name][BAGGAGE_MAPPED_TAGS_METADATA_KEY] = baggage_stamp ## CREDENTIAL NAME AS TAG credential_name: Final = deployment.get("litellm_params", {}).get("litellm_credential_name") @@ -15292,7 +15296,7 @@ class Router: if not candidates: return None - request_tags: Final = get_tags_from_request_kwargs(request_kwargs) + request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) if request_tags: for tagged in candidates: if tagged.tags and is_valid_deployment_tag( @@ -15525,7 +15529,7 @@ class Router: llm_router=self, model_alias=registered_model_name, request_kwargs=request_kwargs, - request_tags=get_tags_from_request_kwargs(request_kwargs), + request_tags=get_routing_tags_from_request_kwargs(request_kwargs), ) # Shared compression already ran in the pre-call hook, so reuse it rather than # compressing twice. Conditional on arming having actually happened: only the @@ -15574,7 +15578,7 @@ class Router: value=self._consumed_request_tags_stamp( selected_strategy=selected_strategy, pre_routing_hook_response=pre_routing_hook_response, - request_tags=get_tags_from_request_kwargs(request_kwargs), + request_tags=get_routing_tags_from_request_kwargs(request_kwargs), ), ) diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 73236294020..f13fd3fc630 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -10,13 +10,18 @@ Use this to route requests between Teams import re from collections.abc import Mapping, Sequence +from dataclasses import replace from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload from litellm._logging import verbose_logger -from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY +from litellm.constants import ( + BAGGAGE_MAPPED_TAGS_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, +) from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs -from litellm.types.router import ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors +from litellm.types.router import BaggageMappedTagsStamp, ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors if TYPE_CHECKING: from litellm.router import Router as _Router @@ -484,6 +489,64 @@ def _request_tags_after_router_consumption(metadata: object, model: str) -> Sequ return tuple(dict.fromkeys((*leftover, *(tag for tag in typed_inherited_tags if isinstance(tag, str))))) +def without_baggage_mapped_tags( + request_tags: Sequence[str] | None, metadata: object, *, keep_serving_deployment_tags: bool = False +) -> Sequence[str] | None: + """Drop baggage-mapped tags that neither key, team or project policy nor the caller itself asked for.""" + if request_tags is None or not isinstance(metadata, Mapping): + return request_tags + typed_metadata: Final[Mapping[str, object]] = metadata + stamp: Final = typed_metadata.get(BAGGAGE_MAPPED_TAGS_METADATA_KEY) + if not isinstance(stamp, BaggageMappedTagsStamp): + return request_tags + asked_for_tags: Final = ( + *_metadata_tags(typed_metadata, "inherited_tags"), + *_metadata_tags(typed_metadata, "caller_tags"), + *(stamp.serving_deployment_tags if keep_serving_deployment_tags else ()), + ) + return tuple(tag for tag in request_tags if tag not in stamp.tags or tag in asked_for_tags) or None + + +def baggage_stamp_for_serving_deployment(metadata: object, deployment_tags: object) -> BaggageMappedTagsStamp | None: + if not isinstance(metadata, Mapping): + return None + typed_metadata: Final[Mapping[str, object]] = metadata + stamp: Final = typed_metadata.get(BAGGAGE_MAPPED_TAGS_METADATA_KEY) + if not isinstance(stamp, BaggageMappedTagsStamp): + return None + return replace( + stamp, serving_deployment_tags=tuple(tag for tag in _tag_sequence(deployment_tags) if isinstance(tag, str)) + ) + + +def tags_with_serving_deployment_tags(metadata: object, deployment_tags: object) -> tuple[object, ...] | None: + """Merge the serving deployment's tags ahead of baggage-mapped tags, so baggage never changes tag precedence.""" + serving_tags: Final = _tag_sequence(deployment_tags) + if not isinstance(metadata, Mapping) or not serving_tags: + return None + typed_metadata: Final[Mapping[str, object]] = metadata + existing_tags: Final = _metadata_tags(typed_metadata, "tags") + stamp: Final = typed_metadata.get(BAGGAGE_MAPPED_TAGS_METADATA_KEY) + mapped_tags: Final = stamp.tags if isinstance(stamp, BaggageMappedTagsStamp) else () + kept_tags: Final = tuple(tag for tag in existing_tags if tag not in mapped_tags) + added_serving_tags: Final = tuple( + tag for index, tag in enumerate(serving_tags) if tag not in kept_tags and tag not in serving_tags[:index] + ) + readded_mapped_tags: Final = tuple( + dict.fromkeys(tag for tag in existing_tags if tag in mapped_tags and tag not in added_serving_tags) + ) + return (*kept_tags, *added_serving_tags, *readded_mapped_tags) + + +def _metadata_tags(metadata: Mapping[str, object], key: str) -> Sequence[object]: + return _tag_sequence(metadata.get(key)) + + +def _tag_sequence(tags: object) -> Sequence[object]: + typed_tags: Final[Sequence[object]] = tags if isinstance(tags, (list, tuple)) else () + return typed_tags + + async def get_deployments_for_tag( llm_router_instance: LitellmRouter, model: str, # used to raise the correct error @@ -524,7 +587,9 @@ async def get_deployments_for_tag( verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name)) if metadata_variable_name in request_kwargs: metadata: Final = request_kwargs[metadata_variable_name] - request_tags: Final = _request_tags_after_router_consumption(metadata, model) + request_tags: Final = without_baggage_mapped_tags( + _request_tags_after_router_consumption(metadata, model), metadata + ) match_any: Final = llm_router_instance.tag_filtering_match_any routing_prefix: Final = llm_router_instance.tag_routing_prefix or "" @@ -707,3 +772,21 @@ def get_tags_from_request_kwargs( _get_tags_from_request_kwargs = get_tags_from_request_kwargs + + +def _request_metadata(request_kwargs: Mapping[str, object]) -> object: + metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_kwargs) + if metadata_variable_name in request_kwargs: + return request_kwargs[metadata_variable_name] + litellm_params: Final = request_kwargs.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return None + typed_litellm_params: Final[Mapping[str, object]] = litellm_params + return typed_litellm_params.get(metadata_variable_name) + + +def get_routing_tags_from_request_kwargs(request_kwargs: Mapping[str, object]) -> Sequence[str]: + return ( + without_baggage_mapped_tags(get_tags_from_request_kwargs(request_kwargs), _request_metadata(request_kwargs)) + or () + ) diff --git a/litellm/types/proxy/request_tag_mappings.py b/litellm/types/proxy/request_tag_mappings.py new file mode 100644 index 00000000000..17cb4f4048d --- /dev/null +++ b/litellm/types/proxy/request_tag_mappings.py @@ -0,0 +1,27 @@ +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Self + + +class RequestTagMapping(BaseModel): + """An exact baggage value mapping, or a prefix for any value of a key.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + header: Literal["baggage"] = "baggage" + key: str = Field(min_length=1, pattern=r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") + value: str | None = Field(None, min_length=1) + tag: str | None = Field(None, min_length=1) + tag_prefix: str | None = Field(None, min_length=1) + required: bool = False + require_budgeted_tag: bool = False + + @model_validator(mode="after") + def validate_mapping(self) -> Self: + if self.value is not None: + if self.tag is None or self.tag_prefix is not None: + raise ValueError("An exact value mapping requires tag and cannot set tag_prefix") + elif self.tag_prefix is None or self.tag is not None: + raise ValueError("An any-value mapping requires tag_prefix and cannot set tag") + return self diff --git a/litellm/types/router.py b/litellm/types/router.py index 47dafcf275d..6cd758d864e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1332,6 +1332,18 @@ class ConsumedRequestTagsStamp: tags: tuple[str, ...] +@dataclass(frozen=True, slots=True) +class BaggageMappedTagsStamp: + """Request tags added by baggage mappings, which classify spend and budgets but never select deployments. + + The serving deployment's own tags are kept beside them, so a mapped tag that the deployment also carries + still picks that deployment's guardrail mode. + """ + + tags: tuple[str, ...] + serving_deployment_tags: tuple[str, ...] = () + + @runtime_checkable class PreRoutingStrategy(Protocol): """Structural interface shared by the auto / complexity / adaptive / quality routers.""" diff --git a/tests/integration/spend/test_baggage_tag_mappings.py b/tests/integration/spend/test_baggage_tag_mappings.py new file mode 100644 index 00000000000..1bf90269484 --- /dev/null +++ b/tests/integration/spend/test_baggage_tag_mappings.py @@ -0,0 +1,847 @@ +import itertools +import json +import uuid +from collections.abc import Generator, Iterator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + eventually, + gateway_from_environment, + list_value, + object_value, +) +from integration._support.database import scratch_database +from integration._support.database_relay import dropped_connection_relay, plaintext_database_url +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter +from websockets.exceptions import InvalidStatus +from websockets.sync.client import connect + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +PRICED: Final[dict[str, JsonValue]] = {"input_cost_per_token": 0.01, "output_cost_per_token": 0.01} +WORKER_PROBES: Final = 6 +JSON_LIST: Final = TypeAdapter(list[JsonValue]) +GUARDRAIL_VERDICT: Final = b'{"action":"GUARDRAIL_INTERVENED","texts":["masked baggage prompt"]}' +GUARDRAIL_PATH: Final = "/beta/litellm_basic_guardrail_api" +NO_FIELDS: Final[Mapping[str, JsonValue]] = MappingProxyType({}) + +Route = Literal["chat", "chat_litellm_metadata", "messages", "responses"] +CallerTag = Literal["none", "header", "metadata_json_with_header", "litellm_metadata"] +RejectedClientTag = Literal["none", "header", "body", "metadata_json", "litellm_metadata"] + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + provider: Wire + callbacks: Wire + marker: str + + @property + def chat_model(self) -> str: + return f"chat-{self.marker}" + + @property + def exact_tag(self) -> str: + return f"usecase-a-{self.marker}" + + def usecase_tag(self, value: str) -> str: + return f"usecase-{self.marker}:{value}" + + def tenant_tag(self, value: str) -> str: + return f"tenant-{self.marker}:{value}" + + def budgeted_tag(self, value: str) -> str: + return f"budgeted-{self.marker}:{value}" + + def team_tag(self, value: str) -> str: + return f"team-{self.marker}:{value}" + + @property + def guarded_tag(self) -> str: + return f"guarded-{self.marker}:on" + + @property + def policy_tag(self) -> str: + return f"policy-{self.marker}:sensitive" + + +def _chat_reply() -> Reply: + marker: Final = uuid.uuid4().hex + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + ).encode() + ) + + +def _responses_reply() -> Reply: + marker: Final = uuid.uuid4().hex + return Reply( + body=json.dumps( + { + "id": f"resp_{marker}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, + } + ).encode() + ) + + +def _provider(request: Request) -> Reply: + if request.target == GUARDRAIL_PATH: + return Reply(body=b'{"action":"NONE"}') + if request.target == "/v1/responses": + return _responses_reply() + return _chat_reply() + + +def _deployment(model_name: str, provider: Wire, **parameters: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider.url}/v1", + "api_key": "synthetic-baggage-key", + **PRICED, + **parameters, + }, + } + + +def _write_config( + directory: Path, + model_list: list[JsonValue], + general_settings: Mapping[str, JsonValue], + litellm_settings: Mapping[str, JsonValue], + guardrails: Sequence[JsonValue] = (), +) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(PROXY_CONFIG.read_text())) + config: Final = { + **base, + "model_list": model_list, + "general_settings": {**object_value(base["general_settings"]), **general_settings}, + "litellm_settings": {**object_value(base["litellm_settings"]), **litellm_settings}, + **({"guardrails": list(guardrails)} if guardrails else {}), + } + path: Final = directory / "baggage-tag-mappings.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _routed_deployment(marker: str, provider: Wire, name: str, tag: str) -> dict[str, JsonValue]: + return { + **_deployment(f"routed-{marker}", provider, tags=[tag]), + "model_info": {"id": f"{name}-{marker}", "enable_tag_filtering": True}, + } + + +def _mapped_models(marker: str, provider: Wire, upstream_url: str) -> list[JsonValue]: + routed: Final = ( + _routed_deployment(marker, provider, "tagged", f"usecase-a-{marker}"), + _routed_deployment(marker, provider, "default", "default"), + ) + realtime: Final[dict[str, JsonValue]] = { + "model_name": f"realtime-{marker}", + "litellm_params": { + "model": "openai/gpt-realtime-2.1", + "api_base": f"{upstream_url}/v1", + "api_key": f"realtime-{marker}", + }, + } + guarded: Final = _deployment(f"guarded-{marker}", provider, tags=[f"guarded-{marker}:on"]) + pre_guarded: Final = _deployment(f"guarded-pre-{marker}", provider, tags=[f"guarded-{marker}:pre"]) + return [_deployment(f"chat-{marker}", provider), guarded, pre_guarded, *routed, realtime] + + +def _mapped_rules(marker: str, *, tenant_required: bool) -> list[JsonValue]: + return [ + {"key": "entrypoint", "value": "a", "tag": f"usecase-a-{marker}"}, + {"key": "entrypoint", "tag_prefix": f"usecase-{marker}:"}, + {"key": "tenant", "tag_prefix": f"tenant-{marker}:", "required": tenant_required}, + {"key": "budgeted", "tag_prefix": f"budgeted-{marker}:", "require_budgeted_tag": True}, + {"key": "team+id", "tag_prefix": f"team-{marker}:"}, + {"key": "guard", "tag_prefix": f"guarded-{marker}:"}, + ] + + +def _deployment_tag_mode_guardrail(marker: str, provider: Wire) -> dict[str, JsonValue]: + return { + "guardrail_name": f"deployment-tag-mode-{marker}", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": { + "tags": { + f"guarded-{marker}:on": "post_call", + f"guarded-{marker}:pre": "pre_call", + f"policy-{marker}:sensitive": "post_call", + } + }, + "default_on": True, + "api_base": f"{provider.url}{GUARDRAIL_PATH}", + "api_key": "synthetic-guardrail-key", + }, + } + + +@contextmanager +def _rig( + tmp_path_factory: pytest.TempPathFactory, + general_settings: Mapping[str, JsonValue], + *, + tenant_required: bool, + workers: int, + tag_mode_guardrail: bool = False, +) -> Generator[Rig]: + marker: Final = uuid.uuid4().hex[:12] + directory: Final = tmp_path_factory.mktemp("baggage-tag-mappings") + with ( + gateway_from_environment() as base, + wire_server(_provider) as provider, + wire_server(lambda _request: Reply()) as callbacks, + ): + config: Final = _write_config( + directory, + _mapped_models(marker, provider, base.upstream_url), + {"request_tag_mappings": _mapped_rules(marker, tenant_required=tenant_required), **general_settings}, + {"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}, + [_deployment_tag_mode_guardrail(marker, provider)] if tag_mode_guardrail else (), + ) + with owned_proxy( + base, directory, {"GENERIC_LOGGER_ENDPOINT": callbacks.url}, config=config, workers=workers + ) as proxy: + yield Rig(proxy, provider, callbacks, marker) + + +@pytest.fixture(scope="module") +def mapped(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig(tmp_path_factory, {}, tenant_required=True, workers=2) as rig: + yield rig + + +@pytest.fixture(scope="module") +def optional(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig(tmp_path_factory, {}, tenant_required=False, workers=1, tag_mode_guardrail=True) as rig: + yield rig + + +@pytest.fixture(scope="module") +def rejecting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig(tmp_path_factory, {"reject_clientside_metadata_tags": True}, tenant_required=False, workers=1) as rig: + yield rig + + +def _digest(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _caller_tags(raw: JsonValue) -> frozenset[str]: + tags: Final = JSON_LIST.validate_json(raw) if isinstance(raw, str) else list_value(raw) + return frozenset(tag for tag in tags if isinstance(tag, str) and not tag.startswith("User-Agent: ")) + + +def _spend_tags(proxy: Gateway, key: str) -> frozenset[str]: + rows: Final = eventually( + lambda: proxy.rows('SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (_digest(key),)), + lambda values: len(values) == 1, + seconds=70, + ) + return _caller_tags(rows[0]["request_tags"]) + + +def _batch_events(batch: Request) -> Iterator[dict[str, JsonValue]]: + return (object_value(item) for item in JSON_LIST.validate_json(batch.body)) + + +def _logged_event(callbacks: Wire, call_id: str) -> dict[str, JsonValue]: + def delivered() -> tuple[dict[str, JsonValue], ...]: + events: Final = itertools.chain.from_iterable(_batch_events(batch) for batch in callbacks.drain()) + return tuple(event for event in events if event.get("litellm_call_id") == call_id) + + events: Final = eventually(delivered, lambda values: len(values) == 1, seconds=30) + return events[0] + + +def _inference_body(route: Route, model: str, text: str, caller_tag: str) -> dict[str, JsonValue]: + match route: + case "chat": + return {"model": model, "messages": [{"role": "user", "content": text}], "metadata": {"tags": [caller_tag]}} + case "chat_litellm_metadata": + return { + "model": model, + "messages": [{"role": "user", "content": text}], + "litellm_metadata": {"tags": [caller_tag]}, + } + case "messages": + return { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": text}], + "litellm_metadata": {"tags": [caller_tag]}, + } + case "responses": + return {"model": model, "input": text, "litellm_metadata": {"tags": [caller_tag]}} + + +ROUTE_PATHS: Final[Mapping[Route, str]] = { + "chat": "/v1/chat/completions", + "chat_litellm_metadata": "/v1/chat/completions", + "messages": "/v1/messages", + "responses": "/v1/responses", +} + + +def _guardrail_inputs(rig: Rig) -> tuple[JsonValue, ...]: + return tuple( + JSON_OBJECT.validate_json(request.body)["input_type"] + for request in rig.provider.drain() + if request.target == GUARDRAIL_PATH + ) + + +def _chat( + rig: Rig, + key: str, + baggage: str | None, + *, + model: str | None = None, + fields: Mapping[str, JsonValue] = NO_FIELDS, + **headers: str, +) -> httpx.Response: + return rig.proxy.request( + "POST", + "/v1/chat/completions", + { + "model": model or rig.chat_model, + "messages": [{"role": "user", "content": f"baggage {uuid.uuid4().hex}"}], + "max_tokens": 16, + **fields, + }, + key=key, + headers={**({"baggage": baggage} if baggage is not None else {}), **headers}, + ) + + +@pytest.mark.parametrize("route", ["chat", "chat_litellm_metadata", "messages", "responses"]) +def test_mapped_baggage_tags_reach_spend_and_callbacks_but_not_requester_metadata(mapped: Rig, route: Route) -> None: + tenant: Final = uuid.uuid4().hex + caller_tag: Final = f"caller-{tenant}" + mapped_tags: Final = frozenset({mapped.exact_tag, mapped.tenant_tag(tenant)}) + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key() + mapped.provider.drain() + response: Final = mapped.proxy.request( + "POST", + ROUTE_PATHS[route], + _inference_body(route, mapped.chat_model, f"baggage {tenant}", caller_tag), + key=key, + headers={"baggage": f"entrypoint=a, tenant={tenant}"}, + ) + assert response.status_code == 200, response.text + assert len(mapped.provider.drain()) == 1 + assert _spend_tags(mapped.proxy, key) == mapped_tags | {caller_tag} + event: Final = _logged_event(mapped.callbacks, response.headers["x-litellm-call-id"]) + assert mapped_tags <= _caller_tags(event["request_tags"]), event["request_tags"] + requester: Final = object_value(object_value(event["metadata"]).get("requester_metadata") or {}) + assert not mapped_tags & _caller_tags(requester.get("tags") or []), requester + + +def test_json_string_metadata_gains_exactly_the_mapped_tags(optional: Rig) -> None: + tenant: Final = uuid.uuid4().hex + mapped_tags: Final = frozenset({optional.exact_tag, optional.tenant_tag(tenant)}) + fields: Final[Mapping[str, JsonValue]] = {"metadata": json.dumps({"tags": [f"caller-{tenant}"]})} + with optional.proxy.scenario() as scenario: + plain_key: Final = scenario.key() + key: Final = scenario.key() + optional.provider.drain() + plain: Final = _chat(optional, plain_key, None, fields=fields) + response: Final = _chat(optional, key, f"entrypoint=a,tenant={tenant}", fields=fields) + assert (plain.status_code, response.status_code) == (200, 200), (plain.text, response.text) + assert len(optional.provider.drain()) == 2 + assert _spend_tags(optional.proxy, key) == _spend_tags(optional.proxy, plain_key) | mapped_tags + event: Final = _logged_event(optional.callbacks, response.headers["x-litellm-call-id"]) + assert mapped_tags <= _caller_tags(event["request_tags"]), event["request_tags"] + requester: Final = object_value(object_value(event["metadata"]).get("requester_metadata") or {}) + assert not mapped_tags & _caller_tags(requester.get("tags") or []), requester + + +def test_a2a_spend_row_carries_the_mapped_tags(mapped: Rig) -> None: + marker: Final = "baggagea2a" + uuid.uuid4().hex[:12] + + def card(url: str) -> dict[str, JsonValue]: + return { + "protocolVersion": "0.3", + "name": marker, + "description": "Synthetic peer", + "version": "1.0.0", + "url": url + "/", + "capabilities": {"streaming": True}, + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "skills": [], + } + + def peer(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps(card(wire.url)).encode()) + rpc: Final = JSON_OBJECT.validate_json(request.body) + return Reply( + body=json.dumps( + { + "jsonrpc": "2.0", + "id": rpc["id"], + "result": { + "kind": "message", + "role": "agent", + "messageId": f"{marker}-out", + "parts": [{"kind": "text", "text": "done"}], + }, + } + ).encode() + ) + + tenant: Final = uuid.uuid4().hex + with wire_server(peer) as wire, mapped.proxy.scenario() as scenario: + created: Final = mapped.proxy.post( + "/v1/agents", + {"agent_name": marker, "agent_card_params": card(wire.url), "litellm_params": {"cost_per_query": 0.25}}, + ) + agent: Final = str(created["agent_id"]) + scenario.cleanups.callback(lambda: mapped.proxy.request("DELETE", f"/v1/agents/{agent}")) + key: Final = scenario.key() + sent: Final = mapped.proxy.request( + "POST", + f"/a2a/{agent}", + { + "jsonrpc": "2.0", + "id": f"{marker}-send", + "method": "message/send", + "params": { + "message": { + "kind": "message", + "role": "user", + "messageId": f"{marker}-in", + "parts": [{"kind": "text", "text": "ping"}], + } + }, + }, + key=key, + headers={"baggage": f"entrypoint=a,tenant={tenant}"}, + ) + assert sent.status_code == 200, sent.text + assert {mapped.exact_tag, mapped.tenant_tag(tenant)} <= _spend_tags(mapped.proxy, key) + + +@pytest.mark.parametrize( + ("key_fields", "fields", "headers"), + [ + ({}, NO_FIELDS, {}), + ({}, {"metadata": {"tags": "run-123"}}, {"x-litellm-tags": "caller-tag"}), + ({"metadata": {"tags": ["key-tag"]}}, {"metadata": {"tags": "run-123"}}, {}), + ({}, {"metadata": {"tags": ""}}, {}), + ], + ids=["plain", "string_tags_with_header_tag", "string_tags_with_key_tag", "empty_string_tags"], +) +def test_spent_mapped_tag_budget_rejects_the_next_request_before_the_provider( + mapped: Rig, key_fields: Mapping[str, JsonValue], fields: Mapping[str, JsonValue], headers: Mapping[str, str] +) -> None: + entry: Final = uuid.uuid4().hex + tag: Final = mapped.usecase_tag(entry) + baggage: Final = f"entrypoint={entry},tenant={entry}" + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key(**key_fields) + mapped.proxy.post("/tag/new", {"name": tag, "max_budget": 0.0001}) + scenario.cleanups.callback(lambda: mapped.proxy.request("POST", "/tag/delete", {"name": tag})) + first: Final = _chat(mapped, key, baggage) + assert first.status_code == 200, first.text + eventually(lambda: _chat(mapped, key, baggage).status_code, lambda status: status != 200, seconds=70) + mapped.provider.drain() + blocked: Final = _chat(mapped, key, baggage, fields=fields, **headers) + assert blocked.status_code == 422, blocked.text + assert "budget" in blocked.text.lower(), blocked.text + assert mapped.provider.drain() == () + other: Final = _chat(mapped, key, f"entrypoint=other-{entry},tenant={entry}") + assert other.status_code == 200, other.text + + +def test_required_mapping_rejects_a_request_without_its_baggage_value_before_the_provider(mapped: Rig) -> None: + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key() + mapped.provider.drain() + rejected: Final = _chat(mapped, key, "entrypoint=a") + assert rejected.status_code == 400, rejected.text + assert mapped.provider.drain() == () + admitted: Final = _chat(mapped, key, f"entrypoint=a,tenant={uuid.uuid4().hex}") + assert admitted.status_code == 200, admitted.text + + +def test_require_budgeted_tag_rejects_an_unbudgeted_mapped_tag_until_the_tag_has_a_budget(mapped: Rig) -> None: + value: Final = uuid.uuid4().hex + tag: Final = mapped.budgeted_tag(value) + baggage: Final = f"budgeted={value},tenant={value}" + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key() + mapped.provider.drain() + unbudgeted: Final = tuple(_chat(mapped, key, baggage, Connection="close") for _ in range(WORKER_PROBES)) + assert [response.status_code for response in unbudgeted] == [400] * WORKER_PROBES, [ + response.text for response in unbudgeted + ] + assert mapped.provider.drain() == () + mapped.proxy.post("/tag/new", {"name": tag, "max_budget": 100}) + scenario.cleanups.callback(lambda: mapped.proxy.request("POST", "/tag/delete", {"name": tag})) + budgeted: Final = tuple(_chat(mapped, key, baggage, Connection="close") for _ in range(WORKER_PROBES)) + assert [response.status_code for response in budgeted] == [200] * WORKER_PROBES, [ + response.text for response in budgeted + ] + + +def _routing_caller_tag(rig: Rig, caller_tag: CallerTag) -> tuple[Mapping[str, JsonValue], Mapping[str, str]]: + json_metadata: Final[Mapping[str, JsonValue]] = {"metadata": json.dumps({"tags": [rig.exact_tag]})} + match caller_tag: + case "none": + return NO_FIELDS, {} + case "header": + return NO_FIELDS, {"x-litellm-tags": rig.exact_tag} + case "metadata_json_with_header": + return json_metadata, {"x-litellm-tags": f"other-{rig.marker}"} + case "litellm_metadata": + return {"litellm_metadata": {"tags": [rig.exact_tag]}}, {} + + +@pytest.mark.parametrize( + "caller_tag", + ["none", "header", "metadata_json_with_header", "litellm_metadata"], + ids=["baggage_only", "caller_tag", "metadata_json_tag_with_header", "litellm_metadata_tag"], +) +def test_mapped_tags_never_select_a_tag_routed_deployment(mapped: Rig, caller_tag: CallerTag) -> None: + tenant: Final = uuid.uuid4().hex + expected: Final = f"default-{mapped.marker}" if caller_tag == "none" else f"tagged-{mapped.marker}" + fields, headers = _routing_caller_tag(mapped, caller_tag) + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key() + responses: Final = tuple( + _chat( + mapped, + key, + f"entrypoint=a,tenant={tenant}", + model=f"routed-{mapped.marker}", + fields=fields, + **headers, + ) + for _ in range(3) + ) + assert [response.status_code for response in responses] == [200] * 3, [r.text for r in responses] + assert [response.headers.get("x-litellm-model-id") for response in responses] == [expected] * 3 + + +def test_mapped_tags_never_trigger_a_tag_scoped_policy_attachment(mapped: Rig) -> None: + entry: Final = uuid.uuid4().hex + tag: Final = mapped.usecase_tag(entry) + baggage: Final = f"entrypoint={entry},tenant={entry}" + proxy: Final = mapped.proxy + with wire_server(lambda _request: Reply(body=GUARDRAIL_VERDICT)) as guardrail, proxy.scenario() as scenario: + key: Final = scenario.key() + guardrail_created: Final = proxy.post( + "/guardrails", + { + "guardrail": { + "guardrail_name": f"guardrail-{entry}", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{guardrail.url}/beta/litellm_basic_guardrail_api", + "api_key": "synthetic-guardrail-key", + }, + } + }, + ) + scenario.cleanups.callback(lambda: proxy.request("DELETE", f"/guardrails/{guardrail_created['guardrail_id']}")) + policy: Final = proxy.post( + "/policies", {"policy_name": f"policy-{entry}", "guardrails_add": [f"guardrail-{entry}"]} + ) + scenario.cleanups.callback(lambda: proxy.request("DELETE", f"/policies/{policy['policy_id']}")) + attachment: Final = proxy.post("/policies/attachments", {"policy_name": f"policy-{entry}", "tags": [tag]}) + scenario.cleanups.callback( + lambda: proxy.request("DELETE", f"/policies/attachments/{attachment['attachment_id']}") + ) + caller_tagged: Final = _chat(mapped, key, baggage, **{"x-litellm-tags": tag}) + assert caller_tagged.status_code == 200, caller_tagged.text + assert caller_tagged.headers.get("x-litellm-applied-policies") == f"policy-{entry}", caller_tagged.text + assert len(guardrail.drain()) == 1 + baggage_only: Final = _chat(mapped, key, baggage) + assert baggage_only.status_code == 200, baggage_only.text + assert baggage_only.headers.get("x-litellm-applied-policies") is None, baggage_only.text + assert guardrail.drain() == () + + +def _handshake_status(url: str, headers: Mapping[str, str]) -> int: + try: + with connect(url, additional_headers=headers, open_timeout=30): + return 101 + except InvalidStatus as refusal: + return refusal.response.status_code + + +def test_required_mapping_refuses_websocket_inference_before_the_upstream(mapped: Rig) -> None: + proxy_url: Final = str(mapped.proxy.client.base_url).rstrip("/") + upstream_api_key: Final = f"Bearer realtime-{mapped.marker}" + with ( + mapped.proxy.scenario() as scenario, + httpx.Client(base_url=mapped.proxy.upstream_url, timeout=5, trust_env=False) as upstream, + ): + key: Final = scenario.key() + upstream.get("/__observations").raise_for_status() + status: Final = _handshake_status( + f"ws{proxy_url.removeprefix('http')}/v1/realtime?model=realtime-{mapped.marker}", + {"Authorization": f"Bearer {key}", "baggage": f"entrypoint=a,tenant={uuid.uuid4().hex}"}, + ) + assert status == 403, status + observed: Final = list_value(JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]) + upgrades: Final = tuple( + request + for request in (object_value(item) for item in observed) + if request["method"] == "WEBSOCKET" and request["authorization"] == upstream_api_key + ) + assert upgrades == (), upgrades + + +@pytest.mark.parametrize( + ("route", "body"), + [ + ("/utils/token_counter", {"messages": [{"role": "user", "content": "count me"}]}), + ("/v1/messages/count_tokens", {"messages": [{"role": "user", "content": "count me"}]}), + ], + ids=["token_counter", "messages_count_tokens"], +) +def test_required_mapping_skips_unbilled_routes(mapped: Rig, route: str, body: dict[str, JsonValue]) -> None: + with mapped.proxy.scenario() as scenario: + key: Final = scenario.key() + response: Final = mapped.proxy.request("POST", route, {"model": mapped.chat_model, **body}, key=key) + assert response.status_code == 200, response.text + + +def test_custom_passthrough_spend_row_carries_the_mapped_tags(mapped: Rig) -> None: + tenant: Final = uuid.uuid4().hex + path: Final = f"/baggage-passthrough-{tenant}" + with mapped.proxy.scenario() as scenario: + created: Final = mapped.proxy.post( + "/config/pass_through_endpoint", {"path": path, "target": f"{mapped.provider.url}/echo", "auth": True} + ) + endpoint_id: Final = str(object_value(list_value(created["endpoints"])[0])["id"]) + scenario.cleanups.callback( + lambda: mapped.proxy.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id}) + ) + key: Final = scenario.key(allowed_passthrough_routes=[path]) + response: Final = mapped.proxy.request( + "POST", + path, + {"model": mapped.chat_model, "messages": [{"role": "user", "content": tenant}]}, + key=key, + headers={"baggage": f"entrypoint=a,tenant={tenant}"}, + ) + assert response.status_code == 200, response.text + assert {mapped.exact_tag, mapped.tenant_tag(tenant)} <= _spend_tags(mapped.proxy, key) + + +def _rejected_client_tag(client_tag: RejectedClientTag, tag: str) -> tuple[Mapping[str, JsonValue], Mapping[str, str]]: + match client_tag: + case "none": + return NO_FIELDS, {} + case "header": + return NO_FIELDS, {"x-litellm-tags": tag} + case "body": + return {"metadata": {"tags": [tag]}}, {} + case "metadata_json": + return {"metadata": json.dumps({"tags": [tag]})}, {} + case "litellm_metadata": + return {"litellm_metadata": {"tags": [tag]}}, {} + + +@pytest.mark.parametrize( + ("client_tag", "baggage_maps", "plain_statuses"), + [ + ("none", True, (200,)), + ("header", True, (400,)), + ("body", True, (400,)), + ("metadata_json", True, (200, 400)), + ("metadata_json", False, (200, 400)), + ("litellm_metadata", True, (200,)), + ], + ids=[ + "baggage_only", + "header_tag", + "body_tag", + "metadata_json_tag", + "metadata_json_tag_unmapped_baggage", + "litellm_metadata_tag", + ], +) +def test_reject_clientside_metadata_tags_with_baggage( + rejecting: Rig, client_tag: RejectedClientTag, baggage_maps: bool, plain_statuses: tuple[int, ...] +) -> None: + tenant: Final = uuid.uuid4().hex + fields, headers = _rejected_client_tag(client_tag, f"caller-{tenant}") + baggage: Final = f"entrypoint=a,tenant={tenant}" if baggage_maps else f"region={tenant}" + mapped_tags: Final = frozenset({rejecting.exact_tag, rejecting.tenant_tag(tenant)} if baggage_maps else ()) + with rejecting.proxy.scenario() as scenario: + plain_key: Final = scenario.key() + key: Final = scenario.key() + plain: Final = _chat(rejecting, plain_key, None, fields=fields, **headers) + response: Final = _chat(rejecting, key, baggage, fields=fields, **headers) + assert plain.status_code in plain_statuses, plain.text + assert response.status_code == plain.status_code, (plain.text, response.text) + if response.status_code != 200: + return + assert _spend_tags(rejecting.proxy, key) == _spend_tags(rejecting.proxy, plain_key) | mapped_tags + + +@pytest.mark.parametrize( + ("baggage_keys", "maps_team"), + [(("team+id",), True), (("team%2Bid",), True), (("team+id", "team%2Bid"), False)], + ids=["plus", "percent_encoded", "both_spellings"], +) +def test_plus_sign_rule_key_matches_either_wire_spelling_but_not_both( + optional: Rig, baggage_keys: tuple[str, ...], maps_team: bool +) -> None: + value: Final = uuid.uuid4().hex + baggage: Final = ",".join(("entrypoint=a", *(f"{baggage_key}={value}" for baggage_key in baggage_keys))) + with optional.proxy.scenario() as scenario: + key: Final = scenario.key() + response: Final = _chat(optional, key, baggage) + assert response.status_code == 200, response.text + team_tags: Final = {optional.team_tag(value)} if maps_team else set() + assert _spend_tags(optional.proxy, key) == {optional.exact_tag} | team_tags + + +@pytest.mark.parametrize("deployment_tagged", [True, False], ids=["tagged_deployment", "untagged_deployment"]) +def test_serving_deployment_tag_selects_its_guardrail_mode_when_baggage_maps_the_same_tag( + optional: Rig, deployment_tagged: bool +) -> None: + model: Final = f"guarded-{optional.marker}" if deployment_tagged else optional.chat_model + with optional.proxy.scenario() as scenario: + key: Final = scenario.key() + optional.provider.drain() + response: Final = _chat(optional, key, "guard=on", model=model) + assert response.status_code == 200, response.text + guardrail_inputs: Final = _guardrail_inputs(optional) + assert guardrail_inputs == (("response",) if deployment_tagged else ()), guardrail_inputs + + +def test_baggage_mapping_a_deployment_tag_keeps_the_team_tag_guardrail_mode_first(optional: Rig) -> None: + model: Final = f"guarded-pre-{optional.marker}" + with optional.proxy.scenario() as scenario: + team: Final = scenario.team(metadata={"tags": [optional.policy_tag]}) + key: Final = scenario.key(team_id=team) + optional.provider.drain() + plain: Final = _chat(optional, key, None, model=model) + plain_inputs: Final = _guardrail_inputs(optional) + response: Final = _chat(optional, key, "guard=pre", model=model) + assert (plain.status_code, response.status_code) == (200, 200), (plain.text, response.text) + assert (plain_inputs, _guardrail_inputs(optional)) == (("response",), ("response",)) + + +@pytest.mark.parametrize("metadata", ["run-123", {"tags": "run-123"}], ids=["non_object", "string_tags"]) +@pytest.mark.parametrize( + ("header_tag", "baggage"), + [(True, "region=sg"), (True, "entrypoint=a"), (False, "region=sg"), (False, "entrypoint=a")], + ids=["header_tag_unmapped_baggage", "header_tag_mapped_baggage", "unmapped_baggage", "mapped_baggage"], +) +def test_malformed_metadata_keeps_its_status_under_baggage( + optional: Rig, header_tag: bool, baggage: str, metadata: JsonValue +) -> None: + headers: Final = {"x-litellm-tags": f"caller-{uuid.uuid4().hex}"} if header_tag else {} + fields: Final[Mapping[str, JsonValue]] = {"metadata": metadata} + with optional.proxy.scenario() as scenario: + key: Final = scenario.key() + optional.provider.drain() + plain: Final = _chat(optional, key, None, fields=fields, **headers) + plain_calls: Final = len(optional.provider.drain()) + response: Final = _chat(optional, key, baggage, fields=fields, **headers) + assert plain.status_code < 500, plain.text + assert plain_calls == (1 if plain.status_code == 200 else 0) + assert (response.status_code, len(optional.provider.drain())) == (plain.status_code, plain_calls), ( + plain.text, + response.text, + ) + + +@pytest.mark.parametrize("outage_flag", [False, True], ids=["outage_flag_off", "outage_flag_on"]) +def test_budgeted_mapping_on_a_failed_tag_read(gateway: Gateway, tmp_path: Path, outage_flag: bool) -> None: + marker: Final = uuid.uuid4().hex[:12] + value: Final = uuid.uuid4().hex + tag: Final = f"budgeted-{marker}:{value}" + with ExitStack() as stack: + database_url: Final = stack.enter_context(scratch_database()) + relay, relayed_url = stack.enter_context(dropped_connection_relay(database_url, tag.encode())) + provider: Final = stack.enter_context(wire_server(_provider)) + config: Final = _write_config( + tmp_path, + [_deployment(f"chat-{marker}", provider)], + { + "request_tag_mappings": [ + {"key": "budgeted", "tag_prefix": f"budgeted-{marker}:", "require_budgeted_tag": True} + ], + "allow_requests_on_db_unavailable": outage_flag, + }, + {}, + ) + proxy: Final = stack.enter_context( + owned_proxy( + gateway, + tmp_path, + {"DATABASE_URL": plaintext_database_url(relayed_url), "PRISMA_HEALTH_WATCHDOG_ENABLED": "false"}, + config=config, + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) + ) + scenario: Final = stack.enter_context(proxy.scenario()) + key: Final = scenario.key(max_budget=50) + proxy.post("/tag/new", {"name": tag, "max_budget": 100}) + rig: Final = Rig(proxy, provider, provider, marker) + warmed: Final = _chat(rig, key, None) + assert warmed.status_code == 200, warmed.text + provider.drain() + relay.arm() + response: Final = _chat(rig, key, f"budgeted={value}") + relay.disarm() + assert relay.dropped.is_set(), response.text + if not outage_flag: + assert response.status_code == 503, response.text + assert provider.drain() == () + return + assert response.status_code == 200, response.text + assert response.headers.get("x-litellm-key-max-budget") == "50.0", dict(response.headers) + assert len(provider.drain()) == 1 diff --git a/tests/integration/spend/test_tag_budget_enforcement.py b/tests/integration/spend/test_tag_budget_enforcement.py index 0c0db2943c6..5430f77a3ec 100644 --- a/tests/integration/spend/test_tag_budget_enforcement.py +++ b/tests/integration/spend/test_tag_budget_enforcement.py @@ -105,6 +105,46 @@ def test_spend_over_a_tag_max_budget_rejects_the_next_request(gateway: Gateway) assert control.status_code == 200, control.text +def test_tag_new_counts_spend_from_creation_not_before(gateway: Gateway) -> None: + tag: Final = f"tag-created-late-{uuid.uuid4().hex}" + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": tag}) + + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.01, output_cost_per_token=0.01) + + def tagged(text: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"{text} {uuid.uuid4().hex}"}], + "metadata": {"tags": [tag]}, + }, + ) + + untracked: Final = tagged("spend before the tag exists") + assert untracked.status_code == 200, untracked.text + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + counted: Final = eventually( + lambda: float(cache.get(f"spend:tag:{tag}") or 0), + lambda spend: spend > 0.0001, + seconds=70, + ) + assert counted > 0.0001, counted + gateway.post("/tag/new", {"name": tag, "max_budget": 0.0001}) + scenario.cleanups.callback(delete_tag) + admitted: Final = tagged("first request under the new budget") + assert admitted.status_code == 200, admitted.text + blocked: Final = eventually( + lambda: tagged("budget probe"), lambda response: response.status_code != 200, seconds=70 + ) + assert blocked.status_code == 422, blocked.text + assert "budget" in blocked.text.lower(), blocked.text + + def test_tag_object_rpm_limit_rejects_the_second_request_across_keys(gateway: Gateway) -> None: tag: Final = f"tag-rpm-{uuid.uuid4().hex}" @@ -446,9 +486,7 @@ def test_tag_budget_duration_can_be_cleared_and_limits_are_read_back(gateway: Ga "rpm_limit": budget_info["rpm_limit"], "budget_duration": budget_info["budget_duration"], "budget_reset_at": budget_info["budget_reset_at"], - } == {"max_budget": 100, "rpm_limit": 1, "budget_duration": None, "budget_reset_at": None}, ( - info_response.text - ) + } == {"max_budget": 100, "rpm_limit": 1, "budget_duration": None, "budget_reset_at": None}, info_response.text tag_rows: Final = read_rows('SELECT budget_id FROM "LiteLLM_TagTable" WHERE tag_name = %s', (tag,)) assert len(tag_rows) == 1, tag_rows @@ -457,9 +495,9 @@ def test_tag_budget_duration_can_be_cleared_and_limits_are_read_back(gateway: Ga "WHERE budget_id = %s", (str(tag_rows[0]["budget_id"]),), ) - assert budget_rows == [ - {"max_budget": 100, "rpm_limit": 1, "budget_duration": None, "budget_reset_at": None} - ], budget_rows + assert budget_rows == [{"max_budget": 100, "rpm_limit": 1, "budget_duration": None, "budget_reset_at": None}], ( + budget_rows + ) @pytest.mark.parametrize("tag_source", _TAG_SOURCES, ids=_TAG_SOURCE_IDS) diff --git a/tests/unit/enterprise/integrations/test_custom_guardrail.py b/tests/unit/enterprise/integrations/test_custom_guardrail.py index c6e48061698..84f34144d99 100644 --- a/tests/unit/enterprise/integrations/test_custom_guardrail.py +++ b/tests/unit/enterprise/integrations/test_custom_guardrail.py @@ -1,3 +1,5 @@ +import pytest + from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks, Mode @@ -249,3 +251,82 @@ def test_custom_guardrail_with_mode(monkeypatch): ) is False ) + + +@pytest.mark.parametrize("caller_tags,runs_pre_call", [((), True), (("test_tag",), False)]) +def test_custom_guardrail_mode_ignores_baggage_mapped_tags_the_caller_did_not_send( + monkeypatch, caller_tags, runs_pre_call +): + from types import SimpleNamespace + from typing import cast + + from fastapi import Request + + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + cg = CustomGuardrail( + guardrail_name="test_guardrail", + supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only], + event_hook=Mode(tags={"test_tag": "logging_only"}, default="pre_call"), + default_on=True, + ) + request = cast(Request, SimpleNamespace(state=SimpleNamespace(litellm_baggage_mapped_tags=("test_tag",)))) + data = {"messages": [{"role": "user", "content": "test"}], "metadata": {"tags": []}} + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) + data["metadata"]["caller_tags"] = caller_tags + assert cg.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call) is runs_pre_call + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "team_tags,deployment_tags,runs_post_call", + [ + (["policy:sensitive"], ["usecase:dev"], True), + ([], ["policy:sensitive", "usecase:dev"], True), + (["policy:sensitive", "usecase:dev"], [], True), + ([], ["usecase:dev"], False), + ([], [], True), + ], +) +async def test_baggage_never_changes_the_guardrail_mode_key_team_and_deployment_tags_select( + monkeypatch, team_tags, deployment_tags, runs_post_call +): + from types import SimpleNamespace + from unittest.mock import MagicMock + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + cg = CustomGuardrail( + guardrail_name="test_guardrail", + supported_event_hooks=[GuardrailEventHooks.post_call, GuardrailEventHooks.logging_only], + event_hook=Mode(tags={"policy:sensitive": "post_call", "usecase:dev": "logging_only"}, default="post_call"), + default_on=True, + ) + router = litellm.Router( + model_list=[{"model_name": "gpt-4", "litellm_params": {"model": "gpt-4o", "tags": deployment_tags}}] + ) + + async def runs_post_call_guardrail(baggage_tags: tuple[str, ...]) -> bool: + request = MagicMock() + request.url.path = "/chat/completions" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client.host = "127.0.0.1" + request.state = SimpleNamespace(litellm_baggage_mapped_tags=baggage_tags) + data = await add_litellm_data_to_request( + data={"model": "gpt-4", "messages": [{"role": "user", "content": "test"}]}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", team_metadata={"tags": team_tags}), + proxy_config=MagicMock(), + general_settings={}, + ) + await router.acompletion(**data, mock_response="hi") + return cg.should_run_guardrail(data=data, event_type=GuardrailEventHooks.post_call) + + assert await runs_post_call_guardrail(()) is runs_post_call + assert await runs_post_call_guardrail(("usecase:dev",)) is runs_post_call diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index c583407e61e..effdd9725c2 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -3753,6 +3753,85 @@ async def test_tag_max_budget_check_still_enforces_registered_tag_over_budget(): assert [call.kwargs["where"]["tag_name"]["in"] for call in batch_calls] == [["paid-tag"]] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("client_tags", "key_metadata"), + [ + (b'"run-123"', {"tags": ["engineering"]}), + (b'""', {}), + (b"{}", {}), + (b"false", {}), + (b"0", {}), + ], + ids=["string_replaced_by_key_tags", "empty_string", "empty_object", "false", "zero"], +) +async def test_common_checks_rejects_a_spent_mapped_tag_whatever_non_list_tags_the_client_sends( + client_tags: bytes, key_metadata: dict[str, list[str]] +): + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.utils import ProxyLogging + + async def fake_find_many(**kwargs): + if "where" not in kwargs: + return [_tag_registry_row("usecase:spent")] + return [_tag_db_row(name, max_budget=1.0) for name in kwargs["where"]["tag_name"]["in"]] + + async def spent_tag_spend(counter_key, fallback_spend, max_budget=None, **kwargs): + return 1.5 if counter_key == "spend:tag:usecase:spent" else fallback_spend + + async def receive(): + return { + "type": "http.request", + "body": b'{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"metadata":{"tags":' + + client_tags + + b"}}", + } + + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_tagtable.find_many = AsyncMock(side_effect=fake_find_many) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=spent")], + }, + receive, + ) + request_body: Final = await read_request_body(request) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + request_body, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.get_current_spend", spent_tag_spend), + pytest.raises(litellm.BudgetExceededError) as exc_info, + ): + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=UserAPIKeyAuth(token="test-token", metadata=key_metadata), + request=request, + ) + assert exc_info.value.entity_id == "usecase:spent" + + @pytest.mark.asyncio async def test_get_team_object_raises_404_when_not_found(): from unittest.mock import AsyncMock, MagicMock @@ -11541,3 +11620,84 @@ async def test_authoritative_group_grants_propagate_policy_outages( await get_agent_ids_from_access_groups(["group"], check_db_only=True) else: assert await get_agent_ids_from_access_groups(["group"]) == [] + + +def _client_tag_request(client_tags, baggage): + return Request( + { + "type": "http", + "headers": [ + *([(b"baggage", b"entrypoint=a")] if baggage else []), + *([(b"x-litellm-tags", b"usecase:a")] if client_tags == "header" else []), + ], + } + ) + + +def _client_tag_body(client_tags, client_slot): + if client_tags is None or client_tags == "header": + return {} + return {client_slot: client_tags if isinstance(client_tags, str) else {"tags": client_tags}} + + +def _client_tag_rejection(route, data, request): + from litellm.proxy.auth.auth_checks import _reject_clientside_metadata_tags_check + + try: + _reject_clientside_metadata_tags_check({"reject_clientside_metadata_tags": True}, data, route, request) + except ProxyException as error: + return error.code + return None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/messages", "/v1/responses"]) +@pytest.mark.parametrize("client_tags", [None, [], ["usecase:a"], '{"tags":["usecase:a"]}', "header"]) +@pytest.mark.parametrize("client_slot", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("baggage", [True, False]) +async def test_baggage_mapping_rejects_exactly_the_client_tags_rejected_without_it( + route, client_tags, client_slot, baggage +): + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + unmapped_request = _client_tag_request(client_tags, baggage=False) + unmapped = _client_tag_body(client_tags, client_slot) + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(unmapped, route) + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth(unmapped_request, unmapped, UserAPIKeyAuth(api_key="qa-key")) + + mapped_request = _client_tag_request(client_tags, baggage=baggage) + mapped = _client_tag_body(client_tags, client_slot) + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(mapped, route) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + mapped_request, + mapped, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth(mapped_request, mapped, UserAPIKeyAuth(api_key="qa-key")) + + assert _client_tag_rejection(route, mapped, mapped_request) == _client_tag_rejection( + route, unmapped, unmapped_request + ) + if client_tags is None and baggage: + assert "usecase:a" in mapped["metadata" if "chat/completions" in route else "litellm_metadata"]["tags"] + + +@pytest.mark.asyncio +async def test_baggage_mapping_still_rejects_client_metadata_tags(): + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + request = _client_tag_request(["usecase:b"], baggage=True) + data = {"metadata": {"tags": ["usecase:b"]}} + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + data, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + assert _client_tag_rejection("/v1/chat/completions", data, request) == "400" + from litellm.utils import get_request_guardrails + + assert get_request_guardrails(data) == [] diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 3660019cc2c..60a8dd73a34 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -2102,3 +2102,183 @@ async def test_jwt_path_skips_user_model_budget_only_for_zero_spend_mcp_methods( assert result.user_id == "jwt-user" assert user_model_budget_check.await_count == (1 if expect_user_model_budget_check else 0) + + +def _qa_websocket(messages): + async def receive(): + return {"type": "websocket.connect"} + + async def send(message): + messages.append(message) + + return WebSocket( + { + "type": "websocket", + "path": "/v1/responses", + "scheme": "ws", + "server": ("localhost", 4000), + "query_string": b"model=qa-model", + "headers": [(b"authorization", b"Bearer qa-key")], + }, + receive=receive, + send=send, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("close_on_auth_error", [True, False]) +async def test_websocket_inference_is_denied_when_a_required_baggage_mapping_is_configured(close_on_auth_error): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket_for_model + + messages = [] + with ( + patch.dict( + litellm.proxy.proxy_server.general_settings, + { + "request_tag_mappings": [ + {"key": "region", "tag_prefix": "region:"}, + {"key": "entrypoint", "tag_prefix": "usecase:", "required": True}, + ] + }, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", + new=AsyncMock(return_value=UserAPIKeyAuth(api_key="qa-key")), + ), + ): + with pytest.raises(HTTPException, match="WebSocket inference is disabled") as error: + await user_api_key_auth_websocket_for_model( + _qa_websocket(messages), "qa-model", close_on_auth_error=close_on_auth_error + ) + assert error.value.status_code == 403 + assert messages == ([{"type": "websocket.close", "code": 1008, "reason": ""}] if close_on_auth_error else []) + + +@pytest.mark.asyncio +async def test_websocket_invalid_key_is_rejected_before_the_required_baggage_mapping_denial(): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket_for_model + + with ( + patch.dict( + litellm.proxy.proxy_server.general_settings, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "required": True}]}, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", + new=AsyncMock(side_effect=HTTPException(status_code=401, detail="Invalid proxy server token passed")), + ), + ): + with pytest.raises(HTTPException) as error: + await user_api_key_auth_websocket_for_model(_qa_websocket([]), "qa-model", close_on_auth_error=False) + assert error.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_websocket_inference_is_allowed_when_no_baggage_mapping_is_required(): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket_for_model + + messages = [] + authorized = UserAPIKeyAuth(api_key="qa-key") + with ( + patch.dict( + litellm.proxy.proxy_server.general_settings, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]}, + ), + patch( + "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", new=AsyncMock(return_value=authorized) + ) as http_auth, + ): + result = await user_api_key_auth_websocket_for_model(_qa_websocket(messages), "qa-model") + assert result is authorized + assert http_auth.await_args.kwargs["api_key"] == "Bearer qa-key" + assert messages == [] + + +async def _auth_then_serve_chat(monkeypatch, general_settings, body, headers): + import copy + + from fastapi import Request as FastAPIRequest + from fastapi import Response + + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.proxy_server import chat_completion + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["usecase:a"], "mock_response": "hi"}, + "model_info": {"id": "usecase-deployment"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["default"], "mock_response": "hi"}, + "model_info": {"id": "default-deployment"}, + }, + ], + enable_tag_filtering=True, + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", general_settings) + + async def receive(): + return {"type": "http.request", "body": json.dumps(body).encode()} + + request = FastAPIRequest( + { + "type": "http", + "method": "POST", + "path": "/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), *headers], + }, + receive, + ) + auth = await user_api_key_auth(request=request, api_key=f"Bearer {MASTER_KEY}") + cached_body = copy.deepcopy(await read_request_body(request)) + response = Response() + await chat_completion(request=request, fastapi_response=response, user_api_key_dict=auth) + return cached_body, response.headers["x-litellm-model-id"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "general_settings", + [{}, {"request_tag_mappings": [{"key": "region", "tag_prefix": "region:"}]}], + ids=["no_mappings", "mapped_baggage"], +) +async def test_auth_header_tag_merge_keeps_json_string_metadata_tags_routable(monkeypatch, general_settings): + body = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "metadata": json.dumps({"tags": ["usecase:a"]}), + } + cached_body, served_by = await _auth_then_serve_chat( + monkeypatch, general_settings, body, [(b"x-litellm-tags", b"other"), (b"baggage", b"region=sg")] + ) + assert served_by == "usecase-deployment" + assert cached_body["metadata"] == {"tags": ["usecase:a", "other"]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("client_metadata", ["run-123", 42, ["x"]]) +@pytest.mark.parametrize( + "baggage", + [[], [(b"baggage", b"region=sg")], [(b"baggage", b"entrypoint=a")]], + ids=["no_baggage", "unmapped_baggage", "mapped_baggage"], +) +async def test_non_object_metadata_is_served_like_unset_mappings_whatever_the_baggage_maps( + monkeypatch, client_metadata, baggage +): + body = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "metadata": client_metadata} + headers = [(b"x-litellm-tags", b"usecase:a"), *baggage] + unset = await _auth_then_serve_chat(monkeypatch, {}, body, headers) + mapped = await _auth_then_serve_chat( + monkeypatch, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, body, headers + ) + assert mapped == unset + assert mapped[1] == "usecase-deployment" diff --git a/tests/unit/proxy/common_utils/test_request_tag_mappings.py b/tests/unit/proxy/common_utils/test_request_tag_mappings.py new file mode 100644 index 00000000000..e1644544c46 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_request_tag_mappings.py @@ -0,0 +1,473 @@ +import json +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import Request +from pydantic import ValidationError + +from litellm.models.budget import LiteLLM_BudgetTable +from litellm.constants import BAGGAGE_MAPPED_TAGS_METADATA_KEY +from litellm.models.tag import LiteLLM_TagTable +from litellm.proxy._types import ConfigGeneralSettings, ProxyException +from litellm.proxy.common_utils.http_parsing_utils import read_request_body +from litellm.proxy.common_utils.request_tag_mappings import is_baggage_mapped_route, resolve_baggage_tag_mappings +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, tag_cache_key +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request +from litellm.types.proxy.request_tag_mappings import RequestTagMapping + + +def test_accounting_enrichment_without_request_state_preserves_legacy_metadata(): + request = cast(Request, SimpleNamespace(headers={})) + data = {"metadata": {"tags": ["existing"]}} + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) + assert data == {"metadata": {"tags": ["existing"]}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("header", [b"", b"entrypoint=a,entrypoint=b"]) +@pytest.mark.parametrize("client_metadata", [None, '{"requester_metadata":{"guardrails":["qa-guard"]}}']) +async def test_mapping_rejection_keeps_string_metadata_safe_for_auth_failure_logging(header, client_metadata): + from litellm.utils import get_request_guardrails + + request = Request({"type": "http", "headers": [(b"baggage", header)]}) + data = {"metadata": client_metadata} + with pytest.raises(ProxyException) as error: + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + data, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "required": True}]}, + None, + MagicMock(), + ) + assert error.value.code == "400" + assert get_request_guardrails(data) == (["qa-guard"] if client_metadata is not None else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("slot", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("invalid_metadata", [[], 42, "not-json", '"not-an-object"']) +async def test_invalid_mapper_metadata_maps_like_empty_metadata_and_keeps_the_client_cache(slot, invalid_metadata): + from litellm.proxy.auth.user_api_key_auth import _read_request_body_deferring_parse_failure + from litellm.utils import get_request_guardrails + + body = {"metadata": '{"tags":["client"]}', "litellm_metadata": '{"tags":["alternate"]}'} + body[slot] = invalid_metadata + + async def receive(): + return {"type": "http.request", "body": json.dumps(body).encode()} + + request = Request( + {"type": "http", "method": "POST", "path": "/v1/responses", "headers": [(b"baggage", b"entrypoint=a")]}, + receive, + ) + data, parse_error = await _read_request_body_deferring_parse_failure(request) + assert parse_error is None + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + data, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + assert "usecase:a" in data["litellm_metadata"]["tags"] + assert "tags" not in data[slot] or slot == "litellm_metadata" + assert get_request_guardrails(data) == [] + assert await read_request_body(request) == body + + +@pytest.mark.parametrize( + "header,value", + [ + ("entrypoint=a", "a"), + ("entrypoint = a", "a"), + ("region=sg, entrypoint=customer%20support;version=1", "customer support"), + ], +) +def test_variable_mapper_uses_decoded_baggage_value(header, value): + rule = RequestTagMapping(key="entrypoint", tag_prefix="usecase:") + assert resolve_baggage_tag_mappings(header, [rule]) == ((f"usecase:{value}", False),) + + +@pytest.mark.parametrize( + "header,expected", + [("entrypoint=a", "department:support"), ("entrypoint=b", "department:sales"), ("entrypoint=x", "usecase:x")], +) +def test_exact_rules_override_variable_fallback_independent_of_order(header, expected): + rules = [ + RequestTagMapping(key="entrypoint", tag_prefix="usecase:"), + RequestTagMapping(key="entrypoint", value="a", tag="department:support"), + RequestTagMapping(key="entrypoint", value="b", tag="department:sales"), + ] + assert resolve_baggage_tag_mappings(header, rules) == ((expected, False),) + assert resolve_baggage_tag_mappings(header, list(reversed(rules))) == ((expected, False),) + + +@pytest.mark.parametrize("header", ["entrypoint=%00", "entrypoint=a%0Ab", "entrypoint=b,entry%70oint=a"]) +def test_variable_mapper_skips_control_characters_and_percent_encoded_duplicate_keys(header): + rule = RequestTagMapping(key="entrypoint", tag_prefix="usecase:") + assert resolve_baggage_tag_mappings(header, [rule]) == () + + +@pytest.mark.parametrize( + "header,expected", + [ + ("entry+point=a", (("usecase:a", False),)), + ("entry%2Bpoint=a", (("usecase:a", False),)), + ("entry+point=a,entry%2Bpoint=b", ()), + ], +) +def test_key_with_a_plus_matches_both_wire_spellings_and_rejects_both_at_once(header, expected): + rule = RequestTagMapping(key="entry+point", tag_prefix="usecase:") + assert resolve_baggage_tag_mappings(header, [rule]) == expected + + +def test_exact_rules_work_without_a_variable_rule(): + rule = RequestTagMapping(key="entrypoint", value="a", tag="usecase:manual") + assert resolve_baggage_tag_mappings("entrypoint=a", [rule]) == (("usecase:manual", False),) + assert resolve_baggage_tag_mappings("entrypoint=x", [rule]) == () + + +@pytest.mark.parametrize( + "header", ["", "region=sg", "entrypoint=", "entrypoint=a,entrypoint=b", "entrypoint=b,entry%70oint=a", "entrypoint=x"] +) +def test_required_mapping_rejects_missing_ambiguous_or_unmapped_values(header): + rule = RequestTagMapping(key="entrypoint", value="a", tag="usecase:a", required=True) + with pytest.raises(ProxyException) as error: + resolve_baggage_tag_mappings(header, [rule]) + assert error.value.code == "400" + + +@pytest.mark.parametrize( + "rule", + [ + {"key": "entrypoint", "tag": "usecase:a"}, + {"key": "entrypoint", "value": "a", "tag_prefix": "usecase:"}, + {"key": "entrypoint", "value": "a", "tag": "usecase:a", "tag_prefix": "usecase:"}, + {"key": "", "tag_prefix": "usecase:"}, + ], +) +def test_invalid_rule_is_rejected_during_config_validation(rule): + with pytest.raises(ValidationError): + ConfigGeneralSettings.model_validate({"request_tag_mappings": [rule]}) + + +def test_duplicate_fallbacks_are_rejected(): + with pytest.raises(ValidationError, match="Only one mapping"): + ConfigGeneralSettings.model_validate( + { + "request_tag_mappings": [ + {"key": "entrypoint", "tag_prefix": "one:"}, + {"key": "entrypoint", "tag_prefix": "two:"}, + ] + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route,slot", + [("/chat/completions", "metadata"), ("/v1/messages", "litellm_metadata"), ("/v1/responses", "litellm_metadata")], +) +async def test_mapped_tag_merges_with_existing_tags_in_the_budget_metadata_slot(route, slot): + data = {slot: {"tags": ["existing", "usecase:a"]}} + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(data, route) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + data, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + assert data[slot]["tags"] == ["existing", "usecase:a"] + + +@pytest.mark.asyncio +async def test_budgeted_mapper_requires_a_database(): + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + with pytest.raises(ProxyException) as error: + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, {}, config, None, MagicMock() + ) + assert error.value.code == "503" + + +@pytest.mark.asyncio +async def test_budgeted_mapper_answers_503_when_the_tag_budget_lookup_fails(): + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(side_effect=Exception("database unreachable")) + config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + with pytest.raises(ProxyException, match="could not read tag budgets") as error: + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, {"metadata": {}}, config, prisma_client, UserApiKeyCache() + ) + assert error.value.code == "503" + + +@pytest.mark.asyncio +async def test_budgeted_mapper_serves_through_a_db_outage_when_requests_are_allowed_on_db_unavailable(): + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(side_effect=Exception("database unreachable")) + data = {"metadata": {}} + config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + with patch.dict("litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": True}): + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, data, config, prisma_client, UserApiKeyCache() + ) + assert data["metadata"]["tags"] == ["usecase:a"] + + +async def _cache_with_tag(tag: LiteLLM_TagTable) -> UserApiKeyCache: + cache = UserApiKeyCache() + await cache.async_set_cache(key=tag_cache_key(tag.tag_name), value=tag, model_type=LiteLLM_TagTable) + return cache + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "budget_table", + [None, LiteLLM_BudgetTable(budget_id="qa-budget", max_budget=None, tpm_limit=10)], +) +async def test_budgeted_mapper_rejects_a_tag_without_a_max_budget(budget_table): + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a")]}) + data = {"metadata": {}} + cache = await _cache_with_tag(LiteLLM_TagTable(tag_name="usecase:a", litellm_budget_table=budget_table)) + config = {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}]} + with pytest.raises(ProxyException, match="usecase:a requires an existing tag budget") as error: + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, data, config, MagicMock(), cache + ) + assert error.value.code == "400" + assert data == {"metadata": {}} + + +@pytest.mark.asyncio +async def test_budgeted_mapper_admits_a_tag_with_a_max_budget_and_unbudgeted_rules_need_none(): + request = Request({"type": "http", "headers": [(b"baggage", b"entrypoint=a,region=sg")]}) + data = {"metadata": {}} + cache = await _cache_with_tag( + LiteLLM_TagTable( + tag_name="usecase:a", litellm_budget_table=LiteLLM_BudgetTable(budget_id="qa-budget", max_budget=1.0) + ) + ) + config = { + "request_tag_mappings": [ + {"key": "entrypoint", "tag_prefix": "usecase:", "require_budgeted_tag": True}, + {"key": "region", "tag_prefix": "region:"}, + ] + } + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, data, config, MagicMock(), cache + ) + assert data["metadata"]["tags"] == ["usecase:a", "region:sg"] + + +@pytest.mark.parametrize( + "route", + ["/v1/chat/completions", "/chat/completions", "/v1/messages", "/v1/responses", "/a2a/qa-agent/message/send"], +) +def test_inference_routes_take_baggage_mappings(route): + assert is_baggage_mapped_route(route) + + +@pytest.mark.parametrize( + "route", + [ + "/utils/token_counter", + "/utils/transform_request", + "/v1/messages/count_tokens", + "/v1/responses/input_tokens", + "/v1beta/models/gemini-pro:countTokens", + "/mcp", + ], +) +def test_unbilled_and_mcp_routes_skip_baggage_mappings(route): + assert not is_baggage_mapped_route(route) + + +@pytest.mark.parametrize("registered", [True, False]) +def test_custom_pass_through_routes_take_baggage_mappings_once_registered(registered): + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import _registered_pass_through_routes + + endpoints = {"qa-endpoint:exact:/billing-api": {"target": "http://example.com", "headers": {}}} + with patch.dict(_registered_pass_through_routes, endpoints if registered else {}, clear=True): + assert is_baggage_mapped_route("/billing-api") is registered + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route,slot", + [("/chat/completions", "metadata"), ("/v1/messages", "litellm_metadata"), ("/v1/responses", "litellm_metadata")], +) +@pytest.mark.parametrize("query", [b"api-version=qa-version", b"key=qa-virtual-key"]) +@pytest.mark.parametrize("client_metadata", [None, '{"requester_metadata":{"guardrails":[]}}']) +async def test_mapping_survives_cached_body_read_without_leaking_auth_query_fields(route, slot, query, client_metadata): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import ProxyConfig + + body = {"model": "qa-model"} + if client_metadata is not None: + body["metadata"] = client_metadata + + async def receive(): + return {"type": "http.request", "body": json.dumps(body).encode()} + + request = Request( + { + "type": "http", + "method": "POST", + "path": route, + "scheme": "http", + "server": ("localhost", 4000), + "query_string": query, + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + }, + receive, + ) + from litellm.proxy.auth.user_api_key_auth import _read_request_body_deferring_parse_failure + + data, parse_error = await _read_request_body_deferring_parse_failure(request) + assert parse_error is None + LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(data, route) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + data, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + downstream = await read_request_body(request) + assert downstream == body + enriched = await add_litellm_data_to_request( + data=downstream, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="qa-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert enriched[slot]["tags"] == ["usecase:a"] + assert "key" not in enriched + assert "api-version" not in enriched + assert "key" not in enriched["proxy_server_request"]["body"] + assert "api-version" not in enriched["proxy_server_request"]["body"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_metadata", + [{"trace_name": "qa"}, {"trace_name": "qa", "tags": ["client:x"]}, {"trace_name": "qa", "tags": ["usecase:a"]}], +) +async def test_mapped_tags_stay_out_of_requester_metadata(client_metadata): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import _read_request_body_deferring_parse_failure + from litellm.proxy.proxy_server import ProxyConfig + + body = {"model": "qa-model", "metadata": client_metadata} + + async def receive(): + return {"type": "http.request", "body": json.dumps(body).encode()} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + }, + receive, + ) + data, parse_error = await _read_request_body_deferring_parse_failure(request) + assert parse_error is None + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, data, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, None, MagicMock() + ) + enriched = await add_litellm_data_to_request( + data=await read_request_body(request), + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="qa-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + requester_metadata = enriched["metadata"]["requester_metadata"] + assert "usecase:a" in enriched["metadata"]["tags"] + assert requester_metadata["trace_name"] == "qa" + assert requester_metadata.get("tags") == client_metadata.get("tags") + assert BAGGAGE_MAPPED_TAGS_METADATA_KEY not in requester_metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "headers,body_fields", + [ + ([(b"x-litellm-tags", b"usecase:a")], {}), + ([], {"tags": ["usecase:a"]}), + ([], {"litellm_metadata": {"tags": ["usecase:a"]}}), + ], + ids=["header", "body", "litellm_metadata"], +) +async def test_client_sent_tag_still_routes_when_baggage_maps_to_it(headers, body_fields): + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import _read_request_body_deferring_parse_failure + from litellm.proxy.proxy_server import ProxyConfig + + body = {"model": "gpt-4", **body_fields} + + async def receive(): + return {"type": "http.request", "body": json.dumps(body).encode()} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a"), *headers], + }, + receive, + ) + data, parse_error = await _read_request_body_deferring_parse_failure(request) + assert parse_error is None + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, data, {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, None, MagicMock() + ) + enriched = await add_litellm_data_to_request( + data=await read_request_body(request), + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="qa-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["usecase:a"]}, + "model_info": {"id": "usecase-deployment"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["default"]}, + "model_info": {"id": "default-deployment"}, + }, + ], + enable_tag_filtering=True, + ) + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(enriched["metadata"]), + mock_response="hi", + ) + assert response._hidden_params["model_id"] == "usecase-deployment" diff --git a/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py index 1dfababae55..e5d13a381ae 100644 --- a/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py @@ -283,6 +283,66 @@ async def test_new_tag_persists_a_budget(): assert created_tag.budget_id == "budget-1" +@pytest.mark.asyncio +async def test_new_tag_budget_reads_the_row_spend_not_spend_from_before_the_tag_existed(): + """Spend on an unbudgeted request tag must not count against a budget created later. + + The new row starts at the spend it persists, and a restart reloads only that row, so the + counter the budget check reads has to agree with it from the moment the tag is created. + """ + from litellm.proxy.management_endpoints.tag_management_endpoints import new_tag + from litellm.proxy.proxy_server import get_current_spend, increment_spend_counter_cache + + counter_key: Final = "spend:tag:spent-before-budget-tag" + await increment_spend_counter_cache(counter_key=counter_key, increment=5.0) + + created_tag: Final = SimpleNamespace( + tag_name="spent-before-budget-tag", + description=None, + models=[], + created_at=datetime(2024, 1, 1), + updated_at=datetime(2024, 1, 1), + created_by="admin", + ) + + async def create_tag(data, **_): + created_tag.spend = data["spend"] + return created_tag + + mock_db: Final = Mock() + mock_db.litellm_tagtable.find_unique = AsyncMock(return_value=None) + mock_db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_budgettable.create = AsyncMock(return_value=SimpleNamespace(budget_id="budget-1")) + mock_db.litellm_tagtable.create = create_tag + with ( + patch( # test-quality-ok: endpoint resolves the fake database through proxy_server + "litellm.proxy.proxy_server.prisma_client", + SimpleNamespace(db=mock_db, jsonify_object=lambda data: dict(data)), + ), + patch( # test-quality-ok: endpoint reads the audit actor from proxy_server + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ), + patch( # test-quality-ok: endpoint requires a router before the budget write + "litellm.proxy.proxy_server.llm_router", object() + ), + patch( # test-quality-ok: auth object cache invalidation is outside this counter contract + "litellm.proxy.management_endpoints.tag_management_endpoints._evict_tag_cache_keys", new=AsyncMock() + ), + ): + await new_tag( + tag=TagNewRequest(name="spent-before-budget-tag", max_budget=25.0), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + budget_check_spend: Final = await get_current_spend( + counter_key=counter_key, + fallback_spend=created_tag.spend, + max_budget=25.0, + fallback_authoritative=True, + ) + assert budget_check_spend == created_tag.spend + + @pytest.mark.asyncio @pytest.mark.parametrize( ("budget_fields", "should_update", "expected_max_budget"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py index e658556bd1f..e654c6d444b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py @@ -264,6 +264,42 @@ async def test_gemini_pass_through_endpoint_forwards_key_and_body( assert outbound.url.params["key"] == "gemini-test-key" assert json.loads(outbound.content) == json.loads(body) +@pytest.mark.asyncio +async def test_router_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): + from fastapi import Request + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import get_passthrough_router_request_metadata + from litellm.proxy.utils import ProxyLogging + + async def receive(): + return {"type": "http.request", "body": b'{"model":"qa-model"}'} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/openai/v1/chat/completions", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + }, + receive, + ) + admitted = await read_request_body(request) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + admitted, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + UserApiKeyCache(), + ) + metadata = get_passthrough_router_request_metadata(UserAPIKeyAuth(api_key="qa-key"), request) + assert metadata["tags"] == ["usecase:a"] + assert await read_request_body(request) == {"model": "qa-model"} + _VERTEX_DEPLOYMENT_PATH: Final = ( "v1/projects/deployment-project/locations/global/publishers/anthropic/models/claude-sonnet-4-5:rawPredict" diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 4c3c7ae0f0a..f0baceecdaa 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -10337,3 +10337,39 @@ def test_init_kwargs_no_authenticated_identity_field_is_client_settable(mock_req metadata: Final = result["litellm_params"]["metadata"] survived: Final = {key: metadata.get(key) for key in expected if metadata.get(key) != expected[key]} assert survived == {}, f"client-supplied values survived for: {sorted(survived)}" + +@pytest.mark.asyncio +async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.utils import ProxyLogging + + async def receive(): + return {"type": "http.request", "body": b'{"model":"qa-model"}'} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/openai/v1/chat/completions", + "query_string": b"", + "headers": [ + (b"content-type", b"application/json"), + (b"baggage", b"entrypoint=a"), + (b"x-litellm-tags", b"existing"), + ], + }, + receive, + ) + admitted = await read_request_body(request) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + admitted, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + UserApiKeyCache(), + ) + metadata = _update_metadata_with_tags_in_header(request, {}) + assert metadata["tags"] == ["existing", "usecase:a"] + assert await read_request_body(request) == {"model": "qa-model"} diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 4647952312a..4a97106b3ac 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -4756,6 +4756,43 @@ async def test_add_guardrails_from_policy_engine(): attachment_registry._initialized = False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller_tags,inherited_tags,expected_guardrails", + [((), (), []), (("usecase:a",), (), ["usecase_guard"]), ((), ("usecase:a",), ["usecase_guard"])], +) +async def test_baggage_mapped_tags_match_tag_policies_only_when_the_caller_or_key_asked_for_them( + caller_tags, inherited_tags, expected_guardrails +): + from typing import cast + + from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.types.proxy.policy_engine import Policy, PolicyAttachment, PolicyGuardrails + + request = cast(Request, SimpleNamespace(state=SimpleNamespace(litellm_baggage_mapped_tags=("usecase:a",)))) + data = {"model": "gpt-4", "metadata": {"tags": [], "inherited_tags": inherited_tags}} + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) + data["metadata"]["caller_tags"] = caller_tags + policy_registry = get_policy_registry() + attachment_registry = get_attachment_registry() + policy_registry._policies = {"usecase": Policy(guardrails=PolicyGuardrails(add=["usecase_guard"]))} + policy_registry._initialized = True + attachment_registry._attachments = [PolicyAttachment(policy="usecase", tags=["usecase:a"])] + attachment_registry._initialized = True + try: + await add_guardrails_from_policy_engine( + data=data, metadata_variable_name="metadata", user_api_key_dict=UserAPIKeyAuth(api_key="test-key") + ) + finally: + policy_registry._policies = {} + policy_registry._initialized = False + attachment_registry._attachments = [] + attachment_registry._initialized = False + assert "usecase:a" in data["metadata"]["tags"] + assert data["metadata"].get("guardrails", []) == expected_guardrails + + def test_match_and_track_policies_preserves_attachment_and_request_body_order(): from litellm.proxy.policy_engine.attachment_registry import AttachmentRegistry from litellm.types.proxy.policy_engine import Policy, PolicyMatchContext @@ -10358,3 +10395,112 @@ async def test_per_user_token_resolution_uses_key_identity_without_header_mappin called_user_ids = [call.kwargs["user_id"] for call in load.await_args_list] assert called_user_ids == ["user-a"] assert updated["secret_fields"]["user_provider_credentials"]["copilot-cred"] == "gho_user_a_token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/images/generations", "/v1/rerank"]) +async def test_raw_body_inference_keeps_authenticated_baggage_tags_for_spend(route): + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.proxy_server import ProxyConfig + + async def receive(): + return {"type": "http.request", "body": b'{"model":"qa-model","metadata":{"tags":["existing"]}}'} + + request = Request( + { + "type": "http", + "method": "POST", + "path": route, + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"baggage", b"entrypoint=a")], + }, + receive, + ) + admitted = await read_request_body(request) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + admitted, + {"request_tag_mappings": [{"key": "entrypoint", "tag_prefix": "usecase:"}]}, + None, + MagicMock(), + ) + reparsed = json.loads(await request.body()) + enriched = await add_litellm_data_to_request( + data=reparsed, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="qa-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert "usecase:a" in enriched["metadata"]["tags"] + assert "existing" in enriched["metadata"]["tags"] + assert "usecase:a" not in enriched["metadata"]["inherited_tags"] + assert enriched["metadata"]["tags"].count("usecase:a") == 1 + + +async def _string_tags_outcome(baggage_headers: tuple[tuple[bytes, bytes], ...]) -> str: + from litellm.proxy.common_utils.http_parsing_utils import read_request_body + from litellm.proxy.proxy_server import ProxyConfig + + async def receive(): + return {"type": "http.request", "body": b'{"model":"qa-model","metadata":{"tags":"client-tag"}}'} + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "query_string": b"", + "headers": [(b"content-type", b"application/json"), *baggage_headers], + }, + receive, + ) + admitted: Final = await read_request_body(request) + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + request, + admitted, + {"request_tag_mappings": [{"key": "tenant", "tag_prefix": "tenant:"}]}, + None, + MagicMock(), + ) + try: + enriched: Final = await add_litellm_data_to_request( + data=json.loads(await request.body()), + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="qa-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + except Exception as error: + return type(error).__name__ + return repr(enriched["metadata"].get("tags")) + + +@pytest.mark.asyncio +async def test_string_metadata_tags_get_the_same_outcome_with_or_without_mapped_baggage(): + without_baggage: Final = await _string_tags_outcome(()) + with_baggage: Final = await _string_tags_outcome(((b"baggage", b"tenant=a"),)) + assert with_baggage == without_baggage + + +def test_applying_mapped_baggage_tags_again_leaves_the_metadata_unchanged(): + request: Final = Request({"type": "http", "headers": []}) + request.state.litellm_baggage_mapped_tags = ("tenant:a",) + once: Final[dict[str, object]] = {"metadata": {"tags": ["client-tag"]}} + twice: Final[dict[str, object]] = copy.deepcopy(once) + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, once) + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, twice) + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, twice) + assert twice == once + + +def test_mapped_baggage_tags_keep_non_string_client_tags_as_sent(): + request: Final = Request({"type": "http", "headers": []}) + request.state.litellm_baggage_mapped_tags = ("tenant:a",) + data: Final[dict[str, object]] = {"metadata": {"tags": [1, {"team": "x"}, "client-tag", "client-tag"]}} + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) + assert data["metadata"]["tags"] == [1, {"team": "x"}, "client-tag", "client-tag", "tenant:a"] diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 0dbf3451c3b..7eb3f557971 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -8311,6 +8311,37 @@ async def test_update_general_settings_ignores_an_invalid_user_api_key_cache_max assert cache.get_cache(key="key-0", local_only=True) == {"token": "key-0"} +@pytest.mark.asyncio +async def test_update_general_settings_serves_an_invalid_db_request_tag_mappings_like_unset_until_fixed(monkeypatch): + from fastapi import Request + + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + monkeypatch.setattr(proxy_server_module, "general_settings", proxy_config.settings) + + async def mapped_tags() -> list[str]: + data: Final[dict[str, object]] = {} + await LiteLLMProxyRequestSetup.apply_baggage_tag_mappings_pre_auth( + Request({"type": "http", "headers": [(b"baggage", b"tenant=a")]}), + data, + proxy_server_module.general_settings, + None, + MagicMock(), + ) + metadata: Final = data.get("metadata") + return list(metadata.get("tags", [])) if isinstance(metadata, dict) else [] + + await proxy_config._update_general_settings(db_general_settings={"request_tag_mappings": [{"key": "tenant"}]}) + assert await mapped_tags() == [] + + await proxy_config._update_general_settings( + db_general_settings={"request_tag_mappings": [{"key": "tenant", "tag_prefix": "tenant:", "required": True}]} + ) + assert await mapped_tags() == ["tenant:a"] + + @pytest.mark.asyncio async def test_update_general_settings_user_api_key_cache_max_size_yaml_wins(monkeypatch): """A DB value must not silently override an explicit YAML capacity on reload.""" diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index f0e85208933..097a34cf2e6 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3175,6 +3175,112 @@ async def test_chat_request_carrying_litellm_metadata_still_routes_on_proxy_merg assert deployment["model_info"]["id"] == "team-a-deployment" +def _baggage_mapped_metadata(client_tags: list[str], inherited_tags: list[str]) -> dict: + from types import SimpleNamespace + from typing import cast + + from fastapi import Request + + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + request = cast(Request, SimpleNamespace(state=SimpleNamespace(litellm_baggage_mapped_tags=("usecase:a",)))) + data = {"metadata": {"tags": client_tags, "inherited_tags": inherited_tags}} + LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) + return data["metadata"] + + +@pytest.mark.asyncio() +@pytest.mark.parametrize( + "client_tags,inherited_tags,expected_model_id", + [ + ([], [], "default-deployment"), + (["usecase:a"], [], "usecase-deployment"), + ([], ["usecase:a"], "usecase-deployment"), + ], +) +async def test_baggage_mapped_tags_classify_spend_without_selecting_deployments( + client_tags, inherited_tags, expected_model_id +): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["usecase:a"]}, + "model_info": {"id": "usecase-deployment"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["default"]}, + "model_info": {"id": "default-deployment"}, + }, + ], + enable_tag_filtering=True, + ) + metadata = _baggage_mapped_metadata(client_tags, inherited_tags) + assert "usecase:a" in metadata["tags"] + for _ in range(5): + response = await router.acompletion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + mock_response="hi", + ) + assert response._hidden_params["model_id"] == expected_model_id + + +@pytest.mark.asyncio() +@pytest.mark.parametrize( + "client_tags,inherited_tags,expected_model_id", + [ + ([], [], "tier-b"), + (["usecase:a"], [], "tier-a"), + ([], ["usecase:a"], "tier-a"), + ], +) +async def test_baggage_mapped_tags_never_select_a_tagged_pre_routing_strategy( + client_tags, inherited_tags, expected_model_id +): + from litellm.types.router import TaggedPreRoutingStrategy + + router = litellm.Router( + model_list=[ + { + "model_name": "smart", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "plain-smart"}, + }, + { + "model_name": "tier-a", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "tier-a"}, + }, + { + "model_name": "tier-b", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "tier-b"}, + }, + ], + num_retries=0, + ) + router.auto_routers = { + "smart": [ + TaggedPreRoutingStrategy(tags=("usecase:b",), strategy=_RewriteToTierStrategy("tier-b")), + TaggedPreRoutingStrategy(tags=("usecase:a",), strategy=_RewriteToTierStrategy("tier-a")), + ] + } + metadata = _baggage_mapped_metadata(client_tags, inherited_tags) + assert "usecase:a" in metadata["tags"] + + response = await router.acompletion( + model="smart", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == expected_model_id + + def _tag_router(model_list: list[dict[str, object]], tag_routing_prefix: str = "route:") -> litellm.Router: return litellm.Router(model_list=model_list, enable_tag_filtering=True, tag_routing_prefix=tag_routing_prefix) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index cc25d73b83a..4a9dd199c08 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -9673,6 +9673,26 @@ def test_update_kwargs_with_deployment_propagates_model_tags(): assert "production" in kwargs["metadata"]["tags"] +def test_update_kwargs_with_deployment_merges_many_client_tags_in_linear_time(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key", "tags": ["deployment-tag"]}, + }, + ], + ) + client_tags: Final = [f"client-{index}" for index in range(100_000)] + kwargs: Final = {"metadata": {"tags": client_tags}} + deployment: Final = router.get_deployment_by_model_group_name(model_group_name="gpt-4o-mini") + + started: Final = time.perf_counter() + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) + + assert time.perf_counter() - started < 2 + assert kwargs["metadata"]["tags"] == [*client_tags, "deployment-tag"] + + def test_update_kwargs_with_deployment_merges_tags_without_duplicates(): """ Test that when both request-level and deployment-level tags exist, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.test.tsx new file mode 100644 index 00000000000..b7ecd6bbb64 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.test.tsx @@ -0,0 +1,145 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { RequestTagMapping, RequestTagMappingsEditor, requestTagMappingsAreValid } from "./RequestTagMappings"; + +const exact: RequestTagMapping = { header: "baggage", key: "entrypoint", value: "a", tag: "support" }; +const variable: RequestTagMapping = { header: "baggage", key: "entrypoint", tag_prefix: "usecase:" }; + +function Editor({ + initial, + changed, + saved = vi.fn(), +}: { + initial: RequestTagMapping[]; + changed: (rules: RequestTagMapping[]) => void; + saved?: (rules: RequestTagMapping[]) => void; +}) { + const [rules, setRules] = useState(initial); + return ( + <> + { + setRules(next); + changed(next); + }} + /> + + + ); +} + +describe("RequestTagMappingsEditor", () => { + it("shows YAML-owned rules without allowing edits", () => { + render(); + expect(screen.getByLabelText("Mapping 1 match value")).toBeDisabled(); + expect(screen.getByRole("button", { name: "Add mapping" })).toBeDisabled(); + expect(screen.getByText(/Managed in config.yaml/)).toBeInTheDocument(); + }); + it("shows one-by-one mappings alongside variable prefix mappings", () => { + render(); + expect(screen.getByRole("combobox", { name: "Mapping 1 mode" })).toHaveTextContent("Exact value"); + expect(screen.getByRole("combobox", { name: "Mapping 2 mode" })).toHaveTextContent("Any value"); + expect(screen.getByLabelText("Mapping 1 match value")).toHaveValue("a"); + expect(screen.getByLabelText("Mapping 1 tag")).toHaveValue("support"); + expect(screen.getByLabelText("Mapping 2 tag prefix")).toHaveValue("usecase:"); + expect(screen.getByText("entrypoint=a → support")).toBeInTheDocument(); + expect(screen.getByText("entrypoint=x → usecase:x")).toBeInTheDocument(); + }); + + it("adds and edits a new exact value mapping", async () => { + const changed = vi.fn(); + render(); + await userEvent.click(screen.getByRole("button", { name: "Add mapping" })); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "entrypoint" } }); + fireEvent.change(screen.getByLabelText("Mapping 1 match value"), { target: { value: "b" } }); + fireEvent.change(screen.getByLabelText("Mapping 1 tag"), { target: { value: "sales" } }); + expect(screen.getByText("entrypoint=b → sales")).toBeInTheDocument(); + expect(changed).toHaveBeenLastCalledWith([{ header: "baggage", key: "entrypoint", value: "b", tag: "sales" }]); + }); + + it("changes an exact mapping to any-value mode and removes its exact match fields", async () => { + const changed = vi.fn(); + render(); + await userEvent.click(screen.getByRole("combobox", { name: "Mapping 1 mode" })); + await userEvent.click(await screen.findByRole("option", { name: "Any value" })); + expect(screen.queryByLabelText("Mapping 1 match value")).not.toBeInTheDocument(); + expect(screen.getByLabelText("Mapping 1 tag prefix")).toHaveValue(""); + expect(screen.getByLabelText("Mapping 1 tag prefix")).toHaveAttribute("placeholder", "usecase:"); + expect(changed).toHaveBeenLastCalledWith([ + expect.objectContaining({ value: undefined, tag: undefined, tag_prefix: "" }), + ]); + }); + + it("blocks saving a new mapping until its match value and tag are filled in", async () => { + const saved = vi.fn(); + render(); + await userEvent.click(screen.getByRole("button", { name: "Add mapping" })); + expect(screen.getByLabelText("Mapping 1 baggage key")).toHaveValue(""); + expect(screen.getByLabelText("Mapping 1 baggage key")).toHaveAttribute("placeholder", "entrypoint"); + expect(screen.getByText("Enter a baggage key")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "entrypoint" } }); + expect(screen.getByText("Enter a match value")).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Update" })); + fireEvent.change(screen.getByLabelText("Mapping 1 match value"), { target: { value: "b" } }); + expect(screen.getByText("Enter a tag")).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Update" })); + expect(saved).not.toHaveBeenCalled(); + fireEvent.change(screen.getByLabelText("Mapping 1 tag"), { target: { value: "sales" } }); + expect(screen.queryByText(/^Enter a/)).not.toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Update" })); + expect(saved).toHaveBeenCalledWith([{ header: "baggage", key: "entrypoint", value: "b", tag: "sales" }]); + }); + + it("blocks saving an any-value mapping until the admin types a tag prefix", async () => { + const saved = vi.fn(); + render(); + await userEvent.click(screen.getByRole("combobox", { name: "Mapping 1 mode" })); + await userEvent.click(await screen.findByRole("option", { name: "Any value" })); + expect(screen.getByText("Enter a tag prefix")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 1 tag prefix"), { target: { value: "team:" } }); + await userEvent.click(screen.getByRole("button", { name: "Update" })); + expect(saved).toHaveBeenCalledWith([expect.objectContaining({ key: "entrypoint", tag_prefix: "team:" })]); + }); + + it("blocks saving a baggage key that is empty or not a header token", async () => { + render(); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "" } }); + expect(screen.getByText("Enter a baggage key")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "entry point" } }); + expect(screen.getByText(/Use only letters, digits/)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "entry.point" } }); + expect(screen.getByRole("button", { name: "Update" })).toBeEnabled(); + }); + + it("blocks saving a second mapping for the same key and value until it differs", () => { + render(); + expect(screen.getByText("Another mapping already matches this key and value")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 2 match value"), { target: { value: "b" } }); + expect(screen.queryByText("Another mapping already matches this key and value")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeEnabled(); + }); + + it("blocks saving a second any-value mapping for the same key", () => { + render(); + expect(screen.getByText("Another mapping already matches this key and value")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Update" })).toBeDisabled(); + }); + + it("deletes one rule while keeping the other mode intact", async () => { + const changed = vi.fn(); + render(); + await userEvent.click(screen.getByRole("button", { name: "Delete mapping 1" })); + expect(screen.getByLabelText("Mapping 1 tag prefix")).toHaveValue("usecase:"); + expect(changed).toHaveBeenLastCalledWith([variable]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.tsx new file mode 100644 index 00000000000..d1101eaaf27 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/RequestTagMappings.tsx @@ -0,0 +1,189 @@ +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { Trash2 } from "lucide-react"; + +export interface RequestTagMapping { + header: "baggage"; + key: string; + value?: string | null; + tag?: string | null; + tag_prefix?: string | null; + required?: boolean; + require_budgeted_tag?: boolean; +} + +interface Props { + value: unknown; + disabled?: boolean; + onChange: (rules: RequestTagMapping[]) => void; +} + +const isMapping = (value: unknown): value is RequestTagMapping => { + if (typeof value !== "object" || value === null) return false; + return "key" in value && typeof value.key === "string"; +}; + +const MODES = [ + { value: "exact", label: "Exact value" }, + { value: "any", label: "Any value" }, +]; + +const BAGGAGE_KEY_PATTERN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; + +const isExact = (rule: RequestTagMapping) => rule.value !== undefined && rule.value !== null; + +const mappingProblem = (rule: RequestTagMapping): string | null => { + if (!rule.key) return "Enter a baggage key"; + if (!BAGGAGE_KEY_PATTERN.test(rule.key)) return "Use only letters, digits, and !#$%&'*+-.^_`|~ in the baggage key"; + if (isExact(rule)) { + if (!rule.value) return "Enter a match value"; + return rule.tag ? null : "Enter a tag"; + } + return rule.tag_prefix ? null : "Enter a tag prefix"; +}; + +const matchIdentity = (rule: RequestTagMapping) => JSON.stringify([rule.key, isExact(rule) ? rule.value : null]); + +const problemAt = (rules: RequestTagMapping[], index: number): string | null => { + const identity = matchIdentity(rules[index]); + const duplicated = rules.slice(0, index).some((earlier) => matchIdentity(earlier) === identity); + return mappingProblem(rules[index]) ?? (duplicated ? "Another mapping already matches this key and value" : null); +}; + +export const requestTagMappingsAreValid = (value: unknown) => + Array.isArray(value) && value.every(isMapping) && value.every((_, index) => problemAt(value, index) === null); + +export function RequestTagMappingsEditor({ value, onChange, disabled = false }: Props) { + const rules = Array.isArray(value) ? value.filter(isMapping) : []; + const update = (index: number, patch: Partial) => + onChange(rules.map((rule, position) => (position === index ? { ...rule, ...patch } : rule))); + + return ( +
+

+ Exact value mappings take priority over any-value rules for the same key. +

+ {rules.map((rule, index) => { + const exact = isExact(rule); + const problem = problemAt(rules, index); + const label = `Mapping ${index + 1}`; + return ( +
+ {label} +
+ + +
+ + {exact ? ( + <> + + +

+ {rule.key}={rule.value || "value"} → {rule.tag || "tag"} +

+ + ) : ( + <> + +

+ {rule.key}=x → {rule.tag_prefix}x +

+ + )} + {problem ?

{problem}

: null} + + +
+ ); + })} + {disabled ? ( +

Managed in config.yaml. Edit the YAML to change these rules.

+ ) : null} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx index cf5cad564ea..35434830d98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx @@ -253,3 +253,33 @@ it("should delete only the Default setting and retain explicit false and zero", ["token", "synthetic_count", 0], ]); }); + +it("keeps a new baggage mapping unsaved until every field the backend requires is filled in", async () => { + vi.mocked(getGeneralSettingsCall).mockResolvedValue([ + { + field_name: "request_tag_mappings", + field_type: "List", + field_value: [], + field_description: "baggage mappings", + stored_in_db: true, + editable: true, + }, + ]); + vi.mocked(updateConfigFieldSetting).mockClear(); + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("tab", { name: "General" })); + await user.click(await screen.findByRole("button", { name: "Add mapping" })); + const update = within(screen.getByRole("row", { name: /request_tag_mappings/ })).getByRole("button", { + name: "Update", + }); + expect(update).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 1 baggage key"), { target: { value: "entrypoint" } }); + fireEvent.change(screen.getByLabelText("Mapping 1 match value"), { target: { value: "a" } }); + expect(update).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Mapping 1 tag"), { target: { value: "usecase:support" } }); + await user.click(update); + expect(vi.mocked(updateConfigFieldSetting).mock.calls).toEqual([ + ["token", "request_tag_mappings", [{ header: "baggage", key: "entrypoint", value: "a", tag: "usecase:support" }]], + ]); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 3d0a2e438ac..5d28adb7ef3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -14,6 +14,7 @@ import { StatusBadge } from "@/components/shared/table_cells"; import RouterSettings from "@/components/router_settings"; import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; import RoutingGroups from "@/components/routing_groups"; +import { RequestTagMappingsEditor, requestTagMappingsAreValid } from "./RequestTagMappings"; const PROMPT_CACHING_TAB = "prompt_caching"; const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; @@ -34,6 +35,7 @@ export interface generalSettingsItem { field_value: any; field_description: string; stored_in_db: boolean | null; + editable?: boolean; field_options?: string[] | null; field_tab?: string | null; field_default_value?: any; @@ -57,6 +59,15 @@ const SettingValueEditor: React.FC<{ setting: generalSettingsItem; onChange: (fieldName: string, newValue: any) => void; }> = ({ setting, onChange }) => { + if (setting.field_name === "request_tag_mappings") { + return ( + onChange(setting.field_name, rules)} + /> + ); + } if (setting.field_type === "Integer") { return ( = ({ accessToken, user setGeneralSettings(updatedSettings); }; - const handleUpdateField = (fieldName: string) => { + const handleUpdateField = async (fieldName: string) => { if (!accessToken) { return; } @@ -253,7 +264,7 @@ const GeneralSettings: React.FC = ({ accessToken, user return; } try { - updateConfigFieldSetting(accessToken, fieldName, fieldValue); + await updateConfigFieldSetting(accessToken, fieldName, fieldValue); // update value in state const updatedSettings = generalSettings.map((setting) => @@ -354,13 +365,23 @@ const GeneralSettings: React.FC = ({ accessToken, user )} - - handleResetField(value.field_name)} - className="inline-flex shrink-0 cursor-pointer items-center justify-center px-1.5 py-1.5 text-destructive" + + {value.field_name === "request_tag_mappings" && value.editable === false ? null : ( + handleResetField(value.field_name)} + className="inline-flex shrink-0 cursor-pointer items-center justify-center px-1.5 py-1.5 text-destructive" + > + + + )} ))} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bf2fe4300a3..992a7d91b0d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30199,6 +30199,11 @@ export interface components { * @description When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. */ reject_clientside_metadata_tags?: boolean | null; + /** + * Request Tag Mappings + * @description Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks. + */ + request_tag_mappings?: components["schemas"]["RequestTagMapping"][] | null; /** * Responses Websocket Session Limit Seconds * @description Maximum lifetime in seconds of a Responses API WebSocket session, measured from connection accept and covering the idle wait for the first response.create frame. Defaults to 3600, matching OpenAI's documented 60-minute WebSocket connection limit. Must be between 60 and 7200 seconds. @@ -42655,6 +42660,36 @@ export interface components { /** Status Code */ status_code: number; }; + /** + * RequestTagMapping + * @description An exact baggage value mapping, or a prefix for any value of a key. + */ + RequestTagMapping: { + /** + * Header + * @default baggage + * @constant + */ + header: "baggage"; + /** Key */ + key: string; + /** + * Require Budgeted Tag + * @default false + */ + require_budgeted_tag: boolean; + /** + * Required + * @default false + */ + required: boolean; + /** Tag */ + tag?: string | null; + /** Tag Prefix */ + tag_prefix?: string | null; + /** Value */ + value?: string | null; + }; /** * RequestType * @description Fixed v0 taxonomy. User-extensible types come in v1. From 5a1fa1c87950f200a69b25083ad60573ad106176 Mon Sep 17 00:00:00 2001 From: Charles <74543653+anneheartrecord@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:26:25 +0800 Subject: [PATCH 085/124] fix(responses): keep strong refs to stream and websocket logging tasks (GC race) (#31074) * fix(responses): keep strong refs to stream and websocket logging tasks * test(responses): patch _spawn_logging_task instead of asyncio.create_task The success-logging dispatch in _log_completed_response now goes through _spawn_logging_task, so tests that patched asyncio.create_task no longer observed it (and the sync ones hit get_running_loop with no loop). * test(responses): assert stream and websocket logging tasks survive garbage collection Adds regression tests that force a collection while the logging task waits on an otherwise unreferenced future, and types the new test parameters * test(responses): drive drain and GC tests through a real logging object with gated callbacks * test(responses): use asyncio.wait_for so the logging tests run on Python 3.10 * test(guardrails): type the spawn_logging_task tracking wrapper --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/responses/streaming_iterator.py | 16 +- .../base_llm/responses/test_transformation.py | 12 +- .../test_deferred_guardrail_logging.py | 26 ++- .../unit/responses/test_streaming_iterator.py | 165 ++++++++++++++++-- 4 files changed, 190 insertions(+), 29 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index ffde8c66c20..bcd917c69d5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -622,7 +622,9 @@ class BaseResponsesAPIStreamingIterator: # the terminal event; dispatching now would snapshot metadata early. self.logging_obj._deferred_stream_complete_args = (logging_coroutine,) else: - asyncio.create_task(logging_coroutine) + spawn_logging_task( + asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging" + ) else: run_async_function( async_function=self.logging_obj.async_success_handler, @@ -2385,14 +2387,20 @@ class ResponsesWebSocketStreaming: events: Final = list(turn.events) exception: Final = turn.failure(self.authorized_model or "", self.custom_llm_provider or "", self.logging_obj) if exception is None: - asyncio.create_task(turn.logging_obj.dispatch_success_handlers(events, prefer_async_handlers=True)) + spawn_logging_task( + asyncio.get_running_loop(), + turn.logging_obj.dispatch_success_handlers(events, prefer_async_handlers=True), + task_name="Responses WebSocket success logging", + ) return if self._failure is None: self._failure = exception _record_usage_for_failure(turn.logging_obj, events) traceback_exception: Final = "".join(traceback.format_exception(exception)) - asyncio.create_task( - turn.logging_obj.dispatch_failure_handlers(exception, traceback_exception, prefer_async_handlers=True) + spawn_logging_task( + asyncio.get_running_loop(), + turn.logging_obj.dispatch_failure_handlers(exception, traceback_exception, prefer_async_handlers=True), + task_name="Responses WebSocket failure logging", ) def _wrap_response_event(self, response_str: str) -> str: diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index eb01cc8a9ce..8d21af83668 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -24,7 +24,7 @@ from litellm.types.llms.openai import( ResponsesAPIStreamEvents, ) from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from typing import Any, AsyncIterator, List +from typing import Any, AsyncIterator, Final, List from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -438,9 +438,9 @@ class TestBaseResponsesAPIStreamingIterator: # This should NOT raise an exception # Previously it would fail with: TypeError: cannot pickle 'ValidatorIterator' - # Mock asyncio.create_task and executor.submit since we're not in async context with ( - patch("asyncio.create_task") as mock_create_task, + patch("asyncio.get_running_loop"), + patch("litellm.responses.streaming_iterator.spawn_logging_task"), patch("litellm.responses.streaming_iterator.executor") as mock_executor, ): try: @@ -728,7 +728,8 @@ class TestBaseResponsesAPIStreamingIterator: "update_responses_api_response_id_with_model_id", return_value=mock_responses_api_response, ), - patch("asyncio.create_task") as mock_create_task, + patch("asyncio.get_running_loop"), + patch("litellm.responses.streaming_iterator.spawn_logging_task") as mock_spawn_logging_task, patch("litellm.responses.streaming_iterator.executor") as mock_executor, ): result = iterator._process_chunk(json.dumps(test_chunk_data)) @@ -739,7 +740,8 @@ class TestBaseResponsesAPIStreamingIterator: # Success handlers are dispatched as one async task (via _handle_logging_completed_response); # the sync handler must never be submitted to the executor concurrently (LIT-4210) - mock_create_task.assert_called_once() + spawned_task_names: Final = [call.kwargs["task_name"] for call in mock_spawn_logging_task.call_args_list] + assert spawned_task_names.count("Responses stream success logging") == 1 mock_executor.submit.assert_not_called() # Failure handlers should NOT have been called diff --git a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py index 17427e839b9..7e5d0f6342e 100644 --- a/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/unit/proxy/guardrails/test_deferred_guardrail_logging.py @@ -16,7 +16,7 @@ Streaming: CSW.__anext__ stores args on logging_obj at stream end. import asyncio import logging -from collections.abc import Callable, Mapping +from collections.abc import Callable, Coroutine, Mapping from datetime import datetime from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -1442,10 +1442,10 @@ class TestResponsesIteratorDeferredLogging: logging_obj._on_deferred_stream_complete = MagicMock() iterator = self._iterator(logging_obj) - with patch("asyncio.create_task") as mock_create_task: + with patch("litellm.responses.streaming_iterator.spawn_logging_task") as mock_spawn_logging_task: iterator._log_completed_response(is_async=True) - mock_create_task.assert_not_called() + mock_spawn_logging_task.assert_not_called() args = logging_obj._deferred_stream_complete_args assert isinstance(args, tuple) and len(args) == 1 assert "dispatched" not in recorded @@ -1458,15 +1458,25 @@ class TestResponsesIteratorDeferredLogging: logging_obj._on_deferred_stream_complete = None iterator = self._iterator(logging_obj) - created = [] - real_create_task = asyncio.create_task + from litellm.responses import streaming_iterator - def tracking_create_task(coro): - task = real_create_task(coro) + created: Final[list[asyncio.Task[object]]] = [] + real_spawn_logging_task: Final = streaming_iterator.spawn_logging_task + + def tracking_spawn_logging_task( + running_loop: asyncio.AbstractEventLoop, + coroutine: Coroutine[object, object, object], + *, + task_name: str, + ) -> asyncio.Task[object]: + task: Final = real_spawn_logging_task(running_loop, coroutine, task_name=task_name) created.append(task) return task - with patch("asyncio.create_task", side_effect=tracking_create_task): + with patch( + "litellm.responses.streaming_iterator.spawn_logging_task", + side_effect=tracking_spawn_logging_task, + ): iterator._log_completed_response(is_async=True) assert len(created) == 1 diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index e5054ed3cef..d0e07e1cc23 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -5,8 +5,10 @@ completion_start_time = end_time.""" import asyncio, importlib import functools +import gc import json import threading +import weakref from collections.abc import Callable, Mapping from datetime import datetime from typing import Final, Optional @@ -14,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest +import websockets.exceptions from pydantic_core import PydanticSerializationError import litellm @@ -509,16 +512,22 @@ class _OrderRecordingSyncLogger(CustomLogger): def _real_logging_obj( - *, call_type: str = "aresponses", litellm_params: dict[str, object] | None = None + *, + call_type: str = "aresponses", + litellm_params: dict[str, object] | None = None, + callbacks: tuple[CustomLogger, ...] | None = None, + stream: bool = True, ) -> LiteLLMLoggingObj: logging_obj: Final = LiteLLMLoggingObj( model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}], - stream=True, + stream=stream, call_type=call_type, start_time=datetime.now(), litellm_call_id="lit-8678-test", function_id="lit-8678-test", + dynamic_async_success_callbacks=None if callbacks is None else list(callbacks), + dynamic_async_failure_callbacks=None if callbacks is None else list(callbacks), ) logging_obj.model_call_details["litellm_params"] = ( dict(litellm_params) if litellm_params is not None else {"aresponses": True} @@ -2598,24 +2607,156 @@ def test_faked_stream_builds_each_refusal_summary_and_custom_tool_input_once_for @pytest.mark.asyncio async def test_shutdown_drain_waits_for_a_pending_logging_task(): - row_written = asyncio.Event() + release: Final = asyncio.Event() + row: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), release.wait(), task_name="row") - async def write_row(): - await asyncio.sleep(0.05) - row_written.set() + drain_held: Final = await _drain_held_until_released(release) - streaming_module.spawn_logging_task(asyncio.get_running_loop(), write_row(), task_name="row") - - await streaming_module.drain_logging_tasks(timeout=1) - - assert row_written.is_set() + assert drain_held + assert row.done() @pytest.mark.asyncio async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): - stuck = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") + stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) assert not stuck.done() stuck.cancel() + + +async def _drain_held_until_released(release: asyncio.Event) -> bool: + drain: Final = asyncio.create_task(streaming_module.drain_logging_tasks(timeout=5)) + await asyncio.sleep(0) + held: Final = not drain.done() + release.set() + await drain + return held + + +class _GatedLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + self.logged = False + + async def _log(self) -> None: + self.entered.set() + await self.release.wait() + self.logged = True + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + await self._log() + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + await self._log() + + +class _UnreferencedWaitLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.parked: Final = asyncio.Event() + self.waiting_tasks: Final = weakref.WeakSet[asyncio.Task[object]]() + + async def _wait_forever(self) -> None: + self.waiting_tasks.add(asyncio.current_task()) + self.parked.set() + await asyncio.get_running_loop().create_future() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + await self._wait_forever() + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + await self._wait_forever() + + +async def _shutdown_drain_held_for(logger: _GatedLogger) -> bool: + await asyncio.wait_for(logger.entered.wait(), timeout=5) + return await _drain_held_until_released(logger.release) + + +async def _logging_tasks_left_after_garbage_collection(logger: _UnreferencedWaitLogger) -> int: + await asyncio.wait_for(logger.parked.wait(), timeout=5) + gc.collect() + survivors: Final = tuple(logger.waiting_tasks) + for task in survivors: + task.cancel() + await asyncio.gather(*survivors, return_exceptions=True) + return len(survivors) + + +async def _stream_to_completion(logger: CustomLogger) -> None: + iterator: Final = _make_iterator( + sse_events=_COMPLETE_STREAM_EVENTS, logging_obj=_real_logging_obj(callbacks=(logger,)) + ) + async for _ in iterator: + pass + + +_WS_TERMINAL_EVENTS: Final = [ + pytest.param({"type": "response.completed", "response": {}}, id="success"), + pytest.param( + {"type": "response.failed", "response": {"error": {"code": "server_error", "message": "boom"}}}, + id="failure", + ), +] + + +async def _run_websocket_turn(logger: CustomLogger, terminal_event: dict[str, object]) -> None: + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[json.dumps(terminal_event), websockets.exceptions.ConnectionClosedOK(None, None)] + ) + streaming: Final = streaming_module.ResponsesWebSocketStreaming( + websocket=MagicMock(send_text=AsyncMock()), + backend_ws=backend_ws, + logging_obj=_real_logging_obj( + call_type=CallTypes.aresponses_websocket.value, + litellm_params={CallTypes.aresponses_websocket.value: True}, + callbacks=(logger,), + stream=False, + ), + ) + await streaming.backend_to_client() + + +@pytest.mark.asyncio +async def test_shutdown_drain_waits_for_a_completed_streams_success_logging(): + logger: Final = _GatedLogger() + await _stream_to_completion(logger) + + drain_held: Final = await _shutdown_drain_held_for(logger) + + assert drain_held + assert logger.logged + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_event", _WS_TERMINAL_EVENTS) +async def test_shutdown_drain_waits_for_a_websocket_turns_logging(terminal_event: dict[str, object]): + logger: Final = _GatedLogger() + await _run_websocket_turn(logger, terminal_event) + + drain_held: Final = await _shutdown_drain_held_for(logger) + + assert drain_held + assert logger.logged + + +@pytest.mark.asyncio +async def test_completed_streams_success_logging_survives_garbage_collection(): + logger: Final = _UnreferencedWaitLogger() + await _stream_to_completion(logger) + + assert await _logging_tasks_left_after_garbage_collection(logger) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_event", _WS_TERMINAL_EVENTS) +async def test_websocket_turns_logging_survives_garbage_collection(terminal_event: dict[str, object]): + logger: Final = _UnreferencedWaitLogger() + await _run_websocket_turn(logger, terminal_event) + + assert await _logging_tasks_left_after_garbage_collection(logger) == 1 From 8d2e7dca263fe91b5c28900ac4f322a5c47a4124 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:36:49 +0000 Subject: [PATCH 086/124] fix(sagemaker)!: parse every streamed record however SageMaker cuts the payload parts (#47495) * fix(sagemaker): parse every streamed record however SageMaker cuts the payload parts SageMaker cuts a container's response stream into PayloadParts at arbitrary byte offsets, so one part can hold several SSE events or JSON lines, or only part of one. The decoder treated each part as one record, so vLLM's [DONE] was logged at ERROR on every stream and a split at an unlucky byte dropped the rest of the answer. Peel records off a running buffer with JSONDecoder.raw_decode, skipping SSE framing and [DONE] between them. * test(sagemaker): cover streamed records cut or merged across payload parts * fix(sagemaker): decode UTF-8 across parts, skip SSE comment lines, and log container error records A UTF-8 character split across two PayloadParts now decodes whole through one incremental decoder per stream. SSE comment lines (TGI's keep-alive) and event, id and retry field lines are skipped as framing. A container error record is logged at ERROR with its message and skipped, and any other record that is not a chunk is logged and skipped, the same way on the sync and async paths. * fix(sagemaker): raise a container's error record and ModelStreamError on both stream paths A record with a non-null error now raises a 500 SagemakerError carrying the record after the chunks before it, on sync and async. Async no longer swallows a non-200 event, so a ModelStreamError raises as it does on sync. A chunk with a null error field is delivered, and any other record that is not a chunk is logged and skipped on both paths. * fix(sagemaker): answer a streamed error with the container's status and fail a stream cut mid-record A container's error record now raises with the status the container gave it (vLLM's error.code, the older flat vLLM shape, TGI's http_status_code or error_type) instead of a fixed 500 that the router retried as a 503. A ModelStreamError raises a SagemakerError naming its ErrorCode and Message, a stream that ends inside a record raises instead of ending as a stop, and a skipped record is logged by its keys only * fix(sagemaker): trust only 422 and 429 from a streamed error record and fail LMI generation errors * test(sagemaker): drive an LMI failed-generation stream through the proxy on a sagemaker/ deployment * test(sagemaker): fall back to the next model when a stream fails before any content --- litellm/llms/sagemaker/common_utils.py | 258 +++++++---- .../providers/_sagemaker_chat_container.py | 51 ++- .../providers/test_sagemaker_chat_wire.py | 222 ++++++++- .../sagemaker/test_sagemaker_common_utils.py | 426 +++++++++++++++++- 4 files changed, 851 insertions(+), 106 deletions(-) diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 579309035f0..14f0e3ead62 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -1,13 +1,15 @@ +import codecs import functools import json +import re from collections.abc import AsyncIterator, Iterator, Mapping -from typing import Final +from types import MappingProxyType +from typing import Final, Protocol import httpx from pydantic import BaseModel, ConfigDict, Field, RootModel, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, Required, TypedDict -import litellm from litellm import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import GenericStreamingChunk as GChunk @@ -40,12 +42,90 @@ class _SageMakerMessagesChunkDataRoot(RootModel[_SageMakerMessagesChunkData]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) -def _parse_sagemaker_chunk_data(data: str) -> _SageMakerChunkData: - return _SageMakerChunkDataRoot.model_validate(json.loads(data)).root +_STREAM_FRAMING: Final = re.compile(r"(?:\s|data:|\[DONE\]|(?::|event:|id:|retry:)[^\n]*\n)*") +_JSON_DECODER: Final = json.JSONDecoder() +_RAW_DECODED: Final = TypeAdapter(tuple[object, int]) +_NO_RECORD: Final = object() +_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_CLIENT_ERROR_TYPE_STATUS: Final = MappingProxyType( + { + "overloaded": 429, + "validation": 422, + "template_error": 422, + "missing_template_variable": 422, + "tool_error": 422, + } +) +_CLIENT_ERROR_STATUSES: Final = frozenset(_CLIENT_ERROR_TYPE_STATUS.values()) -def _parse_sagemaker_messages_chunk_data(data: str) -> _SageMakerMessagesChunkData: - return _SageMakerMessagesChunkDataRoot.model_validate(json.loads(data)).root +def _framing_end(text: str) -> int: + framing: Final = _STREAM_FRAMING.match(text) + return framing.end() if framing else 0 + + +class _StreamRecordBuffer: + """ + SageMaker cuts a container's stream into PayloadParts at arbitrary points, so one part can hold + several SSE events or JSON lines, or only part of one, down to half of a UTF-8 character + """ + + def __init__(self) -> None: + self._decoder: Final = codecs.getincrementaldecoder("utf-8")(errors="replace") + self._pending: str = "" + + def feed(self, data: bytes) -> tuple[object, ...]: + self._pending += self._decoder.decode(data) + return tuple(iter(self._pop_record, _NO_RECORD)) + + def close(self) -> "SagemakerError | str": + unparsed: Final = self._pending + self._decoder.decode(b"", final=True) + record_start: Final = unparsed[_framing_end(unparsed) :] + if record_start.startswith(("{", "[")): + return SagemakerError( + status_code=500, + message=f"SageMaker stream ended inside a record, {len(record_start)} characters into it", + ) + return unparsed + + def _pop_record(self) -> object: + start: Final = _framing_end(self._pending) + try: + record, end = _RAW_DECODED.validate_python(_JSON_DECODER.raw_decode(self._pending, start)) + except json.JSONDecodeError: + self._pending = self._pending[start:] + return _NO_RECORD + self._pending = self._pending[end:] + return record + + +def _fields(value: object) -> Mapping[str, object]: + return STR_KEYED_MAPPING.validate_python(value) if isinstance(value, Mapping) else _NO_FIELDS + + +def _client_error_status(value: object) -> int | None: + return value if isinstance(value, int) and value in _CLIENT_ERROR_STATUSES else None + + +def _streamed_error_status(record: Mapping[str, object]) -> int: + detail: Final = _fields(record.get("error")) + candidates: Final = ( + detail.get("code"), + detail.get("http_status_code"), + record.get("code"), + _CLIENT_ERROR_TYPE_STATUS.get(str(record.get("error_type"))), + ) + return next(filter(None, map(_client_error_status, candidates)), 500) + + +def _failed_generation(record: Mapping[str, object]) -> "SagemakerError | None": + error_message: Final = _fields(record.get("token")).get("error_msg") + if not error_message and _fields(record.get("details")).get("finish_reason") != "error": + return None + return SagemakerError( + status_code=500, + message=f"SageMaker endpoint streamed a failed generation: {error_message or 'no error message'}", + ) def _load_sagemaker_response_stream_shape(): @@ -110,6 +190,10 @@ def client_error_details(error: Exception) -> tuple[int, str]: return response.response_metadata.http_status_code, response.error.message or str(error) +class _EventStreamMessage(Protocol): + def to_response_dict(self) -> Mapping[str, object]: ... + + class SagemakerError(BaseLLMException): def __init__( self, @@ -120,6 +204,24 @@ class SagemakerError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) +class _StreamExceptionBody(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True, hide_input_in_errors=True) + message: str = Field(default="", alias="Message") + error_code: str | None = Field(default=None, alias="ErrorCode") + + +def _stream_exception_error(headers: Mapping[str, object], body: object) -> SagemakerError: + name: Final = headers.get(":exception-type") or headers.get(":error-code") or "an error" + try: + detail: Final = _StreamExceptionBody.model_validate_json(body if isinstance(body, bytes) else b"") + except ValidationError: + return SagemakerError( + status_code=500, message=f"SageMaker stream failed with {name}: {headers.get(':error-message', '')}" + ) + code: Final = f" ({detail.error_code})" if detail.error_code else "" + return SagemakerError(status_code=500, message=f"SageMaker stream failed with {name}{code}: {detail.message}") + + class AWSEventStreamDecoder: def __init__(self, model: str, is_messages_api: bool | None = None) -> None: from botocore.parsers import EventStreamJSONParser @@ -159,52 +261,55 @@ class AWSEventStreamDecoder: usage=None, ) + def _record_to_chunk(self, record: object) -> GChunk | StreamingChatCompletionChunk | SagemakerError | None: + fields: Final = _fields(record) + if fields.get("error") is not None or fields.get("object") == "error": + return SagemakerError( + status_code=_streamed_error_status(fields), + message=f"SageMaker endpoint streamed an error: {json.dumps(record)}", + ) + if failed_generation := _failed_generation(fields): + return failed_generation + try: + if self.is_messages_api: + return self._chunk_parser_messages_api( + chunk_data=_SageMakerMessagesChunkDataRoot.model_validate(record).root + ) + return self._chunk_parser(chunk_data=_SageMakerChunkDataRoot.model_validate(record).root) + except ValidationError: + verbose_logger.error( + "Skipping a SageMaker stream record that is not a chunk: %s with keys %s", + type(record).__name__, + sorted(fields), + ) + return None + + def _chunks(self, records: tuple[object, ...]) -> Iterator[GChunk | StreamingChatCompletionChunk]: + for outcome in map(self._record_to_chunk, records): + if isinstance(outcome, SagemakerError): + raise outcome + if outcome is not None: + yield outcome + def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GChunk | StreamingChatCompletionChunk | None]: """Given an iterator that yields lines, iterate over it & yield every event encountered""" from botocore.eventstream import EventStreamBuffer event_stream_buffer: Final = EventStreamBuffer() - accumulated_json = "" + records: Final = _StreamRecordBuffer() for chunk in iterator: event_stream_buffer.add_data(chunk) for event in event_stream_buffer: - message = self._parse_message_from_event(event) - if message: - # remove data: prefix and "\n\n" at the end - message = litellm.CustomStreamWrapper.strip_sse_data_from_chunk(message) or "" - message = message.replace("\n\n", "") + if payload := self._parse_message_from_event(event): + yield from self._chunks(records.feed(payload)) - # Accumulate JSON data - accumulated_json += message - - # Try to parse the accumulated JSON - try: - if self.is_messages_api: - yield self._chunk_parser_messages_api( - chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) - ) - else: - yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) - # Reset accumulated_json after successful parsing - accumulated_json = "" - except json.JSONDecodeError: - # If it's not valid JSON yet, continue to the next event - continue - - # Handle any remaining data after the iterator is exhausted - if accumulated_json: - try: - if self.is_messages_api: - yield self._chunk_parser_messages_api( - chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) - ) - else: - yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) - except json.JSONDecodeError: - # Handle or log any unparseable data at the end - verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) - yield None + unparsed: Final = records.close() + if isinstance(unparsed, SagemakerError): + raise unparsed + if unparsed: + verbose_logger.error("Warning: Unparseable JSON data remained, %d characters of it", len(unparsed)) + yield None async def aiter_bytes( self, iterator: AsyncIterator[bytes] @@ -213,58 +318,23 @@ class AWSEventStreamDecoder: from botocore.eventstream import EventStreamBuffer event_stream_buffer: Final = EventStreamBuffer() - accumulated_json = "" + records: Final = _StreamRecordBuffer() async for chunk in iterator: event_stream_buffer.add_data(chunk) for event in event_stream_buffer: - try: - message = self._parse_message_from_event(event) - if message: - verbose_logger.debug("sagemaker parsed chunk bytes %s", message) - # remove data: prefix and "\n\n" at the end - message = litellm.CustomStreamWrapper.strip_sse_data_from_chunk(message) or "" - message = message.replace("\n\n", "") + if payload := self._parse_message_from_event(event): + for parsed_chunk in self._chunks(records.feed(payload)): + yield parsed_chunk - # Accumulate JSON data - accumulated_json += message + unparsed: Final = records.close() + if isinstance(unparsed, SagemakerError): + raise unparsed + if unparsed: + verbose_logger.error("Warning: Unparseable JSON data remained, %d characters of it", len(unparsed)) + yield None - # Try to parse the accumulated JSON - if self.is_messages_api: - yield self._chunk_parser_messages_api( - chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) - ) - else: - yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) - # Reset accumulated_json after successful parsing - accumulated_json = "" - except json.JSONDecodeError: - # If it's not valid JSON yet, continue to the next event - continue - except UnicodeDecodeError as e: - verbose_logger.warning("UnicodeDecodeError: %s. Attempting to combine with next event.", e) - continue - except Exception as e: - verbose_logger.error("Error parsing message: %s. Attempting to combine with next event.", e) - continue - - # Handle any remaining data after the iterator is exhausted - if accumulated_json: - try: - if self.is_messages_api: - yield self._chunk_parser_messages_api( - chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) - ) - else: - yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) - except json.JSONDecodeError: - # Handle or log any unparseable data at the end - verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) - yield None - except Exception as e: - verbose_logger.error("Final error parsing accumulated JSON: %s", e) - - def _parse_message_from_event(self, event) -> str | None: + def _parse_message_from_event(self, event: _EventStreamMessage) -> bytes | None: response_stream_shape: Final = get_sagemaker_response_stream_shape() if response_stream_shape is None: raise SagemakerError( @@ -276,18 +346,14 @@ class AWSEventStreamDecoder: ) response_dict: Final = event.to_response_dict() parsed_response: Final = self.parser.parse(response_dict, response_stream_shape) + body: Final = response_dict.get("body") if response_dict["status_code"] != 200: - raise ValueError(f"Bad response code, expected 200: {response_dict}") + raise _stream_exception_error(_fields(response_dict.get("headers")), body) if "chunk" in parsed_response: chunk = parsed_response.get("chunk") if not chunk: return None - return chunk.get("bytes").decode() - else: - chunk = response_dict.get("body") - if not chunk: - return None - - return chunk.decode() + return chunk.get("bytes") + return body if isinstance(body, bytes) and body else None diff --git a/tests/integration/providers/_sagemaker_chat_container.py b/tests/integration/providers/_sagemaker_chat_container.py index 6a06b92e215..4cec8f8b6fe 100644 --- a/tests/integration/providers/_sagemaker_chat_container.py +++ b/tests/integration/providers/_sagemaker_chat_container.py @@ -16,6 +16,7 @@ SECRET_KEY: Final = "synthetic-secret-key-for-testing" REGION: Final = "us-east-1" PROMPT: Final = "synthetic inference component request" CONTENT: Final = ("sagemaker ", "wire control") +USAGE: Final[Mapping[str, int]] = MappingProxyType({"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}) NO_CACHE: Final[Mapping[str, JsonValue]] = MappingProxyType({"cache": {"no-cache": True}}) JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _MARKER_PREFIX: Final = "marker-" @@ -69,27 +70,61 @@ def _event_header(name: str, value: str) -> bytes: return struct.pack("!B", len(name_bytes)) + name_bytes + b"\x07" + struct.pack("!H", len(value_bytes)) + value_bytes +def _frame(headers: bytes, body: bytes) -> bytes: + prelude: Final = struct.pack("!II", 12 + len(headers) + len(body) + 4, len(headers)) + message: Final = prelude + struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) + headers + body + return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) + + def _payload_part(body: bytes) -> bytes: headers: Final = ( _event_header(":event-type", "PayloadPart") + _event_header(":content-type", "application/octet-stream") + _event_header(":message-type", "event") ) - prelude: Final = struct.pack("!II", 12 + len(headers) + len(body) + 4, len(headers)) - message: Final = prelude + struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) + headers + body - return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) + return _frame(headers, body) + + +def model_stream_error(message: str, error_code: str) -> bytes: + headers: Final = ( + _event_header(":exception-type", "ModelStreamError") + + _event_header(":content-type", "application/json") + + _event_header(":message-type", "exception") + ) + return _frame(headers, json.dumps({"Message": message, "ErrorCode": error_code}).encode()) def container_reply(identity: str, stream: bool) -> Reply: if not stream: return Reply(body=_completion(identity)) - chunks: Final = ( + return event_stream_reply( + ( + _chunk(identity, {"role": "assistant", "content": CONTENT[0]}, None), + _chunk(identity, {"content": CONTENT[1]}, None), + _chunk(identity, {}, "stop", usage=dict(USAGE)), + ) + ) + + +def event_stream_reply(payloads: tuple[bytes, ...], *, then: tuple[bytes, ...] = ()) -> Reply: + return Reply(chunks=(*map(_payload_part, payloads), *then), content_type="application/vnd.amazon.eventstream") + + +def vllm_stream_events(identity: str) -> tuple[bytes, ...]: + usage_event: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": SERVED_MODEL, + "choices": [], + "usage": dict(USAGE), + } + return ( _chunk(identity, {"role": "assistant", "content": CONTENT[0]}, None), _chunk(identity, {"content": CONTENT[1]}, None), - _chunk(identity, {}, "stop", usage={"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}), - ) - return Reply( - chunks=tuple(_payload_part(chunk) for chunk in chunks), content_type="application/vnd.amazon.eventstream" + _chunk(identity, {}, "stop"), + f"data: {json.dumps(usage_event, separators=(',', ':'))}\n\n".encode(), + b"data: [DONE]\n\n", ) diff --git a/tests/integration/providers/test_sagemaker_chat_wire.py b/tests/integration/providers/test_sagemaker_chat_wire.py index 0416404cec8..261787fec38 100644 --- a/tests/integration/providers/test_sagemaker_chat_wire.py +++ b/tests/integration/providers/test_sagemaker_chat_wire.py @@ -1,11 +1,14 @@ import asyncio +import itertools +import json import uuid -from collections.abc import Callable, Mapping +from collections.abc import AsyncIterator, Callable, Mapping from dataclasses import dataclass from types import MappingProxyType from typing import Final, Literal, TypeAlias import httpx +import openai import pytest from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value from integration._support.database import read_rows @@ -26,14 +29,18 @@ from integration.providers._sagemaker_chat_container import ( REGION, SECRET_KEY, SERVED_MODEL, + USAGE, answered_choices, assert_signed, by_marker, container_body, container_reply, + event_stream_reply, marked_prompt, marker, + model_stream_error, only_request, + vllm_stream_events, ) from pydantic import JsonValue @@ -260,6 +267,77 @@ def test_sagemaker_chat_keeps_the_deployment_aws_credentials_out_of_the_containe assert SECRET_KEY not in string_value(row["stored"]), row +def _seven_byte_parts(stream: bytes) -> tuple[bytes, ...]: + return tuple(stream[start : start + 7] for start in range(0, len(stream), 7)) + + +def _event_pairs(events: tuple[bytes, ...]) -> tuple[bytes, ...]: + return tuple(b"".join(events[start : start + 2]) for start in range(0, len(events), 2)) + + +_PART_CUTTINGS: Final[Mapping[str, Callable[[tuple[bytes, ...]], tuple[bytes, ...]]]] = MappingProxyType( + { + "one-event-per-part": lambda events: events, + "usage-and-done-share-a-part": lambda events: (*events[:-2], events[-2] + events[-1]), + "event-pairs-share-a-part": _event_pairs, + "seven-byte-parts": lambda events: _seven_byte_parts(b"".join(events)), + "keep-alive-comments-between-events": lambda events: tuple( + itertools.chain.from_iterable((b":\n\n", event) for event in events) + ), + } +) + + +def _sse(record: Mapping[str, JsonValue]) -> bytes: + return f"data: {json.dumps(record)}\n\n".encode() + + +@dataclass(frozen=True, slots=True) +class _StreamedAnswer: + call_id: str + text: str + usages: tuple[tuple[int, int, int], ...] + + +async def _chat_openai_stream_with_usage(gateway: Gateway, model: str, prompt: str) -> _StreamedAnswer: + async with async_openai_client(gateway) as client: + stream: Final = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + max_tokens=16, + stream=True, + stream_options={"include_usage": True}, + extra_body=dict(NO_CACHE), + ) + chunks: Final = [chunk async for chunk in stream] + return _StreamedAnswer( + stream.response.headers["x-litellm-call-id"], + "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices), + tuple( + (chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens) + for chunk in chunks + if chunk.usage + ), + ) + + +@pytest.mark.parametrize("cutting", tuple(_PART_CUTTINGS)) +def test_sagemaker_chat_vllm_stream_reaches_the_caller_whole_however_sagemaker_cuts_its_payload_parts( + gateway: Gateway, cutting: str +) -> None: + identity: Final = f"sagemaker-chat-{uuid.uuid4().hex}" + parts: Final = _PART_CUTTINGS[cutting](vllm_stream_events(identity)) + expected_usage: Final = (USAGE["prompt_tokens"], USAGE["completion_tokens"], USAGE["total_tokens"]) + + with wire_server(lambda _request: event_stream_reply(parts)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, {}) + answer: Final = asyncio.run(_chat_openai_stream_with_usage(gateway, model, marked_prompt())) + only_request(wire) + assert (answer.text, answer.usages) == ("".join(CONTENT), (expected_usage,)), answer + stored: Final = JSON_OBJECT.validate_json(string_value(_spend_row(answer.call_id)["stored"])) + assert (stored["prompt_tokens"], stored["completion_tokens"], stored["total_tokens"]) == expected_usage, stored + + @dataclass(frozen=True, slots=True) class _Shape: provider: str @@ -388,6 +466,148 @@ def test_sagemaker_chat_container_error_reaches_the_caller_after_one_attempt_wit assert_signed(request, ACCESS_KEY, REGION, None) +def test_sagemaker_chat_container_validation_error_record_answers_422_after_one_attempt(gateway: Gateway) -> None: + failure: Final = _sse({"error": {"message": "synthetic prompt too long", "http_status_code": 422}}) + + with wire_server(lambda _request: event_stream_reply((failure,))) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, {}) + response: Final = _chat(gateway, model, marked_prompt(), {"stream": True}) + only_request(wire) + assert response.status_code == 422, response.text + assert "synthetic prompt too long" in response.text, response.text + + +async def _texts_until_error( + stream: openai.AsyncStream[openai.types.chat.ChatCompletionChunk], +) -> AsyncIterator[str | openai.APIError]: + try: + async for chunk in stream: + yield "".join(choice.delta.content or "" for choice in chunk.choices) + except openai.APIError as error: + yield error + + +async def _chat_openai_stream_until_error( + gateway: Gateway, model: str, prompt: str, extra: Mapping[str, JsonValue] = MappingProxyType({}) +) -> tuple[str | openai.APIError, ...]: + async with async_openai_client(gateway) as client: + stream: Final = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + max_tokens=16, + stream=True, + extra_body={**NO_CACHE, **extra}, + ) + return tuple([item async for item in _texts_until_error(stream)]) + + +_MID_STREAM_FAILURES: Final[Mapping[str, tuple[Callable[[tuple[bytes, ...]], Reply], str]]] = MappingProxyType( + { + "error-record": ( + lambda events: event_stream_reply( + (*events[:2], _sse({"error": {"message": "synthetic decode failure", "code": 500}})) + ), + "synthetic decode failure", + ), + "model-stream-error": ( + lambda events: event_stream_reply( + events[:2], then=(model_stream_error("synthetic stream timeout", "ModelInvocationTimeExceeded"),) + ), + "SageMaker stream failed with ModelStreamError (ModelInvocationTimeExceeded): synthetic stream timeout", + ), + "stream-ends-inside-a-record": ( + lambda events: event_stream_reply((*events[:2], events[2][:30])), + "SageMaker stream ended inside a record", + ), + } +) + + +@pytest.mark.parametrize("failure", tuple(_MID_STREAM_FAILURES)) +def test_sagemaker_chat_stream_failure_after_the_first_chunks_reaches_the_caller_as_an_error( + gateway: Gateway, failure: str +) -> None: + reply, expected = _MID_STREAM_FAILURES[failure] + events: Final = vllm_stream_events(f"sagemaker-chat-{uuid.uuid4().hex}") + + with wire_server(lambda _request: reply(events)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, {}) + outcome: Final = asyncio.run(_chat_openai_stream_until_error(gateway, model, marked_prompt())) + only_request(wire) + errors: Final = tuple(item for item in outcome if isinstance(item, openai.APIError)) + assert "".join(item for item in outcome if isinstance(item, str)) == "".join(CONTENT), outcome + assert len(errors) == 1 and expected in errors[0].message, outcome + + +_PRE_CONTENT_FAILURES: Final[Mapping[str, Reply]] = MappingProxyType( + { + "error-record": event_stream_reply((_sse({"error": {"message": "synthetic decode failure", "code": 500}}),)), + "model-stream-error": event_stream_reply( + (), then=(model_stream_error("synthetic stream timeout", "ModelInvocationTimeExceeded"),) + ), + } +) + + +@pytest.mark.parametrize("failure", tuple(_PRE_CONTENT_FAILURES)) +def test_sagemaker_chat_stream_failure_before_any_content_falls_back_to_the_next_model( + gateway: Gateway, failure: str +) -> None: + events: Final = vllm_stream_events(f"sagemaker-chat-{uuid.uuid4().hex}") + + with ( + wire_server(lambda _request: _PRE_CONTENT_FAILURES[failure]) as primary_wire, + wire_server(lambda _request: event_stream_reply(events)) as fallback_wire, + gateway.scenario() as scenario, + ): + primary: Final = _deployment(scenario, primary_wire, {}) + fallback: Final = _deployment(scenario, fallback_wire, {}) + outcome: Final = asyncio.run( + _chat_openai_stream_until_error(gateway, primary, marked_prompt(), {"fallbacks": [fallback]}) + ) + primary_attempts: Final = primary_wire.drain() + only_request(fallback_wire) + assert outcome and all(isinstance(item, str) for item in outcome), outcome + assert "".join(item for item in outcome if isinstance(item, str)) == "".join(CONTENT), outcome + assert len(primary_attempts) == 1, primary_attempts + + +def _lmi_line(record: Mapping[str, JsonValue]) -> bytes: + return f"{json.dumps(dict(record), ensure_ascii=False)}\n".encode() + + +_LMI_TOKENS: Final = ("sagemaker", " lmi tokens") +_LMI_ERROR: Final = "exception occurred during rolling batch inference" +# djl-serving 8a99e6e5: rolling_batch.py's error Token rendered by output_formatter._jsonlines_output_formatter +_LMI_FAILED_GENERATION: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "token": {"id": -1, "text": "", "log_prob": -1, "special_token": True, "error_msg": _LMI_ERROR}, + "generated_text": "".join(_LMI_TOKENS), + "details": {"finish_reason": "error"}, + } +) + + +def test_sagemaker_lmi_failed_generation_after_the_first_tokens_reaches_the_caller_as_an_error( + gateway: Gateway, +) -> None: + tokens: Final = tuple( + _lmi_line({"token": {"id": index, "text": text, "log_prob": -0.1}}) for index, text in enumerate(_LMI_TOKENS, 1) + ) + reply: Final = event_stream_reply((*tokens, _lmi_line(_LMI_FAILED_GENERATION))) + + with wire_server(lambda _request: reply) as wire, gateway.scenario() as scenario: + model: Final = _deployment( + scenario, wire, {"hf_model_name": "meta-llama/Llama-2-7b-chat-hf"}, provider="sagemaker" + ) + outcome: Final = asyncio.run(_chat_openai_stream_until_error(gateway, model, marked_prompt())) + only_request(wire) + errors: Final = tuple(item for item in outcome if isinstance(item, openai.APIError)) + assert "".join(item for item in outcome if isinstance(item, str)) == "".join(_LMI_TOKENS), outcome + assert len(errors) == 1, outcome + assert f"SageMaker endpoint streamed a failed generation: {_LMI_ERROR}" in errors[0].message, outcome + + @dataclass(frozen=True, slots=True) class _Call: endpoint: Endpoint diff --git a/tests/unit/llms/sagemaker/test_sagemaker_common_utils.py b/tests/unit/llms/sagemaker/test_sagemaker_common_utils.py index 17fac935679..ac1c816f86e 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_common_utils.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_common_utils.py @@ -1,4 +1,9 @@ import json +import logging +import struct +import zlib +from collections.abc import AsyncIterator, Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -6,8 +11,10 @@ import pytest from botocore.exceptions import ClientError, ConnectionClosedError -from litellm.llms.sagemaker.common_utils import AWSEventStreamDecoder, client_error_details +from litellm.llms.sagemaker.common_utils import AWSEventStreamDecoder, SagemakerError, client_error_details from litellm.llms.sagemaker.completion.transformation import SagemakerConfig +from litellm.types.utils import GenericStreamingChunk as GChunk +from litellm.types.utils import StreamingChatCompletionChunk # --------------------------------------------------------------------------- # @@ -232,6 +239,423 @@ async def test_aiter_bytes_valid_chunk_followed_by_unicode_error(): assert chunks[0]["text"] == "hello" # Verify the content of the valid chunk +def _event_header(name: str, value: str) -> bytes: + return struct.pack("!B", len(name)) + name.encode() + b"\x07" + struct.pack("!H", len(value)) + value.encode() + + +_PAYLOAD_PART_HEADERS: Final = ( + _event_header(":event-type", "PayloadPart") + + _event_header(":content-type", "application/octet-stream") + + _event_header(":message-type", "event") +) +_MODEL_STREAM_ERROR_HEADERS: Final = ( + _event_header(":exception-type", "ModelStreamError") + + _event_header(":content-type", "application/json") + + _event_header(":message-type", "exception") +) + + +def _frame(headers: bytes, body: str | bytes) -> bytes: + encoded: Final = body.encode() if isinstance(body, str) else body + prelude: Final = struct.pack("!II", 16 + len(headers) + len(encoded), len(headers)) + message: Final = prelude + struct.pack("!I", zlib.crc32(prelude)) + headers + encoded + return message + struct.pack("!I", zlib.crc32(message)) + + +def _payload_part(body: str | bytes) -> bytes: + return _frame(_PAYLOAD_PART_HEADERS, body) + + +def _cut(stream: str, size: int) -> tuple[bytes, ...]: + encoded: Final = stream.encode() + return tuple(encoded[start : start + size] for start in range(0, len(encoded), size)) + + +def _sse(record: Mapping[str, object]) -> str: + return f"data: {json.dumps(record, ensure_ascii=False)}\n\n" + + +def _vllm_chunk(choices: list[Mapping[str, object]], **extra: object) -> str: + return _sse( + {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": choices, **extra} + ) + + +_VLLM_EVENTS: Final = ( + _vllm_chunk([{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}]), + _vllm_chunk([{"index": 0, "delta": {"content": "Hél"}, "finish_reason": None}]), + _vllm_chunk([{"index": 0, "delta": {"content": "lo 👋"}, "finish_reason": "stop"}]), + _vllm_chunk([], usage={"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10}), + "data: [DONE]\n\n", +) +_VLLM_STREAM: Final = "".join(_VLLM_EVENTS) +_TGI_EVENTS: Final = tuple( + f"data:{json.dumps({'index': 0, 'token': {'text': text}}, ensure_ascii=False)}\n\n" for text in ("Hél", "lo 👋") +) +_TGI_STREAM: Final = "".join(_TGI_EVENTS) +_TGI_KEEP_ALIVE_STREAM: Final = ( + f"{_TGI_EVENTS[0]}:\n\n: keep-alive\n\nevent: message\nid: 2\nretry: 1000\n{_TGI_EVENTS[1]}" +) +_LMI_STREAM: Final = "".join( + f"{json.dumps({'token': {'text': text}}, ensure_ascii=False)}\n" for text in ("Hél", "lo 👋") +) + + +async def _stream_frames( + frames: tuple[bytes, ...], is_messages_api: bool, mode: str +) -> AsyncIterator[GChunk | StreamingChatCompletionChunk | None]: + decoder: Final = AWSEventStreamDecoder(model="m", is_messages_api=is_messages_api) + if mode == "sync": + for chunk in decoder.iter_bytes(iter(frames)): + yield chunk + return + + async def stream(): + for frame in frames: + yield frame + + async for chunk in decoder.aiter_bytes(stream()): + yield chunk + + +async def _decode( + parts: tuple[str | bytes, ...], is_messages_api: bool, mode: str +) -> list[GChunk | StreamingChatCompletionChunk | None]: + return [chunk async for chunk in _stream_frames(tuple(map(_payload_part, parts)), is_messages_api, mode)] + + +def _text(chunk: GChunk | StreamingChatCompletionChunk | None, is_messages_api: bool) -> str | None: + if chunk is None: + return None + return chunk.choices[0].delta.content if is_messages_api else chunk["text"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + "parts", + [ + pytest.param(_VLLM_EVENTS, id="one-event-per-part"), + pytest.param((*_VLLM_EVENTS[:3], _VLLM_EVENTS[3] + _VLLM_EVENTS[4]), id="usage-and-done-share-a-part"), + pytest.param( + (_VLLM_EVENTS[0] + _VLLM_EVENTS[1], _VLLM_EVENTS[2] + _VLLM_EVENTS[3], _VLLM_EVENTS[4]), + id="event-pairs-share-a-part", + ), + pytest.param((*_VLLM_EVENTS[:4], "data: [DO", "NE]\n\n"), id="done-split-across-parts"), + pytest.param((_VLLM_STREAM,), id="whole-stream-in-one-part"), + pytest.param(_cut(_VLLM_STREAM, 7), id="seven-byte-parts"), + pytest.param(_cut(_VLLM_STREAM, 1), id="one-byte-parts-split-every-utf8-character"), + ], +) +async def test_messages_api_stream_keeps_every_chunk_however_sagemaker_cuts_the_parts( + parts: tuple[str | bytes, ...], mode: str, caplog: pytest.LogCaptureFixture +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + chunks: Final = await _decode(parts, is_messages_api=True, mode=mode) + + assert None not in chunks + assert "".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices) == "Héllo 👋" + assert [(chunk.usage.prompt_tokens, chunk.usage.total_tokens) for chunk in chunks if chunk.usage] == [(7, 10)] + assert caplog.records == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + "parts", + [ + pytest.param(_TGI_EVENTS, id="tgi-sse-one-event-per-part"), + pytest.param((_TGI_STREAM,), id="tgi-sse-whole-stream-in-one-part"), + pytest.param(_cut(_TGI_STREAM, 5), id="tgi-sse-five-byte-parts"), + pytest.param(_cut(_TGI_STREAM, 1), id="tgi-sse-one-byte-parts-split-every-utf8-character"), + pytest.param((_TGI_KEEP_ALIVE_STREAM,), id="tgi-sse-comments-and-fields-whole-stream-in-one-part"), + pytest.param(_cut(_TGI_KEEP_ALIVE_STREAM, 1), id="tgi-sse-comments-and-fields-one-byte-parts"), + pytest.param((_LMI_STREAM,), id="json-lines-whole-stream-in-one-part"), + pytest.param(_cut(_LMI_STREAM, 5), id="json-lines-five-byte-parts"), + pytest.param(tuple(_LMI_STREAM.splitlines()), id="undelimited-objects-one-per-part"), + ], +) +async def test_completion_stream_keeps_every_token_however_sagemaker_cuts_the_parts( + parts: tuple[str | bytes, ...], mode: str, caplog: pytest.LogCaptureFixture +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + chunks: Final = await _decode(parts, is_messages_api=False, mode=mode) + + assert None not in chunks + assert "".join(chunk["text"] for chunk in chunks) == "Héllo 👋" + assert caplog.records == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + "last_part", + [ + pytest.param(_VLLM_EVENTS[2][:20], id="cut-between-characters"), + pytest.param( + _VLLM_EVENTS[2].encode()[: _VLLM_EVENTS[2].encode().index("👋".encode()) + 2], id="cut-inside-a-character" + ), + ], +) +async def test_stream_that_ends_inside_a_record_raises_after_the_complete_ones(mode: str, last_part: str | bytes): + stream: Final = _stream_frames(tuple(map(_payload_part, (*_VLLM_EVENTS[:2], last_part))), True, mode) + + assert [_text(await anext(stream), True) for _ in range(2)] == ["", "Hél"] + with pytest.raises(SagemakerError, match="ended inside a record") as exc_info: + await anext(stream) + assert exc_info.value.status_code == 500 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +async def test_text_left_over_that_is_not_a_record_is_logged_once(mode: str, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + chunks: Final = await _decode((*_VLLM_EVENTS[:3], "data: upstream hiccup\n\n"), is_messages_api=True, mode=mode) + + assert chunks[-1] is None + assert "".join(chunk.choices[0].delta.content for chunk in chunks[:-1]) == "Héllo 👋" + assert [record.levelno for record in caplog.records] == [logging.ERROR] + assert "hiccup" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + ("parts", "is_messages_api", "status_code"), + [ + pytest.param( + ( + _VLLM_EVENTS[1], + _sse( + { + "error": { + "message": "CUDA out of memory while decoding", + "type": "InternalServerError", + "code": 500, + } + } + ) + + "data: [DONE]\n\n", + ), + True, + 500, + id="vllm-500-error-event-shares-a-part-with-done", + ), + pytest.param( + ( + _VLLM_EVENTS[1], + _sse( + {"error": {"message": "CUDA out of memory while decoding", "type": "BadRequestError", "code": 400}} + ), + "data: [DONE]\n\n", + ), + True, + 500, + id="vllm-400-error-event-in-its-own-part", + ), + pytest.param( + ( + _VLLM_EVENTS[1], + _sse( + { + "object": "error", + "message": "CUDA out of memory while decoding", + "type": "BadRequestError", + "code": 400, + } + ), + ), + True, + 500, + id="legacy-vllm-flat-error-event", + ), + pytest.param( + ( + _VLLM_EVENTS[1], + _sse({"error": {"message": "CUDA out of memory while decoding", "http_status_code": 422}}), + ), + True, + 422, + id="tgi-messages-api-error-event", + ), + pytest.param( + ( + _VLLM_EVENTS[1], + _sse({"error": {"message": "CUDA out of memory while decoding", "http_status_code": 424}}), + ), + True, + 500, + id="tgi-messages-api-generation-error-event", + ), + pytest.param( + ( + _VLLM_EVENTS[1], + _sse( + {"error": {"message": "CUDA out of memory while decoding", "type": "RateLimitError", "code": 429}} + ), + ), + True, + 429, + id="vllm-429-error-event", + ), + pytest.param( + (_VLLM_EVENTS[1], _sse({"error": {"message": "CUDA out of memory while decoding", "code": "oom"}})), + True, + 500, + id="error-event-without-a-status", + ), + pytest.param( + ( + _TGI_EVENTS[0] + + f"data:{json.dumps({'error': 'CUDA out of memory while decoding', 'error_type': 'generation'})}\n\n", + ), + False, + 500, + id="tgi-generation-error-event-shares-a-part-with-a-token", + ), + pytest.param( + ( + _TGI_EVENTS[0], + f"data:{json.dumps({'error': 'CUDA out of memory while decoding', 'error_type': 'overloaded'})}\n\n", + ), + False, + 429, + id="tgi-overloaded-error-event", + ), + pytest.param( + ( + _TGI_EVENTS[0], + f"data:{json.dumps({'error': 'CUDA out of memory while decoding', 'error_type': 'incomplete_generation'})}\n\n", + ), + False, + 500, + id="tgi-incomplete-generation-error-event", + ), + ], +) +async def test_container_error_event_raises_with_its_message_and_status_after_the_chunks_before_it( + parts: tuple[str, ...], is_messages_api: bool, status_code: int, mode: str +): + stream: Final = _stream_frames(tuple(map(_payload_part, parts)), is_messages_api, mode) + + assert _text(await anext(stream), is_messages_api) == "Hél" + with pytest.raises(SagemakerError, match="CUDA out of memory while decoding") as exc_info: + await anext(stream) + assert exc_info.value.status_code == status_code + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + ("last_record", "message"), + [ + pytest.param( + { + "token": {"id": 2, "text": "", "log_prob": 0.0, "error_msg": "CUDA out of memory while decoding"}, + "generated_text": "Hél", + "details": {"finish_reason": "error"}, + }, + "SageMaker endpoint streamed a failed generation: CUDA out of memory while decoding", + id="token-error-message", + ), + pytest.param( + { + "token": {"id": 2, "text": "", "log_prob": 0.0}, + "generated_text": "Hél", + "details": {"finish_reason": "error"}, + }, + "SageMaker endpoint streamed a failed generation: no error message", + id="error-finish-without-a-message", + ), + ], +) +async def test_lmi_failed_generation_raises_after_the_chunks_before_it( + last_record: Mapping[str, object], message: str, mode: str +): + lines: Final = "".join( + f"{json.dumps(record, ensure_ascii=False)}\n" + for record in ({"token": {"id": 1, "text": "Hél", "log_prob": -0.1}}, last_record) + ) + stream: Final = _stream_frames((_payload_part(lines),), False, mode) + + assert _text(await anext(stream), False) == "Hél" + with pytest.raises(SagemakerError) as exc_info: + await anext(stream) + assert exc_info.value.status_code == 500 + assert exc_info.value.message == message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +async def test_model_stream_error_raises_after_the_chunks_before_it(mode: str): + frames: Final = ( + _payload_part(_VLLM_EVENTS[1]), + _frame( + _MODEL_STREAM_ERROR_HEADERS, + json.dumps({"Message": "container exited", "ErrorCode": "ModelInvocationTimeExceeded"}), + ), + _payload_part(_VLLM_EVENTS[2]), + ) + stream: Final = _stream_frames(frames, True, mode) + + assert _text(await anext(stream), True) == "Hél" + with pytest.raises(SagemakerError) as exc_info: + await anext(stream) + assert exc_info.value.status_code == 500 + assert exc_info.value.message == ( + "SageMaker stream failed with ModelStreamError (ModelInvocationTimeExceeded): container exited" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize( + ("parts", "is_messages_api"), + [ + pytest.param( + tuple( + _vllm_chunk([{"index": 0, "delta": {"content": text}, "finish_reason": None}], error=None) + for text in ("Hél", "lo 👋") + ), + True, + id="vllm-chunks-with-null-error", + ), + pytest.param( + tuple( + f"data:{json.dumps({'index': 0, 'token': {'text': text}, 'error': None}, ensure_ascii=False)}\n\n" + for text in ("Hél", "lo 👋") + ), + False, + id="tgi-tokens-with-null-error", + ), + ], +) +async def test_chunk_with_a_null_error_field_is_delivered( + parts: tuple[str, ...], is_messages_api: bool, mode: str, caplog: pytest.LogCaptureFixture +): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + chunks: Final = await _decode(parts, is_messages_api=is_messages_api, mode=mode) + + assert "".join(_text(chunk, is_messages_api) or "" for chunk in chunks) == "Héllo 👋" + assert caplog.records == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +async def test_record_that_is_not_a_chunk_is_logged_and_skipped(mode: str, caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + chunks: Final = await _decode( + (_VLLM_EVENTS[1], _sse({"object": "keepalive", "detail": "still decoding"}), _VLLM_EVENTS[2]), + is_messages_api=True, + mode=mode, + ) + + assert "".join(chunk.choices[0].delta.content for chunk in chunks) == "Héllo 👋" + assert [record.levelno for record in caplog.records] == [logging.ERROR] + assert "['detail', 'object']" in caplog.records[0].getMessage() + assert "still decoding" not in caplog.records[0].getMessage() + + class TestSagemakerTransform: def setup_method(self): self.config = SagemakerConfig() From 77880a0653c4eb71771ff79ae0c9d81372000e4d Mon Sep 17 00:00:00 2001 From: Max Azatian <66359507+HardMax71@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:48:47 +0200 Subject: [PATCH 087/124] fix(otel)!: leave out span cost when the price lookup failed (#44187) * fix(otel): leave out span cost when the price lookup failed A model missing from the price map gets response_cost 0.0 in the standard logging payload, with response_cost_failure_debug_info set. The OpenInference writer (arize, phoenix, langfuse_otel) and the OTel v2 span data copied that 0.0 into llm.cost.total, litellm.cost.total and langfuse.observation.cost_details, so backends showed the call as free. Skip the cost when the lookup failed, except on cache hits, which are free. * test(otel): keep the guardrail charge on the guardrail span for unpriced calls * refactor(otel): type the Arize cost writer input and reuse the cache_hit flag * fix(otel): keep a billed guardrail fee on an unpriced call's Arize span cost Only the placeholder 0 is left out when the price lookup failed, so a guardrail fee LiteLLM billed still reaches Arize, Phoenix, Langfuse OTel and W&B Weave OTel. Adds unit cases for that fee and a malformed cost. * fix(arize): leave out the span cost on any failed price lookup, matching OTel v2 * fix(logging): clear a stale cost failure flag when a later pricing succeeds Stream-start pricing on the request model name can fail and set the cost failure flag even when the final pricing succeeds (OpenRouter usage.cost, an Azure deployment name priced from the response model). The new span gate then dropped a real cost. Each pricing now resets the flag, and uncached_price restores whatever flag it found. * test(logging): pin the start time in the stale cost flag tests --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/arize/_utils.py | 13 ++- litellm/integrations/otel/model/payloads.py | 3 +- litellm/litellm_core_utils/litellm_logging.py | 8 +- .../integrations/arize/test_arize_utils.py | 55 ++++++++++++- .../integrations/otel/test_otel_v2_emitter.py | 57 +++++++++++++ .../test_litellm_logging.py | 81 +++++++++++++++++++ 6 files changed, 209 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 3f89ac6978f..f719dacd148 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -884,7 +884,7 @@ def _set_session_and_user_attrs(span: "Span", kwargs: dict, standard_logging_pay safe_set_attribute(span, "litellm.key_alias", str(key_alias)) -def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None: +def _set_response_cost_attr(span: "Span", standard_logging_payload: Mapping[str, object] | None) -> None: """Emit cost attributes from the StandardLoggingPayload when present. Uses the OpenInference `llm.cost.total` key so Arize / Phoenix can @@ -892,15 +892,22 @@ def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None: single total in `StandardLoggingPayload.response_cost`, so we cannot split it into prompt/completion. We also keep the legacy `llm.response.cost` key for back-compat with any consumer querying it. + When the price lookup failed, the call's total is unknown, so no cost is + emitted and the backend does not record the call as free. A guardrail fee + in `response_cost` stays on the guardrail span, as in OTel v2. """ if not isinstance(standard_logging_payload, dict): return cost: Final = standard_logging_payload.get("response_cost") - if cost is None: + cost_lookup_failed: Final = ( + standard_logging_payload.get("response_cost_failure_debug_info") is not None + and standard_logging_payload.get("cache_hit") is not True + ) + if not isinstance(cost, (int, float, str)) or cost_lookup_failed: return try: cost_value: Final = float(cost) - except (TypeError, ValueError): + except ValueError: return safe_set_attribute(span, "llm.cost.total", cost_value) safe_set_attribute(span, "llm.response.cost", cost_value) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 5bca8e01aec..e15072c009b 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -481,6 +481,7 @@ class LLMCallSpanData: embedding_output: Final = ( EmbeddingOutput.from_response(response) if operation is GenAIOperation.EMBEDDINGS else None ) + cost_lookup_failed: Final = payload.get("response_cost_failure_debug_info") is not None and not cache_hit return cls( operation=operation, provider=resolve_provider(as_str(payload.get("custom_llm_provider"))), @@ -491,7 +492,7 @@ class LLMCallSpanData: usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), - response_cost=as_float(payload.get("response_cost")), + response_cost=None if cost_lookup_failed else as_float(payload.get("response_cost")), cost=LLMCost.from_breakdown(cast("Mapping[str, object] | None", payload.get("cost_breakdown"))), server=None if cache_hit else ServerInfo.from_api_base(context.api_base), identity=context.identity, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8d5be388eff..e2eed3cf6d5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1939,6 +1939,8 @@ class Logging(LiteLLMLoggingBaseClass): used for consistent cost calculation across response headers + logging integrations. """ + self.model_call_details.pop("response_cost_failure_debug_information", None) + if cache_hit is None: cache_hit = self.model_call_details.get("cache_hit", False) @@ -2099,15 +2101,17 @@ class Logging(LiteLLMLoggingBaseClass): previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates debug_key: Final = "response_cost_failure_debug_information" - had_failure_debug_info: Final = debug_key in self.model_call_details + previous_failure_debug_info: Final = self.model_call_details.get(debug_key) try: cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown) finally: self.cost_breakdown = previous_cost_breakdown self.billed_token_rates = previous_billed_token_rates - if not had_failure_debug_info: + if previous_failure_debug_info is None: self.model_call_details.pop(debug_key, None) + else: + self.model_call_details[debug_key] = previous_failure_debug_info def _record_zero_cost_diagnostic( self, diff --git a/tests/unit/integrations/arize/test_arize_utils.py b/tests/unit/integrations/arize/test_arize_utils.py index 0d833b90148..31699359bbe 100644 --- a/tests/unit/integrations/arize/test_arize_utils.py +++ b/tests/unit/integrations/arize/test_arize_utils.py @@ -1,5 +1,6 @@ import json -from typing import Optional +from collections.abc import Mapping +from typing import Final, Optional # Adds the grandparent directory to sys.path to allow importing project modules @@ -7,6 +8,7 @@ import asyncio import httpx import pytest +from opentelemetry.sdk.trace import TracerProvider import litellm from litellm.integrations._types.open_inference import ( @@ -17,7 +19,7 @@ from litellm.integrations._types.open_inference import ( from litellm.integrations.arize._utils import _coerce_response_obj_for_attrs, _parse_passthrough_response from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import Choices, StandardCallbackDynamicParams +from litellm.types.utils import Choices, ModelResponse, StandardCallbackDynamicParams def test_arize_set_attributes(): @@ -1015,6 +1017,55 @@ def test_arize_emits_response_cost(): assert attrs["llm.response.cost"] == 0.0012345 # legacy key still emitted +_PRICE_LOOKUP_FAILED: Final = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""} + + +@pytest.mark.parametrize( + ("cost_fields", "expected_cost"), + [ + pytest.param({"response_cost": 0.0}, 0.0, id="free-model"), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED}, + None, + id="unpriced", + ), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True}, + 0.0, + id="unpriced-cache-hit", + ), + pytest.param({"response_cost": "not-a-number"}, None, id="malformed-cost"), + pytest.param( + {"response_cost": 0.002, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED}, + None, + id="unpriced-with-guardrail-fee", + ), + ], +) +def test_arize_cost_attrs_tell_unpriced_call_from_free_call( + cost_fields: Mapping[str, object], expected_cost: float | None +) -> None: + span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request") + kwargs: Final = { + "model": "unpriced-model", + "messages": [{"role": "user", "content": "hi"}], + "standard_logging_object": {"model_parameters": {}, "metadata": {}, "call_type": "completion", **cost_fields}, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + } + response_obj: Final = ModelResponse( + usage={"total_tokens": 30, "completion_tokens": 20, "prompt_tokens": 10}, + choices=[Choices(message={"role": "assistant", "content": "ok"})], + model="unpriced-model", + id="r4", + ) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + assert span.attributes.get("llm.cost.total") == expected_cost + assert span.attributes.get("llm.response.cost") == expected_cost + + def test_arize_passthrough_bedrock_anthropic_normalization(): """Bedrock-Anthropic passthrough: input/output text must be set so the span renders something other than raw provider attrs.""" diff --git a/tests/unit/integrations/otel/test_otel_v2_emitter.py b/tests/unit/integrations/otel/test_otel_v2_emitter.py index 0905e9e7c30..6e29b2e9c68 100644 --- a/tests/unit/integrations/otel/test_otel_v2_emitter.py +++ b/tests/unit/integrations/otel/test_otel_v2_emitter.py @@ -2,6 +2,7 @@ legacy dual-emit, hierarchy, error status, and idempotency. Needs the OTel SDK.""" import json +from collections.abc import Mapping from typing import Final import pytest @@ -23,6 +24,7 @@ from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.emitter import SpanEmitter, attribute_budget, span_attribute_limit # noqa: E402 from litellm.integrations.otel.emitter import stamp_error # noqa: E402 +from litellm.integrations.otel.mappers import LangfuseMapper # noqa: E402 # after the opentelemetry importorskip from litellm.integrations.otel.mappers.utils import MAX_TOOL_DEFINITION_ATTRS_PER_SPAN # noqa: E402 from litellm.integrations.otel.model.payloads import ( # noqa: E402 GuardrailSpanData, @@ -91,6 +93,61 @@ def test_llm_call_span_cost_breakdown(): assert f"{LiteLLM.COST_PREFIX}margin_total_amount" not in a +_PRICE_LOOKUP_FAILED: Final = {"error_str": "This model isn't mapped yet. model=openai/unpriced-model", "traceback_str": ""} + + +@pytest.mark.parametrize( + ("cost_fields", "expected_cost"), + [ + pytest.param({"response_cost": 0.0}, 0.0, id="free-model"), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED}, + None, + id="unpriced", + ), + pytest.param( + {"response_cost": 0.0, "response_cost_failure_debug_info": _PRICE_LOOKUP_FAILED, "cache_hit": True}, + 0.0, + id="unpriced-cache-hit", + ), + ], +) +def test_llm_call_span_tells_unpriced_call_from_free_call( + cost_fields: Mapping[str, object], expected_cost: float | None +) -> None: + engine, exporter = _engine() + data: Final = LLMCallSpanData.from_standard_logging_payload(_payload(**cost_fields)) + + engine.emit(SpanRole.LLM_CALL, data) + + (span,) = exporter.get_finished_spans() + assert span.attributes.get(f"{LiteLLM.COST_PREFIX}total") == expected_cost + langfuse_cost_details: Final = LangfuseMapper().map(data).get("langfuse.observation.cost_details") + langfuse_cost: Final = json.loads(langfuse_cost_details)["total"] if langfuse_cost_details is not None else None + assert langfuse_cost == expected_cost + + +def test_unpriced_call_keeps_the_guardrail_charge_on_the_guardrail_span() -> None: + engine, exporter = _engine() + payload: Final = _payload( + response_cost=0.002, + response_cost_failure_debug_info=_PRICE_LOOKUP_FAILED, + cost_breakdown={"guardrail_cost": 0.002, "total_cost": 0.002}, + ) + + engine.emit(SpanRole.LLM_CALL, LLMCallSpanData.from_standard_logging_payload(payload)) + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData.from_logging_entry( + {"guardrail_name": "bedrock", "guardrail_status": "success", "guardrail_cost": 0.002} + ), + ) + + llm_span, guardrail_span = exporter.get_finished_spans() + assert f"{LiteLLM.COST_PREFIX}total" not in llm_span.attributes + assert guardrail_span.attributes[LiteLLM.GUARDRAIL_COST] == 0.002 + + def test_tracer_scope_carries_litellm_version(): from litellm._version import version as litellm_version diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 00c2b1ad8a8..9d651b4ae3e 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11014,6 +11014,87 @@ def test_uncached_price_leaves_no_failure_debug_info_for_unpriced_model(): assert "response_cost_failure_debug_information" not in logging_obj.model_call_details +PR44187_PRICED_MODEL: Final = "lit44187-priced-model" +PR44187_INPUT_COST: Final = 1e-6 +PR44187_OUTPUT_COST: Final = 2e-6 +PR44187_START_TIME: Final = datetime.datetime(2026, 10, 20) + + +@pytest.fixture +def pr44187_priced_model() -> Iterator[str]: + litellm.register_model( + model_cost={ + PR44187_PRICED_MODEL: { + "input_cost_per_token": PR44187_INPUT_COST, + "output_cost_per_token": PR44187_OUTPUT_COST, + "litellm_provider": "openai", + "mode": "chat", + } + }, + persist_across_reloads=False, + ) + try: + yield PR44187_PRICED_MODEL + finally: + litellm.model_cost.pop(PR44187_PRICED_MODEL, None) + + +def _pr44187_logging_obj(model: str) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model=model, + messages=[{"role": "user", "content": "Hi"}], + stream=True, + call_type="completion", + start_time=PR44187_START_TIME, + litellm_call_id="lit44187-late-priced-stream", + function_id="fn", + ) + logging_obj.update_environment_variables( + model=model, user="", optional_params={}, litellm_params={"metadata": {}}, custom_llm_provider="openai" + ) + return logging_obj + + +def _pr44187_response(model: str) -> ModelResponse: + return ModelResponse( + model=model, + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hello"))], + usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30), + ) + + +def test_a_later_successful_price_clears_an_earlier_price_failure(pr44187_priced_model: str): + logging_obj: Final = _pr44187_logging_obj("openai/lit44187-deployment-name") + stream_start_cost: Final = logging_obj.response_cost_calculator( + result=_pr44187_response("lit44187-deployment-name"), litellm_model_name="openai/lit44187-deployment-name" + ) + assert stream_start_cost is None + assert logging_obj.model_call_details["response_cost_failure_debug_information"] is not None + + final_cost: Final = logging_obj.response_cost_calculator( + result=_pr44187_response(pr44187_priced_model), litellm_model_name=pr44187_priced_model + ) + + assert final_cost == pytest.approx(10 * PR44187_INPUT_COST + 20 * PR44187_OUTPUT_COST) + assert logging_obj.model_call_details.get("response_cost_failure_debug_information") is None + + +@pytest.mark.parametrize("uncached_model_is_priced", [True, False], ids=["uncached-priced", "uncached-unpriced"]) +def test_uncached_price_keeps_an_earlier_price_failure(pr44187_priced_model: str, uncached_model_is_priced: bool): + uncached_model: Final = pr44187_priced_model if uncached_model_is_priced else "lit44187-other-unpriced-model" + logging_obj: Final = _pr44187_logging_obj(uncached_model) + logging_obj.response_cost_calculator( + result=_pr44187_response("lit44187-deployment-name"), litellm_model_name="openai/lit44187-deployment-name" + ) + earlier_failure: Final = logging_obj.model_call_details["response_cost_failure_debug_information"] + assert earlier_failure is not None + + uncached: Final = logging_obj.uncached_price(result=_pr44187_response(uncached_model)) + + assert (uncached.cost is not None) is uncached_model_is_priced + assert logging_obj.model_call_details["response_cost_failure_debug_information"] == earlier_failure + + def test_enterprise_alerting_loggers_resolves_real_classes(): from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger From f5d6e89ad8341e4df5400a0bd0e99509b8ae62c8 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:42:43 +0000 Subject: [PATCH 088/124] test(integration): follow #47441 metadata-routed counting in count_tokens wire tests (#47525) * test(integration): follow #47441 metadata-routed Mantle counting in count_tokens wire tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): build the runtime count body without mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_mantle_count_tokens_wire.py | 205 ++++++++++++------ .../providers/test_count_tokens_model_wire.py | 81 ++++++- 2 files changed, 207 insertions(+), 79 deletions(-) diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index ab0c35d0ca4..4fe29fa49d8 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -70,6 +70,18 @@ _SCRIPTED_STATUS: Final = re.compile(r"status=(\d{3})") _STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_LIST: Final = TypeAdapter(list[JsonValue]) +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) + + +def _count_model_source(model: str) -> dict[str, JsonValue]: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + token_count_model: Final = cost_map[model].get("token_count_model") + assert isinstance(token_count_model, dict), model + count_model: Final = token_count_model.get("model") + assert isinstance(count_model, str), token_count_model + return {"litellm_count_source": "token_count_model", "litellm_count_model": count_model} + _SDK_MESSAGES: Final[list[MessageParam]] = [{"role": "user", "content": "Count this message"}] _MESSAGES: Final = _JSON_LIST.validate_json(json.dumps(_SDK_MESSAGES)) @@ -94,6 +106,32 @@ _GEMINI_BODY: Final[dict[str, JsonValue]] = {"contents": [{"role": "user", "part _GEMINI_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this"}] +def _runtime_count_body(prompt: str, include_system_and_tools: bool = False) -> dict[str, JsonValue]: + tool: Final = _SDK_TOOLS[0] + converse: Final[dict[str, JsonValue]] = { + "messages": [{"role": "user", "content": [{"text": prompt}]}], + **( + { + "system": [{"text": _SYSTEM}], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "name": tool["name"], + "description": tool["description"], + "inputSchema": {"json": tool["input_schema"]}, + } + } + ] + }, + } + if include_system_and_tools + else {} + ), + } + return {"input": {"converse": converse}} + + def _mantle_body(**fields: JsonValue) -> dict[str, JsonValue]: return {"model": _OPUS_BASE, "messages": _MESSAGES, **fields} @@ -223,7 +261,11 @@ def _owned_config(path: Path, runtime_url: str, settings: Mapping[str, JsonValue { **config, "model_list": [ - {"model_name": name, "litellm_params": _litellm_params(model, runtime_url)} + { + "model_name": name, + "litellm_params": _litellm_params(model, runtime_url), + "model_info": {"token_count_model": {}}, + } for name, model in _OWNED_MODELS.items() ], "litellm_settings": {**litellm_settings, **settings}, @@ -233,7 +275,11 @@ def _owned_config(path: Path, runtime_url: str, settings: Mapping[str, JsonValue return path -def _deployment(scenario: Scenario, api_base: str, model: str = _OPUS) -> str: +def _fallback_deployment(scenario: Scenario, api_base: str, model: str = _OPUS) -> str: + return scenario.model(model_info={"token_count_model": {}}, api_key=None, **_litellm_params(model, api_base)) + + +def _metadata_deployment(scenario: Scenario, api_base: str, model: str = _OPUS) -> str: return scenario.model(model_info=None, api_key=None, **_litellm_params(model, api_base)) @@ -290,6 +336,14 @@ def _runtime_count_targets(wire: Wire) -> tuple[str, ...]: return tuple(target_of(request) for request in counts) +def _runtime_count_requests(wire: Wire) -> tuple[tuple[str, dict[str, JsonValue]], ...]: + counts: Final = tuple(request for request in wire.drain() if _COUNT_TARGET.match(target_of(request))) + for request in counts: + assert request.method == "POST", request.method + _assert_sigv4(request) + return tuple((target_of(request), _JSON_OBJECT.validate_json(request.body)) for request in counts) + + def _clients(stack: ExitStack, base_url: str, count: int, timeout: float = 30) -> tuple[httpx.Client, ...]: return tuple( stack.enter_context(httpx.Client(base_url=base_url, timeout=timeout, trust_env=False)) for _ in range(count) @@ -376,7 +430,7 @@ def test_messages_count_tokens_counts_through_mantle_when_the_runtime_cannot( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS}) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert _runtime_count_targets(runtime) == (f"/model/{_OPUS_BASE}/count-tokens",) @@ -392,7 +446,7 @@ def test_messages_count_tokens_without_system_or_tools_sends_a_bare_body_to_mant wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, _bare(model)) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -408,7 +462,7 @@ def test_utils_token_counter_call_endpoint_reports_the_mantle_tokenizer( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/utils/token_counter", _full(model), params={"call_endpoint": "true"} ) @@ -427,14 +481,14 @@ def test_gemini_count_tokens_route_reports_the_mantle_total(counting_proxy: Owne wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _metadata_deployment(scenario, runtime.url) response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", _GEMINI_BODY) assert _payload(response) == { "totalTokens": _MANTLE_COUNT, "promptTokensDetails": [], - "litellm_count_source": "provider", + **_count_model_source(_OPUS), }, response.text - assert len(_runtime_count_targets(runtime)) == 1 + assert _runtime_count_requests(runtime) == () assert _mantle_bodies(mantle) == ({"model": _OPUS_BASE, "messages": _GEMINI_MESSAGES},) @@ -447,14 +501,16 @@ def test_gemini_count_tokens_route_reports_the_runtime_total_for_a_model_bedrock wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url, _SONNET) + model: Final = _metadata_deployment(scenario, runtime.url, _SONNET) response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", _GEMINI_BODY) assert _payload(response) == { "totalTokens": _RUNTIME_COUNT, "promptTokensDetails": [], - "litellm_count_source": "provider", + **_count_model_source(_SONNET), }, response.text - assert _runtime_count_targets(runtime) == (f"/model/{_SONNET}/count-tokens",) + assert _runtime_count_requests(runtime) == ( + (f"/model/{_SONNET}/count-tokens", _runtime_count_body("Count this")), + ) assert mantle.drain() == () @@ -466,7 +522,7 @@ def test_runtime_404_on_claude_falls_back_to_mantle(counting_proxy: OwnedProxy, wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS}) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -481,7 +537,7 @@ def test_runtime_404_and_mantle_404_surface_provider_error(counting_proxy: Owned wire_server(_mantle(_rejecting(404)), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "system": system, "tools": _TOOLS}) assert response.status_code == 404, response.text assert _REJECTION in response.text @@ -496,7 +552,7 @@ def test_runtime_404_on_nova_does_not_call_mantle(counting_proxy: OwnedProxy, ma wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url, _NOVA) + model: Final = _fallback_deployment(scenario, runtime.url, _NOVA) body: Final = {**_bare(model), "messages": [{"role": "user", "content": "status=404"}]} response: Final = _count(gateway, body) assert response.status_code == 404, response.text @@ -504,9 +560,7 @@ def test_runtime_404_on_nova_does_not_call_mantle(counting_proxy: OwnedProxy, ma assert mantle.drain() == () -def test_google_count_tokens_forwards_system_and_tools_to_mantle( - counting_proxy: OwnedProxy, mantle_port: int -) -> None: +def test_google_count_tokens_forwards_system_and_tools_to_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None: gateway: Final = counting_proxy.gateway system_text: Final = "Use the provided tool" tool_schema: Final = {"type": "object", "properties": {"location": {"type": "string"}}} @@ -526,7 +580,7 @@ def test_google_count_tokens_forwards_system_and_tools_to_mantle( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body) assert _payload(response) == { "totalTokens": _MANTLE_COUNT, @@ -575,7 +629,7 @@ def test_google_count_tokens_normalizes_uppercase_parameters_for_mantle( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", body) assert _payload(response) == { "totalTokens": _MANTLE_COUNT, @@ -607,7 +661,7 @@ def test_responses_input_tokens_counts_through_mantle(counting_proxy: OwnedProxy wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/responses/input_tokens", {"model": model, "input": "Count this message"} ) @@ -627,7 +681,7 @@ def test_anthropic_sdk_count_tokens_counts_through_mantle(counting_proxy: OwnedP wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = _anthropic_client(gateway).messages.count_tokens( model=model, messages=_SDK_MESSAGES, system=_SYSTEM, tools=_SDK_TOOLS ) @@ -643,7 +697,7 @@ def test_async_anthropic_sdk_count_tokens_counts_through_mantle(counting_proxy: wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = asyncio.run( _async_anthropic_client(gateway).messages.count_tokens(model=model, messages=_SDK_MESSAGES, system=_SYSTEM) ) @@ -659,17 +713,36 @@ def test_model_the_runtime_counts_never_reaches_mantle(counting_proxy: OwnedProx wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url, _SONNET) + model: Final = _metadata_deployment(scenario, runtime.url, _SONNET) assert _payload(_count(gateway, _full(model))) == { "input_tokens": _RUNTIME_COUNT, - "litellm_count_source": "provider", + **_count_model_source(_SONNET), } detailed: Final = gateway.request( "POST", "/utils/token_counter", _full(model), params={"call_endpoint": "true"} ) payload: Final = _payload(detailed) - assert (payload["total_tokens"], payload["tokenizer_type"]) == (_RUNTIME_COUNT, "bedrock_api"), detailed.text - assert _runtime_count_targets(runtime) == (f"/model/{_SONNET}/count-tokens",) * 2 + assert payload == { + "total_tokens": _RUNTIME_COUNT, + "request_model": model, + "model_used": _count_model_source(_SONNET)["litellm_count_model"], + "tokenizer_type": "bedrock_api", + "count_source": "token_count_model", + "original_response": {"input_tokens": _RUNTIME_COUNT}, + "error": False, + "error_message": None, + "status_code": None, + }, detailed.text + assert _runtime_count_requests(runtime) == ( + ( + f"/model/{_SONNET}/count-tokens", + _runtime_count_body("Count this message", include_system_and_tools=True), + ), + ( + f"/model/{_SONNET}/count-tokens", + _runtime_count_body("Count this message", include_system_and_tools=True), + ), + ) assert mantle.drain() == () @@ -682,7 +755,7 @@ def test_non_claude_model_the_runtime_cannot_count_surfaces_the_error_without_ma wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url, _NOVA) + model: Final = _fallback_deployment(scenario, runtime.url, _NOVA) response: Final = _count(gateway, _bare(model)) assert response.status_code == 400, response.text assert _UNSUPPORTED in response.text and "input_tokens" not in response.text, response.text @@ -697,7 +770,7 @@ def test_runtime_403_on_a_claude_model_never_reaches_mantle(counting_proxy: Owne wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final[dict[str, JsonValue]] = { "model": model, "messages": [{"role": "user", "content": "status=403 Count this message"}], @@ -712,7 +785,7 @@ def test_runtime_403_on_a_claude_model_never_reaches_mantle(counting_proxy: Owne def test_unreachable_runtime_surfaces_the_error_without_mantle(counting_proxy: OwnedProxy, mantle_port: int) -> None: gateway: Final = counting_proxy.gateway with wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario: - model: Final = _deployment(scenario, _closed_port_url()) + model: Final = _fallback_deployment(scenario, _closed_port_url()) response: Final = _count(gateway, _full(model)) assert 500 <= response.status_code < 600, response.text assert "input_tokens" not in response.text, response.text @@ -728,7 +801,7 @@ def test_bedrock_passthrough_count_tokens_still_answers_the_runtime_rejection( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request("POST", "/bedrock/v1/messages/count_tokens", _full(model)) assert response.status_code == 400, response.text assert _UNSUPPORTED in response.text and "input_tokens" not in response.text, response.text @@ -745,7 +818,7 @@ def test_responses_input_tokens_with_instructions_counts_through_mantle( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/responses/input_tokens", @@ -770,7 +843,7 @@ def test_messages_count_tokens_lifts_the_leading_system_run_for_mantle( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, count_request(model, case)) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -786,7 +859,7 @@ def test_anthropic_sdk_count_tokens_lifts_the_leading_system_through_mantle( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = _anthropic_client(gateway).messages.count_tokens(model=model, messages=_LIFT_SDK_MESSAGES) assert counted.input_tokens == _MANTLE_COUNT, counted assert len(_runtime_count_targets(runtime)) == 1 @@ -802,7 +875,7 @@ def test_async_anthropic_sdk_count_tokens_lifts_the_leading_system_through_mantl wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = asyncio.run( _async_anthropic_client(gateway).messages.count_tokens(model=model, messages=_LIFT_SDK_MESSAGES) ) @@ -820,7 +893,7 @@ def test_utils_token_counter_call_endpoint_counts_a_leading_system_through_mantl wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/utils/token_counter", count_request(model, STRING_CASE), params={"call_endpoint": "true"} ) @@ -841,7 +914,7 @@ def test_openai_sdk_input_tokens_lifts_instructions_through_mantle( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = openai_client(gateway).responses.input_tokens.count( model=model, input=USER_TEXT, instructions=INSTRUCTION ) @@ -859,7 +932,7 @@ def test_async_openai_sdk_input_tokens_lifts_instructions_through_mantle( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) counted: Final = asyncio.run( async_openai_client(gateway).responses.input_tokens.count( model=model, input=USER_TEXT, instructions=INSTRUCTION @@ -879,7 +952,7 @@ def test_responses_input_tokens_lifts_instructions_ahead_of_a_leading_system_ite wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/responses/input_tokens", @@ -903,7 +976,7 @@ def test_messages_count_tokens_forwards_tools_beside_the_lifted_system_to_mantle wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**count_request(model, STRING_CASE), "tools": _TOOLS}) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -919,7 +992,7 @@ def test_messages_count_tokens_keeps_a_mid_conversation_system_in_place_for_mant wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final[dict[str, JsonValue]] = { "model": model, "messages": [LEADING, USER, MID_SYSTEM, ASSISTANT, FOLLOW_UP], @@ -941,7 +1014,7 @@ def test_messages_count_tokens_surfaces_the_mantle_error_when_every_message_is_s wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final[dict[str, JsonValue]] = {"model": model, "messages": [LEADING]} response: Final = _count(gateway, body) assert response.status_code == 400, response.text @@ -959,7 +1032,7 @@ def test_messages_count_tokens_surfaces_the_mantle_error_beside_a_non_text_syste wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**count_request(model, STRING_CASE), "system": 5}) assert response.status_code == 400, response.text assert "input_tokens" not in response.text, response.text @@ -976,7 +1049,7 @@ def test_messages_count_tokens_repeated_request_lifts_the_leading_system_each_ti wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) answers: Final = tuple(_payload(_count(gateway, count_request(model, STRING_CASE))) for _ in range(2)) assert answers == ({"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"},) * 2 assert _runtime_count_targets(runtime) == (f"/model/{_OPUS_BASE}/count-tokens",) * 2 @@ -992,7 +1065,7 @@ def test_messages_count_tokens_surfaces_the_counting_error_for_a_leading_system_ wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final[dict[str, JsonValue]] = {"model": model, "messages": [{"role": "system"}, USER]} response: Final = _count(gateway, body) assert 500 <= response.status_code < 600, response.text @@ -1015,7 +1088,7 @@ def test_messages_count_tokens_duplicate_messages_key_lifts_the_last_value_for_m wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) first: Final = json.dumps([USER]) last: Final = json.dumps([LEADING, USER]) response: Final = gateway.client.post( @@ -1064,7 +1137,7 @@ def test_mantle_outage_between_concurrent_lifted_waves_errors_then_recovers( pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) runtime: Final = stack.enter_context(wire_server(_runtime)) scenario: Final = stack.enter_context(gateway.scenario()) - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final = count_request(model, STRING_CASE) def count(client: httpx.Client) -> tuple[int, JsonValue]: @@ -1092,7 +1165,7 @@ def test_messages_count_tokens_surfaces_the_mantle_status_when_mantle_errors( wire_server(_mantle(_rejecting(status)), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, _full(model)) assert response.status_code == status, response.text assert "input_tokens" not in response.text, response.text @@ -1114,7 +1187,7 @@ def test_messages_count_tokens_errors_when_mantle_answers_without_input_tokens( wire_server(_mantle(lambda _request: Reply(body=body)), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, _bare(model)) assert 500 <= response.status_code < 600, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -1124,16 +1197,14 @@ def test_messages_count_tokens_errors_when_mantle_answers_without_input_tokens( def test_messages_count_tokens_errors_when_mantle_is_unreachable(counting_proxy: OwnedProxy) -> None: gateway: Final = counting_proxy.gateway with wire_server(_runtime) as runtime, gateway.scenario() as scenario: - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, _full(model)) assert 500 <= response.status_code < 600, response.text assert "input_tokens" not in response.text, response.text assert len(_runtime_count_targets(runtime)) == 1 -def test_messages_count_tokens_errors_when_mantle_never_answers( - counting_proxy: OwnedProxy, mantle_port: int -) -> None: +def test_messages_count_tokens_errors_when_mantle_never_answers(counting_proxy: OwnedProxy, mantle_port: int) -> None: gateway: Final = counting_proxy.gateway held: Final[SimpleQueue[str]] = SimpleQueue() release: Final = threading.Event() @@ -1150,7 +1221,7 @@ def test_messages_count_tokens_errors_when_mantle_never_answers( mantle: Final = stack.enter_context(wire_server(_mantle(hold), port=mantle_port)) stack.callback(release.set) scenario: Final = stack.enter_context(gateway.scenario()) - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final = _full(model) future: Final = pool.submit(_counted_on, client, gateway.key, body) eventually(held.qsize, lambda size: size == 1, seconds=30) @@ -1172,7 +1243,7 @@ def test_messages_count_tokens_surfaces_the_mantle_error_for_a_non_text_system( wire_server(_mantle(_strict), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "system": 5}) assert response.status_code == 400, response.text assert "input_tokens" not in response.text, response.text @@ -1194,7 +1265,7 @@ def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_pe wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) refused: Final = _count(gateway, {**_bare(model), "tools": tools}) assert refused.status_code == 400, refused.text assert object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["type"] == "invalid_request_error" @@ -1213,7 +1284,7 @@ def test_messages_count_tokens_without_messages_is_rejected( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {"model": model, "system": _SYSTEM, "tools": _TOOLS, **fields}) assert response.status_code == 400, response.text assert "messages parameter is required" in response.text, response.text @@ -1234,7 +1305,7 @@ def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/messages/count_tokens", {**_full(model), **fields}, key="sk-not-a-key-this-proxy-issued" ) @@ -1253,7 +1324,7 @@ def test_messages_count_tokens_forwards_a_5kb_system_verbatim_to_mantle( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "system": system}) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -1269,7 +1340,7 @@ def test_messages_count_tokens_duplicate_system_and_tools_keys_forward_one_value wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) fields: Final = f'"system": {json.dumps(_SYSTEM)}, "tools": {json.dumps(_TOOLS)}' response: Final = gateway.client.post( "/v1/messages/count_tokens", @@ -1292,7 +1363,7 @@ def test_messages_count_tokens_leaves_empty_tools_and_system_out_of_the_mantle_b wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = _count(gateway, {**_bare(model), "tools": [], "system": ""}) assert _payload(response) == {"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"}, response.text assert len(_runtime_count_targets(runtime)) == 1 @@ -1308,7 +1379,7 @@ def test_messages_count_tokens_repeated_request_reaches_both_peers_each_time( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) answers: Final = tuple(_payload(_count(gateway, _bare(model))) for _ in range(2)) assert answers == ({"input_tokens": _MANTLE_COUNT, "litellm_count_source": "provider"},) * 2 assert _runtime_count_targets(runtime) == (f"/model/{_OPUS_BASE}/count-tokens",) * 2 @@ -1364,7 +1435,7 @@ def test_mantle_outage_between_concurrent_waves_errors_then_recovers( pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) runtime: Final = stack.enter_context(wire_server(_runtime)) scenario: Final = stack.enter_context(gateway.scenario()) - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) body: Final = _full(model) def generate_then_count(client: httpx.Client) -> tuple[int, int, JsonValue]: @@ -1398,7 +1469,7 @@ def test_slow_mantle_holds_concurrent_counts_without_stalling_the_proxy( scenario: Final = stack.enter_context(gateway.scenario()) pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) stack.callback(release.set) - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) futures: Final = tuple( pool.submit(_generated_then_counted, client, gateway.key, model, _full(model)) for client in clients ) @@ -1495,7 +1566,7 @@ def test_chat_completions_on_the_same_deployment_still_generate( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/chat/completions", @@ -1523,7 +1594,7 @@ def test_messages_endpoint_on_the_same_deployment_still_generates(counting_proxy wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request( "POST", "/v1/messages", @@ -1546,7 +1617,7 @@ def test_responses_endpoint_on_the_same_deployment_still_generates( wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) response: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": f"marker-{marker}"}) assert response.status_code == 200, response.text assert answer(marker) in response.text, response.text @@ -1562,6 +1633,6 @@ def test_utils_token_counter_local_mode_never_calls_either_peer(counting_proxy: wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - model: Final = _deployment(scenario, runtime.url) + model: Final = _fallback_deployment(scenario, runtime.url) assert _local_count(gateway, _full(model)) > 0 assert runtime.drain() == () and mantle.drain() == () diff --git a/tests/integration/providers/test_count_tokens_model_wire.py b/tests/integration/providers/test_count_tokens_model_wire.py index 46e5396ef64..b6ace7ddef3 100644 --- a/tests/integration/providers/test_count_tokens_model_wire.py +++ b/tests/integration/providers/test_count_tokens_model_wire.py @@ -31,7 +31,6 @@ _PROFILE_ID: Final = "lit9563count" _PROFILE_ARN: Final = f"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/{_PROFILE_ID}" _COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" _REGION: Final = "us-west-2" -_COUNT_REGION: Final = "us-east-1" _ACCESS_KEY: Final = "AKIAINTEGRATIONCOUNT" _SECRET_KEY: Final = "integration-count-secret" _PROVIDER_COUNT: Final = 1345 @@ -39,12 +38,37 @@ _COUNT_MODEL_COUNT: Final = 2718 _FRAMING: Final = 50 _UNSUPPORTED: Final = "The provided model doesn't support counting tokens." _COUNT_TARGET: Final = re.compile(r"^/model/(.+)/count-tokens$") +_MANTLE_COUNT_TARGET: Final = "/anthropic/v1/messages/count_tokens" +_MANTLE_VERSION: Final = "2023-06-01" _SCRIPTED_STATUS: Final = re.compile(r"status=(\d{3})") _SCRIPTED_COUNT_MODEL_STATUS: Final = re.compile(r"count_model=(\d{3})") _CREDENTIAL: Final = re.compile(r"Credential=([^/]+)/\d+/([^/]+)/") _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) + +def _shipped_token_count_model(model: str) -> dict[str, JsonValue]: + token_count_model: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes())[model].get("token_count_model") + assert isinstance(token_count_model, dict), model + return token_count_model + + +def _shipped_token_count_model_field(model: str, field: str) -> str: + value: Final = _shipped_token_count_model(model).get(field) + assert isinstance(value, str), (model, field, value) + return value + + +def _count_model_source(model: str) -> dict[str, JsonValue]: + return { + "litellm_count_source": "token_count_model", + "litellm_count_model": _shipped_token_count_model_field(model, "model"), + } + + +_COUNT_REGION: Final = _shipped_token_count_model_field(_SONNET, "region") +_COUNT_MODEL_SOURCE: Final = _count_model_source(_SONNET) + _SYSTEM: Final = "You are a terse assistant that answers in one sentence" _WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { "type": "object", @@ -62,7 +86,6 @@ _SDK_TOOLS: Final[list[MessageCountTokensToolParam]] = [ }, } ] -_COUNT_MODEL_SOURCE: Final = {"litellm_count_source": "token_count_model", "litellm_count_model": _OPUS} _OPENAI_PRIMARY: Final = "gpt-count-primary" _OPENAI_COUNT_MODEL: Final = "gpt-count-model" _OPENAI_COUNT_TARGET: Final = "/v1/responses/input_tokens" @@ -140,8 +163,17 @@ def _runtime(request: Request) -> Reply: def _mantle(request: Request) -> Reply: model: Final = _JSON_OBJECT.validate_json(request.body).get("model") + status: Final = _scripted(_SCRIPTED_COUNT_MODEL_STATUS, request, 200) + if status == 200: + return _json_reply(200, {"input_tokens": _COUNT_MODEL_COUNT}) + error_type: Final = { + 401: "authentication_error", + 404: "not_found_error", + 429: "rate_limit_error", + }[status] return _json_reply( - 404, {"type": "error", "error": {"type": "not_found_error", "message": f"scripted mantle 404 for {model}"}} + status, + {"type": "error", "error": {"type": error_type, "message": f"scripted mantle {status} for {model}"}}, ) @@ -165,6 +197,24 @@ def _mantle_models(wire: Wire) -> tuple[JsonValue, ...]: return tuple(_JSON_OBJECT.validate_json(request.body).get("model") for request in wire.drain()) +def _mantle_count_requests(wire: Wire) -> tuple[tuple[tuple[str, str], dict[str, JsonValue]], ...]: + requests: Final = tuple(wire.drain()) + for request in requests: + assert (request.method, request.target) == ("POST", _MANTLE_COUNT_TARGET), request.target + assert request.headers["anthropic-version"] == _MANTLE_VERSION, request.headers + assert request.headers["content-type"] == "application/json", request.headers + return tuple((_signer(request), _JSON_OBJECT.validate_json(request.body)) for request in requests) + + +def _mantle_body(prompt: str) -> dict[str, JsonValue]: + return { + "model": _OPUS_BASE, + "messages": [{"role": "user", "content": prompt}], + "system": _SYSTEM, + "tools": _SDK_TOOLS, + } + + def _url(port: int) -> str: return f"http://127.0.0.1:{port}" @@ -255,15 +305,15 @@ def test_shipped_token_count_model_counts_the_request_in_its_region_without_tryi ): counted: Final = _count(gateway, _deployment(scenario, runtime_port), 404) assert (counted.input_tokens, counted.model_extra) == (_COUNT_MODEL_COUNT, _COUNT_MODEL_SOURCE), counted - assert _runtime_counts(runtime) == ((_OPUS_BASE, (_ACCESS_KEY, _COUNT_REGION), _converse_body(_prompt(404))),) - assert _mantle_models(mantle) == () + assert _runtime_counts(runtime) == () + assert _mantle_count_requests(mantle) == (((_ACCESS_KEY, _COUNT_REGION), _mantle_body(_prompt(404))),) @pytest.mark.parametrize( ("count_model_status", "error", "message"), [ - (401, anthropic.AuthenticationError, f"scripted runtime 401 for {_OPUS_BASE}"), - (429, anthropic.RateLimitError, f"scripted runtime 429 for {_OPUS_BASE}"), + (401, anthropic.AuthenticationError, f"scripted mantle 401 for {_OPUS_BASE}"), + (429, anthropic.RateLimitError, f"scripted mantle 429 for {_OPUS_BASE}"), (404, anthropic.NotFoundError, f"scripted mantle 404 for {_OPUS_BASE}"), ], ) @@ -283,9 +333,16 @@ def test_token_count_model_failure_answers_its_error_and_names_the_region_to_ena ): with pytest.raises(error) as refused: _count(gateway, _deployment(scenario, runtime_port), 200, count_model_status=count_model_status) - assert message in str(refused.value) and f"{_OPUS} in {_COUNT_REGION}" in str(refused.value), refused.value - assert _runtime_models(runtime) == (_OPUS_BASE,) - assert _mantle_models(mantle) == ((_OPUS_BASE,) if count_model_status == 404 else ()) + assert message in str(refused.value) and ( + f"{_COUNT_MODEL_SOURCE['litellm_count_model']} in {_COUNT_REGION}" in str(refused.value) + ), refused.value + assert _runtime_counts(runtime) == () + assert _mantle_count_requests(mantle) == ( + ( + (_ACCESS_KEY, _COUNT_REGION), + _mantle_body(_prompt(200, count_model_status)), + ), + ) def test_framing_is_added_to_the_local_estimate_when_the_runtime_cannot_count( @@ -359,8 +416,8 @@ def test_inference_profile_arn_counts_with_the_token_count_model_of_its_base_mod ) counted: Final = _count(gateway, model, 404) assert (counted.input_tokens, counted.model_extra) == (_COUNT_MODEL_COUNT, _COUNT_MODEL_SOURCE), counted - assert _runtime_models(runtime) == (_OPUS_BASE,) - assert _mantle_models(mantle) == () + assert _runtime_counts(runtime) == () + assert _mantle_count_requests(mantle) == (((_ACCESS_KEY, _COUNT_REGION), _mantle_body(_prompt(404))),) @pytest.mark.parametrize( From 963333d654d3622f74b0bff0ecb41e9d62397135 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:58:42 +0000 Subject: [PATCH 089/124] fix(proxy)!: let proxy admin keys read raw cloud storage file ids on GET /files/{id}/content (#40621) * fix(proxy): let proxy admin keys read raw cloud storage file ids on GET /files/{id}/content Since #30595 the endpoint answered 400 for every raw gs:// or s3:// id, which broke the documented unmanaged Vertex batch flow at its last step, while a double-encoded id slipped past the guard entirely. Raw ids are now readable by proxy admin keys only, every other key gets 403, and the guard decodes the id fully before deciding, so encoding depth no longer matters * fix(proxy): decode nested percent-encoded file ids iteratively An id percent-encoded a thousand or more times sent the recursive decoder past the interpreter's recursion limit, so a proxy admin key got a 500 instead of a verdict on the id. The decoder now walks the encodings in a loop, so any depth resolves * fix(proxy): cap the decode passes the raw cloud storage file id guard makes Any authenticated key could send a file id wrapped in tens of thousands of percent-encoding layers and the guard would unwrap every one of them, each pass walking the whole id, so a 120 KB request line held a worker for about five seconds and a 2 MB one for minutes. The guard now unwraps at most eight layers, and an id still changing after that is treated as a raw cloud storage id, so it stays admin-only instead of being decoded any further * refactor(proxy): type the decode step the raw cloud storage file id guard folds over * test(proxy): return a streaming result from the admin raw id test mock now that vertex_ai file content streams * test(proxy): send raw cloud ids to the content route the way uvicorn delivers them Starlette's TestClient decodes the path twice, so the twice-encoded cases reached the route fully decoded and never hit the decode loop. The cases now name the id the route receives, and the helper tests take a typed quote helper and Final constants. * test(proxy): mark the raw cloud id test locals Final --- .../cloud_storage_security.py | 23 +++- .../openai_files_endpoints/files_endpoints.py | 10 +- .../test_cloud_storage_security.py | 42 +++++++ .../test_files_endpoint.py | 118 +++++++++++++++--- 4 files changed, 170 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index c913e4de02e..e9689bd7d8a 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -1,6 +1,7 @@ import posixpath import re from collections.abc import Mapping, Sequence +from itertools import accumulate, repeat from types import MappingProxyType from typing import Any, Final, cast from urllib.parse import quote, unquote @@ -20,15 +21,33 @@ MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://") _MAPPING_PROXY_TYPE: Final[type] = type(MappingProxyType({})) +MAX_FILE_ID_DECODE_PASSES: Final = 8 + + +def _unquote_once(current: str, _: str) -> str: + return unquote(current) + + +def _decodings(value: str) -> tuple[str, ...]: + return tuple(accumulate(repeat(value, MAX_FILE_ID_DECODE_PASSES + 2), _unquote_once)) + + def is_managed_cloud_storage_uri(file_id: str) -> bool: """ - True if file_id is a raw cloud-storage object URI (e.g. ``s3://bucket/key``). + True if file_id is a raw cloud-storage object URI (e.g. ``s3://bucket/key``), + up to ``MAX_FILE_ID_DECODE_PASSES`` layers of percent-encoding deep, or an id + encoded deeper than that, which no client produces and which is treated as raw + rather than decoded any further. These are internal provider artifacts. On the multi-tenant proxy they must be retrieved through their managed unified file id so owner/team access is enforced; a raw URI supplied by a caller bypasses that check. """ - return isinstance(file_id, str) and file_id.startswith(MANAGED_CLOUD_STORAGE_SCHEMES) + if not isinstance(file_id, str): + return False + decodings: Final = _decodings(file_id) + settled: Final = decodings[-1] == decodings[-2] + return decodings[-1].startswith(MANAGED_CLOUD_STORAGE_SCHEMES) or not settled _SAFE_OBJECT_COMPONENT_PATTERN: Final = re.compile(r"[^A-Za-z0-9._-]+") diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index bdc71908c7c..8823f01d781 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -1102,14 +1102,10 @@ async def get_file_content( } ) else: - # A raw cloud-storage URI (s3://, gs://) supplied here would skip the - # managed-file owner/team check that only runs for unified ids, letting - # a caller read another tenant's object by its key. Such objects are only - # reachable through their managed unified id. - if is_managed_cloud_storage_uri(file_id): + if is_managed_cloud_storage_uri(file_id) and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( - status_code=400, - detail="Raw cloud storage file ids cannot be retrieved directly. Use the LiteLLM managed file id returned when the file was created.", + status_code=403, + detail="Raw cloud storage file ids can only be retrieved by a proxy admin key. Use the LiteLLM managed file id returned when the file was created.", ) # Check for model-based credential routing ( diff --git a/tests/unit/litellm_core_utils/test_cloud_storage_security.py b/tests/unit/litellm_core_utils/test_cloud_storage_security.py index c3a2511a263..46131aa64e5 100644 --- a/tests/unit/litellm_core_utils/test_cloud_storage_security.py +++ b/tests/unit/litellm_core_utils/test_cloud_storage_security.py @@ -1,4 +1,11 @@ +from functools import reduce +from typing import Final +from urllib.parse import quote + +import pytest + from litellm.litellm_core_utils.cloud_storage_security import ( + MAX_FILE_ID_DECODE_PASSES, is_managed_cloud_storage_uri, ) @@ -13,3 +20,38 @@ def test_is_managed_cloud_storage_uri_ignores_provider_and_unified_ids(): assert not is_managed_cloud_storage_uri("file-abc123") assert not is_managed_cloud_storage_uri("bGl0ZWxsbV9wcm94eQ==") assert not is_managed_cloud_storage_uri("") + + +@pytest.mark.parametrize( + "file_id", + ( + "gs%3A%2F%2Fbucket%2Flitellm-vertex-files%2Fprediction-model%2Fpredictions.jsonl", + "gs%253A%252F%252Fbucket%252Flitellm-vertex-files%252Fprediction-model%252Fpredictions.jsonl", + "s3%3A%2F%2Fbucket%2Flitellm-batch-outputs%2Fx.jsonl.out", + "s3%253A%252F%252Fbucket%252Flitellm-batch-outputs%252Fx.jsonl.out", + ), +) +def test_is_managed_cloud_storage_uri_sees_through_percent_encoding(file_id: str): + assert is_managed_cloud_storage_uri(file_id) + + +def _quote_once(current: str, _: int) -> str: + return quote(current, safe="") + + +def _quoted_times(value: str, times: int) -> str: + return reduce(_quote_once, range(times), value) + + +def test_is_managed_cloud_storage_uri_decodes_up_to_the_pass_cap(): + assert is_managed_cloud_storage_uri(_quoted_times("gs://bucket/x", MAX_FILE_ID_DECODE_PASSES)) + assert not is_managed_cloud_storage_uri(_quoted_times("file-abc/x", MAX_FILE_ID_DECODE_PASSES)) + + +def test_is_managed_cloud_storage_uri_fails_closed_on_an_id_encoded_past_the_pass_cap(): + nested_gs_id: Final = "gs" + "%" + "25" * 5000 + "3A//bucket/x" + nested_plain_id: Final = "%" + "25" * 5000 + + assert is_managed_cloud_storage_uri(_quoted_times("file-abc/x", MAX_FILE_ID_DECODE_PASSES + 1)) + assert is_managed_cloud_storage_uri(nested_gs_id) + assert is_managed_cloud_storage_uri(nested_plain_id) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index 793921255ec..eb222366812 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1,5 +1,6 @@ import json from typing import Final, List, Optional +from urllib.parse import quote from unittest.mock import ANY, AsyncMock, MagicMock import pytest @@ -12,7 +13,7 @@ from pytest_mock import MockerFixture import litellm from litellm import Router from litellm.files.types import FileContentStreamingResult -from litellm.proxy._types import LiteLLM_UserTableFiltered, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_UserTableFiltered, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.hooks import get_proxy_hook from litellm.proxy.management_endpoints.internal_user_endpoints import ui_view_users from litellm.proxy.openai_files_endpoints.file_content_streaming_handler import ( @@ -235,23 +236,112 @@ def test_invalid_purpose(mocker: MockerFixture, monkeypatch, llm_router: Router) assert error["param"] == "purpose" -def test_get_file_content_rejects_raw_cloud_storage_uri(llm_router: Router): - """A raw s3:// file id must be rejected on the proxy content endpoint. +RAW_GCS_OUTPUT_FILE_ID: Final = ( + "gs://my-gcs-bucket/litellm-vertex-files/publishers/google/models/gemini-3.8-flash/" + "prediction-model-2026-09-10T20:27:18.178864Z/predictions.jsonl" +) +RAW_S3_OUTPUT_FILE_ID: Final = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" - Such an id is not a managed unified id, so it would otherwise skip the - owner/team access check and let a caller read another tenant's batch output - object by its key. Callers must use the managed unified file id. - """ - from urllib.parse import quote - s3_file_id = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" - response = client.get( - f"/v1/files/{quote(s3_file_id, safe='')}/content?provider=bedrock", - headers={"Authorization": "Bearer test-key"}, +def _ids_a_server_hands_the_route_for(raw_id: str, provider: str) -> tuple[tuple[str, str], ...]: + encoded_once: Final = quote(raw_id, safe="") + return ((raw_id, provider), (encoded_once, provider), (quote(encoded_once, safe=""), provider)) + + +RAW_CLOUD_FILE_IDS_AS_THE_ROUTE_RECEIVES_THEM: Final = _ids_a_server_hands_the_route_for( + RAW_GCS_OUTPUT_FILE_ID, "vertex_ai" +) + _ids_a_server_hands_the_route_for(RAW_S3_OUTPUT_FILE_ID, "bedrock") + + +def _testclient_path_segment_delivering(route_file_id: str) -> str: + return quote(quote(route_file_id, safe=""), safe="") + + +def _override_auth( + monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, llm_router: Router, user_role: LitellmUserRoles +) -> ProxyLogging: + import litellm.proxy.proxy_server as ps + + proxy_logging_obj: Final = setup_proxy_logging_object(monkeypatch, llm_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", user_role=user_role, user_id="test-user" ) + return proxy_logging_obj - assert response.status_code == 400 - assert "managed file id" in response.json()["error"]["message"].lower() + +@pytest.mark.parametrize(("route_file_id", "provider"), RAW_CLOUD_FILE_IDS_AS_THE_ROUTE_RECEIVES_THEM) +def test_get_file_content_answers_403_for_a_raw_cloud_id_from_a_non_admin_key( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + llm_router: Router, + route_file_id: str, + provider: str, +): + import litellm.proxy.proxy_server as ps + + _override_auth(monkeypatch, mocker, llm_router, LitellmUserRoles.INTERNAL_USER) + afile_content: Final = mocker.AsyncMock() + monkeypatch.setattr(litellm, "afile_content", afile_content) + + try: + response: Final = client.get( + f"/v1/files/{_testclient_path_segment_delivering(route_file_id)}/content?provider={provider}", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 403, response.text + assert "proxy admin" in response.json()["error"]["message"] + afile_content.assert_not_called() + + +@pytest.mark.parametrize(("route_file_id", "provider"), RAW_CLOUD_FILE_IDS_AS_THE_ROUTE_RECEIVES_THEM) +def test_get_file_content_forwards_a_raw_cloud_id_from_a_proxy_admin_key( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + llm_router: Router, + route_file_id: str, + provider: str, +): + import litellm.proxy.proxy_server as ps + + _override_auth(monkeypatch, mocker, llm_router, LitellmUserRoles.PROXY_ADMIN) + content: Final = b'{"custom_id": "request-1"}\n' + + async def _stream(): + yield content + + async def _provider_file_content(**kwargs: object) -> FileContentStreamingResult | HttpxBinaryResponseContent: + if kwargs.get("stream"): + return FileContentStreamingResult(stream_iterator=_stream(), headers={"content-length": str(len(content))}) + return HttpxBinaryResponseContent( + response=httpx.Response( + status_code=200, content=content, headers={"content-type": "application/octet-stream"} + ) + ) + + afile_content: Final = mocker.AsyncMock(side_effect=_provider_file_content) + monkeypatch.setattr(litellm, "afile_content", afile_content) + + try: + response: Final = client.get( + f"/v1/files/{_testclient_path_segment_delivering(route_file_id)}/content?provider={provider}", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert response.content == content + afile_content.assert_called_once() + assert afile_content.call_args.kwargs["custom_llm_provider"] == provider + assert afile_content.call_args.kwargs["file_id"] == route_file_id def test_mock_create_audio_file(mocker: MockerFixture, monkeypatch, llm_router: Router): From 808333682ed2f0a5fff5277d862fd224935accc5 Mon Sep 17 00:00:00 2001 From: Vincent Gimenes <147169146+VincentG1234@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:35:18 +0200 Subject: [PATCH 090/124] feat(embeddings): enforce per-request input batch limit (#44574) * feat(embeddings): add max_batch_inputs pre-call validation Signed-off-by: Vincent Gimenes * add unit tests for max embedding batch size Signed-off-by: Vincent Gimenes * feat(embeddings): enforce batch limit on extra_body input override Signed-off-by: Vincent Gimenes * chore(dashboard): update schema.d.ts for embedding batch size model field Signed-off-by: Vincent Gimenes * fix(embeddings): close Bedrock batch bypass and satisfy ruff PT012 Count max(top-level input, extra_body.input) for batch limits so providers that ignore extra_body cannot bypass max_embedding_batch_size. Add router tests and refactor pytest.raises blocks for ruff-tests. Signed-off-by: Vincent Gimenes * fix(embeddings): drop unchecked casts in router batch-size check Signed-off-by: Vincent Gimenes * fix(embeddings): route oversized batches to a group deployment that admits them * fix(embeddings): count extra_body input for any top-level input and prefer admitting deployments before cooldown * fix(embeddings): log cap rejections to failure callbacks and scope batch routing to embedding calls * fix(embeddings): apply the batch cap after tag and pre-call filters and fall back on cap rejections * fix(router): keep embedding cap rejections out of deployment health and hold only for routable cooling deployments * fix(router): clear the embedding cap rejection mark before a fallback's provider call A fallback or retry shares the request's logging object, so the mark a rejected attempt set made telemetry skip the fallback deployment's real call. Unit tests now pass a fixed logging start time instead of reading the clock. * fix(router): log embedding batch rejections once, before a fallback runs * refactor(router): drop docstrings that restate the code --------- Signed-off-by: Vincent Gimenes Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/router.py | 116 +- .../embedding_batch_size_check.py | 199 ++++ litellm/types/router.py | 2 + .../test_embedding_batch_size_check.py | 1024 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 5 files changed, 1327 insertions(+), 16 deletions(-) create mode 100644 litellm/router_utils/embedding_batch_size_check.py create mode 100644 tests/unit/router_utils/test_embedding_batch_size_check.py diff --git a/litellm/router.py b/litellm/router.py index aadbdaa8520..ac0933e158b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -242,6 +242,15 @@ from litellm.router_utils.cooldown_handlers import ( is_video_lookup_not_found, set_cooldown_deployments, ) +from litellm.router_utils.embedding_batch_size_check import ( + async_log_rejection_before_provider_call, + clear_rejected_before_provider_call, + deployments_admitting_embedding_batch, + deployments_when_none_admits_embedding_batch, + embedding_batch_too_large_error, + embedding_deployment_selection, + log_rejection_before_provider_call, +) from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, AttemptedFallbackTargets, @@ -6861,19 +6870,33 @@ class Router: except Exception as e: raise e - def _embedding(self, input: str | list, model: str, **kwargs): + def _embedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: verbose_router_logger.debug("Inside embedding()- model: %s; kwargs: %s", model, kwargs) - deployment: Final = self.get_available_deployment( - model=model, - input=input, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with embedding_deployment_selection(): + deployment: Final = self.get_available_deployment( + model=model, + input=input, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() model_name = data["model"] + batch_error: Final = embedding_batch_too_large_error( + deployment=_OPAQUE_VALUE_ADAPTER.validate_python(deployment), + input=input, + request_kwargs=_OPAQUE_VALUE_ADAPTER.validate_python(kwargs), + model=model, + ) + logging_obj: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if batch_error is not None: + if isinstance(logging_obj, LiteLLMLogging): + log_rejection_before_provider_call(logging_obj, batch_error) + raise batch_error + if isinstance(logging_obj, LiteLLMLogging): + clear_rejected_before_provider_call(logging_obj) potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs, client_type="sync") # check if provided keys == client keys # @@ -6936,20 +6959,34 @@ class Router: ) raise e - async def _aembedding(self, input: str | list, model: str, **kwargs): + async def _aembedding(self, input: str | list[object], model: str, **kwargs): model_name = None try: verbose_router_logger.debug("Inside _aembedding()- model: %s; kwargs: %s", model, kwargs) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( - model=model, - input=input, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with embedding_deployment_selection(): + deployment: Final = await self.async_get_available_deployment( + model=model, + input=input, + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() model_name = data["model"] + batch_error: Final = embedding_batch_too_large_error( + deployment=_OPAQUE_VALUE_ADAPTER.validate_python(deployment), + input=input, + request_kwargs=_OPAQUE_VALUE_ADAPTER.validate_python(kwargs), + model=model, + ) + logging_obj: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if batch_error is not None: + if isinstance(logging_obj, LiteLLMLogging): + await async_log_rejection_before_provider_call(logging_obj, batch_error) + raise batch_error + if isinstance(logging_obj, LiteLLMLogging): + clear_rejected_before_provider_call(logging_obj) model_client: Final = self._get_async_openai_model_client( deployment=deployment, kwargs=kwargs, @@ -14680,10 +14717,34 @@ class Router: batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts list[dict[str, object]], healthy_deployments ) + embedding_batch_admitting: Final = deployments_admitting_embedding_batch( + batch_candidates, + _OPAQUE_VALUE_ADAPTER.validate_python(input), + _OPAQUE_VALUE_ADAPTER.validate_python(request_kwargs), + ) + embedding_batch_routed: Final = ( + embedding_batch_admitting + if embedding_batch_admitting is not None + else deployments_when_none_admits_embedding_batch( + batch_candidates, + cast(list[dict[str, object]], _pre_cooldown_deployments), # cast-ok: router deployments are plain dicts + await self._async_routable_cooled_deployment_ids( + model=model, + request_deployments=request_deployments, + cooldown_deployments=cooldown_deployments, + request_kwargs=cast(Mapping[str, object], request_kwargs), # cast-ok: str-keyed request kwargs + has_request_input=True, + ), + _OPAQUE_VALUE_ADAPTER.validate_python(input), + _OPAQUE_VALUE_ADAPTER.validate_python(request_kwargs), + ) + ) healthy_deployments = litellm.utils.get_order_filtered_deployments( list( self._narrow_to_video_deployment( - without_deployments_that_cannot_batch(batch_candidates, caller_deployments=lambda: batch_candidates) + without_deployments_that_cannot_batch( + embedding_batch_routed, caller_deployments=lambda: batch_candidates + ) ) ), target_order=_target_order, @@ -15890,11 +15951,34 @@ class Router: batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts list[dict[str, object]], healthy_deployments ) + embedding_batch_admitting: Final = deployments_admitting_embedding_batch( + batch_candidates, + _OPAQUE_VALUE_ADAPTER.validate_python(input), + routing_kwargs, + ) + embedding_batch_routed: Final = ( + embedding_batch_admitting + if embedding_batch_admitting is not None + else deployments_when_none_admits_embedding_batch( + batch_candidates, + cast(list[dict[str, object]], _pre_cooldown_deployments), # cast-ok: router deployments are plain dicts + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + True, + ) + ) + & frozenset(cooldown_deployments), + _OPAQUE_VALUE_ADAPTER.validate_python(input), + routing_kwargs, + ) + ) healthy_deployments = litellm.utils.get_order_filtered_deployments( list( self._narrow_to_video_deployment( without_deployments_that_cannot_batch( - batch_candidates, + embedding_batch_routed, caller_deployments=lambda: deployments_the_caller_may_use(batch_candidates, routing_kwargs), ) ) diff --git a/litellm/router_utils/embedding_batch_size_check.py b/litellm/router_utils/embedding_batch_size_check.py new file mode 100644 index 00000000000..6000bc9766d --- /dev/null +++ b/litellm/router_utils/embedding_batch_size_check.py @@ -0,0 +1,199 @@ +import traceback +from collections.abc import Generator, Iterable, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from typing import TYPE_CHECKING, Final + +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 + +import litellm +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY +from litellm.types.utils import CallTypes + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +_SELECTING_FOR_EMBEDDING: Final[ContextVar[bool]] = ContextVar("selecting_for_embedding", default=False) + + +def _is_str_keyed_mapping( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: model_info, litellm_params and extra_body are str-keyed config objects + return isinstance(value, Mapping) + + +def _is_embedding_input( + value: object, +) -> TypeIs[str | Sequence[object]]: # guard-ok: only the length and first item are read + return isinstance(value, Sequence) + + +def count_embedding_inputs(input: str | Sequence[object]) -> int: + if isinstance(input, str): + return 1 + if input and isinstance(input[0], int): + return 1 + return len(input) + + +def effective_embedding_input( + input: object, + extra_body: object, +) -> str | Sequence[object] | None: + """Return the input shape used for batch-size checks, or None when neither is an embedding input. + + Some providers merge ``extra_body`` over the request body (OpenAI-compatible paths); + others ignore ``extra_body.input`` and use the top-level input (e.g. Bedrock Titan). + Use whichever of the two yields the larger input count so limits cannot be bypassed. + """ + override: Final = extra_body.get("input") if _is_str_keyed_mapping(extra_body) else None + candidates: Final = tuple(value for value in (input, override) if _is_embedding_input(value)) + return max(candidates, key=count_embedding_inputs, default=None) + + +def _litellm_param(deployment: Mapping[str, object], key: str) -> object: + litellm_params: Final = deployment.get("litellm_params") + return litellm_params.get(key) if _is_str_keyed_mapping(litellm_params) else None + + +def _embedding_batch_limit(deployment: Mapping[str, object]) -> int | None: + model_info: Final = deployment.get("model_info") + limit: Final = model_info.get("max_embedding_batch_size") if _is_str_keyed_mapping(model_info) else None + return limit if isinstance(limit, int) else None + + +def _requested_embedding_count( + deployment: Mapping[str, object], + input: object, + request_kwargs: Mapping[str, object], +) -> int: + extra_body: Final = request_kwargs.get("extra_body", _litellm_param(deployment, "extra_body")) + effective_input: Final = effective_embedding_input(input, extra_body) + return 0 if effective_input is None else count_embedding_inputs(effective_input) + + +def _deployment_admits_embedding_batch( + deployment: Mapping[str, object], + input: object, + request_kwargs: Mapping[str, object], +) -> bool: + limit: Final = _embedding_batch_limit(deployment) + return limit is None or _requested_embedding_count(deployment, input, request_kwargs) <= limit + + +def _str_keyed_or_empty(value: object) -> Mapping[str, object]: + return value if _is_str_keyed_mapping(value) else {} + + +def _deployment_id(deployment: Mapping[str, object]) -> str | None: + model_info: Final = deployment.get("model_info") + deployment_id: Final = model_info.get("id") if _is_str_keyed_mapping(model_info) else None + return None if deployment_id is None else str(deployment_id) + + +def _is_paused(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return _is_str_keyed_mapping(model_info) and model_info.get("blocked") is True + + +@contextmanager +def embedding_deployment_selection() -> Generator[None]: + """Apply the input limit to deployment selection inside this block, which only embedding calls enter.""" + token: Final = _SELECTING_FOR_EMBEDDING.set(True) + try: + yield + finally: + _SELECTING_FOR_EMBEDDING.reset(token) + + +def deployments_admitting_embedding_batch( + deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + input: object, + request_kwargs: object, +) -> list[dict[str, object]] | None: # mutable-ok: returns a mutable deployment list + """Keep the deployments whose input limit admits the request, all of them outside embedding selection, else None""" + if not _SELECTING_FOR_EMBEDDING.get(): + return deployments + kwargs: Final = _str_keyed_or_empty(request_kwargs) + admitting: Final = [ + deployment for deployment in deployments if _deployment_admits_embedding_batch(deployment, input, kwargs) + ] + return admitting or None + + +def deployments_when_none_admits_embedding_batch( + deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + before_cooldown: Iterable[Mapping[str, object]], + routable_cooling_ids: frozenset[str], + input: object, + request_kwargs: object, +) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + """None while an unpaused cooling deployment the request routes to admits it, so it waits out the cooldown, + else the deployments with the largest limit, so the 400 names the most the group accepts""" + kwargs: Final = _str_keyed_or_empty(request_kwargs) + routable_cooling: Final = ( + deployment + for deployment in before_cooldown + if _deployment_id(deployment) in routable_cooling_ids and not _is_paused(deployment) + ) + if any(_deployment_admits_embedding_batch(deployment, input, kwargs) for deployment in routable_cooling): + return [] + largest_limit: Final = max((_embedding_batch_limit(deployment) or 0 for deployment in deployments), default=0) + return [deployment for deployment in deployments if _embedding_batch_limit(deployment) == largest_limit] + + +def _mark_litellm_params(logging_obj: "LiteLLMLoggingObj", *keys: str) -> None: + litellm_params: Final = logging_obj.model_call_details.get("litellm_params") + if not isinstance(litellm_params, dict): + return + for key in keys: + litellm_params[key] = True + + +def log_rejection_before_provider_call(logging_obj: "LiteLLMLoggingObj", error: Exception) -> None: + """Mark the request as never reaching a provider, so deployment health metrics skip its failure, then log it""" + _mark_litellm_params(logging_obj, PROXY_REJECTED_BEFORE_ROUTING_KEY) + logging_obj.failure_handler(error, "".join(traceback.format_exception(error))) + + +async def async_log_rejection_before_provider_call(logging_obj: "LiteLLMLoggingObj", error: Exception) -> None: + """Log it like an async provider failure: sync handlers inline, before a fallback swaps the deployment they read, + and the call marked async, so a logger with both hooks records it once""" + _mark_litellm_params(logging_obj, PROXY_REJECTED_BEFORE_ROUTING_KEY, CallTypes.aembedding.value) + traceback_text: Final = "".join(traceback.format_exception(error)) + logging_obj.failure_handler(error, traceback_text) + await logging_obj.async_failure_handler(error, traceback_text) + + +def clear_rejected_before_provider_call(logging_obj: "LiteLLMLoggingObj") -> None: + """Drop the mark an earlier rejected attempt left, since a fallback or retry shares the logging object""" + litellm_params: Final = logging_obj.model_call_details.get("litellm_params") + if isinstance(litellm_params, dict) and PROXY_REJECTED_BEFORE_ROUTING_KEY in litellm_params: + del litellm_params[PROXY_REJECTED_BEFORE_ROUTING_KEY] + + +def embedding_batch_too_large_error( + deployment: object, + input: object, + request_kwargs: object, + model: str, +) -> litellm.BadRequestError | None: + if not _is_str_keyed_mapping(deployment): + return None + limit: Final = _embedding_batch_limit(deployment) + if limit is None: + return None + + input_count: Final = _requested_embedding_count(deployment, input, _str_keyed_or_empty(request_kwargs)) + if input_count <= limit: + return None + + return litellm.BadRequestError( + message=( + f"Embedding request has {input_count} inputs, which exceeds " + f"max_embedding_batch_size={limit} for model={model}. " + "Split the input into smaller batches." + ), + model=model, + llm_provider="", + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 6cd758d864e..1f159cb8b16 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -323,6 +323,8 @@ class ModelInfo(MirroredPricingParams): # in the spend log row's metadata. Set it on every deployment of the group. internal_router_model: bool | None = None + max_embedding_batch_size: Annotated[int, Field(gt=0)] | None = None + def __init__(self, id: str | int | None = None, **params) -> None: if id is None: id = str(uuid.uuid4()) # Generate a UUID if id is None or not provided diff --git a/tests/unit/router_utils/test_embedding_batch_size_check.py b/tests/unit/router_utils/test_embedding_batch_size_check.py new file mode 100644 index 00000000000..d5986d9e6f7 --- /dev/null +++ b/tests/unit/router_utils/test_embedding_batch_size_check.py @@ -0,0 +1,1024 @@ +import datetime +import json +import threading +from collections.abc import Iterator +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import respx +from prometheus_client import REGISTRY + +import litellm +from litellm import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.types.router import RouterRateLimitError +from litellm.router_utils.embedding_batch_size_check import ( + count_embedding_inputs, + deployments_admitting_embedding_batch, + deployments_when_none_admits_embedding_batch, + effective_embedding_input, + embedding_batch_too_large_error, + embedding_deployment_selection, +) + +_MODEL: Final = "embed-group" +_PROVIDER: Final = "openai" +_OPENAI_EMBED_BASE: Final = "https://batch-limit-embed.local/v1" +_LOGGING_START: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) +_VLLM_EMBED_BASE: Final = "https://batch-limit-vllm.local/v1" +_UNCAPPED_EMBED_BASE: Final = "https://batch-limit-uncapped.local/v1" + + +def _model_info(max_embedding_batch_size: int | None) -> dict[str, object]: + if max_embedding_batch_size is None: + return {} + return {"max_embedding_batch_size": max_embedding_batch_size} + + +def _deployment( + limit: int | None, + extra_body: dict[str, object] | None = None, + deployment_id: str = "embed-deployment", +) -> dict[str, object]: + return { + "model_info": {"id": deployment_id, **_model_info(limit)}, + "litellm_params": {} if extra_body is None else {"extra_body": extra_body}, + } + + +def _admitted( + deployments: list[dict[str, object]], + input_value: object, + request_kwargs: dict[str, object] | None = None, + cooling: tuple[dict[str, object], ...] = (), + routable_cooling_ids: frozenset[str] = frozenset(), +) -> list[dict[str, object]]: + admitting: Final = deployments_admitting_embedding_batch(deployments, input_value, request_kwargs) + if admitting is not None: + return admitting + return deployments_when_none_admits_embedding_batch( + deployments, (*deployments, *cooling), routable_cooling_ids, input_value, request_kwargs + ) + + +def _batch_error( + input_value: object, + limit: int | None, + request_kwargs: dict[str, object] | None = None, +) -> litellm.BadRequestError | None: + return embedding_batch_too_large_error( + deployment={"model_info": _model_info(limit), "litellm_params": {"custom_llm_provider": _PROVIDER}}, + input=input_value, + request_kwargs=request_kwargs or {}, + model=_MODEL, + ) + + +def _embedding_router( + *, + provider_model: str, + api_base: str, + max_embedding_batch_size: int | None, + deployment_id: str, + litellm_params_extra: dict[str, object] | None = None, +) -> Router: + litellm_params: dict[str, object] = { + "model": provider_model, + "api_key": "sk-fake", + "api_base": api_base, + } + if litellm_params_extra: + litellm_params.update(litellm_params_extra) + return Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": litellm_params, + "model_info": {"id": deployment_id, **_model_info(max_embedding_batch_size)}, + } + ], + num_retries=0, + ) + + +async def _call_embedding(router: Router, *, sync: bool, **kwargs: object) -> None: + if sync: + router.embedding(**kwargs) + else: + await router.aembedding(**kwargs) + + +def _vllm_embedding_response(input_count: int) -> httpx.Response: + return httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": i, "embedding": [0.1, 0.2]} for i in range(input_count)], + "model": "bge-m3", + "usage": {"prompt_tokens": input_count, "total_tokens": input_count}, + }, + ) + + +@pytest.mark.parametrize( + ("input_value", "expected_count"), + [ + ("hello", 1), + (["a", "b"], 2), + ([1, 2, 3, 4, 5], 1), + ([[1, 2, 3], [4, 5, 6]], 2), + ], +) +def test_openai_input_shapes(input_value: object, expected_count: int) -> None: + assert count_embedding_inputs(input_value) == expected_count + + +def test_batch_count_is_independent_of_string_length() -> None: + long_text: Final = "word " * 500 + assert count_embedding_inputs([long_text, long_text]) == 2 + + +_MIXED_EXTRA_BODY: Final = { + "truncate": "END", + "encoding_format": "float", + "input": ["a", "b", "c"], + "user": "batch-job", +} + + +@pytest.mark.parametrize( + ("top_level", "extra_body", "expected"), + [ + ("ok", None, "ok"), + ("ok", {"input": ["a", "b", "c"]}, ["a", "b", "c"]), + (["a", "b"], {"truncate": "END"}, ["a", "b"]), + ("ignored-top-level", _MIXED_EXTRA_BODY, ["a", "b", "c"]), + ( + ["would", "be", "three"], + {"dimensions": 256, "input": "single-string", "model": "ignored"}, + ["would", "be", "three"], + ), + ("ok", {"truncate": "NONE", "input": ("a", "b")}, ("a", "b")), + (["x", "y", "z"], {"input": [[1, 2], [3, 4]], "foo": "bar"}, ["x", "y", "z"]), + ( + ["a", "b", "c"], + {"input": "ok"}, + ["a", "b", "c"], + ), + (["a", "b"], {}, ["a", "b"]), + (["a", "b", "c"], {"input": None, "truncate": "END"}, ["a", "b", "c"]), + (["a", "b"], {"input": {"nested": "dict"}, "truncate": "END"}, ["a", "b"]), + (["a", "b"], {"input": 42, "dimensions": 128}, ["a", "b"]), + (None, {"input": ["a", "b"]}, ["a", "b"]), + (7, {"input": ["a", "b"]}, ["a", "b"]), + (None, None, None), + ], +) +def test_resolves_provider_bound_input( + top_level: object, + extra_body: dict[str, object] | None, + expected: object, +) -> None: + assert effective_embedding_input(top_level, extra_body) == expected + + +@pytest.mark.parametrize( + "extra_body", + ["not-a-mapping", ["input", "list"]], +) +def test_non_mapping_extra_body_keeps_top_level(extra_body: object) -> None: + top_level: Final = ["a", "b", "c"] + assert effective_embedding_input(top_level, extra_body) == top_level + + +def test_readonly_mapping_extra_body_with_input() -> None: + extra_body: Final = MappingProxyType({"input": ["a", "b"], "encoding_format": "float"}) + assert effective_embedding_input("ok", extra_body) == ["a", "b"] + + +class TestDeploymentsAdmittingEmbeddingBatch: + @pytest.fixture(autouse=True) + def _embedding_selection(self) -> Iterator[None]: + with embedding_deployment_selection(): + yield + + @pytest.mark.parametrize( + ("input_value", "limit"), + [ + ("hello", 1), + (["a", "b"], 2), + (["a", "b", "c"], 3), + ([1, 2, 3, 4, 5], 1), + ([[1, 2, 3], [4, 5, 6]], 2), + ], + ) + def test_keeps_deployments_whose_limit_admits_the_batch(self, input_value: str | list, limit: int) -> None: + capped: Final = _deployment(limit) + uncapped: Final = _deployment(None) + assert _admitted([capped, uncapped], input_value) == [capped, uncapped] + + def test_drops_deployments_whose_limit_is_below_the_batch(self) -> None: + capped: Final = _deployment(2) + uncapped: Final = _deployment(None) + assert _admitted([capped, uncapped], ["a", "b", "c"]) == [uncapped] + + def test_keeps_the_largest_limit_deployments_when_none_admits_the_batch(self) -> None: + smaller: Final = _deployment(1, deployment_id="smaller") + small: Final = _deployment(2, deployment_id="small") + other_small: Final = _deployment(2, deployment_id="other-small") + assert _admitted([small, smaller, other_small], ["a", "b", "c"]) == [small, other_small] + + def test_request_without_embedding_input_keeps_every_deployment(self) -> None: + capped: Final = _deployment(1) + uncapped: Final = _deployment(None) + assert _admitted([capped, uncapped], None) == [capped, uncapped] + + def test_request_extra_body_replaces_the_deployment_extra_body(self) -> None: + capped: Final = _deployment(2, extra_body={"input": ["a", "b", "c"]}) + uncapped: Final = _deployment(None) + assert _admitted([capped, uncapped], "ok") == [uncapped] + assert _admitted([capped, uncapped], "ok", {"extra_body": {}}) == [capped, uncapped] + + @pytest.mark.parametrize("top_level", [None, 7, {"text": "a"}]) + def test_extra_body_input_routes_when_top_level_input_is_not_a_list(self, top_level: object) -> None: + capped: Final = _deployment(2) + uncapped: Final = _deployment(None) + request_kwargs: Final = {"extra_body": {"input": ["a", "b", "c"]}} + assert _admitted([capped, uncapped], top_level, request_kwargs) == [uncapped] + + def test_keeps_no_deployment_while_only_a_cooling_one_admits_the_batch(self) -> None: + capped: Final = _deployment(2, deployment_id="capped") + cooling: Final = _deployment(None, deployment_id="cooling") + assert _admitted([capped], ["a", "b", "c"], cooling=(cooling,), routable_cooling_ids=frozenset({"cooling"})) == [] + + def test_cooling_deployment_the_request_cannot_route_to_does_not_hold_the_batch(self) -> None: + capped: Final = _deployment(2, deployment_id="capped") + cooling: Final = _deployment(None, deployment_id="cooling") + assert _admitted([capped], ["a", "b", "c"], cooling=(cooling,)) == [capped] + + def test_paused_cooling_deployment_does_not_hold_the_batch(self) -> None: + capped: Final = _deployment(2, deployment_id="capped") + paused: Final = {"model_info": {"id": "cooling", "blocked": True}, "litellm_params": {}} + assert _admitted([capped], ["a", "b", "c"], cooling=(paused,), routable_cooling_ids=frozenset({"cooling"})) == [ + capped + ] + + def test_keeps_every_deployment_when_the_cooling_one_also_rejects_the_batch(self) -> None: + capped: Final = _deployment(2, deployment_id="capped") + cooling: Final = _deployment(1, deployment_id="cooling") + assert _admitted([capped], ["a", "b", "c"], cooling=(cooling,), routable_cooling_ids=frozenset({"cooling"})) == [ + capped + ] + + +@pytest.mark.parametrize( + ("input_value", "limit", "message_match"), + [ + (["a", "b", "c", "d"], 3, "4 inputs"), + ([[1], [2], [3]], 2, None), + ( + effective_embedding_input( + "ok", + {"truncate": "END", "input": ["a", "b", "c", "d"], "user": "x"}, + ), + 3, + "4 inputs", + ), + ( + effective_embedding_input( + ["a", "b", "c"], + {"input": {"bad": "shape"}, "encoding_format": "float"}, + ), + 2, + None, + ), + ], +) +def test_rejects_inputs_above_limit( + input_value: str | list, + limit: int, + message_match: str | None, +) -> None: + error: Final = _batch_error(input_value, limit) + assert isinstance(error, litellm.BadRequestError) + assert error.status_code == 400 + if message_match is not None: + assert message_match in error.message + assert f"max_embedding_batch_size={limit}" in error.message + + +@pytest.mark.parametrize("top_level", [None, 7, {"text": "a"}]) +def test_extra_body_override_counts_when_top_level_input_is_not_a_list(top_level: object) -> None: + error: Final = _batch_error(top_level, 2, {"extra_body": {"input": ["a", "b", "c"]}}) + assert error is not None + assert "3 inputs" in error.message + + +class TestRouterEmbeddingBatchLimit: + @pytest.fixture(autouse=True) + def _disable_aiohttp(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_top_level_input_rejects_before_provider_call(self, sync: bool) -> None: + router: Final = _embedding_router( + provider_model="openai/text-embedding-3-small", + api_base=_OPENAI_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="embed-batch-capped", + ) + embeddings_url: Final = f"{_OPENAI_EMBED_BASE}/embeddings" + empty_ok: Final = httpx.Response(200, json={"object": "list", "data": [], "model": "x", "usage": {}}) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=empty_ok) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await _call_embedding( + router, + sync=sync, + model="embed", + input=["one", "two", "three"], + ) + assert route.call_count == 0 + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_rejection_reaches_failure_callbacks(self, sync: bool) -> None: + failures: Final[list[tuple[Exception, object]]] = [] + + class FailureRecorder(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + failures.append((kwargs["exception"], kwargs["litellm_params"].get(PROXY_REJECTED_BEFORE_ROUTING_KEY))) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + failures.append((kwargs["exception"], kwargs["litellm_params"].get(PROXY_REJECTED_BEFORE_ROUTING_KEY))) + + recorder: Final = FailureRecorder() + logging_obj: Final = LitellmLogging( + model="embed", + messages=[], + stream=False, + call_type="embedding" if sync else "aembedding", + start_time=_LOGGING_START, + litellm_call_id="embed-batch-rejected", + function_id="embed-batch-rejected", + dynamic_failure_callbacks=[recorder] if sync else None, + dynamic_async_failure_callbacks=None if sync else [recorder], + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": {}, "proxy_server_request": {}} + router: Final = _embedding_router( + provider_model="openai/text-embedding-3-small", + api_base=_OPENAI_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="embed-batch-capped", + ) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(f"{_OPENAI_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [], "model": "x", "usage": {}}) + ) + with pytest.raises(litellm.BadRequestError, match="3 inputs") as excinfo: + await _call_embedding( + router, + sync=sync, + model="embed", + input=["one", "two", "three"], + litellm_logging_obj=logging_obj, + ) + assert route.call_count == 0 + assert failures == [(excinfo.value, True)] + + @pytest.mark.asyncio + async def test_async_rejection_logs_each_failure_hook_once_before_raising(self) -> None: + hook_events: Final[list[str]] = [] + legacy_callback_threads: Final[list[int]] = [] + legacy_callback_ran: Final = threading.Event() + + class BothHooks(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + hook_events.append("sync") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + hook_events.append("async") + + def legacy_failure_callback(kwargs, completion_response, start_time, end_time) -> None: + legacy_callback_threads.append(threading.get_ident()) + legacy_callback_ran.set() + + both_hooks: Final = BothHooks() + logging_obj: Final = LitellmLogging( + model="embed", + messages=[], + stream=False, + call_type="aembedding", + start_time=_LOGGING_START, + litellm_call_id="embed-batch-logged-once", + function_id="embed-batch-logged-once", + dynamic_failure_callbacks=[legacy_failure_callback, both_hooks], + dynamic_async_failure_callbacks=[both_hooks], + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": {}, "proxy_server_request": {}} + router: Final = _embedding_router( + provider_model="openai/text-embedding-3-small", + api_base=_OPENAI_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="embed-batch-capped", + ) + + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await router.aembedding(model="embed", input=["one", "two", "three"], litellm_logging_obj=logging_obj) + + assert legacy_callback_ran.wait(timeout=5) + assert legacy_callback_threads == [threading.get_ident()] + assert hook_events == ["async"] + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_fallback_after_a_rejection_logs_its_provider_call_as_reached(self, sync: bool) -> None: + litellm_params: Final[dict[str, object]] = {"model": "openai/text-embedding-3-small", "api_key": "sk-fake"} + router: Final = Router( + model_list=[ + { + "model_name": "embed-capped", + "litellm_params": {**litellm_params, "api_base": _OPENAI_EMBED_BASE}, + "model_info": {"id": "capped-box", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed-fallback", + "litellm_params": {**litellm_params, "api_base": _VLLM_EMBED_BASE}, + "model_info": {"id": "fallback-box"}, + }, + ], + fallbacks=[{"embed-capped": ["embed-fallback"]}], + num_retries=0, + ) + logging_obj: Final = LitellmLogging( + model="embed-capped", + messages=[], + stream=False, + call_type="embedding" if sync else "aembedding", + start_time=_LOGGING_START, + litellm_call_id="embed-batch-fallback", + function_id="embed-batch-fallback", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {}, "proxy_server_request": {}}, optional_params={} + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_OPENAI_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + fallback_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + await _call_embedding( + router, + sync=sync, + model="embed-capped", + input=["one", "two", "three"], + litellm_logging_obj=logging_obj, + ) + assert (capped_route.call_count, fallback_route.call_count) == (0, 1) + assert PROXY_REJECTED_BEFORE_ROUTING_KEY not in logging_obj.model_call_details["litellm_params"] + + @pytest.fixture + def _isolated_prometheus_registry(self) -> Iterator[None]: + def unregister_all() -> None: + for collector in tuple(REGISTRY._collector_to_names): + REGISTRY.unregister(collector) + + unregister_all() + yield + unregister_all() + + @pytest.mark.asyncio + @pytest.mark.usefixtures("_isolated_prometheus_registry") + async def test_oversized_batch_rejection_leaves_the_deployment_healthy_in_prometheus(self) -> None: + prometheus: Final = PrometheusLogger() + metadata: Final[dict[str, object]] = {} + logging_obj: Final = LitellmLogging( + model="embed", + messages=[], + stream=False, + call_type="aembedding", + start_time=_LOGGING_START, + litellm_call_id="embed-batch-prometheus", + function_id="embed-batch-prometheus", + dynamic_async_failure_callbacks=[prometheus], + ) + logging_obj.model_call_details["litellm_params"] = {"metadata": metadata, "proxy_server_request": {}} + router: Final = _embedding_router( + provider_model="openai/text-embedding-3-small", + api_base=_OPENAI_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="embed-batch-capped", + ) + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.post(f"{_OPENAI_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [], "model": "x", "usage": {}}) + ) + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await router.aembedding( + model="embed", input=["one", "two", "three"], metadata=metadata, litellm_logging_obj=logging_obj + ) + assert prometheus.litellm_deployment_state._metrics == {} + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_no_limit_configured_forwards_any_batch(self, sync: bool) -> None: + router: Final = _embedding_router( + provider_model="hosted_vllm/bge-m3", + api_base=_VLLM_EMBED_BASE, + max_embedding_batch_size=None, + deployment_id="vllm-embed-uncapped", + ) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(100) + ) + await _call_embedding(router, sync=sync, model="embed", input=["a"] * 100) + assert len(json.loads(route.calls.last.request.content)["input"]) == 100 + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_goes_to_the_group_deployment_that_admits_it(self, sync: bool) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + "weight": 1000, + }, + "model_info": {"id": "vllm-embed-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + "weight": 1, + }, + "model_info": {"id": "vllm-embed-uncapped"}, + }, + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + for _ in range(5): + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 5) + + @pytest.mark.asyncio + async def test_non_embedding_call_with_input_ignores_embedding_batch_limit(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-fake", + "api_base": _OPENAI_EMBED_BASE, + "order": 1, + }, + "model_info": {"id": "capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + "order": 2, + }, + "model_info": {"id": "uncapped"}, + }, + ], + num_retries=0, + ) + responses_input: Final = [{"role": "user", "content": str(turn)} for turn in range(3)] + + async_pick: Final = await router.async_get_available_deployment( + model="embed", request_kwargs={}, input=responses_input + ) + sync_pick: Final = router.get_available_deployment(model="embed", request_kwargs={}, input=responses_input) + + assert (async_pick["model_info"]["id"], sync_pick["model_info"]["id"]) == ("capped", "capped") + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_waits_for_a_cooling_down_deployment_that_admits_it(self, sync: bool) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + }, + "model_info": {"id": "vllm-embed-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + }, + "model_info": {"id": "vllm-embed-uncapped"}, + }, + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limited"}}) + ) + with pytest.raises(litellm.RateLimitError): + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + with pytest.raises(RouterRateLimitError): + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 1) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_names_the_cap_while_a_paused_admitting_deployment_cools_down( + self, sync: bool + ) -> None: + uncapped_params: Final = {"model": "hosted_vllm/bge-m3", "api_key": "sk-fake", "api_base": _UNCAPPED_EMBED_BASE} + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + }, + "model_info": {"id": "vllm-embed-capped", "max_embedding_batch_size": 2}, + }, + {"model_name": "embed", "litellm_params": uncapped_params, "model_info": {"id": "vllm-embed-uncapped"}}, + ], + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limited"}}) + ) + with pytest.raises(litellm.RateLimitError): + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + router.upsert_deployment( + Deployment( + model_name="embed", + litellm_params=LiteLLM_Params(**uncapped_params), + model_info=ModelInfo(id="vllm-embed-uncapped", blocked=True), + ) + ) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 1) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_extra_body_input_override_rejects_before_provider_call(self, sync: bool) -> None: + router: Final = _embedding_router( + provider_model="hosted_vllm/bge-m3", + api_base=_VLLM_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="vllm-embed-batch-capped", + ) + embeddings_url: Final = f"{_VLLM_EMBED_BASE}/embeddings" + extra_body: Final = {"truncate": "END", "input": ["a", "b", "c"]} + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=_vllm_embedding_response(3)) + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await _call_embedding( + router, + sync=sync, + model="embed", + input="ok", + extra_body=extra_body, + ) + assert route.call_count == 0 + + @pytest.mark.asyncio + async def test_deployment_extra_body_input_counts_toward_batch_limit(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + "extra_body": {"input": ["a", "b", "c"]}, + }, + "model_info": { + "id": "vllm-embed-deployment-extra-body", + "max_embedding_batch_size": 2, + }, + } + ], + num_retries=0, + ) + embeddings_url: Final = f"{_VLLM_EMBED_BASE}/embeddings" + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=_vllm_embedding_response(3)) + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await router.aembedding(model="embed", input="ok") + assert route.call_count == 0 + + @pytest.mark.asyncio + async def test_extra_body_input_at_limit_reaches_provider(self) -> None: + router: Final = _embedding_router( + provider_model="hosted_vllm/bge-m3", + api_base=_VLLM_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="vllm-embed-batch-capped", + ) + embeddings_url: Final = f"{_VLLM_EMBED_BASE}/embeddings" + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=_vllm_embedding_response(2)) + await router.aembedding(model="embed", input="ok", extra_body={"input": ["a", "b"]}) + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content)["input"] == ["a", "b"] + + @pytest.mark.asyncio + async def test_unrelated_extra_body_does_not_bypass_top_level_batch_limit(self) -> None: + router: Final = _embedding_router( + provider_model="hosted_vllm/bge-m3", + api_base=_VLLM_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="vllm-embed-batch-capped", + ) + embeddings_url: Final = f"{_VLLM_EMBED_BASE}/embeddings" + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=_vllm_embedding_response(3)) + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await _call_embedding( + router, + sync=False, + model="embed", + input=["a", "b", "c"], + extra_body={"truncate": "END", "encoding_format": "float"}, + ) + assert route.call_count == 0 + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_bedrock_ignores_extra_body_input_override_for_provider_but_guard_uses_top_level( + self, + sync: bool, + ) -> None: + router: Final = _embedding_router( + provider_model="bedrock/amazon.titan-embed-text-v1", + api_base="", + max_embedding_batch_size=2, + deployment_id="bedrock-titan-embed-batch-capped", + litellm_params_extra={ + "aws_access_key_id": "AKIAFAKE", + "aws_secret_access_key": "fake-secret", + "aws_region_name": "us-east-1", + "custom_llm_provider": "bedrock", + }, + ) + + with pytest.raises(litellm.BadRequestError, match="3 inputs"): + await _call_embedding( + router, + sync=sync, + model="embed", + input=["a", "b", "c"], + extra_body={"input": "ok"}, + ) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_explicit_custom_llm_provider_in_deployment_rejects_oversized_batch( + self, + sync: bool, + ) -> None: + router: Final = _embedding_router( + provider_model="openai/text-embedding-3-small", + api_base=_OPENAI_EMBED_BASE, + max_embedding_batch_size=2, + deployment_id="embed-batch-capped-explicit-provider", + litellm_params_extra={"custom_llm_provider": "openai"}, + ) + embeddings_url: Final = f"{_OPENAI_EMBED_BASE}/embeddings" + empty_ok: Final = httpx.Response(200, json={"object": "list", "data": [], "model": "x", "usage": {}}) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = respx_mock.post(embeddings_url).mock(return_value=empty_ok) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await _call_embedding( + router, + sync=sync, + model="embed", + input=["one", "two", "three"], + ) + assert route.call_count == 0 + + @pytest.mark.asyncio + async def test_oversized_batch_names_the_cap_when_tags_exclude_the_admitting_deployment(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + "tags": ["team-a"], + }, + "model_info": {"id": "team-a-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + "tags": ["team-b"], + }, + "model_info": {"id": "team-b-uncapped"}, + }, + ], + enable_tag_filtering=True, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await router.aembedding(model="embed", input=["a", "b", "c"], metadata={"tags": ["team-a"]}) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 0) + + @pytest.mark.asyncio + async def test_oversized_batch_names_the_cap_while_another_tags_admitting_deployment_cools_down(self) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + "tags": ["team-a"], + }, + "model_info": {"id": "team-a-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + "tags": ["team-b"], + }, + "model_info": {"id": "team-b-uncapped"}, + }, + ], + enable_tag_filtering=True, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limited"}}) + ) + with pytest.raises(litellm.RateLimitError): + await router.aembedding(model="embed", input=["a", "b", "c"], metadata={"tags": ["team-b"]}) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await router.aembedding(model="embed", input=["a", "b", "c"], metadata={"tags": ["team-a"]}) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 1) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_names_the_cap_while_another_regions_admitting_deployment_cools_down( + self, sync: bool + ) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _VLLM_EMBED_BASE, + "region_name": "eu", + }, + "model_info": {"id": "eu-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + "region_name": "us", + }, + "model_info": {"id": "us-uncapped"}, + }, + ], + enable_pre_call_checks=True, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_VLLM_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + uncapped_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limited"}}) + ) + with pytest.raises(litellm.RateLimitError): + await _call_embedding( + router, sync=sync, model="embed", input=["a", "b", "c"], allowed_model_region="us" + ) + with pytest.raises(litellm.BadRequestError, match="max_embedding_batch_size=2"): + await _call_embedding( + router, sync=sync, model="embed", input=["a", "b", "c"], allowed_model_region="eu" + ) + assert (capped_route.call_count, uncapped_route.call_count) == (0, 1) + + @pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) + @pytest.mark.asyncio + async def test_oversized_batch_rejection_falls_back_with_explicit_provider(self, sync: bool) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-fake", + "api_base": _OPENAI_EMBED_BASE, + "custom_llm_provider": "openai", + }, + "model_info": {"id": "explicit-provider-capped", "max_embedding_batch_size": 2}, + }, + { + "model_name": "embed-fallback", + "litellm_params": { + "model": "hosted_vllm/bge-m3", + "api_key": "sk-fake", + "api_base": _UNCAPPED_EMBED_BASE, + }, + "model_info": {"id": "fallback-uncapped"}, + }, + ], + fallbacks=[{"embed": ["embed-fallback"]}], + fallback_on_unsupported_params=False, + num_retries=0, + ) + + with respx.mock(assert_all_called=False) as respx_mock: + capped_route: Final = respx_mock.post(f"{_OPENAI_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + fallback_route: Final = respx_mock.post(f"{_UNCAPPED_EMBED_BASE}/embeddings").mock( + return_value=_vllm_embedding_response(3) + ) + await _call_embedding(router, sync=sync, model="embed", input=["a", "b", "c"]) + assert (capped_route.call_count, fallback_route.call_count) == (0, 1) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 992a7d91b0d..2b6898e39d9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -50107,6 +50107,8 @@ export interface components { input_cost_per_token?: number | null; /** Internal Router Model */ internal_router_model?: boolean | null; + /** Max Embedding Batch Size */ + max_embedding_batch_size?: number | null; /** * Member Auto Router * @default false From 06e3e32c88326930645e3aabc2e454d7dfc4bfdd Mon Sep 17 00:00:00 2001 From: Ondra Zahradnik Date: Tue, 20 Oct 2026 13:37:46 +0200 Subject: [PATCH 091/124] fix(vertex_ai): reuse known context caches instead of listing on every request (#46857) Vertex AI and Gemini context caching now remember each cache they create or find until 10s before its expireTime, so a repeat cache-marked request goes straight to generateContent instead of listing every cachedContents first. Concurrent async requests for one prefix share one lookup or create When generation rejects a remembered cache with a 400, 403 or 404 that names the cached content, the request rebuilds the cache and sends once more. Any other 400, 403 or 404 drops the record without a retry Fixes #36395 --- .../context_caching/known_cached_contents.py | 156 +++++ .../context_caching/rejected_cache_retry.py | 74 ++ .../vertex_ai_context_caching.py | 179 +++-- .../vertex_and_google_ai_studio_gemini.py | 251 ++++--- .../vertex_ai/context_caching/conftest.py | 10 + .../test_known_cached_contents.py | 663 ++++++++++++++++++ 6 files changed, 1173 insertions(+), 160 deletions(-) create mode 100644 litellm/llms/vertex_ai/context_caching/known_cached_contents.py create mode 100644 litellm/llms/vertex_ai/context_caching/rejected_cache_retry.py create mode 100644 tests/unit/llms/vertex_ai/context_caching/conftest.py create mode 100644 tests/unit/llms/vertex_ai/context_caching/test_known_cached_contents.py diff --git a/litellm/llms/vertex_ai/context_caching/known_cached_contents.py b/litellm/llms/vertex_ai/context_caching/known_cached_contents.py new file mode 100644 index 00000000000..6b7f098d304 --- /dev/null +++ b/litellm/llms/vertex_ai/context_caching/known_cached_contents.py @@ -0,0 +1,156 @@ +import asyncio +import contextlib +import re +import threading +import time +import weakref +from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime, timezone +from typing import Final, Literal + +KNOWN_CACHE_EXPIRY_MARGIN_SECONDS: Final = 10.0 +MAX_KNOWN_CACHED_CONTENTS: Final = 10_000 +_CACHE_REJECTED_STATUS_CODES: Final = frozenset({400, 403, 404}) +_NAMES_CACHED_CONTENT: Final = re.compile(r"cached?\s*content", re.IGNORECASE) +_FORBIDS_FIELDS_BESIDE_A_CACHE: Final = re.compile(r"system[_\s]instruction", re.IGNORECASE) + +_RFC3339_FRACTION: Final = re.compile(r"\.(\d{1,6})\d*") + + +def names_rejected_cache(status_code: int, message: str) -> bool: + return ( + status_code in _CACHE_REJECTED_STATUS_CODES + and _NAMES_CACHED_CONTENT.search(message) is not None + and _FORBIDS_FIELDS_BESIDE_A_CACHE.search(message) is None + ) + + +def parse_expire_time(expire_time: object) -> float | None: + """Parse a Vertex RFC 3339 ``expireTime`` (nanosecond precision, ``Z`` suffix) to epoch seconds.""" + if not isinstance(expire_time, str) or not expire_time: + return None + normalized: Final = _RFC3339_FRACTION.sub(lambda m: "." + m.group(1), expire_time.replace("Z", "+00:00"), count=1) + try: + parsed: Final = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc).timestamp() + return parsed.timestamp() + + +class KnownCachedContents: + """Process-local record of the cachedContents this process created or found. + + Vertex meters every cachedContents list page against a per-minute, per-location quota, + so listing on every cache-marked request exhausts it under load. A cache is remembered + until ``KNOWN_CACHE_EXPIRY_MARGIN_SECONDS`` before the ``expireTime`` Google reported, + and after that requests list again, as they did before the record existed. + """ + + def __init__(self, clock: Callable[[], float] = time.time) -> None: + self._clock: Final = clock + self._entries: dict[tuple[tuple[str, str], str], tuple[str, float]] = {} # mutable-ok: guarded by _lock + self._lock: Final = threading.Lock() + self._inflight: weakref.WeakKeyDictionary[ # mutable-ok: private, guarded by _lock + asyncio.AbstractEventLoop, dict[tuple[tuple[str, str], str], asyncio.Future[str | None]] + ] = weakref.WeakKeyDictionary() + + @staticmethod + def scope( + base_url: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + headers: Mapping[str, str], + vertex_project: str | None, + vertex_location: str | None, + ) -> tuple[str, str]: + """Where a cache name is valid: a Vertex project and location, or the project a Gemini key belongs to. + + A custom ``api_base`` can replace the whole Vertex path, so the project and location are kept beside the URL. + The Gemini key is read from the headers actually sent, so an ``extra_headers`` override scopes the + cache to the caller's project. It is kept as-is in this in-memory scope and never logged or persisted. + """ + if custom_llm_provider != "gemini": + return (base_url, f"{vertex_project}/{vertex_location}") + sent_keys: Final = [value for name, value in headers.items() if name.lower() == "x-goog-api-key"] + return (base_url, sent_keys[-1] if sent_keys else "") + + def has_not_expired(self, expire_time: object) -> bool: + expires_at: Final = parse_expire_time(expire_time) + return expires_at is None or expires_at > self._clock() + + def get(self, scope: tuple[str, str], cache_key: str) -> str | None: + with self._lock: + entry: Final = self._entries.get((scope, cache_key)) + if entry is None: + return None + name, reuse_until = entry + if reuse_until > self._clock(): + return name + del self._entries[(scope, cache_key)] + return None + + def remember(self, scope: tuple[str, str], cache_key: str, name: str | None, expire_time: object) -> None: + expires_at: Final = parse_expire_time(expire_time) + if not name or expires_at is None: + return + reuse_until: Final = expires_at - KNOWN_CACHE_EXPIRY_MARGIN_SECONDS + now: Final = self._clock() + if reuse_until <= now: + return + with self._lock: + if len(self._entries) >= MAX_KNOWN_CACHED_CONTENTS: + self._entries = {k: v for k, v in self._entries.items() if v[1] > now} + while len(self._entries) >= MAX_KNOWN_CACHED_CONTENTS: + del self._entries[next(iter(self._entries))] + self._entries[(scope, cache_key)] = (name, reuse_until) + + def forget_rejected(self, cached_content: str | None, status_code: int) -> None: + if cached_content is None or status_code not in _CACHE_REJECTED_STATUS_CODES: + return + with self._lock: + self._entries = {key: entry for key, entry in self._entries.items() if entry[0] != cached_content} + + async def single_flight( + self, + scope: tuple[str, str], + cache_key: str, + resolve: Callable[[], Awaitable[str | None]], + wait_seconds: float | None, + ) -> str | None: + """Run ``resolve`` once per (scope, cache_key) at a time within this event loop. + + Concurrent requests for the same prefix wait for the first one's lookup-or-create + instead of each listing (and creating a duplicate cache). A waiter resolves on its + own when that first attempt fails or has not finished within ``wait_seconds``, and + never cancels it. + """ + loop: Final = asyncio.get_running_loop() + own: Final[asyncio.Future[str | None]] = loop.create_future() + with self._lock: + inflight: Final = self._inflight.setdefault(loop, {}) + leader: Final = inflight.setdefault((scope, cache_key), own) + if leader is not own: + with contextlib.suppress(asyncio.TimeoutError): + if name := await asyncio.wait_for(asyncio.shield(leader), wait_seconds): + return name + return await resolve() + try: + resolved: Final = await resolve() + except BaseException: + own.set_result(None) + raise + else: + own.set_result(resolved) + return resolved + finally: + with self._lock: + inflight.pop((scope, cache_key), None) + + def clear(self) -> None: + with self._lock: + self._entries.clear() + self._inflight.clear() + + +known_cached_contents: Final = KnownCachedContents() diff --git a/litellm/llms/vertex_ai/context_caching/rejected_cache_retry.py b/litellm/llms/vertex_ai/context_caching/rejected_cache_retry.py new file mode 100644 index 00000000000..43e4ee8dc84 --- /dev/null +++ b/litellm/llms/vertex_ai/context_caching/rejected_cache_retry.py @@ -0,0 +1,74 @@ +from collections.abc import Awaitable, Callable +from copy import deepcopy +from functools import partial +from typing import Final, TypeVar + +import httpx + +from litellm.litellm_core_utils.core_helpers import independent_snapshot +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.vertex_ai import RequestBody +from litellm.utils import is_cached_message + +from .known_cached_contents import names_rejected_cache + +Built: Final = TypeVar("Built") +Sent: Final = TypeVar("Sent") + + +def cache_rebuild( + build: Callable[..., Built], messages: list[AllMessageValues], optional_params: dict +) -> Callable[[], Built] | None: + """Copies the inputs before the first transform, which pops keys from ``optional_params`` and ``messages``.""" + if not any(map(is_cached_message, messages)): + return None + return partial(build, messages=deepcopy(messages), optional_params=independent_snapshot(optional_params)) + + +def _status_and_message(error: httpx.HTTPStatusError | BaseLLMException) -> tuple[int, str]: + match error: + case httpx.HTTPStatusError(): + return error.response.status_code, error.response.text + case BaseLLMException(): + return error.status_code, error.message + + +def _cache_rejected(body: RequestBody, error: httpx.HTTPStatusError | BaseLLMException) -> bool: + return body.get("cachedContent") is not None and names_rejected_cache(*_status_and_message(error)) + + +async def post_with_fresh_cache_async( + body: RequestBody, + rebuild: Callable[[], Awaitable[RequestBody]] | None, + post: Callable[[RequestBody], Awaitable[Sent]], + log_request: Callable[[RequestBody], None], +) -> tuple[RequestBody, Sent]: + try: + return body, await post(body) + except (httpx.HTTPStatusError, BaseLLMException) as error: + if rebuild is None or not _cache_rejected(body, error): + raise + fresh_body: Final = await rebuild() + log_request(fresh_body) + if fresh_body.get("cachedContent") == body.get("cachedContent"): + raise + return fresh_body, await post(fresh_body) + + +def post_with_fresh_cache_sync( + body: RequestBody, + rebuild: Callable[[], RequestBody] | None, + post: Callable[[RequestBody], Sent], + log_request: Callable[[RequestBody], None], +) -> tuple[RequestBody, Sent]: + try: + return body, post(body) + except (httpx.HTTPStatusError, BaseLLMException) as error: + if rebuild is None or not _cache_rejected(body, error): + raise + fresh_body: Final = rebuild() + log_request(fresh_body) + if fresh_body.get("cachedContent") == body.get("cachedContent"): + raise + return fresh_body, post(fresh_body) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 06b8f3a3fbb..5bff43828d6 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -20,6 +20,7 @@ from litellm.utils import is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url from ..vertex_llm_base import VertexBase +from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, separate_cached_messages, @@ -35,6 +36,7 @@ MAX_PAGINATION_PAGES: Final = 100 # Reasonable upper bound for pagination class _CachedContentItem(TypedDict): displayName: ReadOnly[NotRequired[object]] name: ReadOnly[NotRequired[str | None]] + expireTime: ReadOnly[NotRequired[str | None]] @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) @@ -51,6 +53,7 @@ class _CachedContentUsageMetadata(TypedDict): @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) class _CreatedCachedContent(TypedDict): name: ReadOnly[NotRequired[str | None]] + expireTime: ReadOnly[NotRequired[str | None]] usageMetadata: ReadOnly[NotRequired[_CachedContentUsageMetadata | None]] @@ -67,6 +70,14 @@ def _record_cache_creation_tokens(logging_obj: Logging, created: _CreatedCachedC logging_obj.vertex_ai_cache_creation_input_tokens = max(tokens, 0) # rebind-ok: set provider count +def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: + match timeout: + case httpx.Timeout(): + return timeout.read + case _: + return timeout + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -74,8 +85,9 @@ class ContextCachingEndpoints(VertexBase): v0: covers Google AI Studio """ - def __init__(self) -> None: + def __init__(self, known_caches: KnownCachedContents = known_cached_contents) -> None: super().__init__() + self._known_caches: Final = known_caches def _get_token_and_url_context_caching( self, @@ -129,7 +141,7 @@ class ContextCachingEndpoints(VertexBase): self, cache_key: str, client: HTTPHandler, - headers: dict, + headers: dict[str, str], # mutable-ok: passed through to the HTTP client, which takes a dict api_key: str, api_base: str | None, logging_obj: Logging, @@ -159,6 +171,12 @@ class ContextCachingEndpoints(VertexBase): vertex_auth_header=vertex_auth_header, model=model, ) + scope: Final = KnownCachedContents.scope( + base_url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + ) + known_name: Final = self._known_caches.get(scope, cache_key) + if known_name is not None: + return known_name page_token: str | None = None @@ -201,8 +219,14 @@ class ContextCachingEndpoints(VertexBase): # Check current page for matching cache_key for cached_item in raw_response["cachedContents"]: display_name = cached_item.get("displayName") - if display_name is not None and display_name == cache_key: - return cached_item.get("name") + if display_name is None or display_name != cache_key: + continue + expire_time = cached_item.get("expireTime") + if not self._known_caches.has_not_expired(expire_time): + continue + name = cached_item.get("name") + self._known_caches.remember(scope, cache_key, name, expire_time) + return name # Check if there are more pages page_token = raw_response.get("nextPageToken") @@ -216,7 +240,7 @@ class ContextCachingEndpoints(VertexBase): self, cache_key: str, client: AsyncHTTPHandler, - headers: dict, + headers: dict[str, str], # mutable-ok: passed through to the HTTP client, which takes a dict api_key: str, api_base: str | None, logging_obj: Logging, @@ -246,6 +270,12 @@ class ContextCachingEndpoints(VertexBase): vertex_auth_header=vertex_auth_header, model=model, ) + scope: Final = KnownCachedContents.scope( + base_url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + ) + known_name: Final = self._known_caches.get(scope, cache_key) + if known_name is not None: + return known_name page_token: str | None = None @@ -288,8 +318,14 @@ class ContextCachingEndpoints(VertexBase): # Check current page for matching cache_key for cached_item in raw_response["cachedContents"]: display_name = cached_item.get("displayName") - if display_name is not None and display_name == cache_key: - return cached_item.get("name") + if display_name is None or display_name != cache_key: + continue + expire_time = cached_item.get("expireTime") + if not self._known_caches.has_not_expired(expire_time): + continue + name = cached_item.get("name") + self._known_caches.remember(scope, cache_key, name, expire_time) + return name # Check if there are more pages page_token = raw_response.get("nextPageToken") @@ -451,6 +487,14 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root _record_cache_creation_tokens(logging_obj, raw_response_cached) + self._known_caches.remember( + KnownCachedContents.scope( + url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + ), + generated_cache_key, + raw_response_cached.get("name"), + raw_response_cached.get("expireTime"), + ) return ( non_cached_messages, optional_params, @@ -548,68 +592,77 @@ class ContextCachingEndpoints(VertexBase): generated_cache_key: Final = local_cache_obj.get_cache_key( messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) - google_cache_name: Final = await self.async_check_cache( - cache_key=generated_cache_key, - client=client, - headers=headers, - api_key=api_key, - api_base=api_base, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - vertex_auth_header=vertex_auth_header, - model=model, + scope: Final = KnownCachedContents.scope( + url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location ) - if google_cache_name: - return non_cached_messages, optional_params, google_cache_name - - ## TRANSFORM REQUEST - cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( - model=model, - messages=cached_messages, - cache_key=generated_cache_key, - custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, - ) - - cached_content_request_body["tools"] = tools - if tool_choice is not None: - cached_content_request_body["toolConfig"] = tool_choice - - ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": cached_content_request_body, - "api_base": url, - "headers": headers, - }, - ) - - try: - response: Final = await client.post( - url=url, + async def lookup_or_create() -> str | None: + google_cache_name: Final = await self.async_check_cache( + cache_key=generated_cache_key, + client=client, headers=headers, - json=cached_content_request_body, + api_key=api_key, + api_base=api_base, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_auth_header=vertex_auth_header, + model=model, ) - response.raise_for_status() - except httpx.HTTPStatusError as err: - error_code: Final = err.response.status_code - raise VertexAIError(status_code=error_code, message=err.response.text) - except httpx.TimeoutException: - raise VertexAIError(status_code=408, message="Timeout error occurred.") - raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root - _record_cache_creation_tokens(logging_obj, raw_response_cached) - return ( - non_cached_messages, - optional_params, - raw_response_cached.get("name"), + if google_cache_name: + return google_cache_name + + ## TRANSFORM REQUEST + cached_content_request_body: Final = transform_openai_messages_to_gemini_context_caching( + model=model, + messages=cached_messages, + cache_key=generated_cache_key, + custom_llm_provider=custom_llm_provider, + vertex_project=vertex_project, + vertex_location=vertex_location, + ) + + cached_content_request_body["tools"] = tools + if tool_choice is not None: + cached_content_request_body["toolConfig"] = tool_choice + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": cached_content_request_body, + "api_base": url, + "headers": headers, + }, + ) + + try: + response: Final = await client.post( + url=url, + headers=headers, + json=cached_content_request_body, + ) + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code: Final = err.response.status_code + raise VertexAIError(status_code=error_code, message=err.response.text) + except httpx.TimeoutException: + raise VertexAIError(status_code=408, message="Timeout error occurred.") + + raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root + _record_cache_creation_tokens(logging_obj, raw_response_cached) + self._known_caches.remember( + scope, generated_cache_key, raw_response_cached.get("name"), raw_response_cached.get("expireTime") + ) + return raw_response_cached.get("name") + + cache_name: Final = await self._known_caches.single_flight( + scope, generated_cache_key, lookup_or_create, _wait_seconds(timeout) ) + return non_cached_messages, optional_params, cache_name def get_cache(self): pass diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 257fafbb6b8..114b9cd80ae 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -96,6 +96,12 @@ from ..common_utils import ( build_vertex_schema, supports_response_json_schema, ) +from ..context_caching.known_cached_contents import known_cached_contents +from ..context_caching.rejected_cache_retry import ( + cache_rebuild, + post_with_fresh_cache_async, + post_with_fresh_cache_sync, +) from ..vertex_llm_base import VertexBase from .grounding_requests import calculate_grounding_requests from .long_decode import ( @@ -2786,7 +2792,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): litellm_params: dict, api_key: str | dict | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: default_headers: Final = { "Content-Type": "application/json", } @@ -2800,6 +2806,20 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return default_headers +def _log_generate_request( + logging_obj: LoggingClass, + messages: Sequence[AllMessageValues], + api_base: str, + headers: Mapping[str, str], + body: RequestBody, +) -> None: + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={"complete_input_dict": body, "api_base": api_base, "headers": headers}, + ) + + async def make_call( client: AsyncHTTPHandler | None, # module-level client gemini_client: AsyncHTTPHandler | None, # if passed by user @@ -2832,6 +2852,7 @@ async def make_call( response.raise_for_status() except httpx.HTTPStatusError as e: exception_string: Final = str(await e.response.aread()) + known_cached_contents.forget_rejected(request_body.get("cachedContent"), e.response.status_code) raise VertexAIError( status_code=e.response.status_code, message=VertexGeminiConfig().translate_exception_str(exception_string), @@ -2845,14 +2866,18 @@ async def make_call( ) async def open_hop(body: RequestBody) -> httpx.Response: - return await client.post( - url_for(api_base, body), - headers=dict(await hop_headers()), - data=json.dumps(body), - stream=True, - logging_obj=logging_obj, - timeout=timeout, - ) + try: + return await client.post( + url_for(api_base, body), + headers=dict(await hop_headers()), + data=json.dumps(body), + stream=True, + logging_obj=logging_obj, + timeout=timeout, + ) + except httpx.HTTPStatusError as e: + known_cached_contents.forget_rejected(body.get("cachedContent"), e.response.status_code) + raise completion_stream: Final = ModelResponseIterator( streaming_response=response.aiter_lines(), @@ -2891,14 +2916,18 @@ def make_sync_call( if client is None: client = HTTPHandler() # Create a new client if none provided - response: Final = client.post( - url_for(api_base, request_body), - headers=headers, - data=data, - stream=True, - logging_obj=logging_obj, - timeout=timeout, - ) + try: + response: Final = client.post( + url_for(api_base, request_body), + headers=headers, + data=data, + stream=True, + logging_obj=logging_obj, + timeout=timeout, + ) + except httpx.HTTPStatusError as e: + known_cached_contents.forget_rejected(request_body.get("cachedContent"), e.response.status_code) + raise if response.status_code != 200 and response.status_code != 201: raise VertexAIError( @@ -2908,14 +2937,18 @@ def make_sync_call( ) def open_hop(body: RequestBody) -> httpx.Response: - return client.post( - url_for(api_base, body), - headers=dict(hop_headers()), - data=json.dumps(body), - stream=True, - logging_obj=logging_obj, - timeout=timeout, - ) + try: + return client.post( + url_for(api_base, body), + headers=dict(hop_headers()), + data=json.dumps(body), + stream=True, + logging_obj=logging_obj, + timeout=timeout, + ) + except httpx.HTTPStatusError as e: + known_cached_contents.forget_rejected(body.get("cachedContent"), e.response.status_code) + raise completion_stream: Final = ModelResponseIterator( streaming_response=response.iter_lines(), @@ -2983,14 +3016,14 @@ class VertexLLM(VertexBase): custom_llm_provider: Literal[ "vertex_ai", "vertex_ai_beta", "gemini" ], # if it's vertex_ai or gemini (google ai studio) - messages: list, + messages: list[AllMessageValues], model_response: ModelResponse, print_verbose: Callable, data: dict, timeout: float | httpx.Timeout | None, encoding: object, - logging_obj, - stream, + logging_obj: LoggingClass, + stream: bool | None, optional_params: dict, litellm_params: dict, logger_fn=None, @@ -3033,6 +3066,17 @@ class VertexLLM(VertexBase): litellm_params=litellm_params, ) + rebuild: Final = cache_rebuild( + partial( + async_transform_request_body, + **data, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_auth_header=auth_header, + ), + messages, + optional_params, + ) request_body: Final = await async_transform_request_body( **data, vertex_project=vertex_project, @@ -3041,39 +3085,37 @@ class VertexLLM(VertexBase): ) ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": request_body, - "api_base": api_base, - "headers": headers, - }, + log_request: Final = partial(_log_generate_request, logging_obj, messages, api_base, headers) + log_request(request_body) + + open_stream: Final = partial( + make_call, + gemini_client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None), + api_base=api_base, + headers=headers, + model=model, + messages=messages, + logging_obj=logging_obj, + hop_headers=partial( + self._fresh_headers_async, + headers, + extra_headers, + vertex_credentials, + vertex_project, + custom_llm_provider, + ), + timeout=timeout, ) - request_body_str: Final = json.dumps(request_body) + async def open_with_fresh_cache(client: AsyncHTTPHandler | None) -> ModelResponseIterator: + async def post(body: RequestBody) -> ModelResponseIterator: + return await open_stream(client=client, data=json.dumps(body), request_body=body) + + return (await post_with_fresh_cache_async(request_body, rebuild, post, log_request))[1] + streaming_response: Final = CustomStreamWrapper( completion_stream=None, - make_call=partial( - make_call, - gemini_client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None), - api_base=api_base, - headers=headers, - data=request_body_str, - model=model, - messages=messages, - logging_obj=logging_obj, - request_body=request_body, - hop_headers=partial( - self._fresh_headers_async, - headers, - extra_headers, - vertex_credentials, - vertex_project, - custom_llm_provider, - ), - timeout=timeout, - ), + make_call=open_with_fresh_cache, model=model, custom_llm_provider="vertex_ai_beta", logging_obj=logging_obj, @@ -3083,7 +3125,7 @@ class VertexLLM(VertexBase): async def async_completion( self, model: str, - messages: list, + messages: list[AllMessageValues], model_response: ModelResponse, print_verbose: Callable, data: dict, @@ -3093,7 +3135,7 @@ class VertexLLM(VertexBase): timeout: float | httpx.Timeout | None, encoding: object, logging_obj: LoggingClass, - stream, + stream: bool | None, optional_params: dict, litellm_params: dict, logger_fn=None, @@ -3136,6 +3178,17 @@ class VertexLLM(VertexBase): litellm_params=litellm_params, ) + rebuild: Final = cache_rebuild( + partial( + async_transform_request_body, + **data, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_auth_header=auth_header, + ), + messages, + optional_params, + ) request_body: Final = await async_transform_request_body( **data, vertex_project=vertex_project, @@ -3151,15 +3204,8 @@ class VertexLLM(VertexBase): else: client = client ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": request_body, - "api_base": api_base, - "headers": headers, - }, - ) + log_request: Final = partial(_log_generate_request, logging_obj, messages, api_base, headers) + log_request(request_body) fresh_headers: Final = partial( self._fresh_headers_async, headers, extra_headers, vertex_credentials, vertex_project, custom_llm_provider @@ -3175,6 +3221,7 @@ class VertexLLM(VertexBase): ) response.raise_for_status() except httpx.HTTPStatusError as err: + known_cached_contents.forget_rejected(body.get("cachedContent"), err.response.status_code) raise VertexAIError( status_code=err.response.status_code, message=err.response.text, @@ -3188,14 +3235,14 @@ class VertexLLM(VertexBase): ) return response - first_response: Final = await post(request_body) + sent_body, first_response = await post_with_fresh_cache_async(request_body, rebuild, post, log_request) return _transform_long_decode_hops( model=model, - hops=await collect_hops_async(parse_body(first_response), request_body, post), + hops=await collect_hops_async(parse_body(first_response), sent_body, post), first_response=first_response, model_response=model_response, logging_obj=logging_obj, - request_body=request_body, + request_body=sent_body, messages=messages, ) @@ -3320,6 +3367,17 @@ class VertexLLM(VertexBase): ) ## TRANSFORMATION ## + rebuild: Final = cache_rebuild( + partial( + sync_transform_request_body, + **transform_request_params, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_auth_header=auth_header, + ), + messages, + optional_params, + ) data: Final = sync_transform_request_body( **transform_request_params, vertex_project=vertex_project, @@ -3328,15 +3386,8 @@ class VertexLLM(VertexBase): ) ## LOGGING - logging_obj.pre_call( - input=messages, - api_key="", - additional_args={ - "complete_input_dict": data, - "api_base": url, - "headers": headers, - }, - ) + log_request: Final = partial(_log_generate_request, logging_obj, messages, url, headers) + log_request(data) fresh_headers: Final = partial( self._fresh_headers, headers, extra_headers, vertex_credentials, vertex_project, custom_llm_provider @@ -3344,22 +3395,27 @@ class VertexLLM(VertexBase): ## SYNC STREAMING CALL ## if stream is True: - request_data_str: Final = json.dumps(data) + open_stream: Final = partial( + make_sync_call, + gemini_client=(client if client is not None and isinstance(client, HTTPHandler) else None), + api_base=url, + model=model, + messages=messages, + logging_obj=logging_obj, + headers=headers, + hop_headers=fresh_headers, + timeout=timeout, + ) + + def open_with_fresh_cache(client: HTTPHandler | None) -> ModelResponseIterator: + def post(body: RequestBody) -> ModelResponseIterator: + return open_stream(client=client, data=json.dumps(body), request_body=body) + + return post_with_fresh_cache_sync(data, rebuild, post, log_request)[1] + streaming_response: Final = CustomStreamWrapper( completion_stream=None, - make_call=partial( - make_sync_call, - gemini_client=(client if client is not None and isinstance(client, HTTPHandler) else None), - api_base=url, - data=request_data_str, - model=model, - messages=messages, - logging_obj=logging_obj, - headers=headers, - request_body=data, - hop_headers=fresh_headers, - timeout=timeout, - ), + make_call=open_with_fresh_cache, model=model, custom_llm_provider="vertex_ai_beta", logging_obj=logging_obj, @@ -3385,6 +3441,7 @@ class VertexLLM(VertexBase): ) response.raise_for_status() except httpx.HTTPStatusError as err: + known_cached_contents.forget_rejected(body.get("cachedContent"), err.response.status_code) raise VertexAIError( status_code=err.response.status_code, message=err.response.text, @@ -3398,14 +3455,14 @@ class VertexLLM(VertexBase): ) return response - first_response: Final = post(data) + sent_body, first_response = post_with_fresh_cache_sync(data, rebuild, post, log_request) return _transform_long_decode_hops( model=model, - hops=collect_hops_sync(parse_body(first_response), data, post), + hops=collect_hops_sync(parse_body(first_response), sent_body, post), first_response=first_response, model_response=model_response, logging_obj=logging_obj, - request_body=data, + request_body=sent_body, messages=messages, ) diff --git a/tests/unit/llms/vertex_ai/context_caching/conftest.py b/tests/unit/llms/vertex_ai/context_caching/conftest.py new file mode 100644 index 00000000000..9c80c964608 --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/conftest.py @@ -0,0 +1,10 @@ +import pytest + +from litellm.llms.vertex_ai.context_caching.known_cached_contents import known_cached_contents + + +@pytest.fixture(autouse=True) +def _forget_known_cached_contents(): + known_cached_contents.clear() + yield + known_cached_contents.clear() diff --git a/tests/unit/llms/vertex_ai/context_caching/test_known_cached_contents.py b/tests/unit/llms/vertex_ai/context_caching/test_known_cached_contents.py new file mode 100644 index 00000000000..0c5c3facb56 --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/test_known_cached_contents.py @@ -0,0 +1,663 @@ +import asyncio +import json +import threading +from datetime import datetime, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import openai +import pytest +import respx + +import litellm + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.common_utils import VertexAIError +from litellm.llms.vertex_ai.context_caching import vertex_ai_context_caching +from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCachedContents, parse_expire_time +from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ContextCachingEndpoints +from litellm.utils import get_prompt_cache_min_tokens + +_REGISTRY_MODULE: Final = "litellm.llms.vertex_ai.context_caching.known_cached_contents" +_SCOPE: Final = ("https://vertex.test/v1/projects/p/locations/us/cachedContents", "") +_NOW: Final = datetime(2026, 10, 14, 18, 0, tzinfo=timezone.utc).timestamp() +_FAR_FUTURE: Final = "2100-01-01T00:00:00.123456789Z" +_GATEWAY: Final = "https://gateway.test/vertex" +_PREFIX_REPEATS: Final = max( + get_prompt_cache_min_tokens(model=f"{provider}/gemini-3.1-pro") for provider in ("vertex_ai", "gemini") +) + + +@pytest.fixture +def clock() -> MagicMock: + return MagicMock(return_value=_NOW) + + +@pytest.fixture +def endpoints(clock: MagicMock) -> ContextCachingEndpoints: + return ContextCachingEndpoints(known_caches=KnownCachedContents(clock=clock)) + + +def _expire_time(seconds_from_now: float) -> str: + expires: Final = datetime.fromtimestamp(_NOW + seconds_from_now, timezone.utc) + return expires.strftime("%Y-%m-%dT%H:%M:%S.%f") + "123Z" + + +def _messages(label: str = "shared") -> list: + text: Final = f"{label} cacheable prefix " * _PREFIX_REPEATS + return [ + {"role": "system", "content": [{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "question"}, + ] + + +def _response(body: dict) -> MagicMock: + return MagicMock(json=lambda: body) + + +def _created(name: str, expire_time: str | None) -> dict: + body: Final = {"name": name, "usageMetadata": {"totalTokenCount": 2048}} + return body if expire_time is None else {**body, "expireTime": expire_time} + + +def _sync_call( + endpoints: ContextCachingEndpoints, + client: HTTPHandler, + api_key: str = "k", + custom_llm_provider: str = "vertex_ai", + messages=None, + extra_headers: dict | None = None, + api_base: str | None = None, + vertex_project: str = "p", +): + return endpoints.check_and_create_cache( + extra_headers=extra_headers, + messages=messages or _messages(), + optional_params={}, + api_key=api_key, + api_base=api_base, + model="gemini-3.1-pro", + client=client, + timeout=30.0, + logging_obj=MagicMock(model_call_details={}), + custom_llm_provider=custom_llm_provider, + vertex_project=vertex_project, + vertex_location="us", + vertex_auth_header="token", + ) + + +async def _async_call( + endpoints: ContextCachingEndpoints, + client: AsyncHTTPHandler, + timeout: float | httpx.Timeout = 30.0, + api_base: str | None = None, + vertex_project: str = "p", +): + return await endpoints.async_check_and_create_cache( + messages=_messages(), + optional_params={}, + api_key="k", + api_base=api_base, + model="gemini-3.1-pro", + client=client, + timeout=timeout, + logging_obj=MagicMock(model_call_details={}), + custom_llm_provider="vertex_ai", + vertex_project=vertex_project, + vertex_location="us", + vertex_auth_header="token", + ) + + +def test_created_cache_is_reused_without_listing(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.return_value = _response(_created("cachedContents/1", _expire_time(60))) + + assert _sync_call(endpoints, client)[2] == "cachedContents/1" + assert _sync_call(endpoints, client)[2] == "cachedContents/1" + + assert client.get.call_count == 1 + assert client.post.call_count == 1 + + +def test_listed_cache_is_reused_without_listing_again(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + key: Final = vertex_ai_context_caching.local_cache_obj.get_cache_key( + messages=_messages()[:1], tools=None, tool_choice=None, model="gemini-3.1-pro" + ) + client.get.return_value = _response( + {"cachedContents": [{"name": "cachedContents/listed", "displayName": key, "expireTime": _expire_time(60)}]} + ) + + assert _sync_call(endpoints, client)[2] == "cachedContents/listed" + assert _sync_call(endpoints, client)[2] == "cachedContents/listed" + + assert client.get.call_count == 1 + client.post.assert_not_called() + + +def test_listed_cache_past_its_expire_time_is_not_used(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + key: Final = vertex_ai_context_caching.local_cache_obj.get_cache_key( + messages=_messages()[:1], tools=None, tool_choice=None, model="gemini-3.1-pro" + ) + client.get.return_value = _response( + {"cachedContents": [{"name": "cachedContents/stale", "displayName": key, "expireTime": _expire_time(-1)}]} + ) + client.post.return_value = _response(_created("cachedContents/fresh", _expire_time(60))) + + assert _sync_call(endpoints, client)[2] == "cachedContents/fresh" + + +def test_short_lived_cache_is_reused_from_the_listing(endpoints: ContextCachingEndpoints, clock: MagicMock): + client: Final = MagicMock(spec=HTTPHandler) + key: Final = vertex_ai_context_caching.local_cache_obj.get_cache_key( + messages=_messages()[:1], tools=None, tool_choice=None, model="gemini-3.1-pro" + ) + client.get.side_effect = [ + _response({}), + _response( + {"cachedContents": [{"name": "cachedContents/1", "displayName": key, "expireTime": _expire_time(8)}]} + ), + ] + client.post.return_value = _response(_created("cachedContents/1", _expire_time(8))) + + assert _sync_call(endpoints, client)[2] == "cachedContents/1" + clock.return_value = _NOW + 1 + assert _sync_call(endpoints, client)[2] == "cachedContents/1" + + assert client.post.call_count == 1 + + +def test_cache_without_expire_time_is_not_remembered(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.return_value = _response(_created("cachedContents/1", None)) + + _sync_call(endpoints, client) + _sync_call(endpoints, client) + + assert client.get.call_count == 2 + + +def test_remembered_cache_is_listed_again_once_its_reuse_window_closes( + endpoints: ContextCachingEndpoints, clock: MagicMock +): + client: Final = MagicMock(spec=HTTPHandler) + key: Final = vertex_ai_context_caching.local_cache_obj.get_cache_key( + messages=_messages()[:1], tools=None, tool_choice=None, model="gemini-3.1-pro" + ) + client.get.side_effect = [ + _response({}), + _response( + {"cachedContents": [{"name": "cachedContents/listed", "displayName": key, "expireTime": _expire_time(600)}]} + ), + ] + client.post.return_value = _response(_created("cachedContents/1", _expire_time(60))) + assert _sync_call(endpoints, client)[2] == "cachedContents/1" + + clock.return_value = _NOW + 55 + assert _sync_call(endpoints, client)[2] == "cachedContents/listed" + + +def test_gemini_caches_are_not_shared_across_api_keys(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.return_value = _response(_created("cachedContents/1", _expire_time(60))) + + _sync_call(endpoints, client, api_key="key-a", custom_llm_provider="gemini") + _sync_call(endpoints, client, api_key="key-b", custom_llm_provider="gemini") + + assert client.post.call_count == 2 + + +def test_gemini_cache_created_with_an_overridden_api_key_is_not_reused_with_the_configured_key( + endpoints: ContextCachingEndpoints, +): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.side_effect = [ + _response(_created("cachedContents/caller-project", _expire_time(60))), + _response(_created("cachedContents/deployment-project", _expire_time(60))), + ] + + caller: Final = _sync_call( + endpoints, + client, + api_key="deployment-key", + custom_llm_provider="gemini", + extra_headers={"X-Goog-Api-Key": "caller-key"}, + ) + configured: Final = _sync_call(endpoints, client, api_key="deployment-key", custom_llm_provider="gemini") + + assert (caller[2], configured[2]) == ("cachedContents/caller-project", "cachedContents/deployment-project") + + +def test_vertex_caches_behind_one_custom_api_base_are_not_shared_across_projects(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.side_effect = [ + _response(_created("cachedContents/project-a", _expire_time(60))), + _response(_created("cachedContents/project-b", _expire_time(60))), + ] + + sent: Final = [ + _sync_call(endpoints, client, api_base=_GATEWAY, vertex_project=project)[2] + for project in ("project-a", "project-b", "project-a") + ] + + assert sent == ["cachedContents/project-a", "cachedContents/project-b", "cachedContents/project-a"] + + +@pytest.mark.asyncio +async def test_async_vertex_caches_behind_one_custom_api_base_are_not_shared_across_projects( + endpoints: ContextCachingEndpoints, +): + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_response({})) + client.post = AsyncMock( + side_effect=[ + _response(_created("cachedContents/project-a", _expire_time(60))), + _response(_created("cachedContents/project-b", _expire_time(60))), + ] + ) + + sent: Final = [ + (await _async_call(endpoints, client, api_base=_GATEWAY, vertex_project=project))[2] + for project in ("project-a", "project-b", "project-a") + ] + + assert sent == ["cachedContents/project-a", "cachedContents/project-b", "cachedContents/project-a"] + + +def test_different_prefixes_do_not_share_a_cache(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _response({}) + client.post.side_effect = [ + _response(_created("cachedContents/a", _expire_time(60))), + _response(_created("cachedContents/b", _expire_time(60))), + ] + + assert _sync_call(endpoints, client, messages=_messages("first"))[2] == "cachedContents/a" + assert _sync_call(endpoints, client, messages=_messages("second"))[2] == "cachedContents/b" + + +@pytest.mark.asyncio +async def test_concurrent_requests_for_one_prefix_list_and_create_once(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=AsyncHTTPHandler) + + async def slow_list(**_): + await asyncio.sleep(0.01) + return _response({}) + + client.get = AsyncMock(side_effect=slow_list) + client.post = AsyncMock(return_value=_response(_created("cachedContents/1", _expire_time(60)))) + + results: Final = await asyncio.gather(*(_async_call(endpoints, client) for _ in range(5))) + + assert {result[2] for result in results} == {"cachedContents/1"} + assert client.get.await_count == 1 + assert client.post.await_count == 1 + + +@pytest.mark.asyncio +async def test_async_created_cache_is_reused_by_a_later_request(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_response({})) + client.post = AsyncMock(return_value=_response(_created("cachedContents/1", _expire_time(60)))) + + first: Final = await _async_call(endpoints, client) + client.get = AsyncMock(side_effect=AssertionError("a known cache must not be listed again")) + second: Final = await _async_call(endpoints, client) + + assert (first[2], second[2]) == ("cachedContents/1", "cachedContents/1") + + +@pytest.mark.asyncio +async def test_async_listed_cache_past_its_expire_time_is_not_used(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=AsyncHTTPHandler) + key: Final = vertex_ai_context_caching.local_cache_obj.get_cache_key( + messages=_messages()[:1], tools=None, tool_choice=None, model="gemini-3.1-pro" + ) + client.get = AsyncMock( + return_value=_response( + {"cachedContents": [{"name": "cachedContents/stale", "displayName": key, "expireTime": _expire_time(-1)}]} + ) + ) + client.post = AsyncMock(return_value=_response(_created("cachedContents/fresh", _expire_time(60)))) + + assert (await _async_call(endpoints, client))[2] == "cachedContents/fresh" + + +@pytest.mark.asyncio +async def test_async_create_http_error_surfaces_its_status_code(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_response({})) + request: Final = httpx.Request("POST", "https://vertex.test/v1/projects/p/locations/us/cachedContents") + failed: Final = httpx.Response(429, request=request, text="RESOURCE_EXHAUSTED") + client.post = AsyncMock(return_value=failed) + + with pytest.raises(VertexAIError) as raised: + await _async_call(endpoints, client) + + assert raised.value.status_code == 429 + + +@pytest.mark.asyncio +async def test_waiters_resolve_on_their_own_when_the_first_request_fails(endpoints: ContextCachingEndpoints): + client: Final = MagicMock(spec=AsyncHTTPHandler) + + async def slow_list(**_): + await asyncio.sleep(0.01) + return _response({}) + + client.get = AsyncMock(side_effect=slow_list) + client.post = AsyncMock( + side_effect=[ + VertexAIError(status_code=500, message="create failed"), + _response(_created("cachedContents/2", _expire_time(60))), + ] + ) + + results: Final = await asyncio.gather( + _async_call(endpoints, client), _async_call(endpoints, client), return_exceptions=True + ) + + assert isinstance(results[0], VertexAIError) + assert results[1][2] == "cachedContents/2" + + +@pytest.mark.parametrize("waiter_timeout", [0.05, httpx.Timeout(30.0, read=0.05)]) +@pytest.mark.asyncio +async def test_waiter_resolves_on_its_own_once_its_timeout_passes( + endpoints: ContextCachingEndpoints, waiter_timeout: float | httpx.Timeout +): + client: Final = MagicMock(spec=AsyncHTTPHandler) + + async def slow_list(**_): + await asyncio.sleep(0.5) + return _response({}) + + client.get = AsyncMock(side_effect=slow_list) + client.post = AsyncMock(return_value=_response(_created("cachedContents/1", _expire_time(60)))) + + first, waiter = await asyncio.gather(_async_call(endpoints, client), _async_call(endpoints, client, waiter_timeout)) + + assert (first[2], waiter[2]) == ("cachedContents/1", "cachedContents/1") + assert client.get.await_count == 2 + + +def test_parse_expire_time_accepts_nanosecond_precision(): + assert parse_expire_time("2026-10-14T18:06:21.123456789Z") == pytest.approx( + datetime(2026, 10, 14, 18, 6, 21, 123456, tzinfo=timezone.utc).timestamp() + ) + assert ( + parse_expire_time("2026-10-14T18:06:21Z") == datetime(2026, 10, 14, 18, 6, 21, tzinfo=timezone.utc).timestamp() + ) + assert parse_expire_time("not a time") is None + assert parse_expire_time(None) is None + + +def test_parse_expire_time_reads_a_missing_offset_as_utc(): + assert ( + parse_expire_time("2026-10-14T18:06:21") == datetime(2026, 10, 14, 18, 6, 21, tzinfo=timezone.utc).timestamp() + ) + + +def test_cache_expiring_within_the_margin_is_not_remembered(): + known: Final = KnownCachedContents(clock=lambda: _NOW) + known.remember(_SCOPE, "key", "cachedContents/soon", _expire_time(3)) + + assert known.get(_SCOPE, "key") is None + + +def test_registry_is_bounded(): + known: Final = KnownCachedContents(clock=lambda: _NOW) + with patch(f"{_REGISTRY_MODULE}.MAX_KNOWN_CACHED_CONTENTS", 3): + for index in range(5): + known.remember(_SCOPE, f"key-{index}", f"cachedContents/{index}", _expire_time(60)) + + assert known.get(_SCOPE, "key-0") is None + assert known.get(_SCOPE, "key-4") == "cachedContents/4" + + +def _generated(stream: bool) -> httpx.Response: + body: Final = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 1, "totalTokenCount": 11}, + } + if stream: + return httpx.Response(200, text=f"data: {json.dumps(body)}\n\n", headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=body) + + +def _rejected(status_code: int, message: str) -> httpx.Response: + return httpx.Response(status_code, json={"error": {"code": status_code, "message": message}}) + + +async def _complete(sync_mode: bool, stream: bool, **kwargs) -> str: + request: Final = {"model": "gemini/gemini-3.1-pro", "messages": _messages(), "stream": stream, **kwargs} + if sync_mode: + response = litellm.completion(**request) + if stream: + return "".join(chunk.choices[0].delta.content or "" for chunk in response) + return response.choices[0].message.content + response = await litellm.acompletion(**request) + if stream: + return "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + return response.choices[0].message.content + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.asyncio +async def test_rejected_cache_is_replaced_when_a_request_param_cannot_be_deep_copied( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + stream: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.get(url__regex=r".*/cachedContents.*").mock(return_value=httpx.Response(200, json={})) + respx_mock.post(url__regex=r".*/cachedContents$").mock( + side_effect=[ + httpx.Response(200, json=_created("cachedContents/deleted", _FAR_FUTURE)), + httpx.Response(200, json=_created("cachedContents/fresh", _FAR_FUTURE)), + ] + ) + generate_route: Final = respx_mock.post(url__regex=r".*:(stream)?[gG]enerateContent.*").mock( + side_effect=[_generated(stream), _rejected(404, "CachedContent not found"), _generated(stream)] + ) + lock: Final = threading.Lock() + + assert await _complete(sync_mode, stream, lock_holder=lock) == "ok" + assert await _complete(sync_mode, stream, lock_holder=lock) == "ok" + + sent: Final = [json.loads(call.request.content)["cachedContent"] for call in generate_route.calls] + assert sent == ["cachedContents/deleted", "cachedContents/deleted", "cachedContents/fresh"] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize( + ("status_code", "message", "replaced"), + [ + (400, "Invalid resource state for cache content 4484391867902853120.", True), + (403, "CachedContent not found (or permission denied)", True), + (404, "CachedContent not found", True), + (400, "Invalid value at 'generation_config.temperature'", False), + (500, "CachedContent not found", False), + ], +) +@pytest.mark.asyncio +async def test_remembered_cache_rejected_by_generate_content_is_replaced_within_the_same_call( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + stream: bool, + status_code: int, + message: str, + replaced: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + list_route: Final = respx_mock.get(url__regex=r".*/cachedContents.*").mock( + return_value=httpx.Response(200, json={}) + ) + respx_mock.post(url__regex=r".*/cachedContents$").mock( + side_effect=[ + httpx.Response(200, json=_created("cachedContents/deleted", _FAR_FUTURE)), + httpx.Response(200, json=_created("cachedContents/fresh", _FAR_FUTURE)), + ] + ) + generate_route: Final = respx_mock.post(url__regex=r".*:(stream)?[gG]enerateContent.*").mock( + side_effect=[_generated(stream), _rejected(status_code, message), _generated(stream)] + ) + logged: Final[list[dict]] = [] + + assert await _complete(sync_mode, stream) == "ok" + if replaced: + assert await _complete(sync_mode, stream, logger_fn=logged.append) == "ok" + else: + with pytest.raises(openai.APIStatusError): + await _complete(sync_mode, stream, logger_fn=logged.append) + + sent: Final = [json.loads(call.request.content)["cachedContent"] for call in generate_route.calls] + assert sent == ["cachedContents/deleted", "cachedContents/deleted"] + (["cachedContents/fresh"] if replaced else []) + assert list_route.call_count == (2 if replaced else 1) + assert "enerateContent" in logged[-1]["litellm_params"]["api_base"] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize( + ("status_code", "message", "forgotten"), + [ + (400, "Invalid value at 'generation_config.temperature'", True), + ( + 400, + "Tool config, tools and system instruction should not be set in therequest when using cached content.", + True, + ), + ( + 400, + "CachedContent can not be used with GenerateContent request setting system_instruction, tools or tool_config.", + True, + ), + (404, "Requested entity was not found.", True), + (429, "Resource has been exhausted (e.g. check quota).", False), + (500, "Internal error encountered.", False), + ], +) +@pytest.mark.asyncio +async def test_error_that_does_not_name_a_missing_cache_is_not_retried( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + stream: bool, + status_code: int, + message: str, + forgotten: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + list_route: Final = respx_mock.get(url__regex=r".*/cachedContents.*").mock( + return_value=httpx.Response(200, json={}) + ) + respx_mock.post(url__regex=r".*/cachedContents$").mock( + side_effect=[ + httpx.Response(200, json=_created("cachedContents/first", _FAR_FUTURE)), + httpx.Response(200, json=_created("cachedContents/second", _FAR_FUTURE)), + ] + ) + generate_route: Final = respx_mock.post(url__regex=r".*:(stream)?[gG]enerateContent.*").mock( + side_effect=[_generated(stream), _rejected(status_code, message), _generated(stream)] + ) + + assert await _complete(sync_mode, stream) == "ok" + with pytest.raises(openai.APIStatusError): + await _complete(sync_mode, stream) + assert await _complete(sync_mode, stream) == "ok" + + third: Final = "cachedContents/second" if forgotten else "cachedContents/first" + assert [json.loads(call.request.content)["cachedContent"] for call in generate_route.calls] == [ + "cachedContents/first", + "cachedContents/first", + third, + ] + assert list_route.call_count == (2 if forgotten else 1) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.asyncio +async def test_rejected_caller_supplied_cached_content_is_not_sent_again( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + stream: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + list_route: Final = respx_mock.get(url__regex=r".*/cachedContents.*").mock( + return_value=httpx.Response(200, json={}) + ) + generate_route: Final = respx_mock.post(url__regex=r".*:(stream)?[gG]enerateContent.*").mock( + side_effect=[_rejected(404, "CachedContent not found"), _generated(stream)] + ) + + with pytest.raises(openai.APIStatusError): + await _complete(sync_mode, stream, cached_content="cachedContents/callers") + + assert [json.loads(call.request.content)["cachedContent"] for call in generate_route.calls] == [ + "cachedContents/callers" + ] + assert list_route.call_count == 0 + + +def _cut_off_stream() -> httpx.Response: + body: Final = { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "ok"}]}, + "finishReason": "CONTINUATION", + "continuationToken": "dG9rLTE=", + } + ], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 1, "totalTokenCount": 11}, + } + return httpx.Response(200, text=f"data: {json.dumps(body)}\n\n", headers={"content-type": "text/event-stream"}) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_cache_rejected_on_a_resume_hop_is_not_sent_again( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + list_route: Final = respx_mock.get(url__regex=r".*/cachedContents.*").mock( + return_value=httpx.Response(200, json={}) + ) + respx_mock.post(url__regex=r".*/cachedContents$").mock( + side_effect=[ + httpx.Response(200, json=_created("cachedContents/deleted", _FAR_FUTURE)), + httpx.Response(200, json=_created("cachedContents/fresh", _FAR_FUTURE)), + ] + ) + generate_route: Final = respx_mock.post(url__regex=r".*:streamGenerateContent.*").mock( + side_effect=[_cut_off_stream(), _rejected(404, "CachedContent not found"), _generated(True)] + ) + + await _complete(sync_mode, True, max_tokens=100_000) + await _complete(sync_mode, True, max_tokens=100_000) + + sent: Final = [json.loads(call.request.content)["cachedContent"] for call in generate_route.calls] + assert sent == ["cachedContents/deleted", "cachedContents/deleted", "cachedContents/fresh"] + assert list_route.call_count == 2 From d776277647c4c6f2ab7f576222d00e14e2927e95 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:59:37 +0000 Subject: [PATCH 092/124] fix(responses): count billed background responses toward the creator's project and per-model budgets (#47504) * test(responses): billed background responses count toward the creator's project and model budgets * fix(responses): carry the creator's project and per-model budgets on the billed background response * fix(responses): count a key's budget tier toward background per-model budgets * fix(responses): bill a background response's project only while the key stays on its team * revert(responses): keep billing the creating key's project after a team move A key can only keep or detach its project, so the billed project always pairs with the team it was created under. This reverts commit ded4f084c5. * fix(responses): bill a background response under the model name the client sent, with its project alias * fix(responses): name the team's public model as a billed background response's model group The model group lookup now passes the creating key's team, so a team model or team wildcard bills under the public name a normal call logs and per-model budgets on that name move --- .../common_utils/check_responses_cost.py | 26 +++ .../managed_object_attribution.py | 55 ++++- .../test_background_response_billed_once.py | 54 +++++ .../common_utils/test_check_responses_cost.py | 209 +++++++++++++++++- 4 files changed, 330 insertions(+), 14 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 00779bfe8ec..1fa461d7ab7 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -59,6 +59,19 @@ if TYPE_CHECKING: from litellm.router import Router _STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) + + +class _RoutedModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: str | None = None + + +class _RoutedDeployment(BaseModel): + model_config = ConfigDict(frozen=True) + model_info: _RoutedModelInfo = _RoutedModelInfo() + + +_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...]) _RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses" @@ -314,10 +327,23 @@ class CheckResponsesCost: INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, **target.model_metadata, **(_deployment_metadata(deployment) if deployment is not None else {}), + **self._requested_model_group(target, job.team_id), } ), ) + def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]: + requested: Final = target.model_metadata.get("model_group") + if target.deployment is None or not isinstance(requested, str): + return MappingProxyType({}) + routed: Final = _ROUTED_DEPLOYMENTS.validate_python( + self.llm_router.get_model_list(model_name=requested, team_id=team_id) or () + ) + serving_id: Final = target.deployment.model_info.id + return MappingProxyType( + {"model_group": requested} if any(row.model_info.id == serving_id for row in routed) else {} + ) + async def _bill_once( self, job: ManagedObjectRow, diff --git a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py index 3c7735b53c0..16b1b0e973c 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py @@ -107,8 +107,10 @@ class ManagedObjectCreatorAttribution: user's alias, because a resolvable name is more useful on the spend row than a null. user_api_key_org_id is resolved too: the spend update writer reads it off this metadata to increment organization spend, so leaving it out drops the cost from org accounting. - A background response also carries its end user and the key, team, and user budgets, the - way the read that billed it before did, and never names a key-scoped creator as its user. + A background response also carries its end user, the creating key's project and its alias, + and the key, team, and user budgets and per-model budgets, the way the read that billed it + before did, and never names a key-scoped creator as its user. A key that sets no budget of + its own takes its budget tier's, the same rule auth applies. """ def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None: @@ -126,9 +128,23 @@ class ManagedObjectCreatorAttribution: "user_api_key_user_id": _creator_user_id(job.created_by), **_shared_metadata(job, rows), **({"user_api_key_end_user_id": end_user_id} if end_user_id else {}), - **_budget_fields(rows.key, "user_api_key_spend", "user_api_key_max_budget"), - **_budget_fields(rows.team, "user_api_key_team_spend", "user_api_key_team_max_budget"), - **_budget_fields(rows.user, "user_api_key_user_spend", "user_api_key_user_max_budget"), + "user_api_key_project_id": getattr(rows.key, "project_id", None), + "user_api_key_project_alias": getattr( + getattr(rows.key, "litellm_project_table", None), "project_alias", None + ), + **_key_budget_fields(rows.key), + **_budget_fields( + rows.team, + "user_api_key_team_spend", + "user_api_key_team_max_budget", + "user_api_key_team_model_max_budget", + ), + **_budget_fields( + rows.user, + "user_api_key_user_spend", + "user_api_key_user_max_budget", + "user_api_key_user_model_max_budget", + ), } async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows: @@ -155,7 +171,9 @@ class ManagedObjectCreatorAttribution: if not api_key or _is_cli_session_key(api_key, created_by): return None try: - return await _token_table(self._prisma_client).find_unique(where={"token": api_key}) + return await _token_table(self._prisma_client).find_unique( + where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True} + ) except Exception as e: verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}") return None @@ -212,7 +230,28 @@ def _organization_id(row: object) -> str | None: return cast("str | None", getattr(row, "organization_id", None)) if row is not None else None -def _budget_fields(row: object, spend_key: str, max_budget_key: str) -> Mapping[str, object]: +def _budget_fields(row: object, spend_key: str, max_budget_key: str, model_max_budget_key: str) -> Mapping[str, object]: if row is None: return {} - return {spend_key: getattr(row, "spend", None), max_budget_key: getattr(row, "max_budget", None)} + return { + spend_key: getattr(row, "spend", None), + max_budget_key: getattr(row, "max_budget", None), + model_max_budget_key: getattr(row, "model_max_budget", None), + } + + +def _key_budget_fields(key: object) -> Mapping[str, object]: + if key is None: + return {} + tier: Final = getattr(key, "litellm_budget_table", None) + return { + "user_api_key_spend": getattr(key, "spend", None), + "user_api_key_max_budget": _own_or_tier_budget(key, tier, "max_budget"), + "user_api_key_model_max_budget": _own_or_tier_budget(key, tier, "model_max_budget"), + } + + +def _own_or_tier_budget(key: object, tier: object, field: str) -> object: + own: Final = getattr(key, field, None) + tier_value: Final = getattr(tier, field, None) + return own if tier_value is None or own not in (None, {}) else tier_value diff --git a/tests/integration/spend/test_background_response_billed_once.py b/tests/integration/spend/test_background_response_billed_once.py index 8a9818a7532..6d481a98fd0 100644 --- a/tests/integration/spend/test_background_response_billed_once.py +++ b/tests/integration/spend/test_background_response_billed_once.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from typing import Final +import httpx import openai import pytest from integration._support.client import Gateway, Scenario, eventually, held, list_value, object_value, string_value @@ -611,6 +612,59 @@ def test_a_background_response_bills_the_end_user_who_created_it_once(gateway: G assert _charged_end_users(upstream.model_id) == [end_user] +def _project_spend(project_id: str) -> float: + rows: Final = read_rows('SELECT spend FROM "LiteLLM_ProjectTable" WHERE project_id = %s', (project_id,)) + return number(rows[0]["spend"]) + + +def _next_create(scenario: Scenario, key: str, model: str) -> httpx.Response: + response: Final = scenario.gateway.request( + "POST", "/v1/responses", {"model": model, "input": PROMPT, "background": True, "store": False}, key=key + ) + if response.status_code == 200: + scenario.cleanups.callback(forget_row, string_value(object_value(response.json())["id"])) + return response + + +def _key_budget(scenario: Scenario, budget_owner: str, model_budget: JsonValue) -> Mapping[str, JsonValue]: + if budget_owner == "key": + return {"model_max_budget": model_budget} + if budget_owner == "user": + return {"user_id": scenario.user(model_max_budget=model_budget)} + if budget_owner == "budget_tier": + return {"budget_id": scenario.budget(model_max_budget=model_budget)} + return {} + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("budget_owner", ["key", "team", "user", "budget_tier"]) +def test_a_billed_background_response_counts_toward_its_creators_project_and_model_budget( + gateway: Gateway, budget_owner: str +) -> None: + with gateway.scenario() as scenario: + upstream: Final = scripted_upstream(scenario, background_routes("completed")) + model_budget: Final[JsonValue] = {upstream.model: {"max_budget": PRICE / 2, "budget_duration": "1d"}} + team_id: Final = scenario.team(**({"model_max_budget": model_budget} if budget_owner == "team" else {})) + project_id: Final = scenario.project(team_id) + key: Final = scenario.key( + team_id=team_id, project_id=project_id, **_key_budget(scenario, budget_owner, model_budget) + ) + response_id: Final = create_background_response(scenario, key, upstream.model) + read: Final = read_response(gateway, key, response_id) + assert (read,) in (settling_reads("completed", 1), free_reads("completed", 1)) + eventually( + lambda: (key_spend(key), _project_spend(project_id)), + lambda spends: spends[0] > 0 and costs(spends[0])(spends[1]), + seconds=SPEND_LANDING_SECONDS, + ) + refused: Final = eventually( + lambda: _next_create(scenario, key, upstream.model), + lambda response: response.status_code != 200, + seconds=SPEND_LANDING_SECONDS, + ) + assert f"exceeded budget for model={upstream.model}" in refused.text, refused.text + + @pytest.mark.timeout(240) def test_the_read_that_settles_a_background_response_splits_its_charge_into_input_and_output( gateway: Gateway, diff --git a/tests/unit/proxy/common_utils/test_check_responses_cost.py b/tests/unit/proxy/common_utils/test_check_responses_cost.py index f606658fa09..d78c3ef6185 100644 --- a/tests/unit/proxy/common_utils/test_check_responses_cost.py +++ b/tests/unit/proxy/common_utils/test_check_responses_cost.py @@ -7,7 +7,7 @@ import dataclasses from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1217,8 +1217,6 @@ class TestCheckResponsesCost: """The poll's billing event is the one billed event of a background job, so its spend row must name the key, team, org, user, and tags of the create it settles, read off the managed object row, never the poller itself.""" - from types import SimpleNamespace - mock_job = MagicMock() mock_job.unified_object_id = "resp_test_attributed" mock_job.id = "job-attributed" @@ -1494,6 +1492,122 @@ async def test_the_billing_event_carries_the_creators_end_user(): assert billed.metadata["user_api_key_end_user_id"] == "customer-42" +@pytest.mark.asyncio +async def test_the_billing_event_carries_the_creating_keys_project_and_every_creator_model_budget(): + key_budget: Final = {"gpt-5": {"max_budget": 1.0, "budget_duration": "1d"}} + team_budget: Final = {"gpt-5": {"max_budget": 2.0, "budget_duration": "7d"}} + user_budget: Final = {"gpt-5": {"max_budget": 3.0, "budget_duration": "30d"}} + table: Final = _InMemoryManagedObjectTable( + rows=( + _StoredResponse( + id="job-budgets", + unified_object_id="resp_budgets", + status="queued", + created_by="alice", + api_key="hash-alice", + team_id="team-alpha", + ), + ) + ) + billed_logger: Final = _RecordingBilledLogger() + poller: Final = _poller_over(table, billed_logger) + poller.prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=SimpleNamespace( + key_alias=None, + organization_id=None, + spend=0.0, + max_budget=None, + project_id="project-1", + litellm_project_table=SimpleNamespace(project_alias="project-one"), + model_max_budget=key_budget, + ) + ) + poller.prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=SimpleNamespace( + team_alias=None, organization_id=None, spend=0.0, max_budget=None, model_max_budget=team_budget + ) + ) + poller.prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=SimpleNamespace( + user_email=None, user_alias=None, spend=0.0, max_budget=None, model_max_budget=user_budget + ) + ) + + await poller.bill_if_finished(response_id="resp_budgets", response=_terminal_response()) + + (billed,) = billed_logger.billed + assert { + field: billed.metadata.get(field) + for field in ( + "user_api_key_project_id", + "user_api_key_project_alias", + "user_api_key_model_max_budget", + "user_api_key_team_model_max_budget", + "user_api_key_user_model_max_budget", + ) + } == { + "user_api_key_project_id": "project-1", + "user_api_key_project_alias": "project-one", + "user_api_key_model_max_budget": key_budget, + "user_api_key_team_model_max_budget": team_budget, + "user_api_key_user_model_max_budget": user_budget, + } + + +_TIER_MODEL_BUDGET: Final = {"gpt-5": {"max_budget": 4.0, "budget_duration": "1d"}} +_KEY_MODEL_BUDGET: Final = {"gpt-5": {"max_budget": 1.0, "budget_duration": "1d"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_max_budget", "key_model_max_budget", "billed_max_budget", "billed_model_max_budget"), + [ + (None, {}, 5.0, _TIER_MODEL_BUDGET), + (None, None, 5.0, _TIER_MODEL_BUDGET), + (2.0, _KEY_MODEL_BUDGET, 2.0, _KEY_MODEL_BUDGET), + ], + ids=["key_with_an_empty_budget_map", "key_with_no_budget_map", "key_with_its_own_budgets"], +) +async def test_the_billing_event_takes_the_keys_budget_tier_only_where_the_key_sets_no_budget_of_its_own( + key_max_budget: float | None, + key_model_max_budget: dict[str, object] | None, + billed_max_budget: float, + billed_model_max_budget: dict[str, object], +): + table: Final = _InMemoryManagedObjectTable( + rows=( + _StoredResponse( + id="job-tier", + unified_object_id="resp_tier", + status="queued", + created_by="key:hash-tier", + api_key="hash-tier", + ), + ) + ) + billed_logger: Final = _RecordingBilledLogger() + poller: Final = _poller_over(table, billed_logger) + poller.prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=SimpleNamespace( + key_alias=None, + organization_id=None, + spend=0.0, + max_budget=key_max_budget, + project_id=None, + model_max_budget=key_model_max_budget, + litellm_budget_table=SimpleNamespace(max_budget=5.0, model_max_budget=_TIER_MODEL_BUDGET), + ) + ) + + await poller.bill_if_finished(response_id="resp_tier", response=_terminal_response()) + + (billed,) = billed_logger.billed + assert (billed.metadata.get("user_api_key_max_budget"), billed.metadata.get("user_api_key_model_max_budget")) == ( + billed_max_budget, + billed_model_max_budget, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("created_by", "billed_user_id"), @@ -1599,6 +1713,10 @@ async def _bill_through_a_priced_deployment( read: ResponsesAPIResponse, litellm_params: Mapping[str, object] = _PRICED_LITELLM_PARAMS, model_info: Mapping[str, object] = MappingProxyType({}), + model_name: str = "priced-gpt", + requested_model: str = "priced-gpt", + model_group_alias: Mapping[str, str] = MappingProxyType({}), + team_id: str | None = None, ) -> _PricedBilling: import litellm from litellm_enterprise.proxy.common_utils.check_responses_cost import CheckResponsesCost @@ -1610,7 +1728,16 @@ async def _bill_through_a_priced_deployment( custom_llm_provider="openai", model_id="deployment-custom", response_id="resp_upstream" ) table: Final = _InMemoryManagedObjectTable( - rows=(_StoredResponse(id="job-priced", unified_object_id=response_id, status="queued", api_key="hash-alice"),) + rows=( + _StoredResponse( + id="job-priced", + unified_object_id=response_id, + status="queued", + api_key="hash-alice", + file_object={"model": requested_model}, + team_id=team_id, + ), + ) ) prisma_client: Final = MagicMock() prisma_client.db.litellm_managedobjecttable = table @@ -1620,11 +1747,12 @@ async def _bill_through_a_priced_deployment( router: Final = litellm.Router( model_list=[ { - "model_name": "priced-gpt", + "model_name": model_name, "litellm_params": dict(litellm_params), "model_info": {"id": "deployment-custom", **model_info}, } - ] + ], + model_group_alias=dict(model_group_alias), ) charge: Final = await CheckResponsesCost( proxy_logging_obj=MagicMock(), prisma_client=prisma_client, llm_router=router @@ -1657,6 +1785,75 @@ async def test_the_billing_event_prices_the_fetched_usage_at_the_deployment_rate assert billing.table.statuses == {"job-priced": "completed"} +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_name", "deployment_model", "requested_model", "model_group_alias", "model_info", "model_group"), + [ + ("openai/*", "openai/*", "openai/gpt-5.6", {}, {}, "openai/gpt-5.6"), + ("priced-gpt", "openai/gpt-5.6", "priced-alias", {"priced-alias": "priced-gpt"}, {}, "priced-alias"), + ("priced-gpt", "openai/gpt-5.6", "gpt-5.6-2026-01-01", {}, {}, "priced-gpt"), + ( + "model_name_team-1_abc", + "openai/gpt-5.6", + "team-gpt", + {}, + {"team_id": "team-1", "team_public_model_name": "team-gpt"}, + "team-gpt", + ), + ( + "model_name_team-1_wild", + "openai/*", + "openai/gpt-5.6", + {}, + {"team_id": "team-1", "team_public_model_name": "openai/*"}, + "openai/gpt-5.6", + ), + ( + "model_name_team-2_abc", + "openai/gpt-5.6", + "team-gpt", + {}, + {"team_id": "team-2", "team_public_model_name": "team-gpt"}, + "model_name_team-2_abc", + ), + ], + ids=[ + "wildcard_route", + "model_group_alias", + "fallback_kept_the_provider_model_name", + "team_model", + "team_wildcard", + "another_teams_model", + ], +) +async def test_the_billing_event_names_the_requested_model_as_its_model_group_only_while_it_routes_to_the_deployment( + monkeypatch: pytest.MonkeyPatch, + model_name: str, + deployment_model: str, + requested_model: str, + model_group_alias: Mapping[str, str], + model_info: Mapping[str, object], + model_group: str, +): + import litellm + + capture: Final = _spend_capture_logger() + monkeypatch.setattr(litellm, "_async_success_callback", [capture]) + + await _bill_through_a_priced_deployment( + _terminal_response(), + litellm_params={**_PRICED_LITELLM_PARAMS, "model": deployment_model}, + model_info=model_info, + model_name=model_name, + requested_model=requested_model, + model_group_alias=model_group_alias, + team_id="team-1", + ) + + (payload,) = capture.payloads + assert payload["model_group"] == model_group + + @pytest.mark.asyncio @pytest.mark.parametrize("base_model_in", ["litellm_params", "model_info"]) async def test_the_billing_event_prices_the_usage_at_the_deployments_base_model( From dd20bb0ae9bd8f59ce4d34aadeac62c4cbda1b9f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:01:45 +0000 Subject: [PATCH 093/124] fix(bedrock): apply the 16 maxTokens floor to Kimi K3 on Converse (#43875) * fix(bedrock): apply the 16 maxTokens floor to Kimi K3 on Converse Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): use lane AWS credentials for the Kimi K3 messages test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): bypass the proxy cache and declare test metadata for the Kimi K3 probe --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 2 +- .../coverage_registry/llm_conversational.yaml | 1 + .../test_bedrock_kimi_k3_messages_e2e.py | 71 +++++++++++++++++++ .../chat/test_converse_transformation.py | 10 +++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/llm_translation/test_bedrock_kimi_k3_messages_e2e.py diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 9deed66d647..b505ae47c61 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -385,7 +385,7 @@ class AmazonConverseConfig(BaseConfig): @staticmethod def _requires_min_max_tokens(model: str) -> bool: - return re.search(r"openai\.gpt-\d|xai\.grok-", model) is not None + return re.search(r"openai\.gpt-\d|xai\.grok-|moonshotai\.kimi-k3", model) is not None def _is_nova_2_model(self, model: str) -> bool: """ diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index b11ae907cfb..c40aeade1c1 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -68,6 +68,7 @@ - {id: llm.messages.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Messages API"} - {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"} - {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"} +- {id: llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works_for_kimi_k3_max_tokens_1], source: "llm_translation/test_bedrock_kimi_k3_messages_e2e.py", rationale: "Bedrock Converse accepts max_tokens=1 on the Anthropic Messages surface after LiteLLM applies the model minimum", fail_before_fix: proven} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven} - {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven} diff --git a/tests/e2e/llm_translation/test_bedrock_kimi_k3_messages_e2e.py b/tests/e2e/llm_translation/test_bedrock_kimi_k3_messages_e2e.py new file mode 100644 index 00000000000..b381fd0d9e1 --- /dev/null +++ b/tests/e2e/llm_translation/test_bedrock_kimi_k3_messages_e2e.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from anthropic.types import Message, MessageParam +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import NO_PROXY_CACHE, SdkClients + +pytestmark = pytest.mark.e2e + +BEDROCK_MODEL: Final = "bedrock/us.moonshotai.kimi-k3" +MODEL_NAME: Final = "claude-kimi-k3" +MODEL_SWITCH_BETA: Final = ( + "claude-code-20250219,context-1m-2025-08-07,interleaved-thinking-2025-05-14," + "redact-thinking-2026-02-12,context-management-2025-06-27," + "prompt-caching-scope-2026-01-05,mid-conversation-system-2026-04-07" +) + + +def _register_model(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: + model_id: Final = proxy.create_model( + MODEL_NAME, + LiteLLMParamsBody( + model=BEDROCK_MODEL, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + drop_params=True, + ), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return MODEL_NAME, resources.key(models=[MODEL_NAME]) + + +class TestBedrockKimiK3Messages: + @pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.MESSAGES, + providers=(Provider.BEDROCK,), + models=(BEDROCK_MODEL,), + mode=Mode.NONSTREAM, + ) + ) + def test_model_switch_probe_max_tokens_1_returns_message( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register_model(proxy, resources) + client: Final = sdk.anthropic(key) + message_param: Final[MessageParam] = { + "role": "user", + "content": [{"type": "text", "text": "Hi", "cache_control": {"type": "ephemeral"}}], + } + message: Final = client.messages.create( + model=model, + max_tokens=1, + stream=False, + system="You are Claude Code, an assistant for software development.", + messages=[message_param], + extra_headers={"anthropic-beta": MODEL_SWITCH_BETA}, + extra_query={"beta": "true"}, + extra_body=NO_PROXY_CACHE, + ) + + assert isinstance(message, Message) + assert message.role == "assistant" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index f0dd40d4bc9..1fb5542c964 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -896,6 +896,16 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto(): ("us.xai.grok-4.6", "max_tokens", 1, 16), ("global.xai.grok-4.6", "max_completion_tokens", 1, 16), ("us.xai.grok-4.6", "max_tokens", 32, 32), + ("us.moonshotai.kimi-k3", "max_tokens", 1, 16), + ("global.moonshotai.kimi-k3", "max_completion_tokens", 1, 16), + ("moonshotai.kimi-k3", "max_tokens", 64, 64), + ( + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.moonshotai.kimi-k3", + "max_tokens", + 1, + 16, + ), + ("moonshotai.kimi-k2.5", "max_tokens", 1, 1), ("anthropic.claude-sonnet-4-5-20250929-v1:0", "max_tokens", 1, 1), ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.openai.gpt-6-astra", "max_tokens", 1, 16), ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.xai.grok-4.6", "max_tokens", 1, 16), From 4818b767d8b4c4e5fa9885ab953d6f5cfe7c7ca5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:51:05 -0700 Subject: [PATCH 094/124] fix(cost-map): registry audit 2026-10-20, add volcengine doubao-seed-2-1 260915 rows and undated azure data zone rows (#47561) * fix(cost-map): add volcengine doubao-seed-2-1-pro-260915 and doubao-seed-2-1-lite-260915 Pro row absorbed from #47372, lite row added from the same Volcengine Ark pricing and model list docs Co-authored-by: donghongfei <1247543758@qq.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost-map): cover volcengine doubao-seed-2-1-pro-260915 row Co-authored-by: donghongfei <1247543758@qq.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): add undated azure/eu and azure/us rows for o1, o3, o3-mini, o4-mini, gpt-4o, gpt-4o-mini Absorbed from #47499 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: donghongfei <1247543758@qq.com> --- ...odel_prices_and_context_window_backup.json | 269 ++++++++++++++++++ model_prices_and_context_window.json | 269 ++++++++++++++++++ tests/unit/test_cost_calculator.py | 34 +++ .../test_model_prices_and_context_window.py | 23 ++ 4 files changed, 595 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8b3771033a5..903210c57e5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4916,6 +4916,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/eu/gpt-4o": { + "deprecation_date": "2027-04-14", + "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, @@ -4936,6 +4956,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/eu/gpt-4o-mini": { + "cache_read_input_token_cost": 8.3e-08, + "deprecation_date": "2027-04-14", + "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", @@ -5247,6 +5286,25 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 2.75e-09 }, + "azure/eu/o1": { + "cache_read_input_token_cost": 8.25e-06, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", @@ -5283,6 +5341,24 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/eu/o3-mini": { + "cache_read_input_token_cost": 6.05e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -11540,6 +11616,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-4o": { + "deprecation_date": "2027-04-14", + "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, @@ -11560,6 +11656,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-4o-mini": { + "cache_read_input_token_cost": 8.3e-08, + "deprecation_date": "2027-04-14", + "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", @@ -11871,6 +11986,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/o1": { + "cache_read_input_token_cost": 8.25e-06, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", @@ -11907,6 +12041,39 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/us/o3": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -11940,6 +12107,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/o3-mini": { + "cache_read_input_token_cost": 6.05e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -11958,6 +12143,27 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/us/o4-mini": { + "cache_read_input_token_cost": 3.03e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o4-mini-2025-04-16": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", @@ -61148,6 +61354,41 @@ "supports_tool_choice": false, "supports_vision": true }, + "volcengine/doubao-seed-2-1-pro-260915": { + "cache_read_input_token_cost": 1.725e-07, + "input_cost_per_token": 8.625e-07, + "litellm_provider": "volcengine", + "max_input_tokens": 1024000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4.3125e-06, + "source": "https://www.volcengine.com/docs/82379/1544106", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "volcengine/doubao-seed-2-1-lite-260915": { + "cache_read_input_audio_token_cost": 3.45e-07, + "cache_read_input_token_cost": 2.3e-08, + "input_cost_per_audio_token": 1.725e-06, + "input_cost_per_token": 1.15e-07, + "litellm_provider": "volcengine", + "max_input_tokens": 1024000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 3.88126e-07, + "source": "https://www.volcengine.com/docs/82379/1544106", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true + }, "volcengine/doubao-seed-2-1-turbo-260628": { "cache_read_input_token_cost": 8.625e-08, "input_cost_per_token": 4.3125e-07, @@ -70141,6 +70382,20 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, + "azure/eu/o3": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -70164,6 +70419,20 @@ "output_cost_per_token": 4.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, + "azure/eu/o4-mini": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 3.03e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "azure/eu/o4-mini-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 3.03e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8b3771033a5..903210c57e5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4916,6 +4916,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/eu/gpt-4o": { + "deprecation_date": "2027-04-14", + "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, @@ -4936,6 +4956,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/eu/gpt-4o-mini": { + "cache_read_input_token_cost": 8.3e-08, + "deprecation_date": "2027-04-14", + "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", @@ -5247,6 +5286,25 @@ "supports_vision": true, "cache_read_input_token_cost_batches": 2.75e-09 }, + "azure/eu/o1": { + "cache_read_input_token_cost": 8.25e-06, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", @@ -5283,6 +5341,24 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/eu/o3-mini": { + "cache_read_input_token_cost": 6.05e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -11540,6 +11616,26 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-4o": { + "deprecation_date": "2027-04-14", + "cache_creation_input_token_cost": 1.38e-06, + "cache_read_input_token_cost": 1.375e-06, + "input_cost_per_token": 2.75e-06, + "input_cost_per_token_batches": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "output_cost_per_token_batches": 5.5e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-2024-11-20": { "deprecation_date": "2027-04-14", "cache_creation_input_token_cost": 1.38e-06, @@ -11560,6 +11656,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/gpt-4o-mini": { + "cache_read_input_token_cost": 8.3e-08, + "deprecation_date": "2027-04-14", + "input_cost_per_token": 1.65e-07, + "input_cost_per_token_batches": 8.3e-08, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 6.6e-07, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 8.3e-08, "deprecation_date": "2027-04-14", @@ -11871,6 +11986,25 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/o1": { + "cache_read_input_token_cost": 8.25e-06, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.65e-05, + "input_cost_per_token_batches": 8.25e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.6e-05, + "output_cost_per_token_batches": 3.3e-05, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "deprecation_date": "2026-11-19", @@ -11907,6 +12041,39 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/us/o3": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -11940,6 +12107,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure/us/o3-mini": { + "cache_read_input_token_cost": 6.05e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -11958,6 +12143,27 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/us/o4-mini": { + "cache_read_input_token_cost": 3.03e-07, + "deprecation_date": "2026-11-19", + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o4-mini-2025-04-16": { "cache_read_input_token_cost": 3.03e-07, "deprecation_date": "2026-11-19", @@ -61148,6 +61354,41 @@ "supports_tool_choice": false, "supports_vision": true }, + "volcengine/doubao-seed-2-1-pro-260915": { + "cache_read_input_token_cost": 1.725e-07, + "input_cost_per_token": 8.625e-07, + "litellm_provider": "volcengine", + "max_input_tokens": 1024000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 4.3125e-06, + "source": "https://www.volcengine.com/docs/82379/1544106", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true + }, + "volcengine/doubao-seed-2-1-lite-260915": { + "cache_read_input_audio_token_cost": 3.45e-07, + "cache_read_input_token_cost": 2.3e-08, + "input_cost_per_audio_token": 1.725e-06, + "input_cost_per_token": 1.15e-07, + "litellm_provider": "volcengine", + "max_input_tokens": 1024000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 3.88126e-07, + "source": "https://www.volcengine.com/docs/82379/1544106", + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true + }, "volcengine/doubao-seed-2-1-turbo-260628": { "cache_read_input_token_cost": 8.625e-08, "input_cost_per_token": 4.3125e-07, @@ -70141,6 +70382,20 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, + "azure/eu/o3": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -70164,6 +70419,20 @@ "output_cost_per_token": 4.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, + "azure/eu/o4-mini": { + "deprecation_date": "2026-11-19", + "cache_read_input_token_cost": 3.03e-07, + "input_cost_per_token": 1.21e-06, + "input_cost_per_token_batches": 6.05e-07, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "output_cost_per_token_batches": 2.42e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + }, "azure/eu/o4-mini-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 3.03e-07, diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 99119799ac0..fee4db586cd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -357,6 +357,40 @@ def test_completion_cost_prices_rule_only_response_model_at_the_azure_deployment assert cost == pytest.approx(usage.prompt_tokens * 0.0000111 + usage.completion_tokens * 0.0000222) +@pytest.mark.parametrize("region", ["eu", "us"]) +@pytest.mark.parametrize( + ("alias", "dated"), + [ + ("o1", "o1-2024-12-17"), + ("o3", "o3-2025-04-16"), + ("o3-mini", "o3-mini-2025-01-31"), + ("o4-mini", "o4-mini-2025-04-16"), + ("gpt-4o", "gpt-4o-2024-11-20"), + ("gpt-4o-mini", "gpt-4o-mini-2024-07-18"), + ], +) +def test_completion_cost_bills_undated_azure_data_zone_base_model_at_its_dated_rate( + _local_model_cost_map: None, region: str, alias: str, dated: str +) -> None: + """LIT-9812: a deployment with base_model azure/eu/o3 (undated, like the global azure/o3) must bill the EU + Data Zone rate of its latest dated row, not $0 and not the global rate""" + dated_info: Final = litellm.model_cost[f"azure/{region}/{dated}"] + usage: Final = Usage(prompt_tokens=17, completion_tokens=9, total_tokens=26) + response: Final = ModelResponse(model=dated, choices=[], usage=usage) + + cost: Final = completion_cost( + completion_response=response, + model="azure/my-data-zone-deployment", + custom_llm_provider="azure", + base_model=f"azure/{region}/{alias}", + ) + + assert cost == pytest.approx( + usage.prompt_tokens * dated_info["input_cost_per_token"] + + usage.completion_tokens * dated_info["output_cost_per_token"] + ) + + def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests( _local_model_cost_map: None, ) -> None: diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 395fe8c9750..20b46e27213 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -719,3 +719,26 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(path: Path): + """Volcengine Ark lists doubao-seed-2-1-pro-260915 at 6 / 30 CNY per million input / output tokens with + cache hits at 1.2 (ark.volcengine.com model detail and docs.volcengine.com/docs/82379/1544106, read + 2026-10-20), a 1024k context window and a 256k output cap. USD rows use the 6.9565 CNY/USD rate of the + sibling 260628 row.""" + rows: Mapping[str, object] = json.loads(path.read_text()) + entry = rows.get("volcengine/doubao-seed-2-1-pro-260915") + assert isinstance(entry, dict) + assert entry["litellm_provider"] == "volcengine" + assert entry["mode"] == "chat" + assert entry["max_input_tokens"] == 1024000 + assert entry["max_output_tokens"] == 256000 + assert entry["max_tokens"] == 256000 + # Volcengine Ark pricing https://www.volcengine.com/docs/82379/1544106 checked 2026-10-20 + assert entry["input_cost_per_token"] == pytest.approx(6.0 / 6.9565 / 1_000_000, rel=1e-4) + assert entry["output_cost_per_token"] == pytest.approx(30.0 / 6.9565 / 1_000_000, rel=1e-4) + assert entry["cache_read_input_token_cost"] == pytest.approx(1.2 / 6.9565 / 1_000_000, rel=1e-4) + sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] + assert isinstance(sibling, dict) + assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) From ea2a842d7d675bfd6387f438b02d09760eabb24a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 06:51:53 -0700 Subject: [PATCH 095/124] test(perf): set the seven new perf ceilings from 124 lane runs across v1.103.0 to v1.106.0 (#47527) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/cases/mixed_size_requests_2rps.py | 14 +++++++------- tests/e2e/perf/cases/small_bodies_5rps.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/e2e/perf/cases/mixed_size_requests_2rps.py b/tests/e2e/perf/cases/mixed_size_requests_2rps.py index 850658a15c5..d4f1fdddc9c 100644 --- a/tests/e2e/perf/cases/mixed_size_requests_2rps.py +++ b/tests/e2e/perf/cases/mixed_size_requests_2rps.py @@ -64,14 +64,14 @@ CASE: Final = PerformanceTestCase( Limit("unexpected_status_count", max=3), Limit("gateway_added_seconds", percentile=50, max=0.58), Limit("gateway_added_seconds", percentile=99, max=6.1), - Limit("litellm_overhead_seconds", percentile=50, max=0.05), - Limit("litellm_overhead_seconds", percentile=99, max=2.2), - Limit("litellm_response_seconds", percentile=50, max=1.2), + Limit("litellm_overhead_seconds", percentile=50, max=0.1), + Limit("litellm_overhead_seconds", percentile=99, max=3.4), + Limit("litellm_response_seconds", percentile=50, max=1.0), Limit("litellm_response_seconds", percentile=99, max=3.8), - Limit("prometheus_redis_latency_mean_seconds", max=0.003), - Limit("postgres_active_sessions", max=0.004), - Limit("postgres_slowest_statement_seconds", max=0.045), - Limit("postgres_peak_connections", max=30), + Limit("prometheus_redis_latency_mean_seconds", max=0.005), + Limit("postgres_active_sessions", max=0.016), + Limit("postgres_slowest_statement_seconds", max=0.25), + Limit("postgres_peak_connections", max=45), Limit("time_to_first_token_seconds", percentile=50, max=2.7), Limit("time_to_first_token_seconds", percentile=99, max=8.8), Limit("memory_growth_megabytes", max=600), diff --git a/tests/e2e/perf/cases/small_bodies_5rps.py b/tests/e2e/perf/cases/small_bodies_5rps.py index bc22a9a96c8..f93c64afba4 100644 --- a/tests/e2e/perf/cases/small_bodies_5rps.py +++ b/tests/e2e/perf/cases/small_bodies_5rps.py @@ -65,14 +65,14 @@ CASE: Final = PerformanceTestCase( limits=( Limit("unexpected_status_count", max=3), Limit("gateway_added_seconds", percentile=50, max=0.29), - Limit("litellm_overhead_seconds", percentile=50, max=0.02), - Limit("litellm_overhead_seconds", percentile=99, max=0.8), - Limit("litellm_response_seconds", percentile=50, max=0.8), - Limit("litellm_response_seconds", percentile=99, max=2.8), + Limit("litellm_overhead_seconds", percentile=50, max=0.012), + Limit("litellm_overhead_seconds", percentile=99, max=0.35), + Limit("litellm_response_seconds", percentile=50, max=1.0), + Limit("litellm_response_seconds", percentile=99, max=2.0), Limit("prometheus_redis_latency_mean_seconds", max=0.006), - Limit("postgres_active_sessions", max=0.006), - Limit("postgres_slowest_statement_seconds", max=0.035), - Limit("postgres_peak_connections", max=30), + Limit("postgres_active_sessions", max=0.01), + Limit("postgres_slowest_statement_seconds", max=0.025), + Limit("postgres_peak_connections", max=45), Limit("time_to_first_token_seconds", percentile=99, max=3.7), Limit("memory_growth_megabytes", max=100), Limit("postgres_calls_per_request", max=1.8), From 5145250d83dbf8e7113f09e626517b8f184fea7f Mon Sep 17 00:00:00 2001 From: Max Azatian <66359507+HardMax71@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:52:26 +0200 Subject: [PATCH 096/124] fix(langfuse_otel)!: send the completion start time (#45597) * fix(langfuse_otel): send the completion start time langfuse_otel never set langfuse.observation.completion_start_time, so Langfuse showed no time to first token for any call, streamed or not. The legacy langfuse callback already sends kwargs["completion_start_time"]. Send the same value as an ISO timestamp with an explicit offset. * fix(langfuse_otel): type kwargs and test against a real span Type the kwargs of _set_langfuse_specific_attributes as dict[str, Any], the declared type of model_call_details, so the new completion start time read adds no unknown-type errors. The tests now check a real SDK span instead of patching safe_set_attribute, and the parameter is typed. * fix(otel): send the completion start time from the v2 Langfuse mapper With LITELLM_OTEL_V2 the langfuse_otel spans come from LangfuseMapper, which had no completion start time either. Carry the payload's completionStartTime on LLMCallSpanData for streamed calls, the same calls that get time_to_first_chunk_seconds, and map it to langfuse.observation.completion_start_time. * fix(langfuse_otel): take the logging kwargs as a read-only Mapping The new kwargs annotation tripped LIT001. Both helpers only read the logging kwargs, so type them Mapping[str, Any] instead of dict. * fix(langfuse_otel): time /v1/messages streams from the call start and send the V2 completion start time like the default emitter * fix(langfuse_otel): time Gemini, Interactions and A2A streams from their first chunk and send the start time as JSON The native Gemini, Interactions and A2A streams never recorded a first chunk, so with completion_start_time now sent both emitters showed TTFT equal to latency on them. They now record it, and Gemini and Interactions take the call's start time like /v1/messages. Both emitters send the completion start time as a JSON string, the form the Langfuse SDK and the langfuse callback send, through one serializer. * fix(langfuse_otel): time A2A streams from the call start and record a cached replay's first event A2A streams took their start time when the iterator was built, so their latency left out the work before it. They now use the call's start time like every other route. Cached /v1/messages stream replays recorded no first event, so their time to first token fell back to the end time. The replay now records it on the first event it yields. The tests use the real Interactions config and typed logger hooks. * fix(a2a): time streamed A2A calls from before the agent card fetch and make the cached replay test clock-free * test(a2a): type the card fetch test and stop rebinding the seed helper's recorder * test(otel): expect completion_start_time and cache_hit in the B3 span keys --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/a2a_protocol/main.py | 4 +- litellm/a2a_protocol/streaming_iterator.py | 4 +- litellm/google_genai/streaming_iterator.py | 4 +- .../integrations/langfuse/langfuse_otel.py | 15 +++- .../langfuse/langfuse_otel_attributes.py | 5 ++ litellm/integrations/langfuse/langfuse_sdk.py | 3 +- litellm/integrations/otel/logger.py | 1 + litellm/integrations/otel/mappers/langfuse.py | 7 ++ litellm/integrations/otel/model/metadata.py | 2 + litellm/integrations/otel/model/payloads.py | 3 + litellm/interactions/streaming_iterator.py | 4 +- .../pass_through/messages/response_cache.py | 3 + .../messages/streaming_iterator.py | 2 +- litellm/types/integrations/langfuse_otel.py | 1 + ...test_arize_otel_v2_openinference_family.py | 3 + .../test_a2a_streaming_iterator.py | 54 ++++++++++++ tests/unit/a2a_protocol/test_main.py | 52 +++++++++++- .../test_google_genai_streaming_iterator.py | 84 +++++++++++++++++-- .../integrations/otel/test_langfuse_logger.py | 53 ++++++++++++ .../otel/test_otel_v2_vendor_mappers.py | 27 ++++++ tests/unit/integrations/test_langfuse_otel.py | 25 ++++++ .../test_interactions_streaming_iterator.py | 51 +++++++++++ .../messages/test_response_cache.py | 57 ++++++++++++- .../messages/test_streaming_iterator.py | 54 +++++++++++- 24 files changed, 497 insertions(+), 21 deletions(-) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index e5bd38bb7ed..f44e3f6a81b 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -602,9 +602,9 @@ def _build_streaming_logging_obj( litellm_params: dict[str, object] | None, metadata: dict[str, object] | None, proxy_server_request: dict[str, object] | None, + start_time: datetime.datetime, ) -> Logging: """Build logging object for streaming A2A requests.""" - start_time: Final = datetime.datetime.now() model: Final = f"a2a_agent/{agent_name}" logging_obj: Final = Logging( @@ -718,6 +718,7 @@ async def asend_message_streaming( if request is None: raise ValueError("request is required") + called_at: Final = datetime.datetime.now() _raw_logging_obj: Final = kwargs.get("litellm_logging_obj") logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None @@ -751,6 +752,7 @@ async def asend_message_streaming( litellm_params=litellm_params, metadata=metadata, proxy_server_request=proxy_server_request, + start_time=called_at, ) verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name) diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 0b649f62a66..328681b8dbe 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -41,7 +41,7 @@ class A2AStreamingIterator: self.request = request self.logging_obj = logging_obj self.agent_name = agent_name - self.start_time = datetime.now() + self.start_time = logging_obj.start_time # Collect chunks for token counting self.chunks: list[SendStreamingMessageResponse] = [] @@ -54,6 +54,8 @@ class A2AStreamingIterator: async def __anext__(self) -> "SendStreamingMessageResponse": try: chunk: Final = await self.stream.__anext__() + if self.logging_obj.completion_start_time is None: + self.logging_obj.update_completion_start_time(completion_start_time=datetime.now()) # Store chunk self.chunks.append(chunk) diff --git a/litellm/google_genai/streaming_iterator.py b/litellm/google_genai/streaming_iterator.py index 37d08dfb59a..a22c47cb1e7 100644 --- a/litellm/google_genai/streaming_iterator.py +++ b/litellm/google_genai/streaming_iterator.py @@ -71,7 +71,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator: ): self.litellm_logging_obj = litellm_logging_obj self.request_body = request_body - self.start_time = datetime.now() + self.start_time = litellm_logging_obj.start_time self.collected_chunks: list[bytes] = [] self.model = model self.custom_llm_provider = custom_llm_provider @@ -192,6 +192,8 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo async def __anext__(self): try: chunk: Final = await _anext_google_genai_sse_chunk(self.stream_iterator) + if self.litellm_logging_obj.completion_start_time is None: + self.litellm_logging_obj.update_completion_start_time(completion_start_time=datetime.now()) self.collected_chunks.append(chunk) return chunk except StopAsyncIteration: diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index ae63ab99929..f195cd97e3c 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,7 +1,7 @@ import base64 import json import os -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Optional @@ -9,6 +9,7 @@ from litellm._logging import verbose_logger from litellm.integrations.arize import _utils from litellm.integrations.langfuse.langfuse_otel_attributes import ( LangfuseLLMObsOTELAttributes, + serialize_langfuse_datetime, ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -55,7 +56,7 @@ class LangfuseOtelLogger(OpenTelemetry): LangfuseOtelLogger._set_langfuse_specific_attributes(span=span, kwargs=kwargs, response_obj=response_obj) @staticmethod - def _extract_langfuse_metadata(kwargs: dict) -> dict: + def _extract_langfuse_metadata(kwargs: Mapping[str, Any]) -> dict: """ Extracts Langfuse metadata from the standard LiteLLM kwargs structure. @@ -221,7 +222,7 @@ class LangfuseOtelLogger(OpenTelemetry): ) @staticmethod - def _set_langfuse_specific_attributes(span: Span, kwargs, response_obj): + def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj): """ Sets Langfuse specific metadata attributes onto the OTEL span. @@ -255,6 +256,14 @@ class LangfuseOtelLogger(OpenTelemetry): safe_dumps(messages), ) + completion_start_time: Final = kwargs.get("completion_start_time") + if isinstance(completion_start_time, datetime): + safe_set_attribute( + span, + LangfuseSpanAttributes.OBSERVATION_COMPLETION_START_TIME.value, + serialize_langfuse_datetime(completion_start_time), + ) + LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj) @staticmethod diff --git a/litellm/integrations/langfuse/langfuse_otel_attributes.py b/litellm/integrations/langfuse/langfuse_otel_attributes.py index 1eb3ce1a9c2..2873dba93ed 100644 --- a/litellm/integrations/langfuse/langfuse_otel_attributes.py +++ b/litellm/integrations/langfuse/langfuse_otel_attributes.py @@ -6,6 +6,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764 import json from collections.abc import Mapping +from datetime import datetime from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel @@ -29,6 +30,10 @@ if TYPE_CHECKING: from opentelemetry.trace import Span +def serialize_langfuse_datetime(value: datetime) -> str: + return json.dumps(value.astimezone().isoformat()) + + def get_output_content_by_type( response_obj: None | dict diff --git a/litellm/integrations/langfuse/langfuse_sdk.py b/litellm/integrations/langfuse/langfuse_sdk.py index 864c0697eb3..51b08cb72df 100644 --- a/litellm/integrations/langfuse/langfuse_sdk.py +++ b/litellm/integrations/langfuse/langfuse_sdk.py @@ -39,6 +39,7 @@ from pydantic import ConfigDict import litellm from litellm._logging import verbose_logger from litellm.integrations.langfuse.langfuse import PROMPT_CACHE_TTL_ENV, parse_langfuse_debug, whole_number +from litellm.integrations.langfuse.langfuse_otel_attributes import serialize_langfuse_datetime from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_httpx_client from litellm.types.llms.base import LiteLLMBaseModel @@ -168,7 +169,7 @@ def _string_or_none(value: object) -> str | None: def _serialize_datetime(value: object) -> str | None: """A datetime the way the SDK's ``EventSerializer`` sends one: a JSON string, naive values read as local time.""" if isinstance(value, datetime): - return safe_dumps(value.astimezone().isoformat()) + return serialize_langfuse_datetime(value) return _serialize(value) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 73448221acb..5092b248eac 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -594,6 +594,7 @@ class OpenTelemetryV2(CustomLogger): payload, capture_content=self._capture_span_content(), time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, + completion_start_seconds=call.completion_start_seconds, request_route=request_root_http_route(), request_purpose=call.purpose, trace=call.trace, diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 68860931b76..e60ea539188 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -12,8 +12,10 @@ the JSON-serialized payloads. ``trace_attributes`` maps the caller's trace contr import json from collections.abc import Callable +from datetime import datetime, timezone from typing import Final +from litellm.integrations.langfuse.langfuse_otel_attributes import serialize_langfuse_datetime from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( collect, @@ -45,6 +47,11 @@ class LangfuseMapper: "langfuse.observation.id": lambda d: d.identity.call_id or None, "langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None, "langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None, + "langfuse.observation.completion_start_time": lambda d: ( + serialize_langfuse_datetime(datetime.fromtimestamp(d.completion_start_seconds, tz=timezone.utc)) + if d.completion_start_seconds is not None + else None + ), } # Sub-tables folded into their respective JSON blobs. diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index a3151c24519..5d37894bb8f 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -258,6 +258,7 @@ class LLMCallEvent: # needs to be reasonable for a span that never gets closed (a leak). provisional_span_name: str time_to_first_chunk_seconds: float | None + completion_start_seconds: float | None trace: TraceControls session_id: str | None @@ -280,6 +281,7 @@ class LLMCallEvent: cache_hit=bool(payload.get("cache_hit")) if payload else False, # pyright: ignore[reportUnknownMemberType] # TypedDict .get on a str key is partially unknown provisional_span_name=f"{operation.value} {model}".strip(), time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs), + completion_start_seconds=_epoch_seconds(kwargs.get("completion_start_time")), trace=trace, session_id=caller_session_id(kwargs, trace), ) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index e15072c009b..d11505ce026 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -429,6 +429,7 @@ class LLMCallSpanData: choices_out: tuple[Mapping[str, object], ...] = () system_fingerprint: str | None = None time_to_first_chunk_seconds: float | None = None + completion_start_seconds: float | None = None # The requested output modality, set only on the routes that pin one (image # generation, speech, transcription, OCR), and the litellm route itself, which # keeps routes the convention folds into one operation distinguishable. @@ -450,6 +451,7 @@ class LLMCallSpanData: payload: StandardLoggingPayload, capture_content: bool = False, time_to_first_chunk_seconds: float | None = None, + completion_start_seconds: float | None = None, request_route: str | None = None, request_purpose: str | None = None, trace: TraceControls | None = None, @@ -503,6 +505,7 @@ class LLMCallSpanData: choices_out=choices_out if capture_content else (), system_fingerprint=as_str(response.get("system_fingerprint")), time_to_first_chunk_seconds=time_to_first_chunk_seconds, + completion_start_seconds=completion_start_seconds, output_type=resolve_output_type(call_type), call_type=call_type or None, request_route=request_route or context.identity.request_route, diff --git a/litellm/interactions/streaming_iterator.py b/litellm/interactions/streaming_iterator.py index 64b599edfe2..a2468cd894f 100644 --- a/litellm/interactions/streaming_iterator.py +++ b/litellm/interactions/streaming_iterator.py @@ -48,7 +48,7 @@ class BaseInteractionsAPIStreamingIterator: self.finished = False self.interactions_api_config = interactions_api_config self.completed_response: InteractionsAPIStreamingResponse | None = None - self.start_time = datetime.now() + self.start_time = logging_obj.start_time # set request kwargs self.litellm_metadata = litellm_metadata @@ -87,6 +87,8 @@ class BaseInteractionsAPIStreamingIterator: # Format as InteractionsAPIStreamingResponse if isinstance(parsed_chunk, dict): + if self.logging_obj.completion_start_time is None: + self.logging_obj.update_completion_start_time(completion_start_time=datetime.now()) streaming_response: Final = self.interactions_api_config.transform_streaming_response( model=self.model, parsed_chunk=parsed_chunk, diff --git a/litellm/llms/anthropic/pass_through/messages/response_cache.py b/litellm/llms/anthropic/pass_through/messages/response_cache.py index 4c5f47bc650..81d01f1437e 100644 --- a/litellm/llms/anthropic/pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/pass_through/messages/response_cache.py @@ -1,5 +1,6 @@ import re from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast @@ -153,6 +154,8 @@ class CachedAnthropicMessagesStreamIterator(BaseAnthropicMessagesStreamingIterat await self._handle_streaming_logging(chunks) raise StopAsyncIteration chunk: Final = self.chunks[self.current_index] + if self.completion_start_time is None: + self.completion_start_time = datetime.now() self.current_index += 1 return chunk diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py index 44db193ea83..bfa309fad83 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py @@ -506,7 +506,7 @@ class BaseAnthropicMessagesStreamingIterator: ): self.litellm_logging_obj = litellm_logging_obj self.request_body = request_body - self.start_time = datetime.now() + self.start_time = litellm_logging_obj.start_time self.completion_start_time: datetime | None = None async def _handle_streaming_logging(self, collected_chunks: list[bytes], *, stream_teardown: bool = False): diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 4bcd0448c3a..5647f28c757 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -29,6 +29,7 @@ class LangfuseSpanAttributes(str, Enum): # ---- Observation input/output ---- OBSERVATION_INPUT = "langfuse.observation.input" OBSERVATION_OUTPUT = "langfuse.observation.output" + OBSERVATION_COMPLETION_START_TIME = "langfuse.observation.completion_start_time" # ---- Trace-level metadata ---- TRACE_USER_ID = "user.id" diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_family.py b/tests/integration/observability/test_arize_otel_v2_openinference_family.py index 242b7bfe8c1..0b0c077cbee 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_family.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_family.py @@ -43,6 +43,7 @@ _GENAI_B3_KEYS_WITHOUT_BAGGAGE: Final = frozenset( "gen_ai.usage.prompt_tokens", "gen_ai.usage.total_tokens", "litellm.api_key.hash", + "litellm.cache_hit", "litellm.call_id", "litellm.call_type", "litellm.cost.discount_amount", @@ -85,6 +86,7 @@ _LANGFUSE_B3_KEYS: Final = frozenset( "gen_ai.usage.output_tokens", "gen_ai.usage.prompt_tokens", "gen_ai.usage.total_tokens", + "langfuse.observation.completion_start_time", "langfuse.observation.cost_details", "langfuse.observation.id", "langfuse.observation.input", @@ -94,6 +96,7 @@ _LANGFUSE_B3_KEYS: Final = frozenset( "langfuse.observation.type", "langfuse.observation.usage_details", "litellm.api_key.hash", + "litellm.cache_hit", "litellm.call_id", "litellm.call_type", "litellm.cost.discount_amount", diff --git a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py index 2e883e91fda..6d685674984 100644 --- a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py +++ b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py @@ -6,6 +6,8 @@ async_success_handler (cross-thread pydantic mutation segfaults pydantic-core). import asyncio import time +from datetime import datetime +from typing import Final from types import SimpleNamespace import pytest @@ -135,3 +137,55 @@ async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch): assert usage.prompt_tokens > 100_000 assert usage.completion_tokens > 100_000 assert_loop_stayed_free(took, lags) + + +class _SuccessRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.logged: list[tuple[datetime, object]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.logged.append((start_time, kwargs.get("completion_start_time"))) + + async def async_log_stream_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.logged.append((start_time, kwargs.get("completion_start_time"))) + + +@pytest.mark.asyncio +async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start(): + call_start: Final = datetime(2026, 10, 9, 12, 0, 0) + recorder: Final = _SuccessRecorder() + logging_obj: Final = LitellmLogging( + model="a2a/test-agent", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="a2a_send_message_streaming", + start_time=call_start, + litellm_call_id="a2a-first-chunk", + function_id="a2a-first-chunk", + dynamic_async_success_callbacks=[recorder], + ) + + async def _stream(): + yield _AgentChunk("Hel") + yield _AgentChunk("Hello") + + iterator: Final = A2AStreamingIterator( + stream=_stream(), + request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})), + logging_obj=logging_obj, + agent_name="test-agent", + ) + tasks_before: Final = asyncio.all_tasks() + + _ = await iterator.__anext__() + first_chunk_at: Final = logging_obj.completion_start_time + _ = [chunk async for chunk in iterator] + await asyncio.gather(*(asyncio.all_tasks() - tasks_before)) + + assert first_chunk_at is not None + assert recorder.logged == [(call_start, first_chunk_at)] diff --git a/tests/unit/a2a_protocol/test_main.py b/tests/unit/a2a_protocol/test_main.py index bab14dd26d6..75fd9eda75b 100644 --- a/tests/unit/a2a_protocol/test_main.py +++ b/tests/unit/a2a_protocol/test_main.py @@ -1,7 +1,9 @@ """Tests for litellm/a2a_protocol/main.py non-streaming send behavior.""" import asyncio +import datetime import json +from typing import Final import httpx import pytest @@ -155,6 +157,7 @@ def test_streaming_logging_obj_carries_call_type_into_model_call_details(): litellm_params=None, metadata=None, proxy_server_request=None, + start_time=datetime.datetime(2026, 10, 20, 12, 0, 0), ) assert logging_obj.model_call_details["call_type"] == "asend_message_streaming" @@ -352,6 +355,7 @@ async def _seed_shared_a2a_client( rpc_reply=_RPC_REPLY, rpc_stream_events=None, provider: str = httpxSpecialProvider.A2AProvider, + recorder: _RequestRecorder | None = None, ) -> _RequestRecorder: """Put the one A2A client the cache will hand out behind a mock transport. @@ -359,10 +363,12 @@ async def _seed_shared_a2a_client( it. The injected client is a real httpx.AsyncClient, so the merge of per-request headers over client defaults, which is what these tests are about, stays real. """ - recorder = _RequestRecorder(card=card, rpc_reply=rpc_reply, rpc_stream_events=rpc_stream_events) + active_recorder: Final = recorder or _RequestRecorder( + card=card, rpc_reply=rpc_reply, rpc_stream_events=rpc_stream_events + ) handler = AsyncHTTPHandler(timeout=DEFAULT_A2A_AGENT_TIMEOUT) owned_client = handler.client - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(recorder)) + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(active_recorder)) await owned_client.aclose() litellm.in_memory_llm_clients_cache.set_cache( @@ -374,8 +380,8 @@ async def _seed_shared_a2a_client( ) assert seeded is handler, "cache key drifted from get_async_httpx_client; these tests would test nothing" - recorder.client = handler.client - return recorder + active_recorder.client = handler.client + return active_recorder @pytest.fixture @@ -679,6 +685,43 @@ class _UsageRecorder(CustomLogger): self.logged.set() +class _CardServedAtRecorder(_RequestRecorder): + def __init__(self) -> None: + super().__init__(rpc_stream_events=_LANGGRAPH_STREAM_EVENTS) + self.card_served_at: datetime.datetime | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + if request.method == "GET" and self.card_served_at is None: + self.card_served_at = datetime.datetime.now() + return super().__call__(request) + + +@pytest.mark.asyncio +async def test_streamed_send_is_timed_from_before_the_agent_card_fetch( + isolated_client_cache: LLMClientCache, monkeypatch: pytest.MonkeyPatch +): + from litellm.a2a_protocol import main as a2a_main + + card_recorder: Final = _CardServedAtRecorder() + await _seed_shared_a2a_client(recorder=card_recorder) + logger: Final = _UsageRecorder() + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "success_callback", [logger]) + monkeypatch.setattr(litellm, "_async_success_callback", [logger]) + request: Final = SendStreamingMessageRequest( + id="reply", + params=MessageSendParams( + message={"messageId": "m-user", "role": "user", "parts": [{"kind": "text", "text": "hi"}]} + ), + ) + + _ = [chunk async for chunk in a2a_main.asend_message_streaming(request=request, api_base="http://127.0.0.1:9")] + await asyncio.wait_for(logger.logged.wait(), timeout=10) + + assert card_recorder.card_served_at is not None + assert logger.payload["startTime"] <= card_recorder.card_served_at.timestamp() + + @pytest.mark.asyncio async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch): from tests.large_text import text @@ -742,6 +785,7 @@ def test_streaming_logging_obj_keeps_agent_credentials_out_of_logging_params(): }, metadata={"user_api_key": "hashed"}, proxy_server_request={"url": "http://localhost:4000"}, + start_time=datetime.datetime(2026, 10, 20, 12, 0, 0), ) expected = { diff --git a/tests/unit/google_genai/test_google_genai_streaming_iterator.py b/tests/unit/google_genai/test_google_genai_streaming_iterator.py index e8ec2848233..120c99b6e42 100644 --- a/tests/unit/google_genai/test_google_genai_streaming_iterator.py +++ b/tests/unit/google_genai/test_google_genai_streaming_iterator.py @@ -1,16 +1,55 @@ +import asyncio import json +from datetime import datetime +from typing import Final from unittest.mock import MagicMock +import httpx import pytest from litellm.google_genai.streaming_iterator import ( AsyncGoogleGenAIGenerateContentStreamingIterator, GoogleGenAIGenerateContentStreamingIterator, ) +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType +class _SuccessRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.logged: list[tuple[datetime, object]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.logged.append((start_time, kwargs.get("completion_start_time"))) + + +def _logging_obj( + start_time: datetime = datetime(2026, 10, 9, 12, 0, 0), recorder: _SuccessRecorder | None = None +) -> LiteLLMLoggingObj: + logging_obj: Final = LiteLLMLoggingObj( + model="gemini-3.1-flash", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="agenerate_content_stream", + start_time=start_time, + litellm_call_id="google-genai-stream", + function_id="google-genai-stream", + dynamic_async_success_callbacks=[recorder] if recorder else None, + ) + logging_obj.update_environment_variables( + model="gemini-3.1-flash", + user="", + optional_params={}, + litellm_params={"custom_llm_provider": "gemini"}, + custom_llm_provider="gemini", + ) + return logging_obj + + @pytest.mark.parametrize( "custom_llm_provider, expected_endpoint_type", [("gemini", EndpointType.GEMINI), ("vertex_ai", EndpointType.VERTEX_AI)], @@ -31,7 +70,7 @@ def test_streaming_logging_targets_the_provider_that_served_the_request( iterator = iterator_cls( response=MagicMock(), model="gemini-3.1-flash-image", - logging_obj=MagicMock(spec=LiteLLMLoggingObj), + logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), litellm_metadata={}, custom_llm_provider=custom_llm_provider, @@ -73,7 +112,7 @@ async def test_async_streaming_iterator_yields_complete_sse_events(): iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( response=mock_response, model="gemini-3.1-flash-image-preview", - logging_obj=MagicMock(spec=LiteLLMLoggingObj), + logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), litellm_metadata={}, custom_llm_provider="gemini", @@ -95,7 +134,7 @@ def test_sync_streaming_iterator_yields_complete_sse_events(): iterator = GoogleGenAIGenerateContentStreamingIterator( response=mock_response, model="gemini-3.1-flash-image-preview", - logging_obj=MagicMock(spec=LiteLLMLoggingObj), + logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), litellm_metadata={}, custom_llm_provider="gemini", @@ -123,7 +162,7 @@ async def test_async_streaming_iterator_preserves_multi_field_sse_event(): iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( response=mock_response, model="gemini-test", - logging_obj=MagicMock(spec=LiteLLMLoggingObj), + logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), litellm_metadata={}, custom_llm_provider="gemini", @@ -146,7 +185,7 @@ async def test_async_streaming_iterator_forwards_sse_comment_events(): iterator = AsyncGoogleGenAIGenerateContentStreamingIterator( response=mock_response, model="gemini-test", - logging_obj=MagicMock(spec=LiteLLMLoggingObj), + logging_obj=_logging_obj(), generate_content_provider_config=MagicMock(), litellm_metadata={}, custom_llm_provider="gemini", @@ -154,3 +193,38 @@ async def test_async_streaming_iterator_forwards_sse_comment_events(): chunk = await iterator.__anext__() assert chunk == b": keepalive\n\n" + + +def _gemini_event(text: str, finish_reason: str | None = None) -> str: + candidate: Final = {"content": {"role": "model", "parts": [{"text": text}]}, "index": 0} + payload: Final = { + "candidates": [{**candidate, "finishReason": finish_reason} if finish_reason else candidate], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}, + "modelVersion": "gemini-3.1-flash", + } + return f"data: {json.dumps(payload)}\n\n" + + +@pytest.mark.asyncio +async def test_async_stream_logs_its_first_chunk_time_measured_from_the_call_start(): + call_start: Final = datetime(2026, 10, 9, 12, 0, 0) + recorder: Final = _SuccessRecorder() + logging_obj: Final = _logging_obj(call_start, recorder) + response: Final = httpx.Response(200, content=(_gemini_event("Hel") + _gemini_event("lo", "STOP")).encode()) + iterator: Final = AsyncGoogleGenAIGenerateContentStreamingIterator( + response=response, + model="gemini-3.1-flash", + logging_obj=logging_obj, + generate_content_provider_config=MagicMock(), + litellm_metadata={}, + custom_llm_provider="gemini", + ) + tasks_before: Final = asyncio.all_tasks() + + _ = await iterator.__anext__() + first_chunk_at: Final = logging_obj.completion_start_time + _ = [chunk async for chunk in iterator] + await asyncio.gather(*(asyncio.all_tasks() - tasks_before)) + + assert first_chunk_at is not None + assert recorder.logged == [(call_start, first_chunk_at)] diff --git a/tests/unit/integrations/otel/test_langfuse_logger.py b/tests/unit/integrations/otel/test_langfuse_logger.py index ff2060d91c2..d55f0577e03 100644 --- a/tests/unit/integrations/otel/test_langfuse_logger.py +++ b/tests/unit/integrations/otel/test_langfuse_logger.py @@ -4,6 +4,7 @@ stamped from the request task while the root span is still recording, so Langfus import asyncio import json from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import datetime, timezone from typing import Final import pytest @@ -11,6 +12,7 @@ import pytest pytest.importorskip("opentelemetry") from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.trace import SpanKind # noqa: E402 import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 @@ -43,6 +45,7 @@ from litellm.types.utils import ( # noqa: E402 INPUT_ATTR: Final = "langfuse.observation.input" OUTPUT_ATTR: Final = "langfuse.observation.output" TRACE_NAME_ATTR: Final = "langfuse.trace.name" +COMPLETION_START_ATTR: Final = "langfuse.observation.completion_start_time" TRACE_CONTROL_ATTRS: Final = (TRACE_NAME_ATTR, "user.id", "session.id", "langfuse.trace.tags") CHAT_DATA: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "ping"}]} @@ -541,3 +544,53 @@ def test_langfuse_otel_preset_builds_a_logger_that_stamps_the_root(monkeypatch): assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs finally: is_otel_v2_enabled.cache_clear() + + +def _llm_call_kwargs(optional_params: Mapping[str, object], completion_start: datetime | None) -> dict[str, object]: + return { + "standard_logging_object": { + "litellm_call_id": "call_1", + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-mini", + "stream": bool(optional_params.get("stream")), + "status": "success", + "completionStartTime": datetime(2026, 10, 9, 12, 0, 9, tzinfo=timezone.utc).timestamp(), + }, + "litellm_params": {"metadata": {}}, + "optional_params": optional_params, + "api_call_start_time": datetime(2026, 10, 9, 12, 0, 0, tzinfo=timezone.utc), + "completion_start_time": completion_start, + } + + +def _generation_attrs(kwargs: dict[str, object], *, fail: bool = False) -> Mapping[str, object]: + logger, exporter = _logger() + logger.log_pre_api_call(model="gpt-5.4-mini", messages=[], kwargs=kwargs) + hook: Final = logger.async_log_failure_event if fail else logger.async_log_success_event + asyncio.run(hook(kwargs, None, None, None)) + (generation,) = [span for span in exporter.get_finished_spans() if span.kind is SpanKind.CLIENT] + return dict(generation.attributes or {}) + + +@pytest.mark.parametrize( + "optional_params", + [ + pytest.param({"stream": True}, id="streamed"), + pytest.param({}, id="not_streamed_or_pass_through_stream"), + ], +) +def test_generation_carries_litellms_completion_start_time_like_the_default_emitter( + optional_params: Mapping[str, object], +) -> None: + completion_start: Final = datetime(2026, 10, 9, 12, 0, 0, 250000, tzinfo=timezone.utc) + + attrs: Final = _generation_attrs(_llm_call_kwargs(optional_params, completion_start)) + + assert datetime.fromisoformat(json.loads(attrs[COMPLETION_START_ATTR])) == completion_start + + +def test_failed_generation_without_a_first_chunk_has_no_completion_start_time() -> None: + attrs: Final = _generation_attrs(_llm_call_kwargs({"stream": True}, None), fail=True) + + assert COMPLETION_START_ATTR not in attrs diff --git a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py index ca7a9b424ad..7e112fcf6a4 100644 --- a/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py @@ -7,6 +7,7 @@ backends, so one trace lights up every configured destination. import json from collections.abc import Mapping +from datetime import datetime, timezone from itertools import chain from typing import Final @@ -580,6 +581,32 @@ def test_langfuse_usage_details_omit_zero_cache_and_reasoning_counts(): assert usage == {"input": 12, "output": 8, "total": 20} +def _langfuse_payload() -> dict[str, object]: + return { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-4o", + "completionStartTime": 1791547299.0, + } + + +def test_langfuse_mapper_sends_the_completion_start_time_as_the_sdks_json_string(): + data: Final = LLMCallSpanData.from_standard_logging_payload( + _langfuse_payload(), completion_start_seconds=1791547200.25 + ) + + attrs: Final = LangfuseMapper().map(data) + + sent: Final = datetime.fromisoformat(json.loads(attrs["langfuse.observation.completion_start_time"])) + assert sent == datetime(2026, 10, 9, 12, 0, 0, 250000, tzinfo=timezone.utc) + + +def test_langfuse_mapper_ignores_the_payloads_end_time_fallback_when_litellm_recorded_no_completion_start(): + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(_langfuse_payload())) + + assert "langfuse.observation.completion_start_time" not in attrs + + def test_langfuse_mapper_names_the_trace_from_the_caller(): named = LangfuseMapper().map(_llm_call(trace=TraceControls(name="nightly-eval"))) assert named["langfuse.trace.name"] == "nightly-eval" diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 13a49f40d7e..9f536ab0dac 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -168,6 +168,31 @@ class TestLangfuseOtelIntegration: assert span.attributes["langfuse.environment"] == "team-a-env" + @pytest.mark.parametrize( + "completion_start_time", + [ + datetime(2026, 10, 9, 12, 0, 0, 250000, tzinfo=timezone.utc), + datetime(2026, 10, 9, 12, 0, 0, 250000), + ], + ids=["aware", "naive_local_time"], + ) + def test_set_langfuse_specific_attributes_completion_start_time(self, completion_start_time: datetime): + """The first-token time goes out the way the Langfuse SDK sends it: a JSON string, with an explicit offset.""" + span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request") + + LangfuseOtelLogger._set_langfuse_specific_attributes(span, {"completion_start_time": completion_start_time}, {}) + + sent: Final = datetime.fromisoformat(json.loads(span.attributes["langfuse.observation.completion_start_time"])) + assert sent.tzinfo is not None + assert sent.timestamp() == completion_start_time.timestamp() + + def test_set_langfuse_specific_attributes_without_completion_start_time(self): + span: Final = TracerProvider().get_tracer(__name__).start_span("litellm_request") + + LangfuseOtelLogger._set_langfuse_specific_attributes(span, {}, {}) + + assert "langfuse.observation.completion_start_time" not in span.attributes + def test_extract_langfuse_metadata_basic(self): """Ensure metadata is correctly pulled from litellm_params.""" metadata_in = {"generation_name": "my-gen", "custom": "data"} diff --git a/tests/unit/interactions/test_interactions_streaming_iterator.py b/tests/unit/interactions/test_interactions_streaming_iterator.py index c52ab36deaa..afcd0d559d7 100644 --- a/tests/unit/interactions/test_interactions_streaming_iterator.py +++ b/tests/unit/interactions/test_interactions_streaming_iterator.py @@ -5,7 +5,10 @@ async_success_handler (cross-thread pydantic mutation segfaults pydantic-core). """ import asyncio +import json import time +from datetime import datetime +from typing import Final import httpx import pytest @@ -16,6 +19,7 @@ from litellm.interactions import streaming_iterator as interactions_streaming_it from litellm.interactions.streaming_iterator import InteractionsAPIStreamingIterator from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig from litellm.types.interactions import InteractionsAPIStreamingResponse @@ -97,3 +101,50 @@ async def test_custom_logger_only_never_submits_sync_success_handler(monkeypatch assert recorder.async_hook_fired is True assert recording_executor.submitted_for(logging_obj) == [] + + +class _SuccessRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.logged: list[tuple[datetime, object]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.logged.append((start_time, kwargs.get("completion_start_time"))) + + +@pytest.mark.asyncio +async def test_stream_logs_its_first_event_time_measured_from_the_call_start(): + call_start: Final = datetime(2026, 10, 9, 12, 0, 0) + recorder: Final = _SuccessRecorder() + logging_obj: Final = LitellmLogging( + model="gemini/gemini-3-pro-preview", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="ainteraction", + start_time=call_start, + litellm_call_id="interactions-stream", + function_id="interactions-stream", + dynamic_async_success_callbacks=[recorder], + ) + events: Final = ( + {"event_type": "step.delta", "id": "interaction-1"}, + {"event_type": "interaction.completed", "id": "interaction-1", "status": "completed"}, + ) + response: Final = httpx.Response(200, content="".join(f"data: {json.dumps(e)}\n\n" for e in events).encode()) + iterator: Final = InteractionsAPIStreamingIterator( + response=response, + model="gemini/gemini-3-pro-preview", + interactions_api_config=GoogleAIStudioInteractionsConfig(), + logging_obj=logging_obj, + ) + tasks_before: Final = asyncio.all_tasks() + + _ = await iterator.__anext__() + first_event_at: Final = logging_obj.completion_start_time + _ = [event async for event in iterator] + await asyncio.gather(*(asyncio.all_tasks() - tasks_before)) + + assert first_event_at is not None + assert recorder.logged == [(call_start, first_event_at)] diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py b/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py index b88a57f440d..1df4ccec02c 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py @@ -1,5 +1,5 @@ import asyncio -from typing import Any, AsyncIterator, Dict, List +from typing import Any, AsyncIterator, Dict, Final, List import pytest @@ -10,10 +10,13 @@ import litellm from litellm._internal_context import in_post_response_phase from litellm.caching.caching import Cache, LiteLLMCacheType from litellm.caching.caching_handler import LLMCachingHandler +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.anthropic.pass_through.messages import handler from litellm.llms.anthropic.pass_through.messages.response_cache import ( AnthropicMessagesStreamCacheWriter, + CachedAnthropicMessagesStreamIterator, ) STREAM_EVENTS: List[bytes] = [ @@ -299,6 +302,58 @@ async def test_cached_stream_replay_logs_once_when_polled_after_exhaustion(): await asyncio.sleep(0) mock_route.assert_called_once() + + +class _ReplayRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged: list[object] = [] + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.logged.append(kwargs.get("completion_start_time")) + + +@pytest.mark.asyncio +async def test_cached_stream_replay_logs_when_its_first_event_was_replayed(): + recorder: Final = _ReplayRecorder() + logging_obj: Final = LiteLLMLoggingObj( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "which greek letter?"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime(2026, 10, 20, 12, 0, 0), + litellm_call_id="cached-replay", + function_id="cached-replay", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="claude-sonnet-4-5", + user="", + optional_params={}, + litellm_params={"custom_llm_provider": "anthropic"}, + custom_llm_provider="anthropic", + ) + iterator: Final = CachedAnthropicMessagesStreamIterator( + events=[event.decode("utf-8") for event in STREAM_EVENTS], + litellm_logging_obj=logging_obj, + request_body={"model": "claude-sonnet-4-5", "stream": True}, + ) + + _ = await iterator.__anext__() + first_event_at: Final = iterator.completion_start_time + _ = await _collect(iterator) + await GLOBAL_LOGGING_WORKER.flush() + + assert first_event_at is not None + assert recorder.logged == [first_event_at] + + class _HeldBackStream: has_buffered_provider_output = True diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py index b995c6dba9b..fd9aa75be74 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py @@ -5,6 +5,7 @@ from datetime import datetime from typing import Final, Literal from unittest.mock import patch +import httpx import pytest @@ -859,13 +860,15 @@ class _SuccessRecorder(CustomLogger): self.success_kwargs.append(kwargs) -def _make_priced_logging_obj(call_id: str, recorder: _SuccessRecorder, model: str) -> LiteLLMLoggingObj: +def _make_priced_logging_obj( + call_id: str, recorder: _SuccessRecorder, model: str, start_time: datetime | None = None +) -> LiteLLMLoggingObj: logging_obj = LiteLLMLoggingObj( model=model, messages=[{"role": "user", "content": "hi"}], stream=True, call_type="anthropic_messages", - start_time=datetime.now(), + start_time=start_time or datetime.now(), litellm_call_id=call_id, function_id=call_id, dynamic_async_success_callbacks=[recorder], @@ -880,6 +883,53 @@ def _make_priced_logging_obj(call_id: str, recorder: _SuccessRecorder, model: st return logging_obj +@pytest.mark.asyncio +async def test_streamed_messages_log_the_call_start_instead_of_when_the_upstream_answered(): + model: Final = "claude-sonnet-5" + call_start: Final = datetime(2026, 10, 9, 12, 0, 0) + recorder: Final = _SuccessRecorder() + logging_obj: Final = _make_priced_logging_obj("logs_call_start", recorder, model, start_time=call_start) + logging_obj._on_deferred_stream_complete = lambda logging_coroutine: logging_coroutine + iterator: Final = BaseAnthropicMessagesStreamingIterator( + litellm_logging_obj=logging_obj, request_body={"model": model, "stream": True} + ) + + async def _full_stream(): + for event in (*_STREAM_PREFIX, *_STREAM_TAIL): + yield event + + _ = [chunk async for chunk in iterator.async_sse_wrapper(_full_stream())] + (logging_coroutine,) = logging_obj._deferred_stream_complete_args + await logging_coroutine + + assert len(recorder.success_kwargs) == 1 + assert recorder.success_kwargs[0]["standard_logging_object"]["startTime"] == call_start.timestamp() + + +@pytest.mark.asyncio +async def test_streamed_messages_from_anthropic_log_the_call_start_instead_of_when_the_upstream_answered(): + model: Final = "claude-sonnet-5" + call_start: Final = datetime(2026, 10, 9, 12, 0, 0) + recorder: Final = _SuccessRecorder() + logging_obj: Final = _make_priced_logging_obj("anthropic_logs_call_start", recorder, model, start_time=call_start) + logging_obj._on_deferred_stream_complete = lambda logging_coroutine: logging_coroutine + request_body: Final = {"model": model, "stream": True} + iterator: Final = BaseAnthropicMessagesStreamingIterator(litellm_logging_obj=logging_obj, request_body=request_body) + sse: Final = "".join(f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in (*_STREAM_PREFIX, *_STREAM_TAIL)) + + _ = [ + chunk + async for chunk in iterator.get_async_streaming_response_iterator( + httpx.Response(200, content=sse.encode()), request_body, logging_obj + ) + ] + (logging_coroutine,) = logging_obj._deferred_stream_complete_args + await logging_coroutine + + assert len(recorder.success_kwargs) == 1 + assert recorder.success_kwargs[0]["standard_logging_object"]["startTime"] == call_start.timestamp() + + class _UpstreamClosedOnDetach: """Upstream that yields its events and then, like a socket read, waits until it is closed.""" From b88f64e65f29a34ee7363a3e2cd8189e850e4387 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:17:07 +0000 Subject: [PATCH 097/124] refactor(types): replace Any with proven types in 6 files (#47268) * refactor(types): replace Any with proven types in 39 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): use valid response fixtures for strict parsing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): revert anthropic chat response seam Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): revert validation seams that changed tolerated inputs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): revert remaining validation seams to keep behavior unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): drop no-op search router annotation and helper docstring Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): keep ip address utils at base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/datadog/datadog_llm_obs.py | 24 ++++++++++--- litellm/llms/base_llm/chat/transformation.py | 2 +- litellm/models/organization_membership.py | 3 +- litellm/proxy/auth/auth_checks.py | 2 +- .../workflow_management_endpoints.py | 35 +++++++++++++++---- .../complexity_router/complexity_router.py | 19 +++++++--- 6 files changed, 66 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 270d799742c..a72023eaedf 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -12,7 +12,7 @@ import os from collections.abc import Mapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import Any, Final, Literal +from typing import Final, Literal import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -54,6 +54,7 @@ from litellm.types.utils import ( PROMPT_CARRYING_GUARDRAIL_FIELDS, PROMPT_QUOTING_ROUTING_DECISION_FIELDS, CallTypes, + StandardLoggingMetadata, StandardLoggingPayload, StandardLoggingPayloadErrorInformation, ) @@ -93,8 +94,12 @@ _ROUTER_DIMENSIONS: Final[tuple[str, ...]] = ("router_tier", "router_cause", "ro _COST_DIMENSIONS: Final[tuple[str, ...]] = ("team", "user", "key_alias", "model_group", *_ROUTER_DIMENSIONS) -def _metadata_of(standard_logging_payload: StandardLoggingPayload) -> Mapping[str, Any]: - metadata: Final = standard_logging_payload.get("metadata") +def _logging_metadata_of(standard_logging_payload: StandardLoggingPayload) -> StandardLoggingMetadata | None: + return standard_logging_payload.get("metadata") + + +def _metadata_of(standard_logging_payload: StandardLoggingPayload) -> Mapping[str, object]: + metadata: Final = _logging_metadata_of(standard_logging_payload) return metadata or _EMPTY_MAPPING @@ -1057,14 +1062,23 @@ class DataDogLLMObsLogger(CustomBatchLogger): # Get budget information from metadata metadata: Final = _metadata_of(standard_logging_payload) + logging_metadata: Final = _logging_metadata_of(standard_logging_payload) # API key max budget - user_api_key_max_budget: Final = metadata.get("user_api_key_max_budget") + user_api_key_max_budget: Final = ( + logging_metadata["user_api_key_max_budget"] + if logging_metadata and "user_api_key_max_budget" in logging_metadata + else None + ) if user_api_key_max_budget is not None: spend_metrics["user_api_key_max_budget"] = float(user_api_key_max_budget) # API key spend - user_api_key_spend: Final = metadata.get("user_api_key_spend") + user_api_key_spend: Final = ( + logging_metadata["user_api_key_spend"] + if logging_metadata and "user_api_key_spend" in logging_metadata + else None + ) if user_api_key_spend is not None: try: spend_metrics["user_api_key_spend"] = float(user_api_key_spend) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 51c8e4b8bef..b2eb4cde5ce 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -428,7 +428,7 @@ class BaseConfig(ABC): streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse"], sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> object: pass def get_model_response_iterator_for_request( diff --git a/litellm/models/organization_membership.py b/litellm/models/organization_membership.py index e9697cf57b6..6a0d22cb63b 100644 --- a/litellm/models/organization_membership.py +++ b/litellm/models/organization_membership.py @@ -6,7 +6,6 @@ Canonical definition for ``litellm_organizationmembership``. Re-exported from """ from datetime import datetime -from typing import Any from pydantic import ConfigDict, model_validator @@ -24,7 +23,7 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): budget_id: str | None = None created_at: datetime updated_at: datetime - user: Any | None = None + user: object | None = None litellm_budget_table: LiteLLM_BudgetTable | None = None user_email: str | None = None diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 5a09a0efbf6..8887d86570a 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2663,7 +2663,7 @@ async def _load_team_membership_on_cache_miss( parent_otel_span: Span | None, proxy_logging_obj: ProxyLogging | None, ) -> LiteLLM_TeamMembership | None: - redis_cached: Final[object] = await user_api_key_cache.async_get_cache(key=cache_key) + redis_cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=cache_key) redis_membership: Final = _membership_from_cached_payload(redis_cached) if not isinstance(redis_membership, _TeamMembershipCacheMiss): return redis_membership diff --git a/litellm/proxy/management_endpoints/workflow_management_endpoints.py b/litellm/proxy/management_endpoints/workflow_management_endpoints.py index 890a344dfe0..649941bc233 100644 --- a/litellm/proxy/management_endpoints/workflow_management_endpoints.py +++ b/litellm/proxy/management_endpoints/workflow_management_endpoints.py @@ -147,6 +147,34 @@ class _MessageCreateData(TypedDict, total=False): session_id: str +class _EventTable(Protocol): + async def create(self, *, data: _EventCreateData) -> object: ... + + +class _RunTable(Protocol): + async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> object: ... + + +class _WorkflowTx(Protocol): + @property + def litellm_workflowevent(self) -> _EventTable: ... + + @property + def litellm_workflowrun(self) -> _RunTable: ... + + +async def _append_event_in_tx( + tx: _WorkflowTx, run_id: str, event_data: _EventCreateData, new_status: str | None +) -> object: + event: Final = await tx.litellm_workflowevent.create(data=event_data) + if new_status: + await tx.litellm_workflowrun.update( + where={"run_id": run_id}, + data={"status": new_status}, + ) + return event + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -387,12 +415,7 @@ async def append_workflow_event( event_data["data"] = _json(data.data) async with prisma_client.db.tx() as tx: - event: object = await tx.litellm_workflowevent.create(data=event_data) - if new_status: - await tx.litellm_workflowrun.update( - where={"run_id": run_id}, - data={"status": new_status}, - ) + event: object = await _append_event_in_tx(tx, run_id, event_data, new_status) return event diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 1638f20c8e3..a475358e1f0 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -22,7 +22,7 @@ import json import random import re import time -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from itertools import accumulate, chain, islice, takewhile from threading import Lock from types import MappingProxyType @@ -30,6 +30,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from pydantic import BaseModel, TypeAdapter, ValidationError, create_model from pydantic_core import ErrorDetails +from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger @@ -411,9 +412,19 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: list[str] return [*base_keywords, *deduped_custom.values()] -def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, Any]: +class _ParentSessionKwargs(TypedDict, total=False): + litellm_session_id: ReadOnly[object] + litellm_trace_id: ReadOnly[object] + + +def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> _ParentSessionKwargs: kwargs: Final = request_kwargs or {} - return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} + session_id: Final = kwargs.get("litellm_session_id") + trace_id: Final = kwargs.get("litellm_trace_id") + return _ParentSessionKwargs( + **({"litellm_session_id": session_id} if session_id is not None else {}), + **({"litellm_trace_id": trace_id} if trace_id is not None else {}), + ) def _response_cost_or_none(response: object) -> float | None: @@ -2828,7 +2839,7 @@ class ComplexityRouter(CustomLogger): "originating_request_masked": masked_originating_request(request_kwargs), "body": {"model": llm_config.model, **payload}, } - classify: Final = ( + classify: Final[Callable[..., Awaitable[ModelResponse | ResponsesAPIResponse]]] = ( self.litellm_router_instance.aresponses if encrypted_task is not None else self.litellm_router_instance.acompletion From 4963ba4d9906170427a82731ba1b26c4f82be1b8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:56:29 -0700 Subject: [PATCH 098/124] test(translation): add sonnet 5.5 anthropic direct cross-model mid row (#47566) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_completions/cross_model/README.md | 4 +- .../chat_completions/cross_model/anthropic.py | 350 ++++++++++++++++++ .../chat_completions/cross_model/case.py | 5 + 3 files changed, 357 insertions(+), 2 deletions(-) diff --git a/tests/integration/translation/chat_completions/cross_model/README.md b/tests/integration/translation/chat_completions/cross_model/README.md index 90710d5597c..99a4225480e 100644 --- a/tests/integration/translation/chat_completions/cross_model/README.md +++ b/tests/integration/translation/chat_completions/cross_model/README.md @@ -8,7 +8,7 @@ Every row sends `CODING_AGENT_REQUEST` to `/v1/chat/completions`, changing only | Mid | Opus 5 and 5.5, Sonnet 5.5, Gemini 3.8 Flash and 3.1 Pro Preview, GPT 5.6 Sol, 6 Sol and 6.1 Sol, GPT 5.6 Terra | | High | Fable 5 and 5.1, Gemini 4 Argon, GPT 6 Astra | -The 51 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented +The 52 rows cover Anthropic direct, OpenAI direct, Vertex, Gemini direct, Azure OpenAI, Azure AI and Bedrock Converse where accessible. Vertex Haiku 5.5 was unavailable during capture and is not represented The request contains system instructions, conversation history, previous assistant tool calls and results, four function definitions, automatic parallel tools, low reasoning effort and an output limit. Each row exercises a tool-call response, raw streaming, SDK stream assembly and a follow-up that replays the returned assistant message and tool-call IDs @@ -33,7 +33,7 @@ Streaming cases store `expected_litellm_message` and `expected_litellm_usage`. M Usage requires nonnegative integer counts, positive prompt and completion counts, and `total_tokens == prompt_tokens + completion_tokens`. The complete normalized usage must equal the independently expected counters, including cached and reasoning tokens and derived text/cache details. Missing, changed or unexpected counters fail. Only null optional fields, the explicitly listed provider metadata/duplicate counters and computed `cost` are excluded. Thus a wrong prompt count with an adjusted total still fails -The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 51 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips +The raw stream check assembles tool names and argument fragments by index. The SDK check separately uses `OpenAI.chat.completions.stream()` without removing reasoning fields. All 52 SDK checks run, including the seven Haiku rows. [LIT-9653](https://linear.app/litellm-ai/issue/LIT-9653) was fixed in [#46611](https://github.com/BerriAI/litellm/pull/46611), so there are no Haiku skips ## Recorded upstreams diff --git a/tests/integration/translation/chat_completions/cross_model/anthropic.py b/tests/integration/translation/chat_completions/cross_model/anthropic.py index 3df7da17ef6..9302103f369 100644 --- a/tests/integration/translation/chat_completions/cross_model/anthropic.py +++ b/tests/integration/translation/chat_completions/cross_model/anthropic.py @@ -11,6 +11,7 @@ from integration.translation.chat_completions.bases.anthropic import ( CLAUDE_HAIKU_5_5_TEST_CASE, CLAUDE_OPUS_5_5_TEST_CASE, CLAUDE_OPUS_5_TEST_CASE, + CLAUDE_SONNET_5_5_TEST_CASE, ) from integration.translation.chat_completions.templates import ( CODING_AGENT_REQUEST, @@ -2098,3 +2099,352 @@ CLAUDE_OPUS_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, }, ) + + +CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEXT: Final = ( + "The bug is that `retries_remaining` in `src/retry.py` returns `limit + attempts` instead of " + "`max(0, limit - attempts)`, so it can return too many retries and can't be clamped at zero. The only other " + "caller is `src/worker.py` line 12 (`retries_remaining(3, attempts)`), and `tests/test_retry.py::test_retries_remaining` " + "fails with `assert 4 == 2` for `retries_remaining(3, 1)`." +) + + +CLAUDE_SONNET_5_5_AGENT_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_TEST_CASE, + scenario="agent_tools", + litellm_request={**CODING_AGENT_REQUEST, "model": "anthropic/claude-sonnet-5-5"}, + expected_provider_endpoint="/v1/messages", + expected_provider_headers=CLAUDE_OPUS_5_5_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_request={ + **ANTHROPIC_AGENT_REQUEST, + "model": "claude-sonnet-5-5", + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "low"}, + }, + mock_provider_response={ + **ANTHROPIC_AGENT_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_011CgDcA1vnTwnTPTcJv9Lb7", + "content": [ + { + "type": "tool_use", + "id": "toolu_01RK8oPY5AnH3K4R8LfPXPTi", + "name": "search_code", + "input": {"query": "retries_remaining"}, + "caller": {"type": "direct"}, + }, + { + "type": "tool_use", + "id": "toolu_013REgHECA7Rceip3FrejMEy", + "name": "run_tests", + "input": {"path": "tests/test_retry.py"}, + "caller": {"type": "direct"}, + }, + ], + "stop_reason": "tool_use", + "usage": { + **object_value(ANTHROPIC_AGENT_RESPONSE["usage"]), + "input_tokens": 1184, + "output_tokens": 105, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "anthropic/claude-sonnet-5-5", + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"query": "retries_remaining"}', + "name": "search_code", + }, + "id": "toolu_01RK8oPY5AnH3K4R8LfPXPTi", + "type": "function", + }, + { + "index": 1, + "caller": {"type": "direct"}, + "function": { + "arguments": '{"path": "tests/test_retry.py"}', + "name": "run_tests", + }, + "id": "toolu_013REgHECA7Rceip3FrejMEy", + "type": "function", + }, + ], + "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + "content": None, + }, + } + ], + "usage": { + "completion_tokens": 105, + "prompt_tokens": 1184, + "total_tokens": 1289, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1184, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "speed": "standard", + "service_tier": "standard", + }, + }, +) + + +CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_5_AGENT_TEST_CASE, + scenario="agent_followup", + litellm_request=followup_request( + CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request, + CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_litellm_response, + ), + expected_provider_endpoint="/v1/messages", + expected_provider_request={ + **ANTHROPIC_AGENT_REQUEST, + "model": "claude-sonnet-5-5", + "messages": [ + *list_value(ANTHROPIC_AGENT_REQUEST["messages"]), + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01RK8oPY5AnH3K4R8LfPXPTi", + "name": "search_code", + "input": {"query": "retries_remaining"}, + }, + { + "type": "tool_use", + "id": "toolu_013REgHECA7Rceip3FrejMEy", + "name": "run_tests", + "input": {"path": "tests/test_retry.py"}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01RK8oPY5AnH3K4R8LfPXPTi", + "content": '{"matches": [{"path": "src/worker.py", "line": 12, "text": "remaining = ' + 'retries_remaining(3, attempts)"}]}', + }, + { + "type": "tool_result", + "tool_use_id": "toolu_013REgHECA7Rceip3FrejMEy", + "content": '{"exit_code": 1, "output": "FAILED tests/test_retry.py::test_retries_remaining - ' + 'assert 4 == 2"}', + }, + { + "type": "text", + "text": "Stop after these diagnostics. Do not call more tools or change files. Summarize the " + "bug, the caller found and the failed test in two short sentences.", + }, + ], + }, + ], + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "low"}, + }, + mock_provider_response={ + **ANTHROPIC_AGENT_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_011CgDcZdb2JGywm8DnwxFvn", + "content": [{"type": "text", "text": CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEXT}], + "stop_reason": "end_turn", + "usage": { + **object_value(ANTHROPIC_AGENT_RESPONSE["usage"]), + "input_tokens": 1492, + "output_tokens": 155, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + expected_litellm_response={ + **CODING_AGENT_RESPONSE, + "model": "anthropic/claude-sonnet-5-5", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEXT, + "role": "assistant", + "provider_specific_fields": {"citations": None, "thinking_blocks": None}, + }, + } + ], + "usage": { + "completion_tokens": 155, + "prompt_tokens": 1492, + "total_tokens": 1647, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 155}, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1492, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "inference_geo": "global", + "speed": "standard", + "service_tier": "standard", + }, + }, +) + + +CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE: Final = StreamingTranslationTestCase( + scenario="agent_streaming", + litellm_endpoint=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_endpoint, + litellm_request={ + **CLAUDE_SONNET_5_5_AGENT_TEST_CASE.litellm_request, + "stream": True, + "stream_options": {"include_usage": True}, + }, + expected_provider_headers=CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_provider_headers, + expected_provider_endpoint="/v1/messages", + expected_provider_request={ + **CLAUDE_SONNET_5_5_AGENT_TEST_CASE.expected_provider_request, + "stream": True, + }, + mock_provider_frames=( + SSEFrame( + data={ + "type": "message_start", + "message": { + **ANTHROPIC_AGENT_RESPONSE, + "model": "claude-sonnet-5-5", + "id": "msg_011CgDcC8agu11ruKbps6qFH", + "content": [], + "stop_reason": None, + "usage": { + **object_value(ANTHROPIC_AGENT_RESPONSE["usage"]), + "input_tokens": 1184, + "output_tokens": 16, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + }, + event="message_start", + ), + SSEFrame( + data={ + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_01LmCacjLUDXcY139vPTCceR", + "name": "search_code", + "input": {}, + "caller": {"type": "direct"}, + }, + }, + event="content_block_start", + ), + SSEFrame(data={"type": "ping"}, event="ping"), + *( + SSEFrame( + data={ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": partial_json}, + }, + event="content_block_delta", + ) + for partial_json in ("", '{"query', '": "ret', "ries_re", "maining", '"}') + ), + SSEFrame(data={"type": "content_block_stop", "index": 0}, event="content_block_stop"), + SSEFrame( + data={ + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "tool_use", + "id": "toolu_011cyQH4o1bcDJYsWQoLSzGT", + "name": "run_tests", + "input": {}, + "caller": {"type": "direct"}, + }, + }, + event="content_block_start", + ), + *( + SSEFrame( + data={ + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": partial_json}, + }, + event="content_block_delta", + ) + for partial_json in ("", '{"path"', ': "test', "s/test_", "retry.p", 'y"}') + ), + SSEFrame(data={"type": "content_block_stop", "index": 1}, event="content_block_stop"), + SSEFrame( + data={ + "type": "message_delta", + "delta": { + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "container": None, + }, + "usage": { + "input_tokens": 1184, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 105, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + event="message_delta", + ), + SSEFrame(data={"type": "message_stop"}, event="message_stop"), + ), + expected_litellm_message=coding_agent_stream_message( + ("toolu_01LmCacjLUDXcY139vPTCceR", "toolu_011cyQH4o1bcDJYsWQoLSzGT") + ), + expected_litellm_usage={ + "prompt_tokens": 1184, + "completion_tokens": 105, + "total_tokens": 1289, + "prompt_tokens_details": { + "cached_tokens": 0, + "text_tokens": 1184, + "cache_write_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + "completion_tokens_details": {"reasoning_tokens": 0, "text_tokens": 105}, + }, +) diff --git a/tests/integration/translation/chat_completions/cross_model/case.py b/tests/integration/translation/chat_completions/cross_model/case.py index 189592b26c2..8dee0fae2ab 100644 --- a/tests/integration/translation/chat_completions/cross_model/case.py +++ b/tests/integration/translation/chat_completions/cross_model/case.py @@ -132,6 +132,11 @@ MID: Final[tuple[AgentCases, ...]] = ( bedrock_converse.CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE, bedrock_converse.CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE, ), + ( + anthropic.CLAUDE_SONNET_5_5_AGENT_TEST_CASE, + anthropic.CLAUDE_SONNET_5_5_AGENT_FOLLOWUP_TEST_CASE, + anthropic.CLAUDE_SONNET_5_5_AGENT_STREAMING_TEST_CASE, + ), ( bedrock_converse.CLAUDE_OPUS_5_TEST_CASE, bedrock_converse.CLAUDE_OPUS_5_AGENT_FOLLOWUP_TEST_CASE, From 9e3d3a87f4016a621d14e808c916f61d59d9e8af Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:00:31 +0000 Subject: [PATCH 099/124] fix(anthropic, bedrock)!: keep a redacted_thinking content item on every Claude route (#47489) * test(anthropic, bedrock): cover a redacted_thinking content item on every Claude route * fix(anthropic, bedrock): keep a redacted_thinking content item on every Claude route * fix(anthropic, bedrock): send each replayed thinking block once, without cache_control or blank data * fix(anthropic, bedrock): keep thinking order and gate replay key drops on drop_params A thinking block that only one of content and thinking_blocks carries now goes back in its original place instead of being prepended. cache_control on a thinking block and blank redacted data are dropped only under drop_params and otherwise reach Claude as sent. Converse drops a cachePoint after reasoning under drop_params, skips a thinking_blocks entry that is not a block, and the cache control hook no longer marks a thinking part. * fix(anthropic, snowflake): match a signed thinking block on its signature and send each block once on Snowflake Cortex A signed thinking block in both content and thinking_blocks is the same block when the signatures match, even if the thinking text differs, so it is sent once from content instead of twice. Snowflake Cortex now merges thinking_blocks into content the same way the Anthropic routes do. A redacted_thinking item with blank data no longer counts as sent for system message placement. * fix(anthropic, bedrock): send a thinking block once when consecutive assistant messages both carry it Consecutive assistant messages merge into one Claude turn, so a block one message holds in content and the next holds in thinking_blocks went out twice and Claude answered 400. The merged turn now keeps the first copy of each redacted or signed block on the Anthropic routes and both Bedrock Converse builders. Snowflake Cortex goes back to its previous message building, since its double send cannot be proven live without a Cortex account and is tracked on its own. * test(anthropic, bedrock): cover a redacted block with null data and a non-block part beside a cache breakpoint * fix(anthropic, bedrock): match only text-valued thinking items when merging replayed blocks --- .../anthropic_cache_control_hook.py | 21 +- .../prompt_templates/common_utils.py | 10 + .../prompt_templates/factory.py | 256 +++++++---- .../mid_conversation_system.py | 19 +- litellm/llms/anthropic/chat/transformation.py | 5 +- .../bedrock/chat/converse_transformation.py | 2 + .../test_anthropic_cache_control_hook.py | 33 ++ ...llm_core_utils_prompt_templates_factory.py | 409 +++++++++++++++++- ...rompt_templates_mid_conversation_system.py | 28 +- 9 files changed, 660 insertions(+), 123 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index abdbfc79d41..b0deec24c1a 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -342,6 +342,20 @@ EXTERNAL_BREAKPOINTS_STAMP: Final = "_litellm_external_breakpoints" CUSTOM_LLM_PROVIDER_STAMP: Final = "_litellm_custom_llm_provider" +def _accepts_cache_control(part: object) -> bool: + match part: + case {"type": "thinking" | "redacted_thinking"}: + return False + case dict(): + return True + case _: + return False + + +def _last_index_accepting_cache_control(parts: Sequence[object]) -> int | None: + return next((index for index in reversed(range(len(parts))) if _accepts_cache_control(parts[index])), None) + + class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _request_value(request_kwargs: object, key: str) -> object: @@ -740,10 +754,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): # 1. if string, insert cache control in the message if isinstance(message_content, str): message["cache_control"] = control - # 2. list of objects - only apply to last item per Anthropic spec + # 2. list of objects - only apply to the last item Claude accepts a breakpoint on elif isinstance(message_content, list): - if len(message_content) > 0 and isinstance(message_content[-1], dict): - message_content[-1]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict + markable_index: Final = _last_index_accepting_cache_control(message_content) + if markable_index is not None: + message_content[markable_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict return message @staticmethod diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b704dfa2675..b89e4d609ad 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2325,6 +2325,16 @@ def is_unsignable_thinking_block(block: object) -> bool: return not (isinstance(signature, str) and len(signature) > 0) +def has_blank_redacted_data(block: object) -> bool: + match block: + case {"type": "redacted_thinking", "data": str() as data}: + return not data.strip() + case {"type": "redacted_thinking"}: + return True + case _: + return False + + def native_redacted_thinking_data(data: str) -> str: """Claude's own `data` string for a redacted_thinking block. diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 481ec6a5076..3d865bd1819 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6,10 +6,10 @@ import json import mimetypes import re import xml.etree.ElementTree as ET -from collections.abc import Container, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence from enum import Enum from types import MappingProxyType -from typing import Annotated, Any, Final, TypeAlias, TypedDict, cast, overload +from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator @@ -61,6 +61,7 @@ from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( converse_redacted_content, convert_content_list_to_str, + has_blank_redacted_data, infer_content_type_from_url_and_content, is_encrypted_reasoning_block, is_non_content_values_set, @@ -2550,22 +2551,125 @@ def _pair_tool_calls_with_results(messages: Sequence[AllMessageValues]) -> tuple def _replayable_thinking_blocks( thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], + *, + drop_params: bool = False, ) -> list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock]: + keys_to_drop: Final = _UNSUPPORTED_THINKING_KEYS if drop_params else _STREAM_ONLY_THINKING_KEYS return [ - _without_stream_index(_with_native_redacted_data(block)) + _without_keys(_with_native_redacted_data(block), keys_to_drop) for block in thinking_blocks - if not is_unsignable_thinking_block(block) + if not (is_unsignable_thinking_block(block) or (drop_params and has_blank_redacted_data(block))) ] -def _without_stream_index( +def _as_thinking_block(item: object) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None: + match item: + case {"type": "thinking" | "redacted_thinking"}: + return cast( # cast-ok: the mapping pattern matched a thinking block's type tag + ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, item + ) + case _: + return None + + +def _thinking_identity(item: object) -> tuple[str | None, ...] | None: + match item: + case {"type": "redacted_thinking", "data": str() as data}: + return ("redacted_thinking", native_redacted_thinking_data(data)) + case {"type": "thinking", "signature": str() as signature} if signature: + return ("thinking", signature) + case {"type": str() as kind} if kind in ("thinking", "redacted_thinking"): + fields: Final = cast(Mapping[str, object], item) # cast-ok: the mapping pattern matched + return _unsigned_thinking_identity(kind, fields.get("thinking"), fields.get("signature")) + case _: + return None + + +def _unsigned_thinking_identity(kind: str, thinking: object, signature: object) -> tuple[str | None, ...] | None: + match thinking, signature: + case (str() | None as text, str() | None as signed): + return (kind, text, signed) + case _: + return None + + +_Replayed: Final = TypeVar("_Replayed") + + +def _with_missing_thinking_blocks( + content: Iterable[_Replayed], thinking_blocks: Sequence[_Replayed] +) -> tuple[_Replayed, ...]: + content_items: Final = tuple(content) + block_identities: Final = tuple(map(_thinking_identity, thinking_blocks)) + content_positions: Final[Mapping[tuple[object, ...] | None, int]] = MappingProxyType( + { + identity: index + for index, identity in reversed(tuple(enumerate(map(_thinking_identity, content_items)))) + if identity is not None + } + ) + anchors: Final = tuple(index for index, identity in enumerate(block_identities) if identity in content_positions) + if not anchors: + return (*thinking_blocks, *content_items) + missing_before: Final = MappingProxyType( + { + content_positions[block_identities[end]]: thinking_blocks[start + 1 : end] + for start, end in zip((-1, *anchors), anchors) + } + ) + missing_after: Final = MappingProxyType( + {content_positions[block_identities[anchors[-1]]]: thinking_blocks[anchors[-1] + 1 :]} + ) + return tuple(_interleave(content_items, missing_before, missing_after)) + + +def _without_repeated_thinking( + items: Sequence[_Replayed], identity: Callable[[_Replayed], tuple[object, ...] | None] +) -> tuple[_Replayed, ...]: + identities: Final = tuple(map(identity, items)) + first_positions: Final[Mapping[tuple[object, ...], int]] = MappingProxyType( + {key: index for index, key in reversed(tuple(enumerate(identities))) if key is not None} + ) + return tuple( + item for index, (item, key) in enumerate(zip(items, identities)) if key is None or first_positions[key] == index + ) + + +def _interleave( + content: Sequence[_Replayed], + before: Mapping[int, Sequence[_Replayed]], + after: Mapping[int, Sequence[_Replayed]], +) -> Iterator[_Replayed]: + for index, item in enumerate(content): + yield from before.get(index, ()) + yield item + yield from after.get(index, ()) + + +def _thinking_in_replay_order( + content: str | Iterable[object] | None, thinking_blocks: Sequence[object] | None +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + content_items: Final = content if isinstance(content, list) else () + return tuple( + block + for item in _with_missing_thinking_blocks(content_items, thinking_blocks or ()) + if (block := _as_thinking_block(item)) is not None + ) + + +_STREAM_ONLY_THINKING_KEYS: Final = frozenset({"index"}) +_UNSUPPORTED_THINKING_KEYS: Final = frozenset({"index", "cache_control"}) + + +def _without_keys( block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, + keys: frozenset[str], ) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock: - if "index" not in block: + if keys.isdisjoint(block): return block - return cast( # cast-ok: dropping one key keeps the block's TypedDict shape + return cast( # cast-ok: dropping keys keeps the block's TypedDict shape ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, - {key: value for key, value in block.items() if key != "index"}, + {key: value for key, value in block.items() if key not in keys}, ) @@ -2629,6 +2733,7 @@ def anthropic_messages_pt( messages: list[AllMessageValues], model: str, llm_provider: str, + drop_params: bool = False, ) -> _AnthropicMessageList: """ format messages for anthropic @@ -2816,7 +2921,9 @@ def anthropic_messages_pt( _raw_thinking_blocks = assistant_content_block.get("thinking_blocks", None) thinking_blocks = ( - _replayable_thinking_blocks(_raw_thinking_blocks) if _raw_thinking_blocks is not None else None + _replayable_thinking_blocks(_raw_thinking_blocks, drop_params=drop_params) + if _raw_thinking_blocks is not None + else None ) # Check if tool_calls contain server tool calls (web search, etc.) @@ -2951,42 +3058,28 @@ def anthropic_messages_pt( # SEQUENTIAL MODE: No server tool calls, or no thinking blocks, # or content is a list. Use the original sequential approach. - # When content is a list, check if it already contains thinking - # blocks inline. If so, skip prepending thinking_blocks to avoid - # duplication and preserve the original interleaved order. - # Fixes the gap where list-content messages bypass INTERLEAVED - # MODE and still get thinking blocks prepended out of order. _content_is_list = "content" in assistant_content_block and isinstance( assistant_content_block["content"], list ) _content_list = assistant_content_block.get("content") if _content_is_list else None - _list_has_thinking = False - if _content_is_list and _content_list is not None: - for _item in _content_list: - if isinstance(_item, dict) and _item.get("type") in ( - "thinking", - "redacted_thinking", - ): - _list_has_thinking = True - break if ( - thinking_blocks is not None and not _list_has_thinking + thinking_blocks is not None and not _content_is_list ): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR assistant_content.extend(thinking_blocks) if _content_is_list and _content_list is not None: - for m in _content_list: + for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()): if not isinstance(m, dict): continue # handle thinking blocks - text_block = cast(str, m.get("text", "")) - if m.get("type", "") == "thinking" and not is_unsignable_thinking_block(m): - anthropic_message: ChatCompletionThinkingBlock | AnthropicMessagesTextParam = cast( - ChatCompletionThinkingBlock, m + if (thinking_block := _as_thinking_block(m)) is not None: + assistant_content.extend( + _replayable_thinking_blocks([thinking_block], drop_params=drop_params) ) - assistant_content.append(anthropic_message) + continue + text_block = cast(str, m.get("text", "")) # handle text - elif ( + if ( m.get("type", "") == "text" and len(text_block) > 0 ): # don't pass empty text blocks. anthropic api raises errors. anthropic_message = AnthropicMessagesTextParam(type="text", text=text_block) @@ -3058,7 +3151,12 @@ def anthropic_messages_pt( msg_i += 1 if assistant_content: - new_messages.append({"role": "assistant", "content": assistant_content}) + new_messages.append( + { + "role": "assistant", + "content": list(_without_repeated_thinking(assistant_content, _thinking_identity)), + } + ) ## MID-CONVERSATION SYSTEM MESSAGES (placement is the caller's job) ## while msg_i < len(messages) and messages[msg_i]["role"] == "system": @@ -4190,6 +4288,16 @@ def _convert_to_bedrock_tool_call_result( return content_block +def _reasoning_identity(block: BedrockContentBlock) -> tuple[object, ...] | None: + match block: + case {"reasoningContent": {"redactedContent": redacted}}: + return ("redacted_thinking", redacted) + case {"reasoningContent": {"reasoningText": {"signature": str() as signature}}} if signature: + return ("thinking", signature) + case _: + return None + + def _deduplicate_bedrock_content_blocks( blocks: list[BedrockContentBlock], block_key: str, @@ -4673,6 +4781,7 @@ class BedrockConverseMessagesProcessor: assistant_continue_message: str | ChatCompletionAssistantMessage | None = None, deployment_model: str | None = None, base_model: str | None = None, + drop_params: bool = False, ) -> list[BedrockMessageBlock]: from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking @@ -4826,33 +4935,21 @@ class BedrockConverseMessagesProcessor: ) _assistant_content = assistant_message_block.get("content", None) thinking_blocks = assistant_message_block.get("thinking_blocks") - - if thinking_blocks is not None: - converted_thinking_blocks = ( - BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks, to_claude=to_claude - ) - ) - assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( - thinking_blocks=converted_thinking_blocks, - assistant_parts=assistant_content, + converted_thinking_blocks = ( + BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( + _thinking_in_replay_order(_assistant_content, thinking_blocks), to_claude=to_claude ) + ) + assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( + thinking_blocks=converted_thinking_blocks, + assistant_parts=assistant_content, + ) if _assistant_content is not None and isinstance(_assistant_content, list): assistants_parts: list[BedrockContentBlock] = [] for element in _assistant_content: if isinstance(element, dict): - if element["type"] == "thinking" or element["type"] == "redacted_thinking": - thinking_block = BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks=[element], to_claude=to_claude - ) - assistants_parts = ( - BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( - thinking_blocks=thinking_block, - assistant_parts=assistants_parts, - ) - ) - elif element["type"] == "text": + if element["type"] == "text": # Skip completely empty strings to avoid blank content blocks if element.get("text", "").strip(): assistants_part = BedrockContentBlock(text=element["text"]) @@ -4871,7 +4968,9 @@ class BedrockConverseMessagesProcessor: model=model, deployment_model=deployment_model, ) - if _cache_point_block is not None: + if _cache_point_block is not None and not ( + drop_params and _as_thinking_block(element) is not None + ): assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): @@ -4899,7 +4998,11 @@ class BedrockConverseMessagesProcessor: msg_i += 1 - assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse") + assistant_content = list( + _without_repeated_thinking( + _deduplicate_bedrock_content_blocks(assistant_content, "toolUse"), _reasoning_identity + ) + ) assistant_content = _sort_bedrock_assistant_content_blocks(assistant_content) if assistant_content: @@ -5083,6 +5186,7 @@ def _bedrock_converse_messages_pt( assistant_continue_message: str | ChatCompletionAssistantMessage | None = None, deployment_model: str | None = None, base_model: str | None = None, + drop_params: bool = False, ) -> list[BedrockMessageBlock]: """ Converts given messages from OpenAI format to Bedrock format @@ -5244,35 +5348,21 @@ def _bedrock_converse_messages_pt( ) _assistant_content = assistant_message_block.get("content", None) thinking_blocks = assistant_message_block.get("thinking_blocks") - - if thinking_blocks is not None: - converted_thinking_blocks = ( - BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks, to_claude=to_claude - ) - ) - assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( - thinking_blocks=converted_thinking_blocks, - assistant_parts=assistant_content, + converted_thinking_blocks = ( + BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( + _thinking_in_replay_order(_assistant_content, thinking_blocks), to_claude=to_claude ) + ) + assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( + thinking_blocks=converted_thinking_blocks, + assistant_parts=assistant_content, + ) if _assistant_content is not None and isinstance(_assistant_content, list): assistants_parts: list[BedrockContentBlock] = [] for element in _assistant_content: if isinstance(element, dict): - if element["type"] == "thinking" or element["type"] == "redacted_thinking": - thinking_block = ( - BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks=[element], to_claude=to_claude - ) - ) - assistants_parts = ( - BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( - thinking_blocks=thinking_block, - assistant_parts=assistants_parts, - ) - ) - elif element["type"] == "text": + if element["type"] == "text": # AWS Bedrock doesn't allow empty or whitespace-only text content # Skip completely empty strings to avoid blank content blocks if element.get("text", "").strip(): @@ -5292,7 +5382,9 @@ def _bedrock_converse_messages_pt( model=model, deployment_model=deployment_model, ) - if _cache_point_block is not None: + if _cache_point_block is not None and not ( + drop_params and _as_thinking_block(element) is not None + ): assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): @@ -5313,7 +5405,11 @@ def _bedrock_converse_messages_pt( msg_i += 1 - assistant_content = _deduplicate_bedrock_content_blocks(assistant_content, "toolUse") + assistant_content = list( + _without_repeated_thinking( + _deduplicate_bedrock_content_blocks(assistant_content, "toolUse"), _reasoning_identity + ) + ) assistant_content = _sort_bedrock_assistant_content_blocks(assistant_content) if assistant_content: diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index fd0096f9f4d..ecf419695d0 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -49,7 +49,7 @@ from litellm.types.llms.openai import ( ChatCompletionUserMessage, ) -from .common_utils import is_unsignable_thinking_block +from .common_utils import has_blank_redacted_data, is_unsignable_thinking_block CONVERTED_SYSTEM_NOTE: Final = ( "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." @@ -267,27 +267,22 @@ def _block_index_by_message(blocks: Sequence[tuple[bool, tuple[int, ...]]]) -> M def _thinking_block_renders(block: object) -> bool: """A thinking block the converter keeps: one Anthropic can verify, so never bridged encrypted reasoning.""" - return message_field(block, "type") in _THINKING_BLOCK_TYPES and not is_unsignable_thinking_block(block) + return message_field(block, "type") in _THINKING_BLOCK_TYPES and not ( + is_unsignable_thinking_block(block) or has_blank_redacted_data(block) + ) def _assistant_part_renders(part: object) -> bool: """A text part always renders: the converter pads empty text with a placeholder.""" part_type: Final = message_field(part, "type") - if part_type == "thinking": + if part_type in _THINKING_BLOCK_TYPES: return _thinking_block_renders(part) return part_type in _RENDERED_ASSISTANT_PART_TYPES or ( isinstance(part_type, str) and part_type.endswith("_tool_result") ) -def _separate_thinking_blocks_render(message: object, parts: Sequence[object]) -> bool: - """``thinking_blocks`` reach the wire only when no inline thinking part claims the slot. - - The converter skips the separate blocks as soon as the content list carries a - ``thinking`` or ``redacted_thinking`` part, whether or not that part itself renders. - """ - if any(message_field(part, "type") in _THINKING_BLOCK_TYPES for part in parts): - return False +def _separate_thinking_blocks_render(message: object) -> bool: return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks"))) @@ -404,7 +399,7 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool: parts: Final = parts_of(content) return ( any(_assistant_part_renders(part) for part in parts) - or _separate_thinking_blocks_render(message, parts) + or _separate_thinking_blocks_render(message) or _tool_calls_render(message, answered) or bool(message_field(message, "function_call")) or bool(message_field(message_field(message, "provider_specific_fields"), "compaction_blocks")) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 867dd935a03..9a63afd5cc0 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2267,6 +2267,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_messages_pt, ) + drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic")) @@ -2293,7 +2295,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): AnthropicConfig._maybe_drop_speed_param( model=model, optional_params=optional_params, - drop_params=litellm.drop_params or litellm_params.get("drop_params") is True, + drop_params=drop_params, custom_llm_provider=self.custom_llm_provider, ) @@ -2351,6 +2353,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model=model, messages=list(conversation), llm_provider=self._resolved_provider, + drop_params=drop_params, ) except Exception as e: raise AnthropicError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b505ae47c61..cd3727401ef 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2127,6 +2127,7 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), + drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True, ), capability_model, ) @@ -2207,6 +2208,7 @@ class AmazonConverseConfig(BaseConfig): llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), + drop_params=litellm.drop_params is True or litellm_params.get("drop_params") is True, ), capability_model, ) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 22db24f6ffd..c70c7080ab8 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -1705,6 +1705,39 @@ def test_cache_control_hook_does_not_overwrite_existing_cache_control(): assert _count_cache_control(processed) == 1 +@pytest.mark.parametrize( + "content, marked_parts", + [ + ([{"type": "text", "text": "Checking."}, {"type": "redacted_thinking", "data": "abc"}], [0]), + ([{"type": "redacted_thinking", "data": "abc"}], []), + ([{"type": "thinking", "thinking": "hm", "signature": "s"}], []), + ([{"type": "text", "text": "Checking."}, "not a content block"], [0]), + ], + ids=["text-before-trailing-redacted-thinking", "redacted-thinking-only", "signed-thinking-only", "text-before-a-string"], +) +def test_assistant_injection_never_marks_a_thinking_part( + content: list[dict[str, str] | str], marked_parts: list[int] +): + """Claude 400s a cache_control on a thinking or redacted_thinking block, and Bedrock Converse 400s a cache + point after reasoning, so the breakpoint goes on the last part that can carry one.""" + messages: Final[list[AllMessageValues]] = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": copy.deepcopy(content)}, + {"role": "user", "content": "again"}, + ] + + _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model="anthropic/claude-sonnet-4-6", + messages=messages, + non_default_params={"cache_control_injection_points": [{"location": "message", "role": "assistant"}]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + + assert [index for index, part in enumerate(processed[1]["content"]) if "cache_control" in part] == marked_parts + + @pytest.mark.asyncio async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four(monkeypatch: pytest.MonkeyPatch): """End-to-end: outgoing Bedrock payload must not exceed 4 cachePoint blocks. diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 046f448046b..fad1c6178dc 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -541,25 +541,66 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): _NATIVE_REDACTED_DATA: Final = "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" _CONVERSE_REDACTED_DATA: Final = base64.b64encode(_NATIVE_REDACTED_DATA.encode()).decode() +REASONING_PLACEMENTS: Final = pytest.mark.parametrize( + "placement", ["thinking_blocks", "content", "both", "first-block-in-content", "last-block-in-content"] +) + + +def _assistant_with_reasoning( + placement: str, reasoning: Sequence[Mapping[str, object]], text: str +) -> Mapping[str, object]: + in_content: Final = {"role": "assistant", "content": [*reasoning, {"type": "text", "text": text}]} + return { + "thinking_blocks": {"role": "assistant", "content": text, "thinking_blocks": list(reasoning)}, + "content": in_content, + "both": {**in_content, "thinking_blocks": list(reasoning)}, + "first-block-in-content": { + "role": "assistant", + "content": [*reasoning[:1], {"type": "text", "text": text}], + "thinking_blocks": list(reasoning), + }, + "last-block-in-content": { + "role": "assistant", + "content": [*reasoning[-1:], {"type": "text", "text": text}], + "thinking_blocks": list(reasoning), + }, + }[placement] + + +SPLIT_REASONING: Final = pytest.mark.parametrize( + ("in_content", "in_thinking_blocks"), + [((0, 2), (0, 1, 2)), ((0, 1, 2), (1,))], + ids=["middle-block-only-in-thinking-blocks", "outer-blocks-only-in-content"], +) + + +COPY_TEXT: Final = pytest.mark.parametrize("copy_text", ["", "First"], ids=["emptied", "trimmed"]) + + +def _assistant_with_split_reasoning( + reasoning: Sequence[Mapping[str, object]], in_content: Sequence[int], in_thinking_blocks: Sequence[int], text: str +) -> Mapping[str, object]: + return { + "role": "assistant", + "content": [*(reasoning[index] for index in in_content), {"type": "text", "text": text}], + "thinking_blocks": [reasoning[index] for index in in_thinking_blocks], + } @pytest.mark.asyncio @pytest.mark.parametrize("use_async", [False, True]) -@pytest.mark.parametrize("placement", ["thinking_blocks", "content"]) +@REASONING_PLACEMENTS async def test_bedrock_converse_replays_redacted_thinking_as_redacted_content(use_async: bool, placement: str): """A redacted_thinking block that Bedrock returned must go back as reasoningContent.redactedContent, - in order, ahead of text and toolUse. It used to collapse into an empty unsigned reasoningText and get dropped.""" + in order, ahead of text and toolUse. It used to collapse into an empty unsigned reasoningText and get dropped. + Sent twice, when the history carries it in content and thinking_blocks, Bedrock 400s that it was modified.""" bedrock_reasoning_blocks: Final[list[BedrockConverseReasoningContentBlock]] = [ {"reasoningText": {"text": "Let me look this up.", "signature": "sig-abc"}}, {"redactedContent": _CONVERSE_REDACTED_DATA}, ] thinking_blocks: Final = litellm.AmazonConverseConfig()._transform_thinking_blocks(bedrock_reasoning_blocks) assistant_text: Final = "Checking the weather." - assistant_message: Final = ( - {"role": "assistant", "content": assistant_text, "thinking_blocks": thinking_blocks} - if placement == "thinking_blocks" - else {"role": "assistant", "content": [*thinking_blocks, {"type": "text", "text": assistant_text}]} - ) + assistant_message: Final = _assistant_with_reasoning(placement, thinking_blocks, assistant_text) messages: Final = [ {"role": "user", "content": "What's the weather in Paris?"}, { @@ -593,6 +634,231 @@ async def test_bedrock_converse_replays_redacted_thinking_as_redacted_content(us ] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@COPY_TEXT +async def test_bedrock_converse_sends_a_signed_block_once_when_thinking_blocks_holds_other_text( + use_async: bool, copy_text: str +): + bedrock_reasoning_blocks: Final[list[BedrockConverseReasoningContentBlock]] = [ + {"reasoningText": {"text": "First.", "signature": "sig-1"}}, + {"redactedContent": _CONVERSE_REDACTED_DATA}, + ] + reasoning: Final = litellm.AmazonConverseConfig()._transform_thinking_blocks(bedrock_reasoning_blocks) + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [*reasoning, {"type": "text", "text": "hello"}], + "thinking_blocks": [{**reasoning[0], "thinking": copy_text}, reasoning[1]], + }, + {"role": "user", "content": "again"}, + ] + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert result[1] == { + "role": "assistant", + "content": [*({"reasoningContent": block} for block in bedrock_reasoning_blocks), {"text": "hello"}], + } + + +_WEATHER_TOOL_CALL: Final = { + "id": "tooluse_weather_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, +} + + +def _block_split_across_assistant_messages(blocks: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "What's the weather in Paris?"}, + {"role": "assistant", "content": list(blocks)}, + {"role": "assistant", "content": None, "thinking_blocks": list(blocks), "tool_calls": [_WEATHER_TOOL_CALL]}, + {"role": "tool", "tool_call_id": "tooluse_weather_1", "content": "18C, sunny"}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_sends_a_block_once_when_two_assistant_messages_carry_it(use_async: bool): + bedrock_reasoning_blocks: Final[list[BedrockConverseReasoningContentBlock]] = [ + {"reasoningText": {"text": "Let me look this up.", "signature": "sig-abc"}}, + {"redactedContent": _CONVERSE_REDACTED_DATA}, + ] + reasoning: Final = litellm.AmazonConverseConfig()._transform_thinking_blocks(bedrock_reasoning_blocks) + messages: Final = _block_split_across_assistant_messages(reasoning) + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert result[1] == { + "role": "assistant", + "content": [ + *({"reasoningContent": block} for block in bedrock_reasoning_blocks), + {"toolUse": {"toolUseId": "tooluse_weather_1", "name": "get_weather", "input": {"city": "Paris"}}}, + ], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@SPLIT_REASONING +async def test_bedrock_converse_keeps_reasoning_order_across_content_and_thinking_blocks( + use_async: bool, in_content: Sequence[int], in_thinking_blocks: Sequence[int] +): + """A reasoning block only one of content and thinking_blocks carries goes back in its original place.""" + bedrock_reasoning_blocks: Final[list[BedrockConverseReasoningContentBlock]] = [ + {"reasoningText": {"text": "First.", "signature": "sig-1"}}, + {"redactedContent": _CONVERSE_REDACTED_DATA}, + {"reasoningText": {"text": "Third.", "signature": "sig-3"}}, + ] + reasoning: Final = litellm.AmazonConverseConfig()._transform_thinking_blocks(bedrock_reasoning_blocks) + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_split_reasoning(reasoning, in_content, in_thinking_blocks, "hello"), + {"role": "user", "content": "again"}, + ] + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert result[1] == { + "role": "assistant", + "content": [*({"reasoningContent": block} for block in bedrock_reasoning_blocks), {"text": "hello"}], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("entry", [None, "not a block"], ids=["none", "string"]) +async def test_bedrock_converse_skips_a_thinking_blocks_entry_that_is_not_a_block(use_async: bool, entry: object): + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello", "thinking_blocks": [entry]}, + {"role": "user", "content": "again"}, + ] + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert result[1] == {"role": "assistant", "content": [{"text": "hello"}]} + + +MALFORMED_THINKING: Final = pytest.mark.parametrize( + "malformed", + [ + {"type": "thinking", "thinking": {"a": 1}}, + {"type": "thinking", "thinking": ["a"], "signature": ""}, + {"type": "thinking", "thinking": "draft", "signature": ["a"]}, + ], + ids=["dict-thinking", "list-thinking", "list-signature"], +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@REASONING_PLACEMENTS +@MALFORMED_THINKING +async def test_bedrock_converse_replays_redacted_thinking_once_beside_a_malformed_thinking_item( + use_async: bool, placement: str, malformed: dict[str, object] +): + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_reasoning( + placement, [{"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA}, malformed], "hello" + ), + {"role": "user", "content": "again"}, + ] + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert result[1]["content"].count({"reasoningContent": {"redactedContent": _CONVERSE_REDACTED_DATA}}) == 1 + + +@REASONING_PLACEMENTS +@MALFORMED_THINKING +def test_anthropic_messages_pt_replays_redacted_thinking_once_beside_a_malformed_thinking_item( + placement: str, malformed: dict[str, object] +): + redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA} + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_reasoning(placement, [redacted, malformed], "hello"), + {"role": "user", "content": "again"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert result[1]["content"].count(redacted) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("drop_params", [False, True]) +async def test_bedrock_converse_drops_a_cache_point_after_reasoning_only_under_drop_params( + use_async: bool, drop_params: bool +): + """Bedrock 400s "Cache point cannot be inserted after reasoning block", so drop_params drops the + breakpoint and without it the request reaches Bedrock as sent.""" + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA, "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "hello"}, + ], + }, + {"role": "user", "content": "again"}, + ] + model: Final = "us.anthropic.claude-sonnet-4-6" + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock", drop_params=drop_params + ) + if use_async + else _bedrock_converse_messages_pt( + messages=messages, model=model, llm_provider="bedrock", drop_params=drop_params + ) + ) + + assert ({"cachePoint": {"type": "default"}} in result[1]["content"]) is not drop_params + + _SENT_REDACTED: Final = [{"reasoningContent": {"redactedContent": _CONVERSE_REDACTED_DATA}}] @@ -813,17 +1079,17 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): ) +@REASONING_PLACEMENTS @pytest.mark.parametrize("data", [_NATIVE_REDACTED_DATA, _CONVERSE_REDACTED_DATA], ids=["native", "legacy-wrapped"]) -def test_anthropic_messages_pt_replays_redacted_thinking_as_claude_native_data(data: str): +def test_anthropic_messages_pt_replays_redacted_thinking_as_claude_native_data(data: str, placement: str): """Histories saved from Converse before its replies were decoded hold the data base64-wrapped, and the - Anthropic API, Vertex and Bedrock Invoke 400 on it with "Invalid `data` in `redacted_thinking` block".""" + Anthropic API, Vertex and Bedrock Invoke 400 on it with "Invalid `data` in `redacted_thinking` block". + A redacted block in the content list, the shape the Anthropic SDK replays, used to be dropped.""" + signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"} + redacted: Final = {"type": "redacted_thinking", "data": data} messages: Final = [ {"role": "user", "content": "hi"}, - { - "role": "assistant", - "content": "hello", - "thinking_blocks": [{"type": "redacted_thinking", "data": data, "cache_control": {"type": "ephemeral"}}], - }, + _assistant_with_reasoning(placement, [signed, redacted], "hello"), {"role": "user", "content": "again"}, ] @@ -832,12 +1098,125 @@ def test_anthropic_messages_pt_replays_redacted_thinking_as_claude_native_data(d assert result[1] == { "role": "assistant", "content": [ - {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA, "cache_control": {"type": "ephemeral"}}, + signed, + {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA}, {"type": "text", "text": "hello"}, ], } +@SPLIT_REASONING +def test_anthropic_messages_pt_keeps_reasoning_order_across_content_and_thinking_blocks( + in_content: Sequence[int], in_thinking_blocks: Sequence[int] +): + """A thinking block only one of content and thinking_blocks carries goes back in its original place.""" + reasoning: Final = ( + {"type": "thinking", "thinking": "First.", "signature": "sig-1"}, + {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA}, + {"type": "thinking", "thinking": "Third.", "signature": "sig-3"}, + ) + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_split_reasoning(reasoning, in_content, in_thinking_blocks, "hello"), + {"role": "user", "content": "again"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert result[1] == {"role": "assistant", "content": [*reasoning, {"type": "text", "text": "hello"}]} + + +@COPY_TEXT +def test_anthropic_messages_pt_sends_a_signed_block_once_when_thinking_blocks_holds_other_text(copy_text: str): + signed: Final = {"type": "thinking", "thinking": "First.", "signature": "sig-1"} + redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA} + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [signed, redacted, {"type": "text", "text": "hello"}], + "thinking_blocks": [{**signed, "thinking": copy_text}, redacted], + }, + {"role": "user", "content": "again"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + + assert result[1] == {"role": "assistant", "content": [signed, redacted, {"type": "text", "text": "hello"}]} + + + +def test_anthropic_messages_pt_sends_a_block_once_when_two_assistant_messages_carry_it(): + signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"} + redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA} + + result: Final = anthropic_messages_pt( + messages=_block_split_across_assistant_messages([signed, redacted]), + model="claude-sonnet-4-6", + llm_provider="anthropic", + ) + + assert result[1] == { + "role": "assistant", + "content": [ + signed, + redacted, + {"type": "tool_use", "id": "tooluse_weather_1", "name": "get_weather", "input": {"city": "Paris"}}, + ], + } + +@REASONING_PLACEMENTS +@pytest.mark.parametrize("drop_params", [False, True]) +def test_anthropic_messages_pt_drops_cache_control_from_thinking_blocks_only_under_drop_params( + placement: str, drop_params: bool +): + """Claude 400s "thinking.cache_control: Extra inputs are not permitted" on a thinking or redacted_thinking + block, so drop_params drops the breakpoint and without it the request reaches Claude as sent. The streamed + list-position `index` is LiteLLM's own field and never reaches Claude.""" + breakpoint_mark: Final = {"cache_control": {"type": "ephemeral"}} + signed: Final = {"type": "thinking", "thinking": "Let me look this up.", "signature": "sig-abc"} + redacted: Final = {"type": "redacted_thinking", "data": _NATIVE_REDACTED_DATA} + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_reasoning( + placement, [{**signed, **breakpoint_mark, "index": 0}, {**redacted, **breakpoint_mark, "index": 1}], "hello" + ), + {"role": "user", "content": "again"}, + ] + + result: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic", drop_params=drop_params + ) + + sent_reasoning: Final = ( + [signed, redacted] if drop_params else [{**signed, **breakpoint_mark}, {**redacted, **breakpoint_mark}] + ) + assert result[1] == {"role": "assistant", "content": [*sent_reasoning, {"type": "text", "text": "hello"}]} + + +@REASONING_PLACEMENTS +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("data", ["", " ", None], ids=["empty", "whitespace", "null"]) +def test_anthropic_messages_pt_drops_a_blank_redacted_thinking_block_only_under_drop_params( + data: str | None, placement: str, drop_params: bool +): + """Claude 400s "Invalid `data` in `redacted_thinking` block" on blank data, so drop_params drops the block + and without it the request reaches Claude as sent.""" + blank: Final = {"type": "redacted_thinking", "data": data} + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_reasoning(placement, [blank], "hello"), + {"role": "user", "content": "again"}, + ] + + result: Final = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic", drop_params=drop_params + ) + + sent_reasoning: Final = [] if drop_params else [blank] + assert result[1] == {"role": "assistant", "content": [*sent_reasoning, {"type": "text", "text": "hello"}]} + + def test_anthropic_messages_pt_keeps_signed_thinking_block(): """A genuine Anthropic round-trip still holds its original signature, so that thinking block must be forwarded unchanged (we only drop unsignable blocks). diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index 91d7acd331e..aab57b87a90 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -308,16 +308,8 @@ EMPTY_ASSISTANT = pytest.mark.parametrize( "content": None, "thinking_blocks": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("abc")}], }, - { - "role": "assistant", - "content": [{"type": "thinking", "thinking": "unsigned"}], - "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}], - }, - { - "role": "assistant", - "content": [{"type": "redacted_thinking", "data": "x"}], - "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}], - }, + {"role": "assistant", "content": [{"type": "redacted_thinking", "data": ""}]}, + {"role": "assistant", "content": None, "thinking_blocks": [{"type": "redacted_thinking", "data": " "}]}, litellm.Message(role="assistant", content=None), {"role": "assistant", "content": None, "tool_calls": [_TOOL_SEARCH_USE]}, ], @@ -327,8 +319,8 @@ EMPTY_ASSISTANT = pytest.mark.parametrize( "unsigned-thinking-part", "encrypted-thinking-part", "encrypted-redacted-thinking-block", - "unsigned-inline-part-hides-signed-block", - "inline-redacted-part-hides-redacted-block", + "blank-data-redacted-part", + "blank-data-redacted-block", "pydantic-none", "unanswered-tool-search-use", ], @@ -464,6 +456,16 @@ def test_flagged_placement_keeps_a_system_whose_empty_assistant_follower_ends_th "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}], }, {"role": "assistant", "content": None, "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}]}, + { + "role": "assistant", + "content": [{"type": "redacted_thinking", "data": "x"}], + "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}], + }, + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "unsigned"}], + "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}], + }, {"role": "assistant", "content": [{"type": "thinking", "thinking": "hm", "signature": "s"}]}, {"role": "assistant", "content": [{"type": "thinking", "thinking": "", "signature": "s"}]}, {"role": "assistant", "content": None, "function_call": {"name": "f", "arguments": "{}"}}, @@ -485,6 +487,8 @@ def test_flagged_placement_keeps_a_system_whose_empty_assistant_follower_ends_th "tool-calls", "signed-thinking-block", "redacted-thinking-block", + "redacted-part-beside-the-same-redacted-block", + "signed-block-beside-an-unsigned-part", "signed-thinking-part", "signed-empty-thinking-part", "function-call", From 422252326153b3006ed17abe0c0549bf795c3526 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 08:02:23 -0700 Subject: [PATCH 100/124] fix(logging): report the pay-as-you-go entry billing used on Azure PTU spillover (#47567) get_standard_logging_object_payload decided custom_pricing from litellm_params alone, so on an Azure PTU spillover response model_map_information kept the deployment's zeroed PTU entry while response_cost already used the standard model rate. Reuse the billing side LitellmLogging._custom_pricing_for so both read the same decision Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 8 ++--- .../test_litellm_logging.py | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e2eed3cf6d5..78ff073fdcb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1990,7 +1990,7 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - custom_pricing: Final = self._custom_pricing_for(priced_result) + custom_pricing: Final = self.custom_pricing_for(priced_result) prompt: Final = self._prompt_for_cost_calculation() @@ -2165,7 +2165,7 @@ class Logging(LiteLLMLoggingBaseClass): model=model, completion_response=result, custom_llm_provider=custom_llm_provider, - custom_pricing=self._custom_pricing_for(result), + custom_pricing=self.custom_pricing_for(result), base_model=get_base_model_from_metadata(model_call_details=self.model_call_details), router_model_id=router_model_id or self.get_router_model_id(), region_name=_resolve_mantle_region_for_cost( @@ -2190,7 +2190,7 @@ class Logging(LiteLLMLoggingBaseClass): usage=usage, ) - def _custom_pricing_for(self, result: object) -> bool: + def custom_pricing_for(self, result: object) -> bool: litellm_params: Final = getattr(self, "litellm_params", None) result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({}) additional_headers: Final = ( @@ -7201,7 +7201,7 @@ def get_standard_logging_object_payload( # Fall back to the actual deployment model set by the router in metadata. if base_model is None: base_model = metadata.get("deployment") - custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params) + custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") llm_response_cost: Final[float] = raw_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 9d651b4ae3e..7aac43298d4 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10095,6 +10095,40 @@ class TestAzurePTUSpilloverCost: finally: self._unregister_models() + def test_spillover_payload_reports_the_pay_as_you_go_entry_billing_used(self, monkeypatch) -> None: + self._register_models() + try: + obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch) + response = self._response() + response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"} + response_cost: Final = obj.response_cost_calculator(result=response) + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": self.SERVED_MODEL, + "custom_llm_provider": "azure", + "litellm_params": obj.litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=obj, + status="success", + ) + + assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST) + assert payload is not None + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"] + assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"] + assert response_cost == pytest.approx( + 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] + ) + finally: + self._unregister_models() + def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None: self._register_models() custom_model_id: Final = "non-ptu-custom-router-model-id" From c38b86cc93badcee75d822d4776721fce824e982 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 08:03:31 -0700 Subject: [PATCH 101/124] fix(router): skip non-dict tool entries in the prompt caching affinity key instead of raising (#47565) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_utils/prompt_caching_cache.py | 6 ++++-- .../router_utils/test_prompt_caching_cache.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index 4ec0bc62e7f..f78b5982186 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -94,8 +94,10 @@ def _is_ephemeral_mark(mark: object) -> bool: return mark_map.get("type") == "ephemeral" -def _tool_carries_ephemeral_mark(tool: ChatCompletionToolParam) -> bool: - tool_map: Final = cast( # cast-ok: TypedDict.get rejects keys the type does not declare +def _tool_carries_ephemeral_mark(tool: object) -> bool: + if not isinstance(tool, dict): + return False + tool_map: Final = cast( # cast-ok: tools are plain JSON dicts Mapping[str, object], tool ) if _is_ephemeral_mark(tool_map.get("cache_control")): diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index 46af2454dea..ce8f04b8c54 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -608,6 +608,23 @@ def test_unmarked_tools_change_the_key_marked_tools_do_not_share(): assert key_a != key_b +def test_a_non_dict_tool_entry_still_derives_a_key_instead_of_raising(): + """ + A client can send `tools: ["not-a-tool"]`. The provider rejects that with its own 400, so the + routing check has to key on what it was given and let the request through, never raise a 500. + """ + messages = cast( + list[AllMessageValues], + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + ) + tools: Final = cast(list[ChatCompletionToolParam], ["not-a-tool", _marked_tool("read", {"type": "ephemeral"})]) + + positions = PromptCachingCache.prefix_positions(messages, tools) + + assert [position.position for position in positions] == [0, 1] + assert PromptCachingCache.get_prompt_caching_cache_key(messages, tools) == positions[-1].cache_key + + def test_empty_and_missing_tools_seed_the_same_key(): """`tools=[]` and no tools at all are the same request to the provider, so they must key alike.""" messages = cast( From 04e9ed4c52505b5a50840843bc8b367aa32d09eb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 08:04:55 -0700 Subject: [PATCH 102/124] fix(litellm_proxy): forward tool_config cache_control_injection_points to the downstream proxy (#47570) PR #47394 withheld cache_control_injection_points from the OpenAI SDK extra_body for every provider on that path, which also covers litellm_proxy. The downstream LiteLLM proxy applies the point itself, so litellm_proxy keeps forwarding it while OpenAI and the other OpenAI-compatible providers still never receive it Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 3 ++- tests/unit/test_utils.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 14ee35e571e..a6108a15bb3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5552,8 +5552,9 @@ def add_provider_specific_params_to_optional_params( # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46 if _should_drop_param(k="extra_body", additional_drop_params=additional_drop_params) is False: extra_body: Final = dict(passed_params.pop("extra_body", None) or {}) + withheld_params: Final = frozenset[str]() if custom_llm_provider == "litellm_proxy" else LITELLM_ONLY_PARAMS for k in passed_params: - if k not in openai_params and k not in LITELLM_ONLY_PARAMS and passed_params[k] is not None: + if k not in openai_params and k not in withheld_params and passed_params[k] is not None: extra_body[k] = passed_params[k] if not isinstance(optional_params.get("extra_body"), dict): optional_params["extra_body"] = {} diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 44939ca7585..0be3ae05e97 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10703,3 +10703,18 @@ def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_e assert optional_params["extra_body"] == {"kept": True} assert "cache_control_injection_points" not in optional_params + + +def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): + """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy + applies cache_control_injection_points itself, so the point must ride along in extra_body.""" + point: Final = {"location": "tool_config", "control": {"type": "ephemeral"}} + optional_params: Final[dict[str, object]] = litellm.utils.get_optional_params( + model="claude-sonnet-4-5", + custom_llm_provider="litellm_proxy", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + cache_control_injection_points=[point], + extra_body={"kept": True}, + ) + + assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} From 91724bb0a7c88e54621f149b5cdb5f133f58dd12 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:13:59 +0000 Subject: [PATCH 103/124] fix(responses)!: count input tokens the way inference sends the request (#47363) * fix(count-tokens): translate chat tool use before Claude token counts Responses and chat tool use (function tools, function_call items, tool messages) is now converted the way Claude chat inference converts it before the Anthropic, Azure AI and Vertex partner counters call count_tokens. Translation errors answer 400 before any provider call. Anthropic-shaped requests and /v1/messages/count_tokens keep their body as sent. * test(count-tokens): compare Claude count bodies with inference for Responses tool use * fix(responses): forward the caller's Responses body to OpenAI token counts OpenAI deployments now get the caller's /v1/responses/input_tokens body as sent, minus LiteLLM-only fields, so OpenAI's own count and errors come back. previous_response_id is checked for ownership like inference and routes to the deployment that served it * fix(responses)!: count Claude Responses bodies the way inference sends them Non-native Responses counts now run the body through the inference bridge, so text.format, tool_choice, reasoning, built-in tools, and file_search count the way inference sends them on Anthropic, Azure AI, Vertex, and Bedrock. The OpenAI native count body uses the drop_params-resolved tool fields. * fix(responses): honor drop_params and replay previous turns in Claude counts Claude input_tokens counts now read the same drop_params setting inference merges from the deployment, router defaults, and request, so a forced tool_choice downgrades to auto under drop_params. previous_response_id replays the stored turns through inference's session loader, and chat messages carrying fields beyond role and content go through the provider's chat translation * fix(responses): keep proxy-only request keys out of OpenAI counts and map chat tools in Bedrock counts * fix(count-tokens): import on Python 3.11 and keep stored prompts in OpenAI counts under drop_params * fix(responses): leave tool_choice out of counts when drop_params drops every tool * fix(count-tokens): count Mantle options, drop cache marks, and reject input inference cannot send The Bedrock Mantle fallback now counts thinking, tool_choice and output_config like the runtime count. Claude Responses counts leave out cache_control marks again, so more than four marks count instead of failing. Counts answer 400 before any provider call for user blocks chat inference refuses and for Responses tools with no Chat Completions equivalent, instead of dropping the block or counting a placeholder tool. Adds the wire tests for these paths * fix(count-tokens): count Responses tools the way the inference bridge sends them A Claude count now hoists additional_tools input items into tools, strips a malformed tool flag under drop_params and rejects it without, and leaves tools out when the deployment lists tools in additional_drop_params, so the count matches the body Claude inference sends for the same request * fix(count-tokens): keep Bedrock runtime counts option-free and send OpenAI the provider file id * test(count-tokens): capture the Anthropic count request at the HTTP layer instead of patching the handler * fix(count-tokens): ignore a caller-sent file id mapping and keep non-string file ids as sent --- .../prompt_templates/common_utils.py | 6 +- .../anthropic/count_tokens/token_counter.py | 34 ++ .../anthropic/count_tokens/transformation.py | 225 ++++++++++- .../anthropic/count_tokens/handler.py | 3 + .../anthropic/count_tokens/token_counter.py | 34 ++ litellm/llms/base_llm/base_utils.py | 2 + litellm/llms/base_llm/files/batch_records.py | 2 + .../count_tokens/bedrock_token_counter.py | 62 ++- .../bedrock/count_tokens/mantle_handler.py | 7 +- .../bedrock/count_tokens/transformation.py | 9 +- litellm/llms/gemini/common_utils.py | 2 + .../openai/responses/count_tokens/handler.py | 16 +- .../responses/count_tokens/token_counter.py | 163 ++++---- .../responses/count_tokens/transformation.py | 36 ++ litellm/llms/vertex_ai/common_utils.py | 66 ++- .../vertex_ai_partner_models/main.py | 4 +- litellm/main.py | 1 + litellm/proxy/proxy_server.py | 141 ++++++- .../proxy/response_api_endpoints/endpoints.py | 93 ++++- .../responses/file_search/emulated_handler.py | 6 +- .../count_request.py | 207 ++++++++++ .../transformation.py | 2 +- litellm/responses/main.py | 4 +- .../providers/_count_tokens_system_lift.py | 81 ++++ ...anthropic_count_tokens_system_lift_wire.py | 176 +++++++- .../test_bedrock_mantle_count_tokens_wire.py | 107 +++++ .../providers/test_malformed_tools_wire.py | 11 +- .../test_openai_count_tokens_tools_wire.py | 74 +++- .../providers/test_responses_core_wire.py | 37 ++ .../test_vertex_gemini_count_tokens_wire.py | 54 +++ .../test_vertex_partner_count_tokens_wire.py | 86 +++- .../test_vertex_partner_count_tokens_sdk.py | 76 +++- ...ore_utils_prompt_templates_common_utils.py | 24 ++ ...t_anthropic_count_tokens_transformation.py | 306 ++++++++++++++ ...est_bedrock_count_tokens_transformation.py | 19 + .../test_bedrock_token_counter.py | 125 ++++++ .../count_tokens/test_token_counter.py | 121 ++++++ tests/unit/llms/test_file_search_responses.py | 16 +- .../vertex_ai/test_vertex_ai_common_utils.py | 149 ++++++- .../response_api_endpoints/test_endpoints.py | 149 ++++++- tests/unit/proxy/test_proxy_token_counter.py | 378 +++++++++++++++++- .../test_count_request.py | 127 ++++++ tests/unit/test_count_tokens_public_api.py | 46 +++ 43 files changed, 3127 insertions(+), 160 deletions(-) create mode 100644 litellm/responses/litellm_completion_transformation/count_request.py create mode 100644 tests/unit/responses/litellm_completion_transformation/test_count_request.py diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b89e4d609ad..60be4d44706 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -624,7 +624,7 @@ def update_messages_with_model_file_ids( def update_responses_input_with_model_file_ids( input: object, model_id: str | None = None, - model_file_id_mapping: dict[str, dict[str, str]] | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, ) -> object: """ Updates responses API input with provider-specific file IDs. @@ -665,7 +665,7 @@ def update_responses_input_with_model_file_ids( for content_item in content: if isinstance(content_item, dict) and content_item.get("type") == "input_file": file_id = content_item.get("file_id") - if file_id: + if isinstance(file_id, str) and file_id: provider_file_id = file_id # Default to original # Check if we have a mapping for this file ID @@ -761,7 +761,7 @@ def _decode_vector_store_ids_in_tools( def update_responses_tools_with_model_file_ids( tools: list[dict[str, object]] | None, model_id: str | None = None, - model_file_id_mapping: dict[str, dict[str, str]] | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, ) -> list[dict[str, object]] | None: """ Updates responses API tools with provider-specific file IDs. diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index ecf050d185e..0764fb41941 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -2,6 +2,8 @@ Anthropic Token Counter implementation using the CountTokens API. """ +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue @@ -10,6 +12,11 @@ import litellm from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler +from litellm.llms.anthropic.count_tokens.transformation import ( + AnthropicCountFields, + ChatCountTranslationError, + anthropic_count_fields, +) from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse @@ -38,6 +45,8 @@ class AnthropicTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ Count tokens using Anthropic's CountTokens API. @@ -57,6 +66,30 @@ class AnthropicTokenCounter(BaseTokenCounter): if not messages: return None + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + match translated := await anthropic_count_fields( + model_to_use, messages, system, tools, messages_format, AnthropicConfig(), chat_params + ): + case ChatCountTranslationError(): + return translated.response(request_model, model_to_use, "anthropic_api") + case AnthropicCountFields(): + return await self.count_tokens( + model_to_use=model_to_use, + messages=[dict(message) for message in translated.messages], + contents=contents, + deployment=deployment, + request_model=request_model, + tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, + system=translated.system, + messages_format="anthropic", + output_format=output_format, + context_management=context_management, + count_options=translated.options, + ) + case None: + pass + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) api_base: Final = litellm_params.get("api_base") @@ -89,6 +122,7 @@ class AnthropicTokenCounter(BaseTokenCounter): api_base=api_base, tools=tools, system=system, + optional_params=count_options, ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 8e2688d4360..592b5fb0f00 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -4,25 +4,37 @@ Anthropic CountTokens API transformation logic. This module handles the transformation of requests to Anthropic's CountTokens API format. """ +import copy from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import chain from types import MappingProxyType -from typing import Final +from typing import ( + Final, + Protocol, + TypeAlias, + cast, # noqa: TID251 # caller chat messages go to chat inference translation as-is +) -from pydantic import JsonValue, TypeAdapter +from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( anthropic_system_blocks, split_leading_system_run, ) from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base -from litellm.types.llms.openai import ChatCompletionImageObject +from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes +from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) _SYSTEM_BLOCKS: Final = TypeAdapter(list[JsonValue]) _IMAGE_BLOCK: Final = TypeAdapter(ChatCompletionImageObject) COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config") +_PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) def _count_image(block: JsonValue) -> JsonValue: @@ -74,6 +86,213 @@ def _lift_leading_system( return conversation, lifted or system +@dataclass(frozen=True, slots=True) +class AnthropicCountFields: + messages: tuple[Mapping[str, JsonValue], ...] + system: JsonValue + tools: tuple[Mapping[str, JsonValue], ...] | None + options: Mapping[str, JsonValue] + + +@dataclass(frozen=True, slots=True) +class ChatCountTranslationError: + message: str + + def response(self, request_model: str, model_used: str, tokenizer_type: str) -> TokenCountResponse: + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_used, + tokenizer_type=tokenizer_type, + error=True, + error_message=self.message, + status_code=400, + rejected_before_provider_call=True, + ) + + +@with_config(ConfigDict(extra="ignore")) +class _AnthropicChatRequest(TypedDict): + messages: ReadOnly[tuple[Mapping[str, JsonValue], ...]] + system: NotRequired[ReadOnly[JsonValue]] + tools: NotRequired[ReadOnly[tuple[Mapping[str, JsonValue], ...]]] + thinking: NotRequired[ReadOnly[JsonValue]] + tool_choice: NotRequired[ReadOnly[JsonValue]] + output_config: NotRequired[ReadOnly[JsonValue]] + output_format: NotRequired[ReadOnly[JsonValue]] + + +_ANTHROPIC_CHAT_REQUEST: Final = TypeAdapter(_AnthropicChatRequest) +_ChatParams: TypeAlias = dict[str, object] +_ChatMessages: TypeAlias = list[AllMessageValues] + + +class _ChatRequestTranslator(Protocol): + def map_openai_params( + self, non_default_params: _ChatParams, optional_params: _ChatParams, model: str, drop_params: bool + ) -> _ChatParams: ... + + def transform_request( + self, + model: str, + messages: _ChatMessages, + optional_params: _ChatParams, + litellm_params: _ChatParams, + headers: _ChatParams, + ) -> _ChatParams: ... + + +def _needs_chat_translation(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" or not _PLAIN_MESSAGE_KEYS.issuperset(message.keys()) + + +def _is_chat_tool(tool: Mapping[str, object]) -> bool: + from litellm.litellm_core_utils.prompt_templates.factory import is_flat_responses_tool + + return is_flat_responses_tool(tool) or (tool.get("type") == "function" and "function" in tool) + + +def _has_non_chat_user_block(message: Mapping[str, object]) -> bool: + content: Final = message.get("content") + if message.get("role") != "user" or not isinstance(content, list): + return False + blocks: Final = _SYSTEM_BLOCKS.validate_python(content) + return any(isinstance(block, dict) and block.get("type") not in ValidUserMessageContentTypes for block in blocks) + + +def _gets_placeholder_name(tool: Mapping[str, object]) -> bool: + from litellm.llms.anthropic.pass_through.adapters.transformation import is_provider_native_tool_dict + + name: Final = tool.get("name") + return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip())) + + +def _untranslatable_input( + messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] +) -> ChatCountTranslationError | None: + invalid_index: Final = next( + (index for index, message in enumerate(messages) if _has_non_chat_user_block(message)), None + ) + if invalid_index is not None: + return ChatCountTranslationError( + message=f"Invalid user message at index {invalid_index}. " + "Please ensure all user messages are valid OpenAI chat completion messages." + ) + unnamed_tool: Final = next(filter(_gets_placeholder_name, tools), None) + if unnamed_tool is not None: + return ChatCountTranslationError( + message=f"Invalid request for token counting: tool type {unnamed_tool.get('type')!r} " + "has no Chat Completions equivalent" + ) + return None + + +def _system_messages(system: object) -> tuple[Mapping[str, object], ...]: + message: Final[Mapping[str, object]] = {"role": "system", "content": system} + return (message,) if isinstance(system, (str, list)) and system else () + + +def _count_options(request: _AnthropicChatRequest) -> Mapping[str, JsonValue]: + """Structured output reaches chat inference as the beta ``output_format``, which count_tokens + refuses; its GA ``output_config.format`` form counts the same schema the same way.""" + output_config: Final = request.get("output_config") + output_format: Final = request.get("output_format") + counted_config: Final = { + **(output_config if isinstance(output_config, dict) else {}), + **({} if output_format is None else {"format": output_format}), + } + return MappingProxyType( + { + key: value + for key, value in ( + ("thinking", request.get("thinking")), + ("tool_choice", request.get("tool_choice")), + ("output_config", counted_config or None), + ) + if value is not None + } + ) + + +def _translated_chat_request( + model: str, + messages: Sequence[Mapping[str, object]], + system: object, + tools: Sequence[Mapping[str, object]], + chat_params: Mapping[str, object], + translator: _ChatRequestTranslator, +) -> _ChatParams: + from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for + + chat_tools: Final = [dict(chat_tool) for chat_tool in chain.from_iterable(map(chat_completion_tools_for, tools))] + openai_params: Final = {name: value for name, value in chat_params.items() if name != "drop_params"} + mapped_params: Final = {**openai_params, **({"tools": chat_tools} if chat_tools else {})} + chat_messages: Final = cast( # cast-ok: chat-shaped request messages, read the way chat inference reads them + _ChatMessages, copy.deepcopy([*_system_messages(system), *messages]) + ) + return translator.transform_request( + model=model, + messages=chat_messages, + optional_params=translator.map_openai_params( + mapped_params, {}, model, drop_params=chat_params.get("drop_params") is True + ) + if mapped_params + else {}, + litellm_params={}, + headers={}, + ) + + +def anthropic_fields_for_chat_tool_use( + model: str, + messages: Sequence[Mapping[str, object]], + system: object, + tools: Sequence[Mapping[str, object]] | None, + translator: _ChatRequestTranslator, + chat_params: Mapping[str, object] = MappingProxyType({}), +) -> AnthropicCountFields | ChatCountTranslationError | None: + counted_tools: Final = tools or () + if ( + not chat_params + and not any(map(_is_chat_tool, counted_tools)) + and not any(map(_needs_chat_translation, messages)) + ): + return None + untranslatable: Final = _untranslatable_input(messages, counted_tools) + if untranslatable is not None: + return untranslatable + from litellm.exceptions import BadRequestError + from litellm.llms.anthropic.common_utils import AnthropicError + + try: + request: Final = _translated_chat_request(model, messages, system, counted_tools, chat_params, translator) + except (AnthropicError, BadRequestError) as e: + return ChatCountTranslationError(message=str(e.message)) + except (ValueError, KeyError) as e: + return ChatCountTranslationError(message=f"Invalid request for token counting: {e}") + translated: Final = _ANTHROPIC_CHAT_REQUEST.validate_python(request) + return AnthropicCountFields( + messages=translated["messages"], + system=translated.get("system"), + tools=translated.get("tools"), + options=_count_options(translated), + ) + + +async def anthropic_count_fields( + model: str, + messages: Sequence[Mapping[str, object]], + system: object, + tools: Sequence[Mapping[str, object]] | None, + messages_format: TokenCountMessagesFormat, + translator: _ChatRequestTranslator, + chat_params: Mapping[str, object] = MappingProxyType({}), +) -> AnthropicCountFields | ChatCountTranslationError | None: + if messages_format != "openai": + return None + return await asyncify(anthropic_fields_for_chat_tool_use)(model, messages, system, tools, translator, chat_params) + + class AnthropicCountTokensConfig: """ Configuration and transformation logic for Anthropic CountTokens API. diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 53734a77f3c..0f8afaae681 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -4,6 +4,7 @@ Azure AI Anthropic CountTokens API handler. Uses httpx for HTTP requests with Azure authentication. """ +from collections.abc import Mapping from typing import Final import httpx @@ -36,6 +37,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): timeout: float | httpx.Timeout | None = None, tools: list[dict[str, JsonValue]] | None = None, system: object = None, + optional_params: Mapping[str, JsonValue] | None = None, ) -> dict[str, object]: """ Handle a CountTokens request using httpx with Azure authentication. @@ -66,6 +68,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): messages=messages, tools=tools, system=system, + optional_params=optional_params, ) verbose_logger.debug("Transformed request: %s", request_body) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index d1f45751d3e..af7cf9d81ce 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -3,12 +3,19 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API. """ import os +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue import litellm from litellm._logging import verbose_logger +from litellm.llms.anthropic.count_tokens.transformation import ( + AnthropicCountFields, + ChatCountTranslationError, + anthropic_count_fields, +) from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) @@ -55,6 +62,8 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ Count tokens using Azure AI Anthropic's CountTokens API. @@ -74,6 +83,30 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): if not messages: return None + from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig + + match translated := await anthropic_count_fields( + model_to_use, messages, system, tools, messages_format, AzureAnthropicConfig(), chat_params + ): + case ChatCountTranslationError(): + return translated.response(request_model, model_to_use, "azure_ai_anthropic_api") + case AnthropicCountFields(): + return await self.count_tokens( + model_to_use=model_to_use, + messages=[dict(message) for message in translated.messages], + contents=contents, + deployment=deployment, + request_model=request_model, + tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, + system=translated.system, + messages_format="anthropic", + output_format=output_format, + context_management=context_management, + count_options=translated.options, + ) + case None: + pass + deployment = deployment or {} litellm_params: Final = deployment.get("litellm_params", {}) @@ -101,6 +134,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): litellm_params=litellm_params, tools=tools, system=system, + optional_params=count_options, ) if result is not None: diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 640930a780d..a9ec1f97d20 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -9,6 +9,7 @@ import re from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Final from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals @@ -40,6 +41,7 @@ class BaseTokenCounter(ABC): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: pass diff --git a/litellm/llms/base_llm/files/batch_records.py b/litellm/llms/base_llm/files/batch_records.py index 6bb98456e69..070cf73cd57 100644 --- a/litellm/llms/base_llm/files/batch_records.py +++ b/litellm/llms/base_llm/files/batch_records.py @@ -18,6 +18,7 @@ def _responses_request_keys() -> frozenset[str]: def responses_batch_body_to_chat_body( openai_request_body: Mapping[str, object], custom_llm_provider: str | None = None, + drop_params: bool | None = None, ) -> dict[str, object]: # mutable-ok: provider transforms take the bridged chat body as a plain dict """ Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body. @@ -50,5 +51,6 @@ def responses_batch_body_to_chat_body( input=chat_input, responses_api_request=responses_request, custom_llm_provider=custom_llm_provider, + drop_params=drop_params, metadata=openai_request_body.get("metadata"), ) diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 068cf4f2e67..21fbdb04896 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -4,12 +4,15 @@ Bedrock Token Counter implementation using the CountTokens API. from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Final from pydantic import JsonValue from typing_extensions import assert_never from litellm._logging import verbose_logger +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler @@ -58,6 +61,17 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: ) +def _request_data( + model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object +) -> Mapping[str, object]: + return { + "model": model, + "messages": [dict(message) for message in messages], + **({"tools": [dict(tool) for tool in tools]} if tools else {}), + **({"system": system} if system else {}), + } + + class BedrockTokenCounter(BaseTokenCounter): """Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model""" @@ -81,13 +95,13 @@ class BedrockTokenCounter(BaseTokenCounter): self, handler: BedrockCountTokensHandler | BedrockMantleCountTokensHandler, tokenizer_type: str, - request_data: dict[str, object], + request_data: Mapping[str, object], litellm_params: dict[str, object], resolved_model: str, ) -> CountTokensOutcome: try: result: Final = await handler.handle_count_tokens_request( - request_data=request_data, + request_data=dict(request_data), litellm_params=litellm_params, resolved_model=resolved_model, client=self._client, @@ -115,30 +129,44 @@ class BedrockTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "anthropic", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: if not messages: return None - litellm_params: Final = (deployment or {}).get("litellm_params", {}) - request_data: Final[dict[str, object]] = { - "model": model_to_use, - "messages": messages, - **({"tools": tools} if tools else {}), - **({"system": system} if system else {}), - } use_mantle: Final = model_to_use.startswith("bedrock_mantle/") - resolved_model: Final = get_bedrock_base_model(model_to_use.removeprefix("bedrock_mantle/")) + counted_model: Final = model_to_use.removeprefix("bedrock_mantle/") + resolved_model: Final = get_bedrock_base_model(counted_model) + primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE + translated: Final = ( + await anthropic_count_fields( + counted_model, messages, system, tools, messages_format, AnthropicConfig(), chat_params + ) + if is_mantle_claude_model(resolved_model) + else None + ) + if isinstance(translated, ChatCountTranslationError): + return translated.response(request_model, model_to_use, primary_tokenizer) + litellm_params: Final = (deployment or {}).get("litellm_params", {}) + runtime_request: Final = ( + _request_data(model_to_use, messages, tools, system) + if translated is None + else _request_data(model_to_use, translated.messages, translated.tools, translated.system) + ) + mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options} - primary: Final = await self._count_with( - self._mantle_handler if use_mantle else self._runtime_handler, - MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE, - request_data, - litellm_params, - resolved_model, + primary: Final = ( + await self._count_with( + self._mantle_handler, primary_tokenizer, mantle_request, litellm_params, resolved_model + ) + if use_mantle + else await self._count_with( + self._runtime_handler, primary_tokenizer, runtime_request, litellm_params, resolved_model + ) ) outcome: Final = ( await self._count_with( - self._mantle_handler, MANTLE_TOKENIZER_TYPE, request_data, litellm_params, resolved_model + self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model ) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) else primary diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 4ea4440a3c4..4b1a6a75a7a 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import httpx @@ -6,7 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.llms.anthropic.count_tokens.transformation import AnthropicCountTokensConfig +from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client @@ -23,6 +24,7 @@ class MantleCountTokensRequest(TypedDict): _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest) _COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue]) +_COUNT_OPTIONS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) class BedrockMantleCountTokensHandler(BaseAWSLLM): @@ -61,6 +63,9 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): messages=request["messages"], tools=request.get("tools"), system=request.get("system"), + optional_params=_COUNT_OPTIONS.validate_python( + {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES} + ), ) verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url) api_key: Final = litellm_params.get("api_key") diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index 48fc41ed12b..7cca4f69acc 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -52,12 +52,15 @@ def _sanitized_bedrock_tool_name(raw_name: JsonValue) -> str: def _to_bedrock_tool_spec(tool: JsonValue) -> dict[str, JsonValue]: fields: Final = _json_dict(tool) - name: Final = _sanitized_bedrock_tool_name(fields.get("name", "")) + chat_function: Final = _json_dict(fields.get("function")) + spec: Final = chat_function or fields + name: Final = _sanitized_bedrock_tool_name(spec.get("name", "")) + schema_field: Final = "parameters" if chat_function else "input_schema" return { "toolSpec": { "name": name, - "description": fields.get("description") or name, - "inputSchema": {"json": fields.get("input_schema", {"type": "object", "properties": {}})}, + "description": spec.get("description") or name, + "inputSchema": {"json": spec.get(schema_field, {"type": "object", "properties": {}})}, } } diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index c5ebd84e00f..b1a726be876 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -3,6 +3,7 @@ import datetime import json import math from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Any, Final import httpx @@ -498,6 +499,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index 6f911e9c40f..739966c95d1 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -34,7 +34,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): tools: Sequence[Mapping[str, object]] | None = None, instructions: str | None = None, text: Mapping[str, object] | None = None, - ) -> dict[str, object]: + ) -> Mapping[str, object]: """ Handle a token counting request to OpenAI's Responses API. @@ -56,7 +56,19 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): instructions=instructions, text=text, ) + except ValueError as e: + verbose_logger.error("Error in CountTokens handler: %s", e) + raise OpenAIError(status_code=500, message=f"CountTokens processing error: {e}") + return await self.post_count_request(request_body, api_key, api_base, timeout) + async def post_count_request( + self, + request_body: Mapping[str, object], + api_key: str, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> Mapping[str, object]: + try: endpoint_url: Final = self.get_openai_count_tokens_endpoint(api_base) verbose_logger.debug("Making request to: %s", endpoint_url) @@ -70,7 +82,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): response: Final = await async_client.post( endpoint_url, headers=headers, - json=request_body, + json=dict(request_body), timeout=request_timeout, ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index b12fdb95765..437f19a4dc6 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -2,7 +2,9 @@ OpenAI Token Counter implementation using the Responses API /input_tokens endpoint. """ -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Final import openai @@ -11,6 +13,8 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -23,6 +27,7 @@ from litellm.llms.openai.responses.count_tokens.handler import ( from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, ResponsesCountRequestParts, + native_count_request, ) from litellm.llms.openai.workload_identity import ( OpenAIWorkloadIdentityConfig, @@ -50,6 +55,46 @@ def _workload_identity_config( return e +@dataclass(frozen=True, slots=True) +class _CountCredential: + api_base: str | None + credential: str | OpenAIWorkloadIdentityConfig + + +def _count_credential(deployment: Mapping[str, object] | None, model_label: str) -> _CountCredential | OpenAIError: + litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} + api_base: Final = litellm_params.get("api_base") + api_key: Final = litellm_params.get("api_key") + credentials: Final = get_openai_credentials( + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params) + if isinstance(credential, OpenAIError): + return credential + if credential is None: + return OpenAIError( + status_code=401, + message=( + f"No OpenAI API key is set for model {model_label}, so tokens can't be counted. Set api_key or " + "workload identity on the deployment, or OPENAI_API_KEY in the proxy environment." + ), + ) + return _CountCredential(api_base=credentials.api_base, credential=credential) + + +def _count_error(model_to_use: str, request_model: str, message: str, status_code: int) -> TokenCountResponse: + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=message, + status_code=status_code, + ) + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -102,6 +147,7 @@ class OpenAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: """ Count tokens using OpenAI's Responses API /input_tokens endpoint. @@ -129,41 +175,9 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) - deployment = deployment or {} - litellm_params: Final = deployment.get("litellm_params", {}) - - credentials: Final = get_openai_credentials( - api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key") - ) - credential: Final = openai_static_api_key(credentials.api_key) or _workload_identity_config(litellm_params) - - if isinstance(credential, OpenAIError): - return TokenCountResponse( - total_tokens=0, - request_model=request_model, - model_used=model_to_use, - tokenizer_type="openai_api", - error=True, - error_message=credential.message, - status_code=credential.status_code, - ) - - if credential is None: - return TokenCountResponse( - total_tokens=0, - request_model=request_model, - model_used=model_to_use, - tokenizer_type="openai_api", - error=True, - error_message=( - f"No OpenAI API key is set for model {request_model or model_to_use}, so tokens can't be " - "counted. Set api_key or workload identity on the deployment, or OPENAI_API_KEY in the proxy " - "environment." - ), - status_code=401, - ) - - api_base: Final = credentials.api_base + resolved: Final = _count_credential(deployment, request_model or model_to_use) + if isinstance(resolved, OpenAIError): + return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) input_items, instructions, counted_tools, text = self._responses_request_parts( model_to_use=model_to_use, @@ -178,13 +192,9 @@ class OpenAITokenCounter(BaseTokenCounter): if not input_items: return None - try: - api_key: Final = ( - credential - if isinstance(credential, str) - else await asyncify(get_workload_identity_bearer_token)(credential) - ) - result: Final = await openai_count_tokens_handler.handle_count_tokens_request( + return await self._counted( + resolved, + lambda api_key, api_base: openai_count_tokens_handler.handle_count_tokens_request( model=model_to_use, input=input_items, api_key=api_key, @@ -192,7 +202,44 @@ class OpenAITokenCounter(BaseTokenCounter): tools=counted_tools, instructions=instructions, text=text, + ), + model_to_use, + request_model, + ) + + async def count_responses_request( + self, + model_to_use: str, + request: Mapping[str, object], + policy: DropPolicy, + deployment: Mapping[str, object] | None, + request_model: str, + ) -> TokenCountResponse | None: + resolved: Final = _count_credential(deployment, request_model or model_to_use) + if isinstance(resolved, OpenAIError): + return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) + body: Final = native_count_request(request, model_to_use, policy) + return await self._counted( + resolved, + lambda api_key, api_base: openai_count_tokens_handler.post_count_request(body, api_key, api_base), + model_to_use, + request_model, + ) + + @staticmethod + async def _counted( + resolved: _CountCredential, + send: Callable[[str, str | None], Awaitable[Mapping[str, object]]], + model_to_use: str, + request_model: str, + ) -> TokenCountResponse | None: + try: + api_key: Final = ( + resolved.credential + if isinstance(resolved.credential, str) + else await asyncify(get_workload_identity_bearer_token)(resolved.credential) ) + result: Final = await send(api_key, resolved.api_base) if result is not None: return TokenCountResponse.model_validate( @@ -206,36 +253,12 @@ class OpenAITokenCounter(BaseTokenCounter): ) except OpenAIError as e: verbose_logger.warning("OpenAI CountTokens API error: status=%s, message=%s", e.status_code, e.message) - return TokenCountResponse( - total_tokens=0, - request_model=request_model, - model_used=model_to_use, - tokenizer_type="openai_api", - error=True, - error_message=e.message, - status_code=e.status_code, - ) + return _count_error(model_to_use, request_model, e.message, e.status_code) except openai.AuthenticationError as e: verbose_logger.warning("OpenAI workload identity token exchange failed: %s", e.message) - return TokenCountResponse( - total_tokens=0, - request_model=request_model, - model_used=model_to_use, - tokenizer_type="openai_api", - error=True, - error_message=e.message, - status_code=401, - ) + return _count_error(model_to_use, request_model, e.message, 401) except Exception as e: verbose_logger.warning("Error calling OpenAI CountTokens API: %s", e) - return TokenCountResponse( - total_tokens=0, - request_model=request_model, - model_used=model_to_use, - tokenizer_type="openai_api", - error=True, - error_message=str(e), - status_code=500, - ) + return _count_error(model_to_use, request_model, str(e), 500) return None diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 438aa05a4e9..6d882e0eeb1 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -4,12 +4,15 @@ OpenAI Responses API token counting transformation logic. This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint. """ +import copy from collections.abc import Mapping, Sequence from typing import Any, Final, Literal, TypeAlias +from openai.types.responses.input_token_count_params import InputTokenCountParams from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( @@ -17,6 +20,8 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor ) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema +from litellm.types.llms.openai import ResponsesAPIRequestParams +from litellm.types.utils import all_litellm_params class ResponsesInputTextPart(TypedDict): @@ -340,3 +345,34 @@ class OpenAICountTokensConfig: input_items: object, instructions: object, tools: object, text: object = None ) -> ResponsesCountRequestParts: return _RESPONSES_COUNT_REQUEST_PARTS.validate_python((input_items, instructions, tools, text)) + + +_COUNT_FIELDS_THE_SDK_TYPE_OMITS: Final = frozenset({"prompt"}) +_COUNT_REQUEST_FIELDS: Final = ( + InputTokenCountParams.__required_keys__ | InputTokenCountParams.__optional_keys__ | _COUNT_FIELDS_THE_SDK_TYPE_OMITS +) +_INFERENCE_REQUEST_FIELDS: Final = ( + ResponsesAPIRequestParams.__required_keys__ | ResponsesAPIRequestParams.__optional_keys__ +) +_LITELLM_REQUEST_FIELDS: Final = frozenset((*all_litellm_params, "drop_params", "additional_drop_params")) + + +def _forwards(field: str, policy: DropPolicy) -> bool: + if field in _LITELLM_REQUEST_FIELDS or field in policy.additional_drop_params: + return False + return field in _COUNT_REQUEST_FIELDS or (not policy.drop_params and field in _INFERENCE_REQUEST_FIELDS) + + +def _restored(field: str, value: object) -> object: + match field, value: + case "input", _: + return ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(value)) + case "previous_response_id", str(): + return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(value) + case _: + return value + + +def native_count_request(request: Mapping[str, object], model: str, policy: DropPolicy) -> Mapping[str, object]: + forwarded: Final = {field: _restored(field, value) for field, value in request.items() if _forwards(field, policy)} + return {**forwarded, "model": model} diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 8298edd6744..f85a5a8cda9 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,5 +1,5 @@ import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from copy import deepcopy from datetime import datetime, timezone from enum import Enum @@ -1263,6 +1263,52 @@ class VertexAITokenCounter(BaseTokenCounter): return custom_llm_provider == LlmProviders.VERTEX_AI.value + async def _count_translated_claude_request( + self, + model_to_use: str, + messages: Sequence[Mapping[str, object]] | None, + deployment: Mapping[str, object], + request_model: str, + tools: Sequence[Mapping[str, object]] | None, + system: object | None, + messages_format: TokenCountMessagesFormat, + output_format: JsonValue, + context_management: CountContextManagement | None, + chat_params: Mapping[str, JsonValue], + ) -> TokenCountResponse | None: + from litellm.llms.anthropic.count_tokens.transformation import ( + AnthropicCountFields, + ChatCountTranslationError, + anthropic_count_fields, + ) + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher + + if count_tokens_publisher(model_to_use) != "anthropic": + return None + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig + + match translated := await anthropic_count_fields( + model_to_use, messages or [], system, tools, messages_format, VertexAIAnthropicConfig(), chat_params + ): + case ChatCountTranslationError(): + return translated.response(request_model, model_to_use, "vertex_ai_partner_models") + case AnthropicCountFields(): + return await self.count_tokens( + model_to_use=model_to_use, + messages=[dict(message) for message in translated.messages], + contents=None, + deployment=dict(deployment), + request_model=request_model, + tools=[dict(tool) for tool in translated.tools] if translated.tools is not None else None, + system=translated.system, + messages_format="anthropic", + output_format=output_format, + context_management=context_management, + count_options=translated.options, + ) + case None: + return None + async def count_tokens( self, model_to_use: str, @@ -1275,6 +1321,8 @@ class VertexAITokenCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), + count_options: Mapping[str, JsonValue] = MappingProxyType({}), ) -> TokenCountResponse | None: import copy @@ -1313,6 +1361,21 @@ class VertexAITokenCounter(BaseTokenCounter): # Check if this is a partner model (Claude, Mistral, etc.) if VertexAIPartnerModels.is_vertex_partner_model(model_to_use): + translated_count: Final = await self._count_translated_claude_request( + model_to_use=model_to_use, + messages=messages, + deployment=deployment, + request_model=request_model, + tools=tools, + system=system, + messages_format=messages_format, + output_format=output_format, + context_management=context_management, + chat_params=chat_params, + ) + if translated_count is not None: + return translated_count + # Use partner models token counter partner_models_handler: Final = VertexAIPartnerModels() @@ -1343,6 +1406,7 @@ class VertexAITokenCounter(BaseTokenCounter): vertex_credentials=vertex_credentials, system=system, tools=tools, + count_options=count_options, ) except (PartnerVertexAIError, httpx.HTTPStatusError) as e: status_code: Final = e.response.status_code diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index c855a073648..bb200fd542d 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -1,6 +1,6 @@ # What is this? ## API Handler for calling Vertex AI Partner Models -from collections.abc import Callable +from collections.abc import Callable, Mapping from enum import Enum from types import MappingProxyType from typing import Final @@ -266,6 +266,7 @@ class VertexAIPartnerModels(VertexBase): vertex_credentials=None, system: object | None = None, tools: list[dict[str, object]] | None = None, + count_options: Mapping[str, object] = MappingProxyType({}), ): """ Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.) @@ -302,6 +303,7 @@ class VertexAIPartnerModels(VertexBase): **MappingProxyType( {key: value for key, value in (("system", system), ("tools", tools)) if value is not None} ), + **count_options, } # Prepare litellm_params with credentials diff --git a/litellm/main.py b/litellm/main.py index 71cb7af1fbb..f9bdf64c667 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -9247,6 +9247,7 @@ async def acount_tokens( request_model=model, tools=tools, system=system, + messages_format="openai", ) if result is not None and not result.error: return result diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5128ddab87..f87b1edc2e5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14701,7 +14701,9 @@ async def run_thread( # dependencies=[Depends(user_api_key_auth)], # ) # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)): +from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy from litellm.llms.base_llm.base_utils import BaseTokenCounter +from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.model_repository import ModelRepository @@ -14714,6 +14716,11 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.responses.litellm_completion_transformation.count_request import ( + ChatCountBridgeError, + chat_count_request, + with_provider_file_ids, +) def _get_provider_token_counter( @@ -14771,6 +14778,18 @@ def _get_provider_token_counter( return None, None, None +def _provider_count_outcome( + result: Union["TokenCountResponse", None], +) -> Union["TokenCountResponse", ProviderCountFailure, None]: + if result is not None and result.error is True: + return ProviderCountFailure( + status_code=result.status_code or 500, + message=result.error_message or "Token counting failed", + rejected_before_provider_call=result.rejected_before_provider_call, + ) + return result + + def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params) try: @@ -14785,6 +14804,12 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: return call_params +def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None: + if deployment is None: + return None + return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))} + + _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -14825,6 +14850,7 @@ async def _try_provider_token_count( messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: CountContextManagement | None = None, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), ) -> Union["TokenCountResponse", ProviderCountFailure, None]: """Attempt provider-specific token counting. Returns None to fall through to local counting.""" if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): @@ -14844,31 +14870,44 @@ async def _try_provider_token_count( model_to_use=model_to_use or "", messages=messages, contents=contents, - deployment=None - if deployment_in_region is None - else { - **deployment_in_region, - "litellm_params": _with_resolved_api_key(deployment_in_region.get("litellm_params", {})), - }, + deployment=_with_resolved_credentials(deployment_in_region), request_model=request_model, tools=tools, system=system, messages_format=messages_format, output_format=output_format, context_management=context_management, + chat_params=chat_params, ) except httpx.HTTPStatusError as e: return ProviderCountFailure( status_code=getattr(e, "status_code", None) or e.response.status_code, message=getattr(e, "message", None) or str(e), ) - if result is not None and result.error is True: - return ProviderCountFailure( - status_code=result.status_code or 500, - message=result.error_message or "Token counting failed", - rejected_before_provider_call=result.rejected_before_provider_call, + return _provider_count_outcome(result) + + +async def _try_native_responses_count( + provider_counter: OpenAITokenCounter, + custom_llm_provider: str | None, + model_to_use: str, + _region: str | None, + responses_request: Mapping[str, object], + policy: DropPolicy, + deployment: Mapping[str, object] | None, + request_model: str, +) -> Union["TokenCountResponse", ProviderCountFailure, None]: + if not provider_counter.should_use_token_counting_api(custom_llm_provider=custom_llm_provider): + return None + return _provider_count_outcome( + await provider_counter.count_responses_request( + model_to_use=model_to_use, + request=responses_request, + policy=policy, + deployment=_with_resolved_credentials(deployment), + request_model=request_model, ) - return result + ) async def _provider_token_count( @@ -15150,6 +15189,7 @@ class _TokenCountDeployment(TypedDict, total=False): _TOKEN_COUNT_DEPLOYMENT: Final = TypeAdapter(_TokenCountDeployment) +_ROUTED_DEPLOYMENT: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) def _model_group_deployment(router: Router | None, model_group: str) -> Mapping[str, object] | None: @@ -15228,6 +15268,43 @@ def _count_context_management( ) +def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> Mapping[str, object]: + previous_response_id: Final = None if responses_request is None else responses_request.get("previous_response_id") + return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {} + + +def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy: + litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} + router_defaults: Final[Mapping[str, object]] = ( + {} if llm_router is None else _router_default_litellm_params(llm_router) + ) + return drop_policy({**litellm_params, **router_defaults, **responses_request}) + + +def _native_responses_count( + provider_counter: BaseTokenCounter, + custom_llm_provider: str | None, + responses_request: Mapping[str, object] | None, + deployment: Mapping[str, object] | None, + request_model: str, +) -> Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]] | None: + if responses_request is None or not isinstance(provider_counter, OpenAITokenCounter): + return None + return partial( + _try_native_responses_count, + provider_counter, + custom_llm_provider, + responses_request=responses_request, + policy=_count_drop_policy(deployment, responses_request), + deployment=deployment, + request_model=request_model, + ) + + +def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None: + return _non_empty_str(deployment.get("litellm_params", {}).get("api_base")) + + def _system_message(system: object) -> ChatCompletionSystemMessage | None: if not isinstance(system, (str, list)) or not system: return None @@ -15260,6 +15337,7 @@ async def count_request_tokens( output_format: JsonValue = None, context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), + responses_request: Mapping[str, object] | None = None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15281,7 +15359,7 @@ async def count_request_tokens( try: deployment = await llm_router.async_get_available_deployment( model=request.model, - request_kwargs={}, + request_kwargs=dict(_responses_routing_kwargs(responses_request)), ) except Exception: verbose_proxy_logger.exception( @@ -15335,23 +15413,54 @@ async def count_request_tokens( request_messages: Final = _counted_request_messages(messages, messages_format, count_deployment) contents_estimable: Final = not contents or (bool(messages) and _estimable_contents(contents)) prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES + routed_deployment: Final = _ROUTED_DEPLOYMENT.validate_python(deployment) + routed_request: Final = ( + None if responses_request is None else with_provider_file_ids(responses_request, routed_deployment) + ) + chat_count: Final = ( + None + if routed_request is None or provider_counter is None or isinstance(provider_counter, OpenAITokenCounter) + else await chat_count_request( + model_to_use, + custom_llm_provider, + _deployment_api_base(deployment_view), + routed_request, + _count_drop_policy(routed_deployment, routed_request), + ) + ) + if isinstance(chat_count, ChatCountBridgeError): + raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) ladder: Final = ( None if provider_counter is None else await _provider_token_count( - count_with=partial( + count_with=_native_responses_count( + provider_counter, + custom_llm_provider, + routed_request, + routed_deployment, + request.model, + ) + or partial( _try_provider_token_count, provider_counter, custom_llm_provider, - messages=[{"role": "user", "content": prompt}] if prompt_as_message else request_messages, + messages=( + [dict(message) for message in chat_count.messages] + if chat_count is not None + else [{"role": "user", "content": prompt}] + if prompt_as_message + else request_messages + ), contents=None if prompt_as_message else contents, deployment=count_deployment, request_model=request.model, - tools=tools, + tools=tools if chat_count is None else [dict(tool) for tool in chat_count.tools or ()] or None, system=system, messages_format=messages_format, output_format=output_format, context_management=_count_context_management(context_management, request.model, request_litellm_params), + chat_params=MappingProxyType({}) if chat_count is None else chat_count.params, ), model_to_use=model_to_use, settings=settings, diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index a0fb2b190e0..c918ea3f1d1 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -32,12 +32,13 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping +from litellm.caching.dual_cache import DualCache from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup -from litellm.litellm_core_utils.tool_fields import first_validation_error +from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, build_blocked_response, @@ -79,6 +80,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.proxy.utils import handle_exception_on_proxy +from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel @@ -89,7 +91,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DeleteResponseResult from litellm.types.token_count import TokenCountSourceFields -from litellm.types.utils import TokenCountResponse +from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1388,13 +1390,15 @@ class _TokenCountPayload(TypedDict): class _TokenCounter(Protocol): - def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ... + def __call__( + self, *, request: TokenCountRequest, call_endpoint: bool, responses_request: Mapping[str, object] | None + ) -> Awaitable[TokenCountResponse]: ... def _proxy_token_counter() -> _TokenCounter: - from litellm.proxy.proxy_server import token_counter + from litellm.proxy.proxy_server import count_request_tokens - return token_counter + return count_request_tokens _token_counter_dep: Final = Depends(_proxy_token_counter) @@ -1409,6 +1413,67 @@ def _proxy_llm_router() -> Router | None: _llm_router_dep: Final = Depends(_proxy_llm_router) +def _proxy_response_id_security() -> ResponsesIDSecurity | None: + from litellm.proxy.proxy_server import proxy_logging_obj + + security_hook: Final = proxy_logging_obj.get_proxy_hook("responses_id_security") + return security_hook if isinstance(security_hook, ResponsesIDSecurity) else None + + +_response_id_security_dep: Final = Depends(_proxy_response_id_security) + + +class _ManagedFileAccess(Protocol): + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], + call_type: Literal["aresponses"], + ) -> object: ... + + +def _proxy_managed_files_hook() -> _ManagedFileAccess | None: + from litellm.proxy.proxy_server import proxy_logging_obj + + return proxy_logging_obj.get_proxy_hook("managed_files") + + +_managed_files_hook_dep: Final = Depends(_proxy_managed_files_hook) + + +async def _with_managed_file_id_mapping( + data: Mapping[str, object], managed_files: _ManagedFileAccess | None, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, object]: + caller_fields: Final = {name: value for name, value in data.items() if name != MODEL_FILE_ID_MAPPING_FIELD} + if managed_files is None: + return caller_fields + from litellm.proxy.proxy_server import user_api_key_cache + + hooked: Final = dict(caller_fields) + try: + await managed_files.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, cache=user_api_key_cache, data=hooked, call_type="aresponses" + ) + except HTTPException as e: + raise ProxyException(message=str(e.detail), type="permission_error", param=None, code=e.status_code) from e + mapping: Final = hooked.get(MODEL_FILE_ID_MAPPING_FIELD) + return caller_fields if mapping is None else {**caller_fields, MODEL_FILE_ID_MAPPING_FIELD: mapping} + + +def _with_authorized_previous_response( + data: Mapping[str, object], security: ResponsesIDSecurity | None, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, object]: + previous_response_id: Final = data.get("previous_response_id") + if security is None or not isinstance(previous_response_id, str) or not previous_response_id: + return data + try: + authorized_id: Final = security.authorize_response_id(previous_response_id, user_api_key_dict) + except HTTPException as e: + raise ProxyException(message=str(e.detail), type="permission_error", param=None, code=e.status_code) from e + return {**data, "previous_response_id": authorized_id, ADDRESSED_RESPONSE_ID_FIELD: previous_response_id} + + def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: body: Final[_ResponsesApiErrorBody] = { "error": { @@ -1546,6 +1611,8 @@ async def responses_input_tokens( token_counter: _TokenCounter = _token_counter_dep, llm_router: Router | None = _llm_router_dep, user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, + response_id_security: ResponsesIDSecurity | None = _response_id_security_dep, + managed_files: _ManagedFileAccess | None = _managed_files_hook_dep, ): """ Count the input tokens of a Responses API request without calling the model. @@ -1579,7 +1646,7 @@ async def responses_input_tokens( ) counted_fields: Final = { - name: data[name] for name in ("model", "tools", "drop_params", "additional_drop_params") if name in data + name: data[name] for name in ("model", "drop_params", "additional_drop_params", *TOOL_FIELDS) if name in data } token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id) if isinstance(token_fields, ProxyException): @@ -1604,8 +1671,20 @@ async def responses_input_tokens( message=f"Invalid request for token counting: {e}", param=None, code=None ) + request_fields: Final = _PROXY_SERVER_REQUEST_ADAPTER.validate_python(data) + responses_request: Final = await _with_managed_file_id_mapping( + _with_authorized_previous_response( + {**{name: value for name, value in request_fields.items() if name not in counted_fields}, **token_fields}, + response_id_security, + user_api_key_dict, + ), + managed_files, + user_api_key_dict, + ) try: - token_response: Final = await token_counter(request=token_request, call_endpoint=True) + token_response: Final = await token_counter( + request=token_request, call_endpoint=True, responses_request=responses_request + ) except ProxyException as e: if e.type != _PROVIDER_COUNT_ERROR_TYPE: raise diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index e2bb2ae40ea..7121908517b 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -45,7 +45,7 @@ class FileSearchToolCallArgs(TypedDict): def should_use_emulated_file_search( - tools: Iterable[ToolParam] | None, + tools: Iterable[object] | None, provider_config: "BaseResponsesAPIConfig | None", ) -> bool: """Return True when there is a file_search tool and the provider can't handle it natively.""" @@ -105,7 +105,7 @@ def _build_function_tool(vector_store_ids: list[str]) -> dict[str, object]: } -def _replace_file_search_tools( +def replace_file_search_tools( tools: Iterable[ToolParam] | None, ) -> tuple[list[object], list[str]]: """ @@ -529,7 +529,7 @@ async def aresponses_with_emulated_file_search( _include_search_results, call_kwargs = _prepare_emulated_file_search_call(kwargs=kwargs) # 1. Replace file_search tools with function tool - transformed_tools, all_vs_ids = _replace_file_search_tools(tools) + transformed_tools, all_vs_ids = replace_file_search_tools(tools) # 2. First provider call — provider will call the file_search function. # Mark as an internal sub-call so wrapper_async skips billing callbacks; diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py new file mode 100644 index 00000000000..56a0b9024c5 --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -0,0 +1,207 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, cast # noqa: TID251 # input items are forwarded as sent, as inference does + +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + update_responses_input_with_model_file_ids, + update_responses_tools_with_model_file_ids, +) +from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body +from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.file_search.emulated_handler import replace_file_search_tools +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.main import previous_response_client_ids +from litellm.types.llms.openai import ResponseInputParam +from litellm.utils import ProviderConfigManager + +_COUNTED_CHAT_PARAMS: Final = ( + "tool_choice", + "parallel_tool_calls", + "response_format", + "web_search_options", + "reasoning_effort", +) +_ITEMS: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_MAPPINGS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(tuple[Mapping[str, object], ...]) +_JSON_PARAMS: Final[TypeAdapter[Mapping[str, JsonValue]]] = TypeAdapter(Mapping[str, JsonValue]) +_ENTRY: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_MARKED_ENTRY_LISTS: Final = frozenset({"content", "tool_calls"}) +_FILE_ID_MAPPING: Final[TypeAdapter[Mapping[str, Mapping[str, str]] | None]] = TypeAdapter( + Mapping[str, Mapping[str, str]] | None +) +MODEL_FILE_ID_MAPPING_FIELD: Final = "model_file_id_mapping" + + +@dataclass(frozen=True, slots=True) +class ChatCountRequest: + messages: tuple[Mapping[str, object], ...] + tools: tuple[Mapping[str, object], ...] | None + params: Mapping[str, JsonValue] + + +@dataclass(frozen=True, slots=True) +class ChatCountBridgeError: + message: str + + +def _plain(item: object) -> object: + return item.model_dump(exclude_none=True) if isinstance(item, BaseModel) else item + + +def _mappings(value: object) -> tuple[Mapping[str, object], ...]: + return _MAPPINGS.validate_python(tuple(_plain(item) for item in _ITEMS.validate_python(value or ()))) + + +def _unmarked(entry: Mapping[str, object]) -> Mapping[str, object]: + return { + key: _unmarked_entries(value) if key in _MARKED_ENTRY_LISTS else value + for key, value in entry.items() + if key != "cache_control" + } + + +def _unmarked_entries(value: object) -> object: + return ( + [ + _unmarked(_ENTRY.validate_python(item)) if isinstance(item, Mapping) else item + for item in _ITEMS.validate_python(value) + ] + if isinstance(value, list) + else value + ) + + +def _routed_model_id(deployment: Mapping[str, object] | None) -> str | None: + model_info: Final = as_str_mapping(None if deployment is None else deployment.get("model_info")) or {} + model_id: Final = model_info.get("id") + return model_id if isinstance(model_id, str) else None + + +def with_provider_file_ids( + responses_request: Mapping[str, object], deployment: Mapping[str, object] | None +) -> Mapping[str, object]: + mapping: Final = _FILE_ID_MAPPING.validate_python(responses_request.get(MODEL_FILE_ID_MAPPING_FIELD)) + model_id: Final = _routed_model_id(deployment) + fields: Final = {name: value for name, value in responses_request.items() if name != MODEL_FILE_ID_MAPPING_FIELD} + tools: Final = fields.get("tools") + return { + **fields, + **( + {"input": update_responses_input_with_model_file_ids(fields["input"], model_id, mapping)} + if "input" in fields + else {} + ), + **( + { + "tools": update_responses_tools_with_model_file_ids( + cast(list[dict[str, object]], tools), # cast-ok: entries that are not tools pass through as sent + model_id, + mapping, + ) + } + if isinstance(tools, list) + else {} + ), + } + + +def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> Mapping[str, object]: + responses_input: Final = responses_request.get("input") + if not isinstance(responses_input, list): + return responses_request + input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does + hoisted: Final = hoist_additional_tools(input_items, None) + if not hoisted.hoisted: + return responses_request + return { + **responses_request, + "input": hoisted.input, + "tools": [*_ITEMS.validate_python(responses_request.get("tools") or ()), *hoisted.hoisted], + } + + +def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: + tools: Final = responses_request.get("tools") + if tools is None: + return responses_request + return {**responses_request, "tools": replace_file_search_tools(_ITEMS.validate_python(tools))[0]} + + +async def _with_previous_turns( + chat_body: Mapping[str, object], + model: str, + custom_llm_provider: str, + responses_request: Mapping[str, object], + policy: DropPolicy, +) -> Mapping[str, object]: + previous_response_id: Final = responses_request.get("previous_response_id") + if not isinstance(previous_response_id, str) or not previous_response_id: + return chat_body + instructions: Final = responses_request.get("instructions") + return await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( # pyright: ignore[reportUnknownMemberType] # the session handler takes a bare dict request + previous_response_id=previous_response_id, + litellm_completion_request={**chat_body, "model": model, "custom_llm_provider": custom_llm_provider}, + instructions=instructions if isinstance(instructions, str) else None, + request_kwargs={"drop_params": policy.drop_params}, + previous_response_client_ids=previous_response_client_ids(previous_response_id, responses_request), + ) + + +def _counted_params( + chat_body: Mapping[str, object], tools: tuple[Mapping[str, object], ...], policy: DropPolicy +) -> Mapping[str, JsonValue]: + sent: Final = { + name: chat_body[name] + for name in _COUNTED_CHAT_PARAMS + if chat_body.get(name) is not None and name not in policy.additional_drop_params + } + return _JSON_PARAMS.validate_python( + {**sent, **({"drop_params": True} if policy.drop_params and (sent or tools) else {})} + ) + + +async def chat_count_request( + model: str, + custom_llm_provider: str | None, + api_base: str | None, + responses_request: Mapping[str, object], + policy: DropPolicy, +) -> ChatCountRequest | ChatCountBridgeError | None: + """The chat request Responses inference sends a provider with no native Responses API, or None + when inference sends this provider the Responses body itself.""" + if custom_llm_provider is None or ( + ProviderConfigManager.get_provider_responses_api_config( + provider=custom_llm_provider, model=model, api_base=api_base + ) + is not None + ): + return None + try: + chat_body: Final = await _with_previous_turns( + responses_batch_body_to_chat_body( + _with_emulated_file_search(_with_hoisted_additional_tools(responses_request)), + custom_llm_provider, + policy.drop_params, + ), + model, + custom_llm_provider, + responses_request, + policy, + ) + tools: Final = ( + () if "tools" in policy.additional_drop_params else tuple(map(_unmarked, _mappings(chat_body.get("tools")))) + ) + return ChatCountRequest( + messages=tuple(map(_unmarked, _mappings(chat_body.get("messages")))), + tools=tools or None, + params=_counted_params(chat_body, tools, policy), + ) + except BadRequestError as e: + return ChatCountBridgeError(message=str(e.message)) + except (ValidationError, ValueError, KeyError) as e: + return ChatCountBridgeError(message=f"Invalid request for token counting: {e}") diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b613b817225..d9ebf50cc1a 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1005,7 +1005,7 @@ class LiteLLMCompletionResponsesConfig: session_lookup: Callable[[str], Awaitable[ChatCompletionSession]] | None = None, request_kwargs: Mapping[str, object] = MappingProxyType({}), previous_response_client_ids: tuple[str, ...] = (), - ) -> dict: + ) -> dict[str, object]: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages """ diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 51cd3be1b63..995921beed1 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -548,7 +548,7 @@ def _bridges_to_chat_completions( _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"}) -def _previous_response_client_ids(previous_response_id: str | None, kwargs: Mapping[str, object]) -> tuple[str, ...]: +def previous_response_client_ids(previous_response_id: str | None, kwargs: Mapping[str, object]) -> tuple[str, ...]: """The forms of the previous response's id the client held, which its spend log row can be stored under.""" if not previous_response_id: return () @@ -1622,7 +1622,7 @@ def responses( if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api): bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params) return litellm_completion_transformation_handler.response_api_handler( - _previous_response_client_ids(previous_response_id, kwargs), + previous_response_client_ids(previous_response_id, kwargs), model=model, input=input, responses_api_request=response_api_optional_params, diff --git a/tests/integration/providers/_count_tokens_system_lift.py b/tests/integration/providers/_count_tokens_system_lift.py index 56619d1c963..dedf6b5006d 100644 --- a/tests/integration/providers/_count_tokens_system_lift.py +++ b/tests/integration/providers/_count_tokens_system_lift.py @@ -49,6 +49,87 @@ TOOLS: Final[list[JsonValue]] = [ } ] +WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string", "description": "City to look up"}}, + "required": ["city"], +} +RESPONSES_TOOLS: Final[list[JsonValue]] = [ + { + "type": "function", + "name": "get_weather", + "description": "Look up the current weather for a city", + "parameters": WEATHER_SCHEMA, + } +] +RESPONSES_TOOL_USE: Final[Mapping[str, dict[str, JsonValue]]] = MappingProxyType( + { + "function-tool": {"input": "Count this message", "instructions": "Be terse", "tools": RESPONSES_TOOLS}, + "tool-round-trip": { + "input": [ + {"role": "user", "content": "What is the weather in Paris?"}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": "Sunny, 21C"}, + ], + "tools": RESPONSES_TOOLS, + }, + } +) + + +def counted_fields(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {field: body[field] for field in ("messages", "system", "tools") if field in body} + + +REPORT_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}, "summary": {"type": "string"}}, + "required": ["city", "summary"], + "additionalProperties": False, +} +WEATHER_QUESTION: Final = "What is the weather in Paris?" +RESPONSES_REQUEST_OPTIONS: Final[Mapping[str, dict[str, JsonValue]]] = MappingProxyType( + { + "text-format": { + "input": WEATHER_QUESTION, + "text": {"format": {"type": "json_schema", "name": "weather_report", "schema": REPORT_SCHEMA}}, + }, + "forced-tool-choice-dropped": { + "input": WEATHER_QUESTION, + "tools": RESPONSES_TOOLS, + "tool_choice": {"type": "function", "name": "get_weather"}, + "parallel_tool_calls": False, + "drop_params": True, + }, + "reasoning": { + "input": WEATHER_QUESTION, + "tools": RESPONSES_TOOLS, + "tool_choice": "auto", + "reasoning": {"effort": "low"}, + }, + } +) +_REQUEST_OPTIONS: Final = ("tool_choice", "thinking") + + +def count_request_fields(count_body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + options: Final = (*_REQUEST_OPTIONS, "output_config", "output_format") + return {**counted_fields(count_body), **{field: count_body[field] for field in options if field in count_body}} + + +def inference_count_fields(inference_body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + output_config: Final = inference_body.get("output_config") + output_format: Final = inference_body.get("output_format") + counted_config: Final = { + **(output_config if isinstance(output_config, dict) else {}), + **({} if output_format is None else {"format": output_format}), + } + return { + **counted_fields(inference_body), + **{field: inference_body[field] for field in _REQUEST_OPTIONS if field in inference_body}, + **({"output_config": counted_config} if counted_config else {}), + } + @dataclass(frozen=True, slots=True) class LiftCase: diff --git a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py index afd25fcfa04..27cbbe81a13 100644 --- a/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py +++ b/tests/integration/providers/test_anthropic_count_tokens_system_lift_wire.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterator, Mapping from concurrent.futures import ThreadPoolExecutor from contextlib import AbstractContextManager, ExitStack from dataclasses import dataclass +from pathlib import Path from queue import SimpleQueue from types import MappingProxyType from typing import Final, cast @@ -14,8 +15,10 @@ import httpcore import httpx import psutil import pytest +import yaml from anthropic.types import MessageParam -from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.ports import allocate_port from integration.providers._count_tokens_system_lift import ( @@ -32,18 +35,24 @@ from integration.providers._count_tokens_system_lift import ( MID_SYSTEM, REJECTION, REMINDER, + RESPONSES_REQUEST_OPTIONS, + RESPONSES_TOOL_USE, STRING_CASE, TOKEN_COUNTING_BETA, TOOLS, USER, USER_TEXT, + WEATHER_QUESTION, LiftCase, accepts_count_body, anthropic_client, async_anthropic_client, async_openai_client, count_request, + count_request_fields, + counted_fields, expected_count_body, + inference_count_fields, openai_client, ) from pydantic import JsonValue, TypeAdapter @@ -480,9 +489,7 @@ def test_messages_count_tokens_unauthenticated_request_never_reaches_the_peer( assert peer.drain() == () -def test_peer_outage_between_concurrent_waves_errors_then_recovers( - deployment: _Deployment, gateway: Gateway -) -> None: +def test_peer_outage_between_concurrent_waves_errors_then_recovers(deployment: _Deployment, gateway: Gateway) -> None: with ExitStack() as stack: clients: Final = _clients(stack, str(gateway.client.base_url), 8) pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) @@ -573,3 +580,164 @@ def test_chat_completions_on_the_same_deployment_keeps_a_mid_conversation_system {"role": "assistant", "content": [{"type": "text", "text": "One."}]}, {"role": "user", "content": [{"type": "text", "text": "Again"}]}, ], body + + +_GENERATED: Final[dict[str, JsonValue]] = { + "id": "msg_generated", + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, +} + + +def _generating(provider: _Provider) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.target == provider.target.removesuffix("/count_tokens"): + return _json_reply(200, _GENERATED) + return _peer(provider)(request) + + return respond + + +@pytest.mark.parametrize("request_body", RESPONSES_TOOL_USE.values(), ids=RESPONSES_TOOL_USE.keys()) +def test_responses_input_tokens_counts_what_responses_inference_sends( + deployment: _Deployment, gateway: Gateway, request_body: dict[str, JsonValue] +) -> None: + with wire_server(_generating(deployment.provider), port=deployment.port) as peer: + generated: Final = gateway.request( + "POST", "/v1/responses", {**request_body, "model": deployment.model, "max_output_tokens": 16} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request( + "POST", "/v1/responses/input_tokens", {**request_body, "model": deployment.model} + ) + assert _payload(response)["input_tokens"] == _COUNT, response.text + inference, counted = (counted_fields(JSON_OBJECT.validate_json(request.body)) for request in peer.drain()) + assert counted == inference, (counted, inference) + + +def _inference_and_count(peer: Wire) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]: + inference, counted = (JSON_OBJECT.validate_json(request.body) for request in peer.drain()) + return inference, counted + + +@pytest.mark.parametrize("request_body", RESPONSES_REQUEST_OPTIONS.values(), ids=RESPONSES_REQUEST_OPTIONS.keys()) +def test_responses_input_tokens_counts_the_request_options_responses_inference_sends( + deployment: _Deployment, gateway: Gateway, request_body: dict[str, JsonValue] +) -> None: + with wire_server(_generating(deployment.provider), port=deployment.port) as peer: + generated: Final = gateway.request( + "POST", "/v1/responses", {**request_body, "model": deployment.model, "max_output_tokens": 4096} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request( + "POST", "/v1/responses/input_tokens", {**request_body, "model": deployment.model} + ) + assert _payload(response)["input_tokens"] == _COUNT, response.text + inference, counted = _inference_and_count(peer) + assert count_request_fields(counted) == inference_count_fields(inference), (counted, inference) + + +def _text_blocks(message: JsonValue) -> JsonValue: + if not isinstance(message, dict) or not isinstance(message.get("content"), str): + return message + return {**message, "content": [{"type": "text", "text": message["content"]}]} + + +def _with_text_blocks(fields: dict[str, JsonValue]) -> dict[str, JsonValue]: + messages: Final = fields["messages"] + assert isinstance(messages, list), fields + return {**fields, "messages": [_text_blocks(message) for message in messages]} + + +def test_responses_input_tokens_leaves_tool_choice_out_when_drop_params_drops_every_tool( + deployment: _Deployment, gateway: Gateway +) -> None: + request_body: Final[dict[str, JsonValue]] = { + "input": WEATHER_QUESTION, + "tools": [5], + "tool_choice": "required", + "drop_params": True, + } + with wire_server(_generating(deployment.provider), port=deployment.port) as peer: + generated: Final = gateway.request( + "POST", "/v1/responses", {**request_body, "model": deployment.model, "max_output_tokens": 16} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request( + "POST", "/v1/responses/input_tokens", {**request_body, "model": deployment.model} + ) + assert _payload(response)["input_tokens"] == _COUNT, response.text + inference, counted = _inference_and_count(peer) + assert "tools" not in counted and "tool_choice" not in counted, counted + assert _with_text_blocks(count_request_fields(counted)) == inference_count_fields(inference), ( + counted, + inference, + ) + + +def _config_storing_prompts(directory: Path) -> Path: + config: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general_settings: Final = object_value(config["general_settings"]) + path: Final = directory / "store-prompts.yaml" + path.write_text( + yaml.safe_dump({**config, "general_settings": {**general_settings, "store_prompts_in_spend_logs": True}}) + ) + return path + + +def _replying(answer: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.target == _ANTHROPIC.target: + return _counted(request) + if request.target == "/v1/messages": + return _json_reply( + 200, + {**_GENERATED, "id": f"msg_{uuid.uuid4().hex}", "content": [{"type": "text", "text": answer}]}, + ) + return _json_reply(404, {"error": f"unscripted target {request.target}"}) + + return respond + + +def _spend_rows_with(prompt_gateway: Gateway, marker: str) -> list[dict[str, JsonValue]]: + return prompt_gateway.rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE proxy_server_request::text LIKE %s', (f"%{marker}%",) + ) + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_responses_input_tokens_counts_the_turns_previous_response_id_replays(gateway: Gateway, tmp_path: Path) -> None: + first_prompt: Final = f"Remember the number {uuid.uuid4().hex}" + answer: Final = f"Remembered {uuid.uuid4().hex}" + follow_up: Final[dict[str, JsonValue]] = {"input": "What number did I ask you to remember?"} + port: Final = allocate_port() + with ( + wire_server(_replying(answer), port=port) as peer, + owned_proxy(gateway, tmp_path, {}, config=_config_storing_prompts(tmp_path), workers=1) as prompt_gateway, + prompt_gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model_info=None, model=f"anthropic/{_MODEL}", api_base=f"http://127.0.0.1:{port}", api_key=_API_KEY + ) + first: Final = prompt_gateway.request( + "POST", "/v1/responses", {"model": model, "input": first_prompt, "max_output_tokens": 16} + ) + assert first.status_code == 200, first.text + previous: Final = { + **follow_up, + "model": model, + "previous_response_id": JSON_OBJECT.validate_json(first.content)["id"], + } + eventually(lambda: _spend_rows_with(prompt_gateway, first_prompt), lambda rows: len(rows) == 1, seconds=70) + response: Final = prompt_gateway.request("POST", "/v1/responses/input_tokens", previous) + assert _payload(response)["input_tokens"] == _COUNT, response.text + generated: Final = prompt_gateway.request("POST", "/v1/responses", {**previous, "max_output_tokens": 16}) + assert generated.status_code == 200, generated.text + _, counted, inference = (JSON_OBJECT.validate_json(request.body) for request in peer.drain()) + assert first_prompt in json.dumps(inference["messages"]) and answer in json.dumps(inference["messages"]) + assert counted_fields(counted) == counted_fields(inference), (counted, inference) diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index 4fe29fa49d8..063b0e46479 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -32,9 +32,12 @@ from integration.providers._count_tokens_system_lift import ( LIFTED, MID_SYSTEM, REMINDER, + RESPONSES_TOOLS, STRING_CASE, USER, USER_TEXT, + WEATHER_QUESTION, + WEATHER_SCHEMA, LiftCase, accepts_count_body, async_openai_client, @@ -1636,3 +1639,107 @@ def test_utils_token_counter_local_mode_never_calls_either_peer(counting_proxy: model: Final = _fallback_deployment(scenario, runtime.url) assert _local_count(gateway, _full(model)) > 0 assert runtime.drain() == () and mantle.drain() == () + + +_FORCED_TOOL_CHOICE: Final[dict[str, JsonValue]] = { + "input": WEATHER_QUESTION, + "tools": RESPONSES_TOOLS, + "tool_choice": {"type": "function", "name": "get_weather"}, +} +_CHAT_TOOLS: Final[list[JsonValue]] = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Look up the current weather for a city", + "parameters": WEATHER_SCHEMA, + }, + } +] +_WEATHER_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": WEATHER_QUESTION}] +_TOOL_USE_FIELDS: Final = ("tools", "tool_choice") + + +def _tool_use_fields(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {field: body[field] for field in _TOOL_USE_FIELDS if field in body} + + +def _converse_count_tools(request: Request) -> JsonValue: + count_input: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(request.body)["input"]) + converse: Final = _JSON_OBJECT.validate_python(count_input["converse"]) + return _JSON_OBJECT.validate_python(converse["toolConfig"])["tools"] + + +def _requests_to(requests: Sequence[Request], pattern: re.Pattern[str]) -> tuple[Request, ...]: + return tuple(request for request in requests if pattern.match(target_of(request))) + + +def test_responses_input_tokens_sends_mantle_the_tool_use_invoke_inference_sends( + counting_proxy: OwnedProxy, mantle_port: int +) -> None: + gateway: Final = counting_proxy.gateway + with ( + wire_server(_runtime) as runtime, + wire_server(_mantle(), port=mantle_port) as mantle, + gateway.scenario() as scenario, + ): + counted_model: Final = _deployment(scenario, runtime.url) + invoked_model: Final = _deployment(scenario, runtime.url, f"invoke/{_OPUS}") + generated: Final = gateway.request( + "POST", "/v1/responses", {**_FORCED_TOOL_CHOICE, "model": invoked_model, "max_output_tokens": 16} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request( + "POST", "/v1/responses/input_tokens", {**_FORCED_TOOL_CHOICE, "model": counted_model} + ) + assert _payload(response)["input_tokens"] == _MANTLE_COUNT, response.text + requests: Final = runtime.drain() + (invoked,) = _requests_to(requests, _INVOKE_TARGET) + (mantle_count,) = _mantle_bodies(mantle) + expected: Final = _tool_use_fields(_JSON_OBJECT.validate_json(invoked.body)) + assert set(expected) == set(_TOOL_USE_FIELDS), expected + assert _requests_to(requests, _COUNT_TARGET) == () + assert _tool_use_fields(mantle_count) == expected, mantle_count + + +def _counting_every_model(request: Request) -> Reply: + if _COUNT_TARGET.match(target_of(request)): + return _json_reply(200, {"inputTokens": _RUNTIME_COUNT}) + return runtime_generation(request) + + +_NOVA_COUNTS: Final[Mapping[str, tuple[str, dict[str, JsonValue], str]]] = MappingProxyType( + { + "utils-token-counter": ( + "/utils/token_counter?call_endpoint=true", + {"messages": _WEATHER_MESSAGES, "tools": _CHAT_TOOLS}, + "total_tokens", + ), + "responses-input-tokens": ( + "/v1/responses/input_tokens", + {"input": WEATHER_QUESTION, "tools": RESPONSES_TOOLS}, + "input_tokens", + ), + } +) + + +@pytest.mark.parametrize("route", _NOVA_COUNTS.keys()) +def test_nova_count_sends_the_tool_specs_converse_inference_sends(counting_proxy: OwnedProxy, route: str) -> None: + path, body, count_field = _NOVA_COUNTS[route] + gateway: Final = counting_proxy.gateway + with wire_server(_counting_every_model) as runtime, gateway.scenario() as scenario: + model: Final = _deployment(scenario, runtime.url, _NOVA) + generated: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _WEATHER_MESSAGES, "tools": _CHAT_TOOLS, "max_tokens": 16}, + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request("POST", path, {**body, "model": model}) + assert _payload(response)[count_field] == _RUNTIME_COUNT, response.text + requests: Final = runtime.drain() + (converse,) = (request for request in requests if target_of(request) == f"/model/{_NOVA}/converse") + (counted,) = _requests_to(requests, _COUNT_TARGET) + inference_tools: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(converse.body)["toolConfig"]) + assert _converse_count_tools(counted) == inference_tools["tools"], counted.body diff --git a/tests/integration/providers/test_malformed_tools_wire.py b/tests/integration/providers/test_malformed_tools_wire.py index 706babac398..910f3faf312 100644 --- a/tests/integration/providers/test_malformed_tools_wire.py +++ b/tests/integration/providers/test_malformed_tools_wire.py @@ -1418,7 +1418,7 @@ def test_count_tokens_rejects_malformed_tools_when_a_request_list_replaces_the_d assert _delivered(rig, marker) == () -def test_responses_input_tokens_drops_the_bad_entry_a_deployment_list_names( +def test_responses_input_tokens_drops_the_tools_a_deployment_list_names( rig: _Rig, field_groups: _FieldGroups ) -> None: marker: Final = _marker() @@ -1428,8 +1428,7 @@ def test_responses_input_tokens_drops_the_bad_entry_a_deployment_list_names( {"model": field_groups.anthropic, "input": marker, "tools": [RESPONSES.tool, 5]}, ) assert response.status_code == 200, response.text - tools: Final = _one_delivery(rig, marker, "/v1/messages/count_tokens")["tools"] - assert isinstance(tools, list) and len(tools) == 1 and "get_weather" in json.dumps(tools), tools + assert "tools" not in _one_delivery(rig, marker, "/v1/messages/count_tokens") def test_responses_input_tokens_rejects_malformed_tools_when_a_request_list_replaces_the_deployment_list( @@ -1669,7 +1668,7 @@ _CHECKED: Final[Mapping[str, _Checked]] = MappingProxyType( send=_input_tokens, target="/v1/messages/count_tokens", answered=_counted({"object": "response.input_tokens", "input_tokens": _COUNT, **_PROVIDER_SOURCE}), - kept_by_a_tools_list=("get_weather",), + kept_by_a_tools_list=(), bad_entry=_RESPONSES_BAD_ENTRY, ), } @@ -2336,14 +2335,14 @@ def test_a_deployment_list_with_a_non_string_entry_drops_tools_it_names_on_messa @_TOKEN_COUNTS -def test_token_counting_with_a_request_list_with_a_non_string_entry_drops_the_bad_entry( +def test_token_counting_with_a_request_list_with_a_non_string_entry_counts_the_tools_its_route_keeps( rig: _Rig, checked: _Checked ) -> None: marker: Final = _marker() response: Final = checked.send( rig, rig.anthropic, marker, {"tools": [checked.tool, 5], "additional_drop_params": ["tools", 5]} ) - _assert_forwarded(rig, checked, response, marker, ("get_weather",)) + _assert_forwarded(rig, checked, response, marker, checked.kept_by_a_tools_list) def _alias(suffix: str, variant: str) -> str: diff --git a/tests/integration/providers/test_openai_count_tokens_tools_wire.py b/tests/integration/providers/test_openai_count_tokens_tools_wire.py index 16a5b8685ad..96410b03a92 100644 --- a/tests/integration/providers/test_openai_count_tokens_tools_wire.py +++ b/tests/integration/providers/test_openai_count_tokens_tools_wire.py @@ -415,4 +415,76 @@ def test_responses_input_tokens_passes_an_openai_count_error_through( response: Final = gateway.request("POST", "/v1/responses/input_tokens", {"model": model, "input": "hello"}) assert response.status_code == 400, response.text assert response.json() == {"error": {"param": None, "code": None, **error}}, response.text - assert _count_body(wire)["input"] == [{"role": "user", "content": "hello"}] + assert _count_body(wire)["input"] == "hello" + + +_NATIVE_TEXT: Final[dict[str, JsonValue]] = { + "format": {"type": "json_schema", "name": "answer", "schema": _ANSWER_SCHEMA}, +} +_NATIVE_BODY: Final[Mapping[str, JsonValue]] = { + "input": [ + {"role": "user", "content": "hi"}, + {"type": "reasoning", "summary": [], "encrypted_content": _WRAPPED_REASONING}, + {"role": "user", "content": "again"}, + ], + "text": _NATIVE_TEXT, + "max_output_tokens": "lots", + "metadata": {"run": "nightly"}, +} + + +@pytest.mark.parametrize( + ("drop_params", "expected_extra"), + [pytest.param(False, {"max_output_tokens": "lots"}, id="as-sent"), pytest.param(True, {}, id="drop-params")], +) +def test_responses_input_tokens_sends_the_callers_body_to_openai( + counting_proxy: OwnedProxy, drop_params: bool, expected_extra: dict[str, JsonValue] +) -> None: + gateway: Final = counting_proxy.gateway + with wire_server(_count_response) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{wire.url}/v1", + api_key="sk-integration-openai", + drop_params=drop_params, + ) + response: Final = gateway.request("POST", "/v1/responses/input_tokens", {"model": model, **_NATIVE_BODY}) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["input_tokens"] == _COUNT + assert _count_body(wire) == { + "model": "gpt-4o-mini", + "input": [ + {"role": "user", "content": "hi"}, + {"type": "reasoning", "summary": [], "encrypted_content": "gAAAAoriginal"}, + {"role": "user", "content": "again"}, + ], + "text": _NATIVE_TEXT, + **expected_extra, + } + + +def test_responses_input_tokens_sends_openai_the_owners_previous_response_id(counting_proxy: OwnedProxy) -> None: + gateway: Final = counting_proxy.gateway + with wire_server(_responses_peer) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire.url) + team: Final = scenario.team(models=[model]) + owner_key: Final = scenario.key(team_id=team, user_id=scenario.member(team), models=[model]) + other_key: Final = scenario.key(team_id=team, user_id=scenario.member(team), models=[model]) + generated: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": "hi"}, key=owner_key) + assert generated.status_code == 200, generated.text + follow_up: Final = { + "model": model, + "input": "again", + "previous_response_id": _JSON_OBJECT.validate_json(generated.content)["id"], + } + _body_for(wire.drain(), "/v1/responses") + + refused: Final = gateway.request("POST", "/v1/responses/input_tokens", follow_up, key=other_key) + assert refused.status_code == 403, refused.text + assert _JSON_OBJECT.validate_json(refused.content)["error"]["type"] == "permission_error", refused.text + assert wire.drain() == () + + counted: Final = gateway.request("POST", "/v1/responses/input_tokens", follow_up, key=owner_key) + assert counted.status_code == 200, counted.text + count_body: Final = _body_for(wire.drain(), "/v1/responses/input_tokens") + assert count_body["previous_response_id"] == _RESPONSES_REPLY["id"] diff --git a/tests/integration/providers/test_responses_core_wire.py b/tests/integration/providers/test_responses_core_wire.py index 791b2b8783f..9a24903a20d 100644 --- a/tests/integration/providers/test_responses_core_wire.py +++ b/tests/integration/providers/test_responses_core_wire.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import os import uuid +from collections.abc import Callable from typing import Final import openai @@ -192,6 +193,42 @@ def test_previous_response_id_uses_the_raw_id_and_affinity( ], [request.body for request in requests] +def _counting_peer(raw_id: str, input_tokens: int) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.target == "/v1/responses/input_tokens": + return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": input_tokens}).encode()) + return Reply(body=_response(raw_id)) + + return respond + + +def test_input_tokens_count_goes_to_the_deployment_that_served_the_previous_response(gateway: Gateway) -> None: + raw_id: Final = f"resp_count_affinity_{uuid.uuid4().hex}" + with ( + wire_server(_counting_peer(raw_id, 37)) as first_wire, + wire_server(_counting_peer(raw_id, 99)) as second_wire, + gateway.scenario() as scenario, + ): + model: Final = f"integration-{uuid.uuid4().hex}" + _enable_responses_affinity(scenario) + assert _register(scenario, first_wire, model) == model + client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0) + first: Final = client.responses.create(model=model, input="turn one") + _register_in_group(scenario, second_wire, model, order=1) + counted: Final = client.responses.input_tokens.count( + model=model, input="turn two", previous_response_id=first.id + ) + assert counted.input_tokens == 37, counted.model_dump_json() + assert drain_contract_requests(second_wire) == (), "the count left the previous response's deployment" + requests: Final = drain_contract_requests(first_wire) + assert [(request.target, _JSON_OBJECT.validate_json(request.body)) for request in requests] == [ + ("/v1/responses", {"model": _MODEL, "input": "turn one"}), + ("/v1/responses/input_tokens", {"model": _MODEL, "input": "turn two", "previous_response_id": raw_id}), + ], [request.body for request in requests] + + @pytest.mark.parametrize("prefix", _ALIASES) def test_codex_responses_request_preserves_all_supported_fields(gateway: Gateway, prefix: str) -> None: response_id: Final = f"resp_codex_{uuid.uuid4().hex}" diff --git a/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py index ec3fb941037..1632e424c55 100644 --- a/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py +++ b/tests/integration/providers/test_vertex_gemini_count_tokens_wire.py @@ -1096,3 +1096,57 @@ def test_utils_token_counter_counts_a_local_prompt_with_system_or_tools_like_its assert as_message.status_code == 200, as_message.text assert _total_tokens(prompted) == _total_tokens(as_message) assert requests == () + + +_RESPONSES_TOOL_ROUND_TRIP: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "instructions": "Follow the system instruction.", + "input": [ + {"role": "user", "content": "Look up the value."}, + {"type": "function_call", "call_id": "call_lookup", "name": "lookup", "arguments": '{"query":"value"}'}, + {"type": "function_call_output", "call_id": "call_lookup", "output": "result"}, + ], + "tools": [ + { + "type": "function", + "name": "lookup", + "description": "Look up a value", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}, + } + ], + } +) + + +@pytest.mark.parametrize("provider", ("vertex_ai", "gemini")) +def test_responses_input_tokens_counts_the_tool_round_trip_contents_responses_inference_sends( + gateway: Gateway, provider: str +) -> None: + peer: Final = _GeminiPeer() + with wire_server(peer) as wire, gateway.scenario() as scenario: + model: Final = ( + _deployment(gateway, scenario, wire.url) + if provider == "vertex_ai" + else _ai_studio_deployment(scenario, wire.url) + ) + body: Final = {"model": model, **_RESPONSES_TOOL_ROUND_TRIP} + generated: Final = gateway.request("POST", "/v1/responses", {**body, "max_output_tokens": 16}) + counted: Final = gateway.request("POST", "/v1/responses/input_tokens", body) + assert generated.status_code == 200, generated.text + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _COUNT, counted.text + requests: Final = peer.drain() + (generated_request,) = (request for request in requests if request.target.endswith(":generateContent")) + (count_request,) = (request for request in requests if request.target.endswith(":countTokens")) + instruction: Final[JsonValue] = {"text": _RESPONSES_TOOL_ROUND_TRIP["instructions"]} + first_turn, *later_turns = _JSON_LIST.validate_python(_counted_contents(_json_body(count_request))) + first_parts: Final = _JSON_LIST.validate_python(_JSON_OBJECT.validate_python(first_turn)["parts"]) + assert instruction in first_parts, count_request.body + counted_turns: Final = [ + { + **_JSON_OBJECT.validate_python(first_turn), + "parts": [part for part in first_parts if part != instruction], + }, + *later_turns, + ] + assert counted_turns == _json_body(generated_request)["contents"], count_request.body diff --git a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py index 1956301f1be..c679311e4cc 100644 --- a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py +++ b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py @@ -19,6 +19,13 @@ from integration._support.client import Gateway, Scenario, eventually from integration._support.process import graceful_stop_seconds, owned_proxy_process from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._count_tokens_system_lift import ( + RESPONSES_REQUEST_OPTIONS, + RESPONSES_TOOL_USE, + count_request_fields, + counted_fields, + inference_count_fields, +) from integration._support.ports import allocate_port from pydantic import JsonValue, TypeAdapter from tokenizers import Regex, Tokenizer, models, pre_tokenizers @@ -427,14 +434,22 @@ def test_utils_token_counter_local_mode_never_calls_the_peer(gateway: Gateway) - assert wire.drain() == () -def test_utils_token_counter_surfaces_the_peer_error_for_openai_tools(gateway: Gateway) -> None: +def _inference_and_count_fields(wire: Wire) -> tuple[dict[str, JsonValue], ...]: + requests: Final = wire.drain() + (inference,) = (request for request in requests if request.target == _MESSAGE_TARGET) + return (counted_fields(_JSON_OBJECT.validate_json(inference.body)), *map(counted_fields, _count_bodies(requests))) + + +def test_utils_token_counter_counts_openai_tools_the_way_chat_inference_sends_them(gateway: Gateway) -> None: with wire_server(_peer(_strict)) as wire, gateway.scenario() as scenario: model: Final = _deployment(gateway, scenario, wire.url) body: Final = {**_bare(model), "tools": _OPENAI_TOOLS} + generated: Final = gateway.request("POST", "/v1/chat/completions", {**body, "max_tokens": 16}) + assert generated.status_code == 200, generated.text response: Final = gateway.request("POST", "/utils/token_counter", body, params={"call_endpoint": "true"}) - assert response.status_code == 400, response.text - assert "total_tokens" not in response.text, response.text - assert _counted_bodies(wire) == ({**_PEER_BARE, "tools": _OPENAI_TOOLS},) + assert _payload(response)["total_tokens"] == _PEER_COUNT, response.text + inference, counted = _inference_and_count_fields(wire) + assert counted == inference, (counted, inference) def test_responses_input_tokens_counts_through_the_partner_peer(gateway: Gateway) -> None: @@ -451,8 +466,24 @@ def test_responses_input_tokens_counts_through_the_partner_peer(gateway: Gateway assert _counted_bodies(wire) == (_PEER_BARE,) -def test_responses_input_tokens_surfaces_the_peer_error(gateway: Gateway) -> None: +@pytest.mark.parametrize("request_body", RESPONSES_TOOL_USE.values(), ids=RESPONSES_TOOL_USE.keys()) +def test_responses_input_tokens_counts_what_responses_inference_sends( + gateway: Gateway, request_body: dict[str, JsonValue] +) -> None: with wire_server(_peer(_strict)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire.url) + generated: Final = gateway.request( + "POST", "/v1/responses", {**request_body, "model": model, "max_output_tokens": 16} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request("POST", "/v1/responses/input_tokens", {**request_body, "model": model}) + assert _payload(response)["input_tokens"] == _PEER_COUNT, response.text + inference, counted = _inference_and_count_fields(wire) + assert counted == inference, (counted, inference) + + +def test_responses_input_tokens_surfaces_the_peer_error(gateway: Gateway) -> None: + with wire_server(_peer(_rejecting(400))) as wire, gateway.scenario() as scenario: model: Final = _deployment(gateway, scenario, wire.url) response: Final = gateway.request( "POST", @@ -462,7 +493,33 @@ def test_responses_input_tokens_surfaces_the_peer_error(gateway: Gateway) -> Non assert response.status_code == 400, response.text assert response.json() == _RESPONSES_ROUTE.rejection(_REJECTION), response.text (sent,) = _counted_bodies(wire) - assert sent["tools"] == _RESPONSES_TOOLS, sent + assert sent["system"] == [{"type": "text", "text": "Be terse"}], sent + assert sent["tools"] == [ + { + "name": "get_weather", + "input_schema": _WEATHER_SCHEMA, + "type": "custom", + "description": "Look up the current weather for a city", + } + ], sent + + +def test_responses_input_tokens_rejects_an_unpaired_tool_output_without_calling_vertex(gateway: Gateway) -> None: + with wire_server(_peer(_strict)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire.url) + body: Final[dict[str, JsonValue]] = { + "model": model, + "input": [ + {"role": "user", "content": "What is the weather in Paris?"}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "Paris?"}, + {"type": "function_call_output", "call_id": "call_1", "output": "Sunny"}, + ], + "tools": _RESPONSES_TOOLS, + } + response: Final = gateway.request("POST", "/v1/responses/input_tokens", body) + assert response.status_code == 400, response.text + assert "get_weather" in _error_message(_JSON_OBJECT.validate_json(response.content)), response.text + assert wire.drain() == () def test_responses_input_tokens_types_a_peer_error_by_its_status(gateway: Gateway) -> None: @@ -890,3 +947,20 @@ def test_responses_endpoint_on_the_same_deployment_still_generates(gateway: Gate (sent,) = wire.drain() assert sent.target == _MESSAGE_TARGET, sent.target assert marker in sent.body.decode(), sent.body + + +def test_responses_input_tokens_counts_the_structured_output_responses_inference_sends(gateway: Gateway) -> None: + request_body: Final = RESPONSES_REQUEST_OPTIONS["text-format"] + with wire_server(_peer(_strict)) as wire, gateway.scenario() as scenario: + model: Final = _deployment(gateway, scenario, wire.url) + generated: Final = gateway.request( + "POST", "/v1/responses", {**request_body, "model": model, "max_output_tokens": 4096} + ) + assert generated.status_code == 200, generated.text + response: Final = gateway.request("POST", "/v1/responses/input_tokens", {**request_body, "model": model}) + assert _payload(response)["input_tokens"] == _PEER_COUNT, response.text + requests: Final = wire.drain() + (inference,) = (request for request in requests if request.target == _MESSAGE_TARGET) + (counted,) = _count_bodies(requests) + expected: Final = inference_count_fields(_JSON_OBJECT.validate_json(inference.body)) + assert count_request_fields(counted) == expected, (counted, expected) diff --git a/tests/integration/sdk/test_vertex_partner_count_tokens_sdk.py b/tests/integration/sdk/test_vertex_partner_count_tokens_sdk.py index e59d3f9264a..2e1205be688 100644 --- a/tests/integration/sdk/test_vertex_partner_count_tokens_sdk.py +++ b/tests/integration/sdk/test_vertex_partner_count_tokens_sdk.py @@ -1,5 +1,5 @@ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import litellm @@ -88,3 +88,77 @@ async def test_acount_tokens_falls_back_to_the_local_tokenizer_when_the_peer_rej assert len(_count_requests(wire.drain())) == 1 assert counted.tokenizer_type == "local_tokenizer", counted assert counted.total_tokens > 0 and counted.total_tokens != _PEER_COUNT, counted + + +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_CHAT_TOOLS: Final[list[dict[str, JsonValue]]] = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Look up the current weather for a city", + "parameters": _WEATHER_SCHEMA, + }, + } +] +_CHAT_TOOL_USE: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 21C"}, +] +_GENERATED: Final[dict[str, JsonValue]] = { + "id": "msg_sdk_count", + "type": "message", + "role": "assistant", + "model": _BACKEND, + "content": [{"type": "text", "text": "It is sunny."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, +} +_COUNTED_FIELDS: Final = ("messages", "system", "tools") + + +def _generating_peer(request: Request) -> Reply: + if request.target == "/_oauth/token": + token: Final = {"access_token": "scripted-token", "token_type": "Bearer", "expires_in": 3600} + return Reply(body=json.dumps(token).encode()) + if "count-tokens" in request.target: + return Reply(body=json.dumps({"input_tokens": _PEER_COUNT}).encode()) + if request.target.endswith(f"/{_BACKEND}:rawPredict"): + return Reply(body=json.dumps(_GENERATED).encode()) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _counted_fields(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {field: body[field] for field in _COUNTED_FIELDS if field in body} + + +async def test_acount_tokens_sends_chat_tool_use_the_way_acompletion_sends_it( + vertex_environment: Callable[[str], None], +) -> None: + with wire_server(_generating_peer) as wire: + vertex_environment(wire.url) + generated: Final = await litellm.acompletion( + model=f"vertex_ai/{_BACKEND}", messages=_CHAT_TOOL_USE, tools=_CHAT_TOOLS, max_tokens=16, api_base=wire.url + ) + assert generated.choices[0].message.content == "It is sunny.", generated + counted: Final = await litellm.acount_tokens( + model=f"vertex_ai/{_BACKEND}", messages=_CHAT_TOOL_USE, tools=_CHAT_TOOLS, api_base=wire.url + ) + assert (counted.total_tokens, counted.tokenizer_type) == (_PEER_COUNT, "vertex_ai_partner_models"), counted + requests: Final = wire.drain() + (inference,) = (request for request in requests if request.target.endswith(f"/{_BACKEND}:rawPredict")) + (count,) = _count_requests(requests) + expected: Final = _counted_fields(_JSON_OBJECT.validate_json(inference.body)) + assert _counted_fields(_JSON_OBJECT.validate_json(count.body)) == expected, (count.body, expected) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c08eeea4bb9..cc0ef062e4b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1085,6 +1085,30 @@ def test_update_responses_input_with_model_file_ids_decodes_xlitellm_encoded_id( assert updated[0]["content"][1]["file_id"] == raw_id + +@pytest.mark.parametrize("unusable_file_id", [[1], {"a": 1}]) +def test_update_responses_input_with_model_file_ids_keeps_a_non_string_file_id_as_sent(unusable_file_id): + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + update_responses_input_with_model_file_ids, + ) + + encoded_id = _xlitellm_encoded("file-ExTuCawUqxEMjVFK6xwR9B", "gpt-5.1") + input_items = [ + { + "role": "user", + "content": [ + {"type": "input_file", "file_id": encoded_id}, + {"type": "input_file", "file_id": unusable_file_id}, + ], + } + ] + + updated = update_responses_input_with_model_file_ids( + input_items, "model-A", {encoded_id: {"model-A": "file-on-a"}} + ) + + assert [item["file_id"] for item in updated[0]["content"]] == ["file-on-a", unusable_file_id] + def test_update_messages_xlitellm_decode_does_not_override_mapping(): """If the call-site already resolved a provider id via the mapping, that wins. The new decode fallback runs only when no mapping match.""" diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 9af7497bc82..63f149fabe4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -1,7 +1,10 @@ import asyncio from base64 import b64encode +from collections.abc import Mapping from copy import deepcopy +from queue import SimpleQueue from threading import get_ident +from types import MappingProxyType from typing import Final import httpx @@ -10,16 +13,20 @@ import respx from pydantic import JsonValue, TypeAdapter import litellm +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) from litellm.llms.azure_ai.anthropic.count_tokens.handler import ( AzureAIAnthropicCountTokensHandler, ) +from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) +from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @pytest.mark.parametrize( @@ -441,3 +448,302 @@ async def test_handler_sends_the_leading_system_run_as_system_not_as_a_message(h "system": [{"type": "text", "text": "Be terse"}], "messages": [{"role": "user", "content": "Hello, how are you?"}], } + + +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {"type": "object", "properties": {"city": {"type": "string"}}} + +_CHAT_TOOL_TURNS: Final[list[dict[str, JsonValue]]] = [ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "SF"}'}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +] + + +_COUNTERS: Final = pytest.mark.parametrize( + ("counter_type", "count_endpoint"), + ( + (AnthropicTokenCounter, "https://gateway.example/v1/messages/count_tokens"), + (AzureAIAnthropicTokenCounter, "https://gateway.example/anthropic/v1/messages/count_tokens"), + ), +) + + +async def _counted_body( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + messages: list[dict[str, JsonValue]], + tools: list[dict[str, JsonValue]] | None, + messages_format: TokenCountMessagesFormat, + chat_params: Mapping[str, JsonValue] = MappingProxyType({}), +) -> tuple[TokenCountResponse | None, tuple[dict[str, JsonValue], ...]]: + sent: Final[SimpleQueue[bytes]] = SimpleQueue() + + def count(request: httpx.Request) -> httpx.Response: + sent.put(request.content) + return httpx.Response(200, json={"input_tokens": 31}) + + with respx.mock: + respx.post(count_endpoint).mock(side_effect=count) + result: Final = await counter_type().count_tokens( + model_to_use="claude-opus-5-5", + messages=messages, + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + request_model="claude-opus-5-5", + tools=tools, + messages_format=messages_format, + chat_params=chat_params, + ) + return result, tuple(TypeAdapter(dict[str, JsonValue]).validate_json(sent.get()) for _ in range(sent.qsize())) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + "tool", + ( + {"type": "function", "name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}, + { + "type": "function", + "function": {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}, + }, + ), +) +async def test_counter_sends_chat_tool_use_in_the_shape_chat_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + tool: dict[str, JsonValue], +) -> None: + """Responses and chat bodies reach the counter as chat messages and tools, which Anthropic rejects + with a 400 unless they are translated the way the chat inference request translates them.""" + original: Final = deepcopy(_CHAT_TOOL_TURNS) + result, bodies = await _counted_body(counter_type, count_endpoint, _CHAT_TOOL_TURNS, [tool], "openai") + + assert result is not None + assert result.error is not True + assert result.total_tokens == 31 + assert bodies == ( + { + "model": "claude-opus-5-5", + "system": [{"type": "text", "text": "Be terse."}], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Weather in SF?"}]}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "SF"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}]}, + ], + "tools": [ + { + "name": "get_weather", + "input_schema": _WEATHER_SCHEMA, + "type": "custom", + "description": "Look up the weather", + } + ], + }, + ) + assert _CHAT_TOOL_TURNS == original + + +_ANTHROPIC_TOOL_USE: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "Weather in SF?"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]}, +] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("messages", "tool", "messages_format"), + ( + (_ANTHROPIC_TOOL_USE, {"name": "get_weather", "input_schema": _WEATHER_SCHEMA}, "openai"), + ( + [{"role": "user", "content": "Weather in SF?"}], + {"type": "function", "name": "get_weather", "parameters": _WEATHER_SCHEMA}, + "anthropic", + ), + ), +) +async def test_counter_sends_anthropic_shaped_and_messages_api_requests_as_sent( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + messages: list[dict[str, JsonValue]], + tool: dict[str, JsonValue], + messages_format: TokenCountMessagesFormat, +) -> None: + _, bodies = await _counted_body(counter_type, count_endpoint, messages, [tool], messages_format) + + assert bodies == ({"model": "claude-opus-5-5", "messages": messages, "tools": [tool]},) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_rejects_a_tool_call_chat_inference_rejects_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "sunny?"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + result, bodies = await _counted_body(counter_type, count_endpoint, messages, None, "openai") + + assert bodies == () + assert result is not None + assert result.error is True + assert result.status_code == 400 + assert result.rejected_before_provider_call is True + assert "Failed to parse tool call arguments for tool 'get_weather'" in (result.error_message or "") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_rejects_a_chat_tool_without_a_name_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + result, bodies = await _counted_body( + counter_type, + count_endpoint, + [{"role": "user", "content": "Weather in SF?"}], + [{"type": "function", "function": {"parameters": _WEATHER_SCHEMA}}], + "openai", + ) + + assert bodies == () + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + + +_CHAT_WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "get_weather", "parameters": _WEATHER_SCHEMA}, +} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +@pytest.mark.parametrize( + ("messages", "tools"), + ( + pytest.param( + [ + { + "role": "user", + "name": "ana", + "content": [ + {"type": "text", "text": "Describe this"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0K"}}, + ], + } + ], + None, + id="native-image-block", + ), + pytest.param( + [ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "SF"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny"}]}, + ], + [_CHAT_WEATHER_TOOL], + id="native-tool-result-block", + ), + pytest.param( + [{"role": "user", "content": "Open the docs"}], + [ + {"type": "computer_use_preview", "display_width": 1024, "display_height": 768, "environment": "browser"}, + _CHAT_WEATHER_TOOL, + ], + id="computer-use-preview-tool", + ), + pytest.param( + [{"role": "user", "content": "Ask the wiki"}], + [{"type": "mcp", "server_label": "wiki", "server_url": "https://mcp.example/mcp"}, _CHAT_WEATHER_TOOL], + id="mcp-tool", + ), + ), +) +async def test_counter_rejects_input_chat_translation_would_leave_out_without_calling_anthropic( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + count_endpoint: str, + messages: list[dict[str, JsonValue]], + tools: list[dict[str, JsonValue]] | None, +) -> None: + result, bodies = await _counted_body(counter_type, count_endpoint, messages, tools, "openai") + + assert bodies == () + assert result is not None + assert (result.error, result.status_code, result.rejected_before_provider_call) == (True, 400, True) + + +_ANSWER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, +} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@_COUNTERS +async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat_inference_sends( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], count_endpoint: str +) -> None: + """count_tokens refuses the beta output_format that chat inference sends, so the count carries the same + schema as output_config.format, next to the inference request's thinking and tool_choice.""" + messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": "Weather in SF?"}] + tool: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "get_weather", "parameters": _WEATHER_SCHEMA}, + } + chat_params: Final[dict[str, JsonValue]] = { + "response_format": {"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + "reasoning_effort": "low", + "tool_choice": "auto", + } + config: Final = AnthropicConfig() + inference: Final = config.transform_request( + model="claude-opus-5-5", + messages=deepcopy(messages), + optional_params=config.map_openai_params( + {**chat_params, "tools": [tool]}, {}, "claude-opus-5-5", drop_params=False + ), + litellm_params={}, + headers={}, + ) + + _, bodies = await _counted_body(counter_type, count_endpoint, messages, [tool], "openai", chat_params) + + (body,) = bodies + assert "output_format" not in body + assert body["output_config"] == {**inference.get("output_config", {}), "format": inference["output_format"]} + assert (body["thinking"], body["tool_choice"], body["tools"]) == ( + inference["thinking"], + inference["tool_choice"], + inference["tools"], + ) diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py index b357c5ac126..052baf601f9 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py @@ -168,6 +168,25 @@ def test_transform_includes_tools(): assert tools[0]["toolSpec"]["inputSchema"]["json"]["type"] == "object" +def test_transform_maps_chat_function_tools(): + parameters = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} + request = { + "model": "amazon.nova-pro-v1:0", + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "description": "Look up the weather", "parameters": parameters}, + } + ], + } + + result = BedrockCountTokensConfig().transform_anthropic_to_bedrock_count_tokens(request) + + assert result["input"]["converse"]["toolConfig"]["tools"] == [ + {"toolSpec": {"name": "get_weather", "description": "Look up the weather", "inputSchema": {"json": parameters}}} + ] + def test_transform_includes_system_and_tools_together(): """Test that both system and tools are included together.""" config = BedrockCountTokensConfig() diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index dd0d6429f8a..6fb47491323 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -1,3 +1,4 @@ +import base64 from collections.abc import Mapping from typing import Final @@ -176,6 +177,130 @@ async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> N } +def _invoke_body(request: httpx.Request) -> dict[str, JsonValue]: + invoke_model: Final = _JSON_BODY.validate_json(request.content)["input"] + assert isinstance(invoke_model, dict) + encoded: Final = invoke_model["invokeModel"] + assert isinstance(encoded, dict) and isinstance(encoded["body"], str) + return _JSON_BODY.validate_json(base64.b64decode(encoded["body"])) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chat_params", + [ + {"tool_choice": "required"}, + { + "response_format": { + "type": "json_schema", + "json_schema": {"name": "city", "schema": {"type": "object", "properties": {"city": {"type": "string"}}}}, + } + }, + {"reasoning_effort": "low"}, + ], +) +async def test_runtime_count_leaves_out_the_options_only_mantle_counts(chat_params: dict[str, JsonValue]) -> None: + runtime_counts: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + mantle_counts: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177})) + request: Final = { + "model_to_use": "global.anthropic.claude-opus-4-8", + "messages": [{"role": "user", "content": "Weather in SF?"}], + "contents": None, + "deployment": DEPLOYMENT, + "request_model": "claude-opus-4-8", + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "messages_format": "openai", + "chat_params": chat_params, + } + + await _counter(runtime_counts).count_tokens(**request) + await _counter(mantle_counts).count_tokens(**request) + + assert runtime_counts.posted_hosts() == (RUNTIME_HOST,) + assert mantle_counts.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) + options: Final = ("thinking", "tool_choice", "output_config") + invoke_body: Final = _invoke_body(runtime_counts.requests[0]) + mantle_body: Final = _JSON_BODY.validate_json(mantle_counts.requests[1].content) + assert [name for name in options if name in invoke_body] == [] + assert [name for name in options if name in mantle_body] != [] + assert {name: mantle_body[name] for name in ("messages", "tools")} == { + name: invoke_body[name] for name in ("messages", "tools") + } + + +@pytest.mark.asyncio +async def test_claude_chat_tool_use_is_counted_in_the_shape_claude_inference_sends() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + schema: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="global.anthropic.claude-opus-4-8", + messages=[ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + contents=None, + deployment=DEPLOYMENT, + request_model="claude-opus-4-8", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": schema}}], + messages_format="openai", + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 412) + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + body: Final = _invoke_body(bedrock.requests[0]) + assert body["tools"] == [{"name": "get_weather", "input_schema": schema, "type": "custom"}] + assert body["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "Weather in SF?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}]}, + ] + +@pytest.mark.asyncio +async def test_claude_chat_tool_use_counted_on_mantle_metadata_route_is_translated_too() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 412}), mantle=(200, {"input_tokens": 2177})) + schema: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="bedrock_mantle/global.anthropic.claude-opus-4-8", + messages=[ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + contents=None, + deployment=DEPLOYMENT, + request_model="claude-opus-4-8", + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": schema}}], + messages_format="openai", + ) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 2177) + assert bedrock.posted_hosts() == (MANTLE_HOST,) + body: Final = _JSON_BODY.validate_json(bedrock.requests[0].content) + assert body["tools"] == [{"name": "get_weather", "input_schema": schema, "type": "custom"}] + assert body["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "Weather in SF?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}]}, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "host", "tokenizer", "expected_tokens"), diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 7495d98b42a..17c1b35e211 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -1,3 +1,4 @@ +import copy import json import operator from collections.abc import Mapping @@ -9,6 +10,7 @@ import respx from pydantic import JsonValue, ValidationError import litellm +from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -521,3 +523,122 @@ async def test_count_sends_the_global_key_only_to_the_global_api_base(monkeypatc assert result is not None assert result.total_tokens == 4 assert route.calls.last.request.headers["authorization"] == "Bearer sk-private-gateway" + + +_KEEP_ALL: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) +_ANSWER_FORMAT: Final[dict[str, object]] = { + "format": {"type": "json_schema", "name": "answer", "schema": {"type": "object", "properties": {}}} +} +_STORED_PROMPT: Final[dict[str, object]] = {"id": "pmpt_abc", "variables": {"city": "Paris"}} +_CALLER_BODY: Final[dict[str, object]] = { + "model": "my-gpt", + "input": "hi", + "text": _ANSWER_FORMAT, + "prompt": _STORED_PROMPT, + "max_output_tokens": "lots", + "metadata": {"run": "nightly"}, + "drop_params": False, + "additional_drop_params": [], + "timeout": 30, + "extra_headers": {"x-trace": "1"}, + "disable_fallbacks": True, + "spend_logs_metadata": {"team": "search"}, + "mock_testing_fallbacks": True, +} + + +async def _native_count( + monkeypatch: pytest.MonkeyPatch, request: Mapping[str, object], policy: DropPolicy, response: httpx.Response +) -> tuple[TokenCountResponse | None, object]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=response) + result: Final = await OpenAITokenCounter().count_responses_request( + model_to_use="gpt-4o", + request=request, + policy=policy, + deployment={"litellm_params": {"api_key": "sk-test"}}, + request_model="my-gpt", + ) + return result, json.loads(route.calls.last.request.content) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "expected_sent"), + [ + pytest.param( + _KEEP_ALL, + { + "model": "gpt-4o", + "input": "hi", + "text": _ANSWER_FORMAT, + "prompt": _STORED_PROMPT, + "max_output_tokens": "lots", + }, + id="keeps-responses-fields", + ), + pytest.param( + DropPolicy(drop_params=True, additional_drop_params=frozenset()), + {"model": "gpt-4o", "input": "hi", "text": _ANSWER_FORMAT, "prompt": _STORED_PROMPT}, + id="drop-params-keeps-count-fields", + ), + pytest.param( + DropPolicy(drop_params=False, additional_drop_params=frozenset({"text"})), + {"model": "gpt-4o", "input": "hi", "prompt": _STORED_PROMPT, "max_output_tokens": "lots"}, + id="additional-drop-params", + ), + ], +) +async def test_native_count_sends_the_callers_body_without_litellm_fields( + monkeypatch: pytest.MonkeyPatch, policy: DropPolicy, expected_sent: dict[str, object] +) -> None: + result, sent = await _native_count( + monkeypatch, _CALLER_BODY, policy, httpx.Response(200, json={"input_tokens": 76}) + ) + + assert sent == expected_sent + assert result is not None + assert (result.total_tokens, result.model_used, result.error) == (76, "gpt-4o", False) + + +@pytest.mark.asyncio +async def test_native_count_restores_provider_ids_on_a_copy_of_the_callers_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + wrapped: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAAoriginal", "deployment-1") + request: Final[dict[str, object]] = { + "model": "my-gpt", + "previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id( + "openai", "deployment-1", "resp_original" + ), + "input": [ + {"type": "reasoning", "summary": [], "encrypted_content": wrapped}, + {"role": "user", "content": "again"}, + ], + } + before: Final = copy.deepcopy(request) + + _, sent = await _native_count(monkeypatch, request, _KEEP_ALL, httpx.Response(200, json={"input_tokens": 1})) + + assert sent == { + "model": "gpt-4o", + "previous_response_id": "resp_original", + "input": [ + {"type": "reasoning", "summary": [], "encrypted_content": "gAAAAoriginal"}, + {"role": "user", "content": "again"}, + ], + } + assert request == before + + +@pytest.mark.asyncio +async def test_native_count_answers_openais_rejection_with_its_status_and_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rejection: Final = json.dumps({"error": {"message": "Unknown parameter: 'max_output_tokens'."}}) + + result, _ = await _native_count(monkeypatch, _CALLER_BODY, _KEEP_ALL, httpx.Response(400, text=rejection)) + + assert result is not None + assert (result.error, result.status_code, result.error_message) == (True, 400, rejection) diff --git a/tests/unit/llms/test_file_search_responses.py b/tests/unit/llms/test_file_search_responses.py index 9d2a5dbaf59..b2cbcd4a6dd 100644 --- a/tests/unit/llms/test_file_search_responses.py +++ b/tests/unit/llms/test_file_search_responses.py @@ -573,11 +573,11 @@ class TestEmulatedFileSearchHandler: def test_H1_file_search_replaced_with_function_tool(self): from litellm.responses.file_search.emulated_handler import ( - _replace_file_search_tools, + replace_file_search_tools, ) tools = [{"type": "file_search", "vector_store_ids": ["vs_abc", "vs_def"]}] - new_tools, vs_ids = _replace_file_search_tools(tools) + new_tools, vs_ids = replace_file_search_tools(tools) assert vs_ids == ["vs_abc", "vs_def"] assert len(new_tools) == 1 @@ -590,14 +590,14 @@ class TestEmulatedFileSearchHandler: def test_H2_non_file_search_tools_preserved(self): from litellm.responses.file_search.emulated_handler import ( - _replace_file_search_tools, + replace_file_search_tools, ) tools = [ {"type": "web_search"}, {"type": "file_search", "vector_store_ids": ["vs_abc"]}, ] - new_tools, vs_ids = _replace_file_search_tools(tools) + new_tools, vs_ids = replace_file_search_tools(tools) assert len(new_tools) == 2 # web_search + generated function tool assert new_tools[0]["type"] == "web_search" @@ -605,22 +605,22 @@ class TestEmulatedFileSearchHandler: def test_H3_no_file_search_tools_returns_unchanged(self): from litellm.responses.file_search.emulated_handler import ( - _replace_file_search_tools, + replace_file_search_tools, ) tools = [{"type": "web_search"}] - new_tools, vs_ids = _replace_file_search_tools(tools) + new_tools, vs_ids = replace_file_search_tools(tools) assert vs_ids == [] assert new_tools == [{"type": "web_search"}] def test_H4_empty_vector_store_ids_no_function_tool(self): from litellm.responses.file_search.emulated_handler import ( - _replace_file_search_tools, + replace_file_search_tools, ) tools = [{"type": "file_search", "vector_store_ids": []}] - new_tools, vs_ids = _replace_file_search_tools(tools) + new_tools, vs_ids = replace_file_search_tools(tools) assert vs_ids == [] assert new_tools == [] # no function tool added without store IDs diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 7d380205ca1..38737df7545 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -28,6 +28,7 @@ from litellm.llms.vertex_ai.common_utils import ( vertex_request_labels_from_litellm_params, ) from litellm.llms.vertex_ai.count_tokens import handler as vertex_count_tokens_handler +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat @@ -64,7 +65,9 @@ async def test_vertex_gemini_rejects_non_text_system_blocks_before_request( "google/gemma-4-26b-a4b-it-maas", ), ) -async def test_vertex_partner_model_without_an_inference_matching_count_answers_400_naming_the_model(model: str) -> None: +async def test_vertex_partner_model_without_an_inference_matching_count_answers_400_naming_the_model( + model: str, +) -> None: result: Final = await VertexAITokenCounter().count_tokens( model_to_use=model, messages=[{"role": "user", "content": "Count this"}], @@ -488,7 +491,6 @@ def test_anyof_conversion_forwards_a_non_object_branch_for_the_provider_to_judge assert schema == {"type": "object", "properties": {"example": {"anyOf": expected}}} - @pytest.mark.parametrize( ("field", "expected"), [ @@ -534,6 +536,7 @@ def test_vertex_schema_folds_every_null_branch_into_nullable(field: JsonValue, e assert schema["properties"]["value"] == expected + def test_anyof_with_excessive_nesting(): """Test conversion with excessive nesting > max levels +1 deep.""" # generate a schema with excessive nesting @@ -1700,6 +1703,148 @@ async def test_vertex_ai_token_counter_forwards_system_and_tools_to_partner_requ assert without_optional_fields.total_tokens == 37 +_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +] +_PARTNER_RESPONSES_TOOL: Final[dict[str, object]] = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, +} + + +async def _partner_counted_bodies( + model: str, chat_params: dict[str, JsonValue] | None = None +) -> tuple[dict[str, object], ...]: + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler + + class FakeResponse: + status_code = 200 + + def json(self) -> dict[str, int]: + return {"input_tokens": 37} + + class FakeHttpClient: + posted_bodies: tuple[dict[str, object], ...] = () + + async def post( + self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float + ) -> FakeResponse: + self.posted_bodies = (*self.posted_bodies, json) + return FakeResponse() + + fake_http_client: Final = FakeHttpClient() + with ( + patch.object(handler, "get_async_httpx_client", return_value=fake_http_client), + patch.object( + handler.VertexAIPartnerModelsTokenCounter, + "_ensure_access_token_async", + new=AsyncMock(return_value=("fake-token", "test-project")), + ), + ): + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use=model, + messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS), + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + tools=[_PARTNER_RESPONSES_TOOL], + chat_params=chat_params or {}, + ) + + assert result is not None + assert result.total_tokens == 37 + return fake_http_client.posted_bodies + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_chat_tool_use_in_the_shape_chat_inference_sends() -> None: + assert await _partner_counted_bodies("claude-opus-5-5") == ( + { + "model": "claude-opus-5-5", + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Weather in SF?"}]}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}]}, + ], + "tools": [ + { + "name": "get_weather", + "input_schema": {"type": "object", "properties": {}}, + "type": "custom", + "description": "", + } + ], + }, + ) + + +@pytest.mark.asyncio +async def test_vertex_claude_count_sends_the_structured_output_vertex_inference_sends() -> None: + model: Final = "claude-haiku-4-5" + chat_params: Final[dict[str, JsonValue]] = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}, + }, + } + } + config: Final = VertexAIAnthropicConfig() + inference: Final = config.transform_request( + model=model, + messages=copy.deepcopy(_PARTNER_CHAT_TOOL_TURNS), + optional_params=config.map_openai_params( + { + **chat_params, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "", + "parameters": _PARTNER_RESPONSES_TOOL["parameters"], + }, + } + ], + }, + {}, + model, + drop_params=False, + ), + litellm_params={}, + headers={}, + ) + + (body,) = await _partner_counted_bodies(model, chat_params) + + assert (body["tools"], body.get("tool_choice"), body.get("output_config")) == ( + inference["tools"], + inference.get("tool_choice"), + inference.get("output_config"), + ) + + +@pytest.mark.asyncio +async def test_vertex_llama_count_sends_chat_tool_use_as_sent() -> None: + assert await _partner_counted_bodies("meta/llama-4-maverick-17b-128e-instruct-maas") == ( + { + "model": "meta/llama-4-maverick-17b-128e-instruct-maas", + "messages": _PARTNER_CHAT_TOOL_TURNS, + "tools": [_PARTNER_RESPONSES_TOOL], + }, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("provider_failure", "expected_status", "expected_message"), diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index e3ce513dbcb..32b77c3b564 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -4171,10 +4171,17 @@ class TestResponsesInputTokens: path: str = "/v1/responses/input_tokens", counter: AsyncMock | None = None, llm_router: litellm.Router | None = None, + response_id_security: object | None = None, + managed_files: object | None = None, ) -> tuple[Response, AsyncMock]: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - from litellm.proxy.response_api_endpoints.endpoints import _proxy_llm_router, _proxy_token_counter + from litellm.proxy.response_api_endpoints.endpoints import ( + _proxy_llm_router, + _proxy_managed_files_hook, + _proxy_response_id_security, + _proxy_token_counter, + ) from litellm.types.utils import TokenCountResponse token_counter_mock = ( @@ -4192,6 +4199,10 @@ class TestResponsesInputTokens: app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path) app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock app.dependency_overrides[_proxy_llm_router] = lambda: llm_router + if response_id_security is not None: + app.dependency_overrides[_proxy_response_id_security] = lambda: response_id_security + if managed_files is not None: + app.dependency_overrides[_proxy_managed_files_hook] = lambda: managed_files try: client = TestClient(app) response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"}) @@ -4200,6 +4211,8 @@ class TestResponsesInputTokens: app.dependency_overrides.pop(user_api_key_auth, None) app.dependency_overrides.pop(_proxy_token_counter, None) app.dependency_overrides.pop(_proxy_llm_router, None) + app.dependency_overrides.pop(_proxy_response_id_security, None) + app.dependency_overrides.pop(_proxy_managed_files_hook, None) def test_string_input_returns_openai_input_tokens_shape(self): response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) @@ -4324,6 +4337,139 @@ class TestResponsesInputTokens: assert response.status_code == 200, response.text assert counter.await_args.kwargs["request"].tools == [{"type": "web_search"}] + assert counter.await_args.kwargs["responses_request"]["tools"] == [{"type": "web_search"}] + + @pytest.mark.parametrize( + ("tools", "expected"), + [ + ([5], {}), + ( + [5, {"type": "web_search"}], + {"tools": [{"type": "web_search"}], "tool_choice": "required", "parallel_tool_calls": False}, + ), + ], + ) + def test_tool_choice_leaves_the_count_with_the_tools_drop_params_drops( + self, tools: list[object], expected: dict[str, object] + ) -> None: + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": "hi", + "tools": tools, + "tool_choice": "required", + "parallel_tool_calls": False, + "drop_params": True, + } + ) + + assert response.status_code == 200, response.text + sent = counter.await_args.kwargs["responses_request"] + assert {name: sent[name] for name in ("tools", "tool_choice", "parallel_tool_calls") if name in sent} == expected + + def test_previous_response_id_reaches_the_counter_authorized_next_to_the_id_the_client_held(self) -> None: + from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD + + class DecodingSecurity: + def authorize_response_id(self, response_id: str, user_api_key_dict: object) -> str: + return {"resp_client_form": "resp_raw"}[response_id] + + response, counter = self._post_input_tokens( + {"model": "gpt-4o", "input": "hi", "previous_response_id": "resp_client_form"}, + response_id_security=DecodingSecurity(), + ) + + assert response.status_code == 200, response.text + responses_request: Final = counter.await_args.kwargs["responses_request"] + assert (responses_request["previous_response_id"], responses_request[ADDRESSED_RESPONSE_ID_FIELD]) == ( + "resp_raw", + "resp_client_form", + ) + + def test_managed_file_ids_reach_the_counter_with_the_mapping_inference_uses(self) -> None: + from litellm.integrations.custom_logger import CustomLogger + + mapping: Final = {"litellm-managed-file": {"deployment-a": "file-on-a"}} + + class ManagedFiles(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + if call_type == "aresponses": + data["model_file_id_mapping"] = mapping + return None + + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-managed-file"}]}], + }, + managed_files=ManagedFiles(), + ) + + assert response.status_code == 200, response.text + assert counter.await_args.kwargs["responses_request"]["model_file_id_mapping"] == mapping + + @pytest.mark.parametrize("caller_mapping", [5, "lots", [1], {"file-x": {"deployment-a": 7}}]) + def test_a_caller_sent_model_file_id_mapping_does_not_break_the_count( + self, monkeypatch: pytest.MonkeyPatch, caller_mapping: object + ) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http + from litellm.proxy.proxy_server import count_request_tokens + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-files", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "api_base": "https://count.test/v1", + }, + "model_info": {"id": "deployment-a"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + sent: Final[list[object]] = [] # mutable-ok: collects what the mock peer receives + + def count_peer(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response(200, json={"object": "response.input_tokens", "input_tokens": 7}, request=request) + + with patch.object( + openai_count_tokens_http, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(count_peer)), + ): + response, _ = self._post_input_tokens( + {"model": "gpt-files", "input": "hi", "model_file_id_mapping": caller_mapping}, + counter=AsyncMock(side_effect=count_request_tokens), + llm_router=router, + ) + + assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text + assert sent == [{"model": "gpt-4o-mini", "input": "hi"}] + + def test_a_managed_file_the_key_cannot_read_answers_403_without_a_count(self) -> None: + from litellm.integrations.custom_logger import CustomLogger + + denied: Final = "User u1 does not have access to the file litellm-managed-file" + + class ManagedFiles(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise HTTPException(status_code=403, detail=denied) + + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-managed-file"}]}], + }, + managed_files=ManagedFiles(), + ) + + assert response.status_code == 403, response.text + assert response.json()["error"]["message"] == denied + counter.assert_not_awaited() def test_malformed_tools_follow_the_counted_deployment_even_when_fallbacks_are_configured(self) -> None: router: Final = litellm.Router( @@ -4345,6 +4491,7 @@ class TestResponsesInputTokens: assert response.status_code == 200, response.text assert counter.await_args.kwargs["request"].tools is None + assert "tools" not in counter.await_args.kwargs["responses_request"] def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None: router: Final = litellm.Router( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 714d8bb77d7..46e88614bb6 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -57,8 +57,10 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.gemini.count_tokens import handler as gemini_count_tokens_handler +from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import VertexAIPartnerModelsTokenCounter from litellm.proxy._types import ProxyException, TokenCountRequest, UserAPIKeyAuth +from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler from litellm.proxy.anthropic_endpoints.endpoints import ( count_tokens as anthropic_count_tokens, ) @@ -77,9 +79,16 @@ from litellm.proxy.proxy_server import ( count_request_tokens, token_counter, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem -from litellm.types.utils import SelectTokenizerResponse, TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, + SelectTokenizerResponse, + SpecialEnums, + TokenCountMessagesFormat, + TokenCountResponse, +) def _hugging_face_offline() -> NoReturn: @@ -270,6 +279,50 @@ async def test_token_counter_reads_an_oidc_api_key_only_when_it_calls_the_provid assert str(tmp_path) not in excinfo.value.message +@pytest.mark.asyncio +async def test_native_openai_responses_count_sends_the_token_an_oidc_api_key_points_to( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path)) + token_file: Final = tmp_path / "token" + token_file.write_text("sa-token-from-file") + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "gpt-oidc", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": f"oidc/file/{token_file}", + "api_base": "https://count.test/v1", + }, + } + ] + ), + ) + authorizations: Final[list[str | None]] = [] # mutable-ok: collects what the mock peer receives + + def count_peer(request: httpx.Request) -> httpx.Response: + authorizations.append(request.headers.get("authorization")) + return httpx.Response(200, json={"object": "response.input_tokens", "input_tokens": 9}, request=request) + + with patch.object( + openai_count_tokens_http, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(count_peer)), + ): + result: Final = await count_request_tokens( + request=TokenCountRequest(model="gpt-oidc", messages=[{"role": "user", "content": "hello"}]), + call_endpoint=True, + responses_request={"model": "gpt-oidc", "input": "hello"}, + ) + + assert (result.error, result.total_tokens) == (False, 9) + assert authorizations == ["Bearer sa-token-from-file"] + + @pytest.mark.asyncio async def test_anthropic_messages_count_tokens_endpoint(): """ @@ -1567,6 +1620,7 @@ class _ScriptedCounter(BaseTokenCounter): messages_format: TokenCountMessagesFormat = "openai", output_format: JsonValue = None, context_management: object = None, + chat_params: object = None, ) -> TokenCountResponse | None: self.calls.append((model_to_use, messages, tools, system)) self.regions.append(None if deployment is None else deployment["litellm_params"].get("aws_region_name")) @@ -3505,3 +3559,325 @@ async def test_call_endpoint_prompt_keeps_the_local_count_on_a_counter_that_need local.total_tokens, local.tokenizer_type, ) + + +_PINNED_COUNTS: Final = MappingProxyType({"deployment-a.test": 11, "deployment-b.test": 22}) + + +def _pinned_count_peer(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("previous_response_id") != "resp_original": + return httpx.Response(400, json={"error": {"message": "previous_response_id was not decoded"}}, request=request) + return httpx.Response(200, json={"input_tokens": _PINNED_COUNTS[request.url.host]}, request=request) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pinned", ["deployment-a", "deployment-b"]) +async def test_responses_count_goes_to_the_deployment_that_served_the_previous_response( + monkeypatch: pytest.MonkeyPatch, pinned: str +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-group", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "api_base": f"https://{identity}.test/v1", + }, + "model_info": {"id": identity}, + } + for identity in ("deployment-a", "deployment-b") + ], + optional_pre_call_checks=["responses_api_deployment_check"], + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_pinned_count_peer)) + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + "openai", pinned, "resp_original" + ) + + with patch.object(openai_count_tokens_http, "get_async_httpx_client", return_value=client): + counts: Final = [ + ( + await count_request_tokens( + request=TokenCountRequest(model="gpt-group", messages=[{"role": "user", "content": "again"}]), + call_endpoint=True, + responses_request={ + "model": "gpt-group", + "input": "again", + "previous_response_id": previous_response_id, + }, + ) + ).total_tokens + for _ in range(8) + ] + + assert counts == [_PINNED_COUNTS[f"{pinned}.test"]] * 8 + + +def _managed_file_id(provider_file_id: str, model_id: str) -> str: + unified: Final = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( + "application/pdf", "3f0c9a", "gpt-files", provider_file_id, model_id + ) + return base64.urlsafe_b64encode(unified.encode()).decode().rstrip("=") + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("with_mapping", "expected_file_id"), [(False, "file-uploaded"), (True, "file-on-a")]) +async def test_responses_count_sends_openai_the_provider_file_id_inference_sends( + monkeypatch: pytest.MonkeyPatch, with_mapping: bool, expected_file_id: str +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-files", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key", "api_base": "https://count.test/v1"}, + "model_info": {"id": "deployment-a"}, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + managed_id: Final = _managed_file_id("file-uploaded", "deployment-a") + bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives + + def count_peer(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"object": "response.input_tokens", "input_tokens": 49}, request=request) + + with patch.object( + openai_count_tokens_http, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(count_peer)), + ): + result: Final = await count_request_tokens( + request=TokenCountRequest(model="gpt-files", messages=[{"role": "user", "content": "Summarize"}]), + call_endpoint=True, + responses_request={ + "model": "gpt-files", + "input": [{"role": "user", "content": [{"type": "input_file", "file_id": managed_id}]}], + **({"model_file_id_mapping": {managed_id: {"deployment-a": "file-on-a"}}} if with_mapping else {}), + }, + ) + + assert (result.error, result.total_tokens) == (False, 49) + (body,) = bodies + assert body["input"] == [{"role": "user", "content": [{"type": "input_file", "file_id": expected_file_id}]}] + assert "model_file_id_mapping" not in body + + +_ANSWER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, +} + + +@pytest.mark.asyncio +async def test_responses_count_on_claude_counts_the_chat_request_responses_inference_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "claude-counted", + "litellm_params": { + "model": "anthropic/claude-opus-5-5", + "api_key": "test-key", + "api_base": "https://count-tokens.test", + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives + + def count_peer(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"input_tokens": 77}, request=request) + + responses_tools: Final[list[dict[str, JsonValue]]] = [ + {"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + {"type": "web_search"}, + {"type": "file_search", "vector_store_ids": ["vs_docs"]}, + ] + with patch.object( + anthropic_count_tokens_handler, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(count_peer)), + ): + result: Final = await count_request_tokens( + request=TokenCountRequest( + model="claude-counted", messages=[{"role": "user", "content": "Weather?"}], tools=responses_tools + ), + call_endpoint=True, + responses_request={ + "model": "claude-counted", + "input": "Weather?", + "tools": responses_tools, + "tool_choice": "auto", + "text": {"format": {"type": "json_schema", "name": "answer", "schema": _ANSWER_SCHEMA}}, + "reasoning": {"effort": "low"}, + }, + ) + + assert (result.error, result.total_tokens) == (False, 77) + (body,) = bodies + tools: Final = body["tools"] + assert isinstance(tools, list) + tool_names: Final = [tool["name"] for tool in tools if isinstance(tool, dict)] + assert sorted(tool_names) == ["get_weather", "litellm_file_search", "web_search"] + assert any(isinstance(tool, dict) and str(tool.get("type")).startswith("web_search") for tool in tools) + assert body["tool_choice"] == {"type": "auto"} + output_config: Final = body["output_config"] + assert isinstance(output_config, dict) + assert output_config["format"] == {"type": "json_schema", "schema": _ANSWER_SCHEMA} + assert "thinking" in body + + +_WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, +} + + +async def _claude_count( + monkeypatch: pytest.MonkeyPatch, + deployment_params: Mapping[str, JsonValue], + responses_request: Mapping[str, object], +) -> tuple[TokenCountResponse, list[dict[str, JsonValue]]]: + router: Final = Router( + model_list=[ + { + "model_name": "claude-counted", + "litellm_params": { + "model": "anthropic/claude-zeta-9", + "api_key": "test-key", + "api_base": "https://count-tokens.test", + **deployment_params, + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + bodies: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: collects what the mock peer receives + + def count_peer(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"input_tokens": 77}, request=request) + + with patch.object( + anthropic_count_tokens_handler, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(count_peer)), + ): + result: Final = await count_request_tokens( + request=TokenCountRequest(model="claude-counted", messages=[{"role": "user", "content": "Weather?"}]), + call_endpoint=True, + responses_request={"model": "claude-counted", **responses_request}, + ) + return result, bodies + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_params", "request_params"), + [({"drop_params": True}, {}), ({}, {"drop_params": True})], + ids=["deployment", "request"], +) +async def test_responses_count_on_claude_downgrades_forced_tool_choice_under_drop_params( + monkeypatch: pytest.MonkeyPatch, + deployment_params: Mapping[str, JsonValue], + request_params: Mapping[str, JsonValue], +) -> None: + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False}) + + result, bodies = await _claude_count( + monkeypatch, + deployment_params, + {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required", **request_params}, + ) + + assert (result.error, result.total_tokens) == (False, 77) + (body,) = bodies + assert body["tool_choice"] == {"type": "auto"} + + +@pytest.mark.asyncio +async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_forced_tool_use": False}) + + with pytest.raises(ProxyException) as rejected: + await _claude_count( + monkeypatch, {}, {"input": "Weather?", "tools": [_WEATHER_TOOL], "tool_choice": "required"} + ) + + assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True) + + +def _texts(message: JsonValue) -> list[str]: + assert isinstance(message, dict) + content: Final = message["content"] + if isinstance(content, str): + return [content] + assert isinstance(content, list) + return [str(block["text"]) for block in content if isinstance(block, dict) and "text" in block] + + +@pytest.mark.asyncio +async def test_responses_count_on_claude_counts_the_turns_previous_response_id_replays( + monkeypatch: pytest.MonkeyPatch, +) -> None: + stored_turn: Final = { + "request_id": "chatcmpl-stored", + "call_type": "aresponses", + "session_id": "session-1", + "proxy_server_request": {"input": "Who won in 1998?", "model": "claude-counted"}, + "response": { + "id": "chatcmpl-stored", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The Bulls.", + "reasoning_content": "Chicago beat Utah in six games.", + }, + "finish_reason": "stop", + } + ], + }, + "status": "success", + } + + with patch.object( + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + return_value=[stored_turn], + ) as spend_logs: + result, bodies = await _claude_count( + monkeypatch, + {}, + { + "input": "And in 1999?", + "previous_response_id": "chatcmpl-stored", + ADDRESSED_RESPONSE_ID_FIELD: "resp_client_form", + }, + ) + + assert (result.error, result.total_tokens) == (False, 77) + (body,) = bodies + messages: Final = body["messages"] + assert isinstance(messages, list) + assert list(itertools.chain.from_iterable(map(_texts, messages))) == [ + "Who won in 1998?", + "The Bulls.", + "And in 1999?", + ] + assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3 + spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py new file mode 100644 index 00000000000..464bdb1a367 --- /dev/null +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -0,0 +1,127 @@ +import json +from typing import Final + +import pytest +from pydantic import JsonValue + +from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.responses.litellm_completion_transformation.count_request import ( + ChatCountBridgeError, + ChatCountRequest, + chat_count_request, +) + +_MARK: Final[dict[str, JsonValue]] = {"type": "ephemeral"} +_MARKED_RESPONSES_REQUEST: Final[dict[str, JsonValue]] = { + "model": "claude-opus-5-5", + "instructions": "Be terse.", + "input": [ + {"type": "message", "role": "user", "content": "Weather in SF?", "cache_control": _MARK}, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "And in LA?", "cache_control": _MARK}], + }, + { + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "SF"}', + "cache_control": _MARK, + }, + {"type": "function_call_output", "call_id": "call_1", "output": "sunny", "cache_control": _MARK}, + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + "cache_control": _MARK, + } + ], +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", (False, True)) +async def test_claude_count_request_leaves_out_every_cache_control_mark(drop_params: bool) -> None: + request: Final = await chat_count_request( + model="claude-opus-5-5", + custom_llm_provider="anthropic", + api_base=None, + responses_request=_MARKED_RESPONSES_REQUEST, + policy=DropPolicy(drop_params=drop_params, additional_drop_params=frozenset()), + ) + + assert isinstance(request, ChatCountRequest) + counted: Final = json.dumps({"messages": request.messages, "tools": request.tools}, default=str) + assert "cache_control" not in counted + for kept in ("Weather in SF?", "And in LA?", "call_1", "sunny", "get_weather"): + assert kept in counted + + +_WEATHER_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, +} +_QUESTION: Final[dict[str, JsonValue]] = {"role": "user", "content": "Weather in SF?"} + + +async def _claude_count( + responses_request: dict[str, JsonValue], policy: DropPolicy +) -> ChatCountRequest | ChatCountBridgeError | None: + return await chat_count_request( + model="claude-opus-5-5", + custom_llm_provider="anthropic", + api_base=None, + responses_request={"model": "claude-opus-5-5", **responses_request}, + policy=policy, + ) + + +@pytest.mark.asyncio +async def test_claude_count_request_counts_additional_tools_input_items_as_tools_like_inference() -> None: + policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) + + hoisted: Final = await _claude_count( + {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION]}, policy + ) + + assert isinstance(hoisted, ChatCountRequest) + assert hoisted.tools is not None + assert hoisted == await _claude_count({"input": [_QUESTION], "tools": [_WEATHER_TOOL]}, policy) + + +@pytest.mark.asyncio +async def test_claude_count_request_drops_a_malformed_tool_flag_under_drop_params_like_inference() -> None: + policy: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) + + flagged: Final = await _claude_count( + {"input": [_QUESTION], "tools": [{**_WEATHER_TOOL, "defer_loading": "yes"}]}, policy + ) + + assert isinstance(flagged, ChatCountRequest) + assert flagged == await _claude_count({"input": [_QUESTION], "tools": [_WEATHER_TOOL]}, policy) + + +@pytest.mark.asyncio +async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_params() -> None: + flagged: Final = await _claude_count( + {"input": [_QUESTION], "tools": [{**_WEATHER_TOOL, "defer_loading": "yes"}]}, + DropPolicy(drop_params=False, additional_drop_params=frozenset()), + ) + + assert isinstance(flagged, ChatCountBridgeError) + assert "defer_loading" in flagged.message + + +@pytest.mark.asyncio +async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None: + policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"})) + + dropped: Final = await _claude_count({"input": [_QUESTION], "tools": [_WEATHER_TOOL]}, policy) + + assert isinstance(dropped, ChatCountRequest) + assert dropped.tools is None + assert dropped == await _claude_count({"input": [_QUESTION]}, policy) diff --git a/tests/unit/test_count_tokens_public_api.py b/tests/unit/test_count_tokens_public_api.py index e33bc1e5907..4eb8bea3427 100644 --- a/tests/unit/test_count_tokens_public_api.py +++ b/tests/unit/test_count_tokens_public_api.py @@ -3,9 +3,13 @@ Tests for litellm.acount_tokens() public API. """ import asyncio +import json import os from unittest.mock import AsyncMock, patch +import httpx +import pytest +import respx import litellm from litellm.types.utils import TokenCountResponse @@ -157,3 +161,45 @@ def test_acount_tokens_no_api_key_falls_back(monkeypatch): assert result.tokenizer_type == "local_tokenizer" + + +@pytest.fixture +def httpx_transport_clients(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +def test_acount_tokens_sends_chat_tool_use_to_anthropic_in_the_shape_claude_inference_sends(httpx_transport_clients): + schema = {"type": "object", "properties": {"city": {"type": "string"}}} + with respx.mock(assert_all_called=True) as router: + route = router.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 52}) + ) + result = asyncio.run( + litellm.acount_tokens( + model="anthropic/claude-opus-5-5", + messages=[ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": schema}}], + api_key="sk-ant-test-key", + ) + ) + + assert (result.error, result.total_tokens) == (False, 52) + sent = json.loads(route.calls.last.request.content) + assert sent["tools"] == [{"name": "get_weather", "input_schema": schema, "type": "custom"}] + assert [message["role"] for message in sent["messages"]] == ["user", "assistant", "user"] + assert sent["messages"][2]["content"] == [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}] From 02199b670bbbda106e6a35afe9263ca6bfb91f6a Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 20 Oct 2026 08:21:49 -0700 Subject: [PATCH 104/124] fix(proxy): share the billing logger across callback registries (#47510) * fix(logging): deduplicate equivalent completed success callbacks * fix(proxy): share the billing logger across callback registries * test(proxy): use fixed timestamps for billing regressions --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 5 +- .../test_litellm_logging.py | 72 +++++++++++++++---- .../unit/proxy/proxy_server/test_lifecycle.py | 72 +++++++++++++------ 3 files changed, 112 insertions(+), 37 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f87b1edc2e5..2c6d2e2736a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3118,8 +3118,9 @@ def cost_tracking(): from litellm.integrations.shadow_eval_logger import ShadowEvalLogger spend_event_producer = build_spend_event_producer(CollectorSettings(), fallback=run_spend_event) - litellm.logging_callback_manager.add_litellm_callback(ProxyDBLogger(spend_event_producer)) - litellm.logging_callback_manager.add_litellm_async_success_callback(ProxyDBLogger(spend_event_producer)) + db_logger: Final = ProxyDBLogger(spend_event_producer) + litellm.logging_callback_manager.add_litellm_callback(db_logger) + litellm.logging_callback_manager.add_litellm_async_success_callback(db_logger) litellm.logging_callback_manager.add_litellm_callback(ShadowEvalLogger()) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 7aac43298d4..981c43a9792 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2462,7 +2462,41 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests( @pytest.mark.asyncio -@pytest.mark.parametrize("registration", ["sync", "async", "overlapping"]) +@pytest.mark.parametrize("different_headers", [False, True]) +async def test_completed_success_preserves_generic_logger_configuration( + monkeypatch: pytest.MonkeyPatch, different_headers: bool +) -> None: + from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger + + first: Final = GenericAPILogger( + endpoint="https://logs.example.test/events", + headers={"Authorization": "Bearer first"}, + event_types=["llm_api_success"], + ) + second: Final = GenericAPILogger( + endpoint="https://logs.example.test/events", + headers={"Authorization": "Bearer second" if different_headers else "Bearer first"}, + event_types=["llm_api_success"] if different_headers else ["llm_api_failure"], + ) + monkeypatch.setattr(litellm, "success_callback", [first]) + monkeypatch.setattr(litellm, "_async_success_callback", [second]) + monkeypatch.setattr(litellm, "generic_api_use_v1", False) + logging_obj: Final = _async_request_logging_obj("distinct-generic-loggers", []) + response: Final = ModelResponse(id="chatcmpl-distinct-generic-loggers") + now: Final = datetime.datetime(2026, 1, 1) + + await logging_obj.async_success_handler(response, now, now) + + assert len(first.log_queue) == 1 + assert first.log_queue[0]["id"] == response.id + if different_headers: + assert second.log_queue == first.log_queue + else: + assert second.log_queue == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("registration", ["sync", "async", "overlapping", "equivalent", "distinct"]) @pytest.mark.parametrize("kind", ["sync", "async", "dual"]) @pytest.mark.parametrize("compatibility_first", [False, True]) async def test_completed_success_selects_one_implemented_hook( @@ -2517,9 +2551,15 @@ async def test_completed_success_selects_one_implemented_hook( ) -> None: record("async", kwargs, response_obj) - callback: Final = {"sync": SyncRecorder, "async": AsyncRecorder, "dual": DualRecorder}[kind]() + recorder: Final = {"sync": SyncRecorder, "async": AsyncRecorder, "dual": DualRecorder}[kind] + callback: Final = recorder() + async_callback: Final = ( + recorder(turn_off_message_logging=registration == "distinct") + if registration in ("equivalent", "distinct") + else callback + ) monkeypatch.setattr(litellm, "success_callback", [callback] if registration != "async" else []) - monkeypatch.setattr(litellm, "_async_success_callback", [callback] if registration != "sync" else []) + monkeypatch.setattr(litellm, "_async_success_callback", [async_callback] if registration != "sync" else []) logging_obj: Final = _async_request_logging_obj("completed-hook", []) response: Final = ModelResponse(id="chatcmpl-completed-hook") now: Final = datetime.datetime.now() @@ -2531,20 +2571,26 @@ async def test_completed_success_selects_one_implemented_hook( await logging_obj.async_success_handler(response, now, now) logging_obj.handle_sync_success_callbacks_for_async_calls(response, now, now) - assert events.qsize() == 1 - hook, response_id, cost, thread_id = events.get_nowait() - assert hook == ("sync" if kind == "sync" else "async") - assert response_id == response.id - assert cost == logging_obj.model_call_details["response_cost"] - assert (thread_id == loop_thread) is (kind != "sync") + expected_count: Final = 2 if registration in ("equivalent", "distinct") else 1 + assert events.qsize() == expected_count + for _ in range(expected_count): + hook, response_id, cost, thread_id = events.get_nowait() + assert hook == ("sync" if kind == "sync" else "async") + assert response_id == response.id + assert cost == logging_obj.model_call_details["response_cost"] + assert (thread_id == loop_thread) is (kind != "sync") -@pytest.mark.parametrize("for_async_request", [False, True]) +@pytest.mark.parametrize("for_async_request,split_registries", [(False, False), (True, False), (True, True)]) @pytest.mark.parametrize("private_first", [False, True]) @pytest.mark.parametrize("response_kind", ["responses", "completion"]) def test_sync_success_redacts_only_opted_out_loggers( - monkeypatch: pytest.MonkeyPatch, for_async_request: bool, private_first: bool, response_kind: str + monkeypatch: pytest.MonkeyPatch, + for_async_request: bool, + split_registries: bool, + private_first: bool, + response_kind: str, ) -> None: from queue import SimpleQueue @@ -2563,8 +2609,8 @@ def test_sync_success_redacts_only_opted_out_loggers( private_logger: Final = PayloadRecorder(turn_off_message_logging=True) public_logger: Final = PayloadRecorder() callbacks: Final = [private_logger, public_logger] if private_first else [public_logger, private_logger] - monkeypatch.setattr(litellm, "success_callback", callbacks) - monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", callbacks[:1] if split_registries else callbacks) + monkeypatch.setattr(litellm, "_async_success_callback", callbacks[1:] if split_registries else []) monkeypatch.setattr(litellm, "turn_off_message_logging", False) logging_obj: Final = LitellmLogging( model="openai/gpt-6.1-sol", diff --git a/tests/unit/proxy/proxy_server/test_lifecycle.py b/tests/unit/proxy/proxy_server/test_lifecycle.py index 3914c337f59..11a44a0ec16 100644 --- a/tests/unit/proxy/proxy_server/test_lifecycle.py +++ b/tests/unit/proxy/proxy_server/test_lifecycle.py @@ -697,33 +697,61 @@ def test_load_from_azure_key_vault_missing_uri_failure_is_swallowed(monkeypatch) # --------------------------------------------------------------------------- -def test_cost_tracking_adds_db_and_shadow_eval_callbacks_when_prisma_set(monkeypatch): +@pytest.mark.asyncio +@pytest.mark.parametrize("setup_count", [1, 2]) +async def test_cost_tracking_adds_db_and_shadow_eval_callbacks_when_prisma_set( + monkeypatch: pytest.MonkeyPatch, setup_count: int +) -> None: + from datetime import datetime + import litellm from litellm.integrations.shadow_eval_logger import ShadowEvalLogger + from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, decode_spend_event + from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer + from litellm.types.utils import ModelResponse, Usage + from litellm.utils import Rules, function_setup - fake_prisma = MagicMock() - monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) - monkeypatch.setattr(litellm, "callbacks", [], raising=False) - monkeypatch.setattr(litellm, "_async_success_callback", [], raising=False) + producer: Final = AsyncMock(spec=SpendEventProducer) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(ps, "spend_event_producer", None) + monkeypatch.setattr(ps, "build_spend_event_producer", lambda settings, fallback: producer) + for name in ( + "callbacks", + "input_callback", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, name, []) - before_callbacks = len(litellm.callbacks) - before_async = len(litellm._async_success_callback) + for _ in range(setup_count): + cost_tracking() - cost_tracking() - cost_tracking() - - observed = { - "added_to_callbacks": len(litellm.callbacks) - before_callbacks, - "added_to_async_success": len(litellm._async_success_callback) - before_async, - "shadow_eval_loggers": sum(isinstance(cb, ShadowEvalLogger) for cb in litellm.callbacks), - "prisma_was_set": True, - } - assert normalize(observed) == { - "added_to_callbacks": 2, - "added_to_async_success": 1, - "shadow_eval_loggers": 1, - "prisma_was_set": True, - } + assert len(litellm.callbacks) == 2 + assert sum(isinstance(cb, ShadowEvalLogger) for cb in litellm.callbacks) == 1 + assert len(litellm._async_success_callback) == 1 + now: Final = datetime(2026, 1, 1) + logging_obj, _ = function_setup( + "acompletion", + Rules(), + now, + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + litellm_call_id="single-accounting-event", + ) + logging_obj.update_environment_variables(litellm_params={"acompletion": True, "metadata": {}}, optional_params={}) + response: Final = ModelResponse( + id="chatcmpl-single-accounting-event", + model="gpt-4o-mini", + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + await logging_obj.async_success_handler(response, now, now) + producer.publish.assert_awaited_once() + event: Final = decode_spend_event(producer.publish.call_args.args[0]) + assert not isinstance(event, SpendEventDecodeError) + assert event.response_cost == logging_obj.model_call_details["response_cost"] + assert event.response_cost > 0 def test_cost_tracking_no_op_when_prisma_missing(monkeypatch): From ce63f48ed2ee3a18b7860b72f292e90ebeb1a986 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 08:22:06 -0700 Subject: [PATCH 105/124] fix(decisions): log non-UTF-8 bytes as base64 so the request reaches validation and gets a 400 (#47564) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/decisions/call.py | 2 +- tests/unit/decisions/test_call.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index b31cb1f69db..b94b3380ad5 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -427,7 +427,7 @@ def _jsonable_fallback(value: object) -> object: def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage: caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python( - to_jsonable_python(caller_request, fallback=_jsonable_fallback) + to_jsonable_python(caller_request, fallback=_jsonable_fallback, bytes_mode="base64") ) sent_fields: Final = json.dumps( dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {})) diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py index 03c135369b8..36a16dd5ad1 100644 --- a/tests/unit/decisions/test_call.py +++ b/tests/unit/decisions/test_call.py @@ -1,3 +1,4 @@ +import base64 import json from collections.abc import Mapping from datetime import datetime, timezone @@ -112,3 +113,22 @@ def test_decisions_request_is_logged_even_when_its_questions_fail_validation(): logged: Final = _logged_request(logging_obj) assert logged["questions"][0]["instructions"] == QUESTION_TEXT + + +def test_non_utf8_bytes_in_the_request_are_logged_and_rejected_with_a_400(): + logging_obj: Final = _logging_obj("adecisions") + raw_input: Final = b"\xff\xfe not utf-8" + + with pytest.raises(litellm.BadRequestError): + _prepare_decisions_call( + **CONNECTION, + decision_input=raw_input, + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert base64.urlsafe_b64decode(logged["input"]) == raw_input + assert logged["questions"][0]["instructions"] == QUESTION_TEXT From 0ede0e10e7ee694c46f14afd1606885d8c369342 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 20 Oct 2026 08:22:19 -0700 Subject: [PATCH 106/124] fix(batch): preserve recoverable costs and prevent poll starvation (#47190) * fix(batch): preserve recoverable costs and rotate polling pages * test(batch): type retirement failure regression * fix(batch): alternate new arrivals with retry polling * test(batch): defer polling row annotation on Python 3.12 * test(batch): reconcile fixtures with atomic metadata claims * refactor(batch): simplify reconciliation to one bounded scan * fix(batch): restore arrival priority during retry scans * fix(batch): preserve recoverable jobs on unrelated resource errors * fix: revisit recent batches before old retry backlog --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../proxy/common_utils/check_batch_cost.py | 128 +++++--- .../managed_object_attribution.py | 7 + .../spend/test_batch_poll_starvation.py | 230 ++++++++++++- .../common_utils/test_check_batch_cost.py | 304 +++++++++++++++--- 4 files changed, 557 insertions(+), 112 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 7449ec9882f..05e3e0d839d 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -4,6 +4,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t import asyncio import json +import re from collections.abc import Mapping, Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta, timezone @@ -39,7 +40,12 @@ if TYPE_CHECKING: from litellm.types.utils import LiteLLMBatch +def _error_names_resource(error: Exception, resource_id: str) -> bool: + return resource_id in re.findall(r"""[^\s"'`]+""", str(error)) + + CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" +RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48) PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = ( "completed", @@ -109,6 +115,11 @@ class CheckBatchCost: # the guaranteed-failing primary query on every subsequent cycle. self._has_batch_processed_column: bool = True self.batch_processed_support_confirmed: bool = False + self._poll_cursor: tuple[datetime, str] | None = None + self._poll_before: datetime = datetime.now(timezone.utc) + self._arrival_cursor: tuple[datetime, str] | None = None + self._arrival_before: datetime = self._poll_before + self._poll_arrivals: bool = False self._line_item_stores: Final[set[asyncio.Task[None]]] = set() self._line_item_store_turn: Final = asyncio.Lock() self._poll_idle: Final = asyncio.Event() @@ -161,29 +172,9 @@ class CheckBatchCost: f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired" ) - if not self._has_batch_processed_column: - return - - # A row already in a terminal status is never rewritten by the sweep above, so - # without this it keeps a poll-page slot forever and starves newer batches. - retired: Final = await _managed_object_table(self.prisma_client).update_many( - where={ - "file_purpose": "batch", - "batch_processed": False, - "status": {"in": ["complete", "completed"]}, - "created_at": {"lt": cutoff}, - }, - data={"batch_processed": True}, - ) - if retired > 0: - verbose_proxy_logger.warning( - f"CheckBatchCost: gave up on {retired} completed managed objects older than " - f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed" - ) - async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]": """Query batch jobs without the batch_processed filter (for older schemas).""" - return await _managed_object_table(self.prisma_client).find_many( + return await self._find_jobs( where={ "file_purpose": "batch", "status": { @@ -197,10 +188,59 @@ class CheckBatchCost: ] }, }, - take=MAX_OBJECTS_PER_POLL_CYCLE, - order={"created_at": "asc"}, ) + async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]: + arrivals: Final = self._poll_arrivals + page: Final = await self._find_job_page(where, arrivals=arrivals) + self._poll_arrivals = not arrivals if page else arrivals + if page: + return page + return await self._find_job_page(where, arrivals=not arrivals) + + async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]: + cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor + if cursor is None: + if arrivals: + self._arrival_before = datetime.now(timezone.utc) + else: + self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW + before: Final = self._arrival_before if arrivals else self._poll_before + page: Final = await _managed_object_table(self.prisma_client).find_many( + where={ + **where, + "created_at": { + "lte": before, + **({"gt": before - RECENT_BATCH_POLL_WINDOW} if arrivals else {}), + }, + **( + { + "OR": [ + {"created_at": {"gt": cursor[0]}}, + {"created_at": cursor[0], "id": {"gt": cursor[1]}}, + ] + } + if cursor is not None + else {} + ), + }, + take=MAX_OBJECTS_PER_POLL_CYCLE, + order=[{"created_at": "asc"}, {"id": "asc"}], + ) + next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None + if arrivals: + self._arrival_cursor = next_cursor + else: + self._poll_cursor = next_cursor + return page + + def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]: + return { + "id": job.id, + "updated_at": job.updated_at, + **({"batch_processed": False} if self._has_batch_processed_column else {}), + } + async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None: """ Take a row that can never be costed out of the poll page. Leaving it selectable @@ -208,24 +248,19 @@ class CheckBatchCost: once enough such rows accumulate no newer batch is ever reached. Older schemas without batch_processed can only be excluded through the status filter. """ - data: Final = ( - {"batch_processed": True} - if self._has_batch_processed_column - else {"status": "stale_expired"} - ) + data: Final = {"batch_processed": True} if self._has_batch_processed_column else {"status": "stale_expired"} try: - await _managed_object_table(self.prisma_client).update( - where={"id": job.id}, + retired: Final = await _managed_object_table(self.prisma_client).update_many( + where=self._unprocessed_snapshot(job), data=data, ) except Exception as db_err: - verbose_proxy_logger.error( - f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}" - ) + verbose_proxy_logger.error(f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}") + return + if retired == 0: return verbose_proxy_logger.warning( - f"CheckBatchCost: job {job.id} can never be costed ({reason}), " - "so it will no longer be polled" + f"CheckBatchCost: job {job.id} can never be costed ({reason}), so it will no longer be polled" ) async def _claim_job_for_costing(self, job: "ManagedObjectRow", response: "LiteLLMBatch") -> bool: @@ -312,13 +347,13 @@ class CheckBatchCost: from litellm.exceptions import NotFoundError - return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error) + return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, batch_id) def _batch_deployment_exists(self, model_id: str) -> bool: """A 404 only proves the batch is gone when it came from the batch's own deployment. Once that deployment leaves the router, default fallbacks can silently send the retrieve to a provider that never saw the batch, so its - 404 must not retire the row; the staleness sweep bounds it instead.""" + 404 must not retire a potentially recoverable row.""" return self.llm_router.get_deployment(model_id=model_id) is not None @staticmethod @@ -333,7 +368,7 @@ class CheckBatchCost: if not output_file_id: return False - return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error) + return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id) async def _finalize_unbilled_terminal_job( self, job: "ManagedObjectRow", response: "LiteLLMBatch" @@ -355,18 +390,18 @@ class CheckBatchCost: db_batch_object=job, unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id), ) - update_data: Final[dict] = { + update_data: Final[dict[str, object]] = { "status": response.status, "file_object": response.model_dump_json(), **({"batch_processed": True} if self._has_batch_processed_column else {}), } - await _managed_object_table(self.prisma_client).update( - where={"id": job.id}, + finalized: Final = await _managed_object_table(self.prisma_client).update_many( + where=self._unprocessed_snapshot(job), data=update_data, ) - verbose_proxy_logger.info( - f"CheckBatchCost: marked job {job.id} as {response.status} in DB" - ) + if finalized == 0: + return False + verbose_proxy_logger.info(f"CheckBatchCost: marked job {job.id} as {response.status} in DB") return True except Exception as db_err: verbose_proxy_logger.error( @@ -1047,6 +1082,7 @@ class CheckBatchCost: """ try: from litellm.integrations.prometheus import PrometheusLogger + prom_logger = PrometheusLogger.get_instance() except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not get Prometheus logger: {e}") @@ -1071,7 +1107,7 @@ class CheckBatchCost: # endpoint may transition a batch to "complete" before # CheckBatchCost runs. The batch_processed=False filter # already prevents reprocessing finished batches. - jobs = await _managed_object_table(self.prisma_client).find_many( + jobs = await self._find_jobs( where={ "file_purpose": "batch", "batch_processed": False, @@ -1084,8 +1120,6 @@ class CheckBatchCost: ] }, }, - take=MAX_OBJECTS_PER_POLL_CYCLE, - order={"created_at": "asc"}, ) self.batch_processed_support_confirmed = True except Exception as query_err: diff --git a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py index 16b1b0e973c..2e2c07fd544 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass +from datetime import datetime from typing import TYPE_CHECKING, Final, Protocol, cast from pydantic import TypeAdapter @@ -27,6 +28,12 @@ if TYPE_CHECKING: class ManagedObjectRow(Protocol): + @property + def created_at(self) -> datetime: ... + + @property + def updated_at(self) -> datetime: ... + @property def id(self) -> str: ... diff --git a/tests/integration/spend/test_batch_poll_starvation.py b/tests/integration/spend/test_batch_poll_starvation.py index f68f10e3d8a..51f5f0871f5 100644 --- a/tests/integration/spend/test_batch_poll_starvation.py +++ b/tests/integration/spend/test_batch_poll_starvation.py @@ -1,14 +1,20 @@ import json import os +from datetime import datetime, timezone +from functools import partial from hashlib import sha256 +from itertools import chain from typing import Final +from uuid import uuid4 +import httpx import pytest from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows -from integration._support.upstream import delete_scenario, register_scenario +from integration._support.database import read_rows, write_rows +from integration._support.prisma_clients import connected_proxy_prisma +from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE @@ -59,17 +65,17 @@ def _accepting_routes() -> dict[str, JsonResponse | TextResponse]: } -def _gone_at_provider_routes() -> RoutedResponse: +def _gone_at_provider_routes(*, retryable: bool = False) -> RoutedResponse: return RoutedResponse( content_type="application/x-routed", routes={ **_accepting_routes(), "GET /batches/batch-$REQUEST_ID": JsonResponse( content_type="application/json", - status=404, + status=503 if retryable else 404, body={ "error": { - "message": "No batch found with id 'batch-$REQUEST_ID'.", + "message": "Temporary failure" if retryable else "No batch found with id 'batch-$REQUEST_ID'.", "type": "invalid_request_error", "param": "id", "code": "batch_not_found", @@ -80,7 +86,7 @@ def _gone_at_provider_routes() -> RoutedResponse: ) -def _completed_routes() -> RoutedResponse: +def _completed_routes(*, ready: bool = True) -> RoutedResponse: output_line: Final = { "id": "batch_req_1", "custom_id": "r1", @@ -106,7 +112,8 @@ def _completed_routes() -> RoutedResponse: routes={ **_accepting_routes(), "GET /batches/batch-$REQUEST_ID": JsonResponse( - content_type="application/json", body=_batch("completed", "file-out-$REQUEST_ID") + content_type="application/json", + body=_batch("completed", "file-out-$REQUEST_ID") if ready else _batch("in_progress", None), ), "GET /files/file-out-$REQUEST_ID/content": TextResponse( content_type="application/jsonl", body=json.dumps(output_line, separators=(",", ":")) + "\n" @@ -115,7 +122,37 @@ def _completed_routes() -> RoutedResponse: ) -def _scripted_deployment(scenario: Scenario, marker: str, routes: RoutedResponse) -> str: +def _batch_was_polled(upstream: ScenarioHandle) -> bool: + response: Final = httpx.get(f"{upstream.control_url}/__observations", timeout=15, trust_env=False) + response.raise_for_status() + observations: Final = TypeAdapter(dict[str, list[dict[str, JsonValue]]]).validate_json(response.content) + return any( + item.get("method") == "GET" and string_value(item["path"]).startswith(f"/{upstream.scenario_id}/batches/") + for item in observations["requests"] + ) + + +def _missing_output_routes() -> RoutedResponse: + return RoutedResponse( + content_type="application/x-routed", + routes={ + **_completed_routes().routes, + "GET /files/file-out-$REQUEST_ID/content": JsonResponse( + content_type="application/json", + status=404, + body={ + "error": { + "message": "No file found with id 'file-out-$REQUEST_ID'.", + "type": "invalid_request_error", + "code": "file_not_found", + } + }, + ), + }, + ) + + +def _scripted_deployment(scenario: Scenario, marker: str, routes: RoutedResponse) -> tuple[str, ScenarioHandle]: scenario_id: Final = f"poll-{marker}-{sha256(os.urandom(16)).hexdigest()[:12]}" handle: Final = register_scenario(scenario_id, routes) scenario.cleanups.callback(delete_scenario, handle) @@ -133,7 +170,7 @@ def _scripted_deployment(scenario: Scenario, marker: str, routes: RoutedResponse }, ) scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) - return string_value(created["model_name"]) + return string_value(created["model_name"]), handle def _submitted_batch_id(gateway: Gateway, key: str, model_name: str) -> str: @@ -173,20 +210,51 @@ def _managed_rows(batch_ids: tuple[str, ...]) -> list[dict[str, JsonValue]]: ) +@pytest.mark.parametrize( + ("retryable", "old_completed", "backlog"), + [(False, False, 0), (False, True, 0), (True, False, 0), (True, True, 0), (True, False, 1000)], +) @pytest.mark.timeout(180) @pytest.mark.covers("quota_management.spend_tracking.batch_costs.uncostable_rows_retire_so_newer_batches_are_costed") -def test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_polling(gateway: Gateway) -> None: +def test_uncostable_batches_do_not_starve_billable_batches( + gateway: Gateway, retryable: bool, old_completed: bool, backlog: int +) -> None: with gateway.scenario() as scenario: - key: Final = scenario.key() + owner: Final = f"batch-backlog-{uuid4().hex}" + if backlog: + _insert_scan_rows(owner, "old", backlog, datetime(2000, 1, 1, tzinfo=timezone.utc)) + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_ManagedObjectTable" WHERE created_by = %s', (owner,) + ) + team_id: Final = scenario.team() + key: Final = scenario.key(team_id=team_id) gone_batch_ids: Final = tuple( _submitted_batch_id( - gateway, key, _scripted_deployment(scenario, f"gone{index}", _gone_at_provider_routes()) + gateway, + key, + _scripted_deployment(scenario, f"gone{index}", _gone_at_provider_routes(retryable=retryable))[0], ) for index in range(MAX_OBJECTS_PER_POLL_CYCLE) ) - costable_batch_id: Final = _submitted_batch_id( - gateway, key, _scripted_deployment(scenario, "costable", _completed_routes()) + missing_output_id: Final = _submitted_batch_id( + gateway, key, _scripted_deployment(scenario, "missing-output", _missing_output_routes())[0] ) + model, upstream = _scripted_deployment(scenario, "costable", _completed_routes(ready=False)) + costable_batch_id: Final = _submitted_batch_id(gateway, key, model) + for batch_id in (*gone_batch_ids, missing_output_id, costable_batch_id): + scenario.cleanups.callback( + write_rows, + 'DELETE FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id = %s', + (batch_id,), + ) + if old_completed: + write_rows( + 'UPDATE "LiteLLM_ManagedObjectTable" SET status = %s, ' + "created_at = NOW() - INTERVAL '30 days' WHERE unified_object_id = %s", + ("completed", costable_batch_id), + ) + assert eventually(lambda: _batch_was_polled(upstream), bool, seconds=120) + register_scenario(upstream.scenario_id, _completed_routes(), control_url=upstream.control_url) spend_rows: Final = eventually( lambda: read_rows( 'SELECT call_type, status, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' @@ -209,4 +277,134 @@ def test_batches_gone_at_provider_do_not_starve_a_newer_batch_out_of_cost_pollin } ] assert _managed_rows((costable_batch_id,)) == [{"batch_processed": True}] - assert _managed_rows(gone_batch_ids) == [{"batch_processed": True}] * MAX_OBJECTS_PER_POLL_CYCLE + assert eventually( + lambda: _managed_rows((missing_output_id,)), + lambda rows: rows == [{"batch_processed": True}], + seconds=120, + ) == [{"batch_processed": True}] + expected_blockers: Final = [{"batch_processed": not retryable}] * MAX_OBJECTS_PER_POLL_CYCLE + blockers: Final = eventually( + lambda: _managed_rows(gone_batch_ids), lambda rows: rows == expected_blockers, seconds=120 + ) + assert blockers == expected_blockers + + expected_spend: Final = BATCH_COST_SHARE * ( + PROMPT_TOKENS * INPUT_COST_PER_TOKEN + COMPLETION_TOKENS * OUTPUT_COST_PER_TOKEN + ) + for table, column, identity in ( + ("LiteLLM_VerificationToken", "token", sha256(key.encode()).hexdigest()), + ("LiteLLM_TeamTable", "team_id", team_id), + ): + balance: Final = eventually( + partial(read_rows, f'SELECT spend FROM "{table}" WHERE {column} = %s', (identity,)), + lambda rows: len(rows) == 1 and rows[0]["spend"] == pytest.approx(expected_spend), + seconds=70, + ) + assert balance == [{"spend": pytest.approx(expected_spend)}] + + +def _insert_scan_rows(owner: str, group: str, size: int, created_at: datetime) -> tuple[str, ...]: + prefix: Final = f"{owner}-{group}-" + write_rows( + 'INSERT INTO "LiteLLM_ManagedObjectTable" ' + "(id, unified_object_id, model_object_id, file_object, file_purpose, status, created_by, created_at, updated_at) " + "SELECT %s || lpad(n::text, 8, '0'), %s || n, %s || n, '{}'::jsonb, 'batch', 'completed', %s, " + "%s::timestamptz, NOW() FROM generate_series(0, %s::int - 1) n", + (prefix, prefix, prefix, owner, created_at.isoformat(), str(size)), + ) + return tuple(f"{prefix}{index:08}" for index in range(size)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("count", [2 * MAX_OBJECTS_PER_POLL_CYCLE, 2 * MAX_OBJECTS_PER_POLL_CYCLE + 1, 1000]) +async def test_arrivals_progress_while_retry_scan_advances_and_wraps(count: int) -> None: + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + from litellm.router import Router + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + owner: Final = f"batch-scan-{uuid4().hex}" + + try: + old_ids: Final = _insert_scan_rows(owner, "old", count, datetime(2000, 1, 1, tzinfo=timezone.utc)) + async with connected_proxy_prisma() as prisma: + logging: Final = ProxyLogging(UserApiKeyCache()) + router: Final = Router() + poller: Final = CheckBatchCost(logging, prisma, router) + where: Final = {"created_by": owner, "batch_processed": False} + first: Final = await poller._find_jobs(where) + assert tuple(row.id for row in first) == old_ids[:MAX_OBJECTS_PER_POLL_CYCLE] + + recent_ids: Final = _insert_scan_rows( + owner, "recent", MAX_OBJECTS_PER_POLL_CYCLE + 1, datetime.now(timezone.utc) + ) + + async def poll_with_arrival(turn: int) -> tuple[str, ...]: + if turn % 4 == 0: + _insert_scan_rows(owner, f"arrival-{turn}", 1, datetime.now(timezone.utc)) + page: Final = await poller._find_jobs(where) + assert len(page) <= MAX_OBJECTS_PER_POLL_CYCLE + write_rows( + 'UPDATE "LiteLLM_ManagedObjectTable" SET batch_processed = true ' + "WHERE created_by = %s AND id = ANY(%s) AND id LIKE %s", + (owner, [row.id for row in page], f"{owner}-arrival-%"), + ) + return tuple(row.id for row in page) + + pages: Final = tuple( + [await poll_with_arrival(turn) for turn in range(4 * (count // MAX_OBJECTS_PER_POLL_CYCLE + 2))] + ) + selected: Final = tuple(chain.from_iterable(pages)) + assert all(selected.count(identity) >= 2 for identity in recent_ids) + selected_old: Final = tuple(identity for identity in selected if identity in old_ids) + assert selected_old[: count - MAX_OBJECTS_PER_POLL_CYCLE] == old_ids[MAX_OBJECTS_PER_POLL_CYCLE:] + assert set(old_ids).issubset(selected_old) + + latest_ids: Final = _insert_scan_rows(owner, "latest", 1, datetime.now(timezone.utc)) + followups: Final = tuple([await poller._find_jobs(where) for _ in range(12)]) + assert any(latest_ids[0] in tuple(row.id for row in page) for page in followups) + + restarted: Final = CheckBatchCost(logging, prisma, router) + restart_page: Final = await restarted._find_jobs(where) + assert tuple(row.id for row in restart_page) == old_ids[:MAX_OBJECTS_PER_POLL_CYCLE] + finally: + write_rows('DELETE FROM "LiteLLM_ManagedObjectTable" WHERE created_by = %s', (owner,)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("steady_arrivals", [False, True]) +@pytest.mark.parametrize("backlog", [0, 1000]) +async def test_recent_batches_are_revisited_without_scanning_old_backlog(steady_arrivals: bool, backlog: int) -> None: + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + from litellm.router import Router + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + owner: Final = f"batch-repeat-{uuid4().hex}" + try: + _insert_scan_rows(owner, "old", backlog, datetime(2000, 1, 1, tzinfo=timezone.utc)) + async with connected_proxy_prisma() as prisma: + poller: Final = CheckBatchCost(ProxyLogging(UserApiKeyCache()), prisma, Router()) + where: Final = {"created_by": owner, "batch_processed": False} + await poller._find_jobs(where) + target: Final = _insert_scan_rows(owner, "target", 1, datetime.now(timezone.utc))[0] + assert target in tuple(row.id for row in await poller._find_jobs(where)) + for cycle in range(3): + if steady_arrivals: + _insert_scan_rows(owner, f"arrival-{cycle}", 1, datetime.now(timezone.utc)) + pages: Final = tuple([await poller._find_jobs(where) for _ in range(4)]) + assert any(target in tuple(row.id for row in page) for page in pages) + assert all(len(page) <= MAX_OBJECTS_PER_POLL_CYCLE for page in pages) + write_rows( + 'UPDATE "LiteLLM_ManagedObjectTable" SET batch_processed = true WHERE created_by = %s AND id != %s', + (owner, target), + ) + write_rows( + 'UPDATE "LiteLLM_ManagedObjectTable" SET created_at = %s WHERE id = %s', + (datetime(2000, 1, 1, tzinfo=timezone.utc), target), + ) + recovery: Final = tuple([await poller._find_jobs(where) for _ in range(4)]) + assert any(target in tuple(row.id for row in page) for page in recovery) + + finally: + write_rows('DELETE FROM "LiteLLM_ManagedObjectTable" WHERE created_by = %s', (owner,)) diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index e3012dfcd13..241cf90eb5d 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -11,7 +11,8 @@ import json import logging from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager -from datetime import datetime +from copy import copy +from datetime import datetime, timezone from hashlib import sha256 from queue import SimpleQueue from types import SimpleNamespace @@ -217,7 +218,7 @@ class TestCheckBatchCost: find_call = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args assert find_call[1]["take"] == MAX_OBJECTS_PER_POLL_CYCLE - assert find_call[1]["order"] == {"created_at": "asc"} + assert find_call[1]["order"] == [{"created_at": "asc"}, {"id": "asc"}] not_in = find_call[1]["where"]["status"]["not_in"] assert "stale_expired" in not_in # "complete"/"completed" are intentionally NOT excluded from the @@ -240,13 +241,13 @@ class TestCheckBatchCost: mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) # First find_many (primary query) raises with a schema error; second (fallback) returns empty mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( - side_effect=[Exception("column batch_processed does not exist"), []] + side_effect=[Exception("column batch_processed does not exist"), [], []] ) await check_batch_cost_instance.check_batch_cost() calls = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args_list - assert len(calls) == 2 + assert len(calls) == 3 fallback_where = calls[1][1]["where"] assert "batch_processed" not in fallback_where assert "stale_expired" in fallback_where["status"]["not_in"] @@ -267,8 +268,7 @@ class TestCheckBatchCost: await check_batch_cost_instance.check_batch_cost() - # Only one find_many call — the fallback directly, no primary query attempt - assert mock_prisma_client.db.litellm_managedobjecttable.find_many.call_count == 1 + assert mock_prisma_client.db.litellm_managedobjecttable.find_many.call_count == 2 fallback_where = mock_prisma_client.db.litellm_managedobjecttable.find_many.call_args[1]["where"] assert "batch_processed" not in fallback_where @@ -744,7 +744,7 @@ class TestCheckBatchCost: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( "Expected update() to be called exactly once for the completed job" ) update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] @@ -877,7 +877,7 @@ class TestCheckBatchCost: mock_update_database.assert_awaited_once() assert mock_update_database.call_args.kwargs["response_cost"] == 0.01 - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( "the job must still be marked processed once cost tracking succeeds" ) @@ -979,10 +979,10 @@ class TestCheckBatchCost: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( f"Expected update() to be called exactly once for a {terminal_status} job" ) - update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[1]["data"] + update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["status"] == terminal_status assert update_data["batch_processed"] is True, ( "terminal-status update() must set batch_processed=True so polling stops" @@ -1085,9 +1085,13 @@ class TestCheckBatchCost: assert owner.user_id == "user-1" assert owner.team_id == "team-1" - assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1 - update_call = mock_prisma_client.db.litellm_managedobjecttable.update.call_args - assert update_call.kwargs["where"] == {"id": "job-terminal-mint-1"} + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2 + update_call = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args + assert update_call.kwargs["where"] == { + "id": "job-terminal-mint-1", + "batch_processed": False, + "updated_at": mock_job.updated_at, + } update_data = update_call.kwargs["data"] assert update_data["status"] == terminal_status assert update_data["batch_processed"] is True @@ -1152,10 +1156,10 @@ class TestCheckBatchCost: ) as mock_afile_content: await check_batch_cost_instance.check_batch_cost() - assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( "a completed batch with no output file must be marked processed exactly once" ) - update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[1]["data"] + update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["status"] == completed_status assert update_data["batch_processed"] is True, ( "completed-without-output update() must set batch_processed=True so polling stops" @@ -1371,7 +1375,7 @@ class TestCheckBatchCost: f"{terminal_status} batch with an output file must fetch results and be billed" ) mock_logging_obj.async_success_handler.assert_awaited_once() - assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count >= 3 + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2 update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == terminal_status, ( @@ -1544,10 +1548,10 @@ class TestCheckBatchCost: assert mock_afile_content.await_count == 1 mock_calculate.assert_not_awaited() - assert mock_prisma_client.db.litellm_managedobjecttable.update.call_count == 1, ( + assert mock_prisma_client.db.litellm_managedobjecttable.update_many.call_count == 2, ( "a terminal batch with a 404ing output file must be retired, not retried forever" ) - update_data = mock_prisma_client.db.litellm_managedobjecttable.update.call_args[1]["data"] + update_data = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["status"] == "failed" assert update_data["batch_processed"] is True @@ -1942,7 +1946,7 @@ class TestUnmanagedVertexRouting: mock_logging_obj.async_success_handler.assert_awaited_once() assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.01 - assert prisma.db.litellm_managedobjecttable.update_many.call_count >= 3 + assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2 update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == "complete" @@ -2159,7 +2163,7 @@ class TestUnmanagedBedrockRouting: mock_logging_obj.async_success_handler.assert_awaited_once() assert mock_logging_obj.async_success_handler.call_args[1]["batch_cost"] == 0.02 - assert prisma.db.litellm_managedobjecttable.update_many.call_count >= 3 + assert prisma.db.litellm_managedobjecttable.update_many.call_count == 2 update_data = prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] assert update_data["batch_processed"] is True assert update_data["status"] == "complete" @@ -2740,9 +2744,10 @@ class TestPollPageStarvation: await self._instance(prisma, llm_router).check_batch_cost() llm_router.aretrieve_batch.assert_not_awaited() - prisma.db.litellm_managedobjecttable.update.assert_awaited_once() - call = prisma.db.litellm_managedobjecttable.update.call_args[1] - assert call["where"] == {"id": "job-no-model"} + prisma.db.litellm_managedobjecttable.update_many.assert_awaited() + call = prisma.db.litellm_managedobjecttable.update_many.call_args[1] + assert call["where"]["id"] == "job-no-model" + assert call["where"]["batch_processed"] is False assert call["data"] == {"batch_processed": True} @pytest.mark.asyncio @@ -2770,8 +2775,8 @@ class TestPollPageStarvation: await self._instance(prisma, llm_router).check_batch_cost() - prisma.db.litellm_managedobjecttable.update.assert_awaited_once() - assert prisma.db.litellm_managedobjecttable.update.call_args[1]["data"] == {"batch_processed": True} + prisma.db.litellm_managedobjecttable.update_many.assert_awaited() + assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"batch_processed": True} @pytest.mark.asyncio async def test_provider_404_with_deployment_gone_keeps_job(self): @@ -2830,24 +2835,7 @@ class TestPollPageStarvation: await instance.check_batch_cost() - assert prisma.db.litellm_managedobjecttable.update.call_args[1]["data"] == {"status": "stale_expired"} - - @pytest.mark.asyncio - async def test_stale_cleanup_gives_up_on_never_costed_completed_rows(self): - """A row already in a terminal status is never rewritten by the staleness sweep, so - it needs its own bound or it starves newer batches indefinitely.""" - prisma = self._prisma([]) - - await self._instance(prisma, MagicMock()).check_batch_cost() - - calls = prisma.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 2, "expected the staleness sweep plus the never-costed sweep" - where = calls[1][1]["where"] - assert where["file_purpose"] == "batch" - assert where["batch_processed"] is False - assert where["status"] == {"in": ["complete", "completed"]} - assert "created_at" in where - assert calls[1][1]["data"] == {"batch_processed": True} + assert prisma.db.litellm_managedobjecttable.update_many.call_args[1]["data"] == {"status": "stale_expired"} @pytest.mark.asyncio async def test_newer_batch_is_polled_once_dead_rows_are_retired(self): @@ -2885,7 +2873,11 @@ class TestPollPageStarvation: await self._instance(prisma, llm_router).check_batch_cost() - retired = [call[1]["where"]["id"] for call in prisma.db.litellm_managedobjecttable.update.call_args_list] + retired = [ + call[1]["where"]["id"] + for call in prisma.db.litellm_managedobjecttable.update_many.call_args_list + if "id" in call[1]["where"] + ] assert retired == ["job-no-model", "job-gone"] assert llm_router.aretrieve_batch.await_args_list[-1][1]["batch_id"] == "batch_live", ( "the newer healthy batch must still be polled in the same cycle" @@ -2933,7 +2925,8 @@ class _FakeManagedObjectRow: self.team_id = None self.api_key = None self.request_tags = None - self.created_at = 1700000000 + self.created_at = datetime(9999, 1, 1, tzinfo=timezone.utc) + self.updated_at = datetime(2000, 1, 1, tzinfo=timezone.utc) self.file_object = json.dumps( { "id": "batch-456", @@ -2963,7 +2956,9 @@ class _FakeManagedObjectTable: def _matches(self, where: dict) -> bool: for key, value in where.items(): if key == "created_at": - return False + if "lt" in value and not self.row.created_at < value["lt"]: + return False + continue if key == "status": if self.row.status in value.get("not_in", []): return False @@ -2976,7 +2971,9 @@ class _FakeManagedObjectTable: async def _update_many(self, *, where: dict, data: dict) -> int: if not self._matches(where): return 0 - if "batch_processed" in where: + if "updated_at" in where: + self.journal.append("finalize") + elif "batch_processed" in where: self.journal.append("claim" if data.get("batch_processed") else "release") for key, value in data.items(): setattr(self.row, key, value) @@ -3224,6 +3221,89 @@ class TestMultiPodBatchCostClaim: ["batch-test-model"] if deployment_model == "openai/batch-test-model" else ["batch-test-model"] * successes ) + @pytest.mark.asyncio + @pytest.mark.parametrize("error_source", ("batch", "output")) + async def test_a_404_for_another_identifier_preserves_recovery(self, error_source: str) -> None: + import base64 + + import httpx + import litellm + import respx + + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import SpecialEnums + + class SpendRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.costs: tuple[float, ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + if kwargs["call_type"] == "aretrieve_batch": + cost: Final = kwargs["response_cost"] + assert isinstance(cost, float) + self.costs += (cost,) + + recorder: Final = SpendRecorder() + litellm.logging_callback_manager.add_litellm_async_success_callback(recorder) + litellm.register_model( + { + "batch-recovery-model": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token_batches": 0.01, + "output_cost_per_token_batches": 0.02, + } + } + ) + row: Final = _FakeManagedObjectRow() + row.unified_object_id = ( + base64.urlsafe_b64encode( + SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format("model-123", "batch-456").encode() + ) + .decode() + .rstrip("=") + ) + router: Final = self._router() + response: Final = router.aretrieve_batch.return_value + error: Final = litellm.NotFoundError( + message=f"No object found with id '{'batch-456' if error_source == 'batch' else _CLAIM_OUTPUT_FILE_ID}0'.", + model="gpt-4", + llm_provider="openai", + ) + router.aretrieve_batch.side_effect = (error, response) if error_source == "batch" else (response, response) + prisma: Final = self._prisma(row, []) + output: Final = json.dumps( + { + "response": { + "status_code": 200, + "body": { + "model": "batch-recovery-model", + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + }, + "error": None, + } + ).encode() + with respx.mock: + route: Final = respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content") + route.mock( + side_effect=error if error_source == "output" else None, + return_value=None if error_source == "output" else httpx.Response(200, content=output), + ) + await self._instance(prisma, router).check_batch_cost() + assert row.batch_processed is False + assert recorder.costs == () + route.mock(return_value=httpx.Response(200, content=output)) + await self._instance(prisma, router).check_batch_cost() + await self._instance(prisma, router).check_batch_cost() + + assert row.batch_processed is True + assert row.status == "complete" + assert recorder.costs == (pytest.approx(10 * 0.01 + 5 * 0.02),) + @staticmethod def _claim_calls(prisma) -> list: return [ @@ -3696,12 +3776,17 @@ class TestBatchLineItemsFromThePoller: assert logger.payloads == [] @pytest.mark.asyncio - async def test_a_batch_that_could_not_be_retired_sends_no_lines_until_it_is(self, monkeypatch): + @pytest.mark.parametrize("database_error", [False, True]) + async def test_a_batch_that_could_not_be_retired_sends_no_lines_until_it_is( + self, monkeypatch: pytest.MonkeyPatch, database_error: bool + ) -> None: import litellm journal: list = [] row, instance = self._setup(journal, status="expired", output_file_id=None, error_file_id="file-err-1") - instance.prisma_client.db.litellm_managedobjecttable.update = AsyncMock(side_effect=RuntimeError("db down")) + instance.prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( + side_effect=RuntimeError("db down") if database_error else None, return_value=0 + ) logger = self._recording_logger(journal) monkeypatch.setattr(litellm, "batch_line_item_callbacks", (logger,)) @@ -4111,6 +4196,127 @@ class TestCreatorTurnedOffMessageLogging: instance.prisma_client.db.litellm_verificationtoken.find_unique.assert_not_called() +@pytest.mark.asyncio +async def test_old_completed_batch_remains_available_for_costing() -> None: + row: Final = _FakeManagedObjectRow() + row.status = "completed" + row.created_at = datetime(2000, 1, 1, tzinfo=timezone.utc) + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + + await poller._cleanup_stale_managed_objects() + + assert row.batch_processed is False + assert row.status == "completed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("progress", ["claimed", "refreshed"]) +async def test_stale_retirement_preserves_concurrent_progress(progress: str) -> None: + row: Final = _FakeManagedObjectRow() + selected: Final = copy(row) + if progress == "claimed": + row.batch_processed = True + row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc) + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + + await poller._retire_job(selected, "provider batch disappeared") + + assert row.batch_processed is (progress == "claimed") + assert row.updated_at > selected.updated_at + + +@pytest.mark.asyncio +@pytest.mark.parametrize("progress", ["claimed", "refreshed"]) +async def test_stale_terminal_response_preserves_concurrent_progress(progress: str) -> None: + from litellm.types.utils import LiteLLMBatch + + row: Final = _FakeManagedObjectRow() + selected: Final = copy(row) + if progress == "claimed": + row.batch_processed = True + row.updated_at = datetime(2000, 1, 2, tzinfo=timezone.utc) + row.file_object = '{"revision":"fresh"}' + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + response: Final = LiteLLMBatch( + id="batch-456", + completion_window="24h", + created_at=1, + endpoint="/v1/chat/completions", + input_file_id="file-input-1", + object="batch", + status="failed", + ) + + assert await poller._finalize_unbilled_terminal_job(selected, response) is False + + assert row.file_object == '{"revision":"fresh"}' + assert row.status == "in_progress" + assert row.batch_processed is (progress == "claimed") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy", [False, True]) +async def test_permanently_lost_batch_retires_unchanged_row(legacy: bool) -> None: + row: Final = _FakeManagedObjectRow() + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + poller._has_batch_processed_column = not legacy + + await poller._retire_job(row, "provider batch disappeared") + + assert row.batch_processed is (not legacy) + assert row.status == ("stale_expired" if legacy else "in_progress") + + +@pytest.mark.asyncio +async def test_failed_retirement_keeps_batch_available() -> None: + row: Final = _FakeManagedObjectRow() + prisma: Final = TestMultiPodBatchCostClaim._prisma(row, []) + prisma.db.litellm_managedobjecttable.update_many = AsyncMock(side_effect=RuntimeError("database unavailable")) + poller: Final = TestMultiPodBatchCostClaim._instance(prisma, TestMultiPodBatchCostClaim._router()) + + await poller._retire_job(row, "provider batch disappeared") + + assert row.batch_processed is False + assert row.status == "in_progress" + + +@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl")) +@pytest.mark.parametrize("other_id", ("{id}4", "prefix-{id}", "{id}/child", "{id}.backup", "{id}?generation=2")) +def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other_id: str) -> None: + from litellm.exceptions import NotFoundError + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + error: Final = NotFoundError( + message=f"No object found with id '{other_id.format(id=resource_id)}'.", model="test", llm_provider="openai" + ) + assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is False + assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is False + + +@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl")) +@pytest.mark.parametrize("message", ("No object found with id '{id}'.", 'File "{id}" does not exist', "{id} not found")) +def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, message: str) -> None: + from litellm.exceptions import NotFoundError + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + error: Final = NotFoundError(message=message.format(id=resource_id), model="test", llm_provider="openai") + assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is True + assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True + + +@pytest.mark.parametrize("output_file_id", (None, "")) +def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None: + from litellm.exceptions import NotFoundError + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + + error: Final = NotFoundError(message="Output file not found", model="test", llm_provider="openai") + assert CheckBatchCost._is_output_file_gone_at_provider(error, output_file_id) is False + + @pytest.mark.asyncio @pytest.mark.parametrize( "kind", ["sync", "async", "custom", "overlapping", "general", "mixed", "sync_only", "executor_shutdown"] From 047cb4a8e2b2c0853baaf9b6a3f082d851d58a9f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:29:30 +0000 Subject: [PATCH 107/124] test(integration): expect the Exa search envelope's provider_metadata after #47423 (#47572) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/authorization/test_search_tool_allowlist.py | 5 ++++- tests/integration/management/test_search_tool_crud.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/authorization/test_search_tool_allowlist.py b/tests/integration/authorization/test_search_tool_allowlist.py index 84f2b7ebaf2..3782f31991b 100644 --- a/tests/integration/authorization/test_search_tool_allowlist.py +++ b/tests/integration/authorization/test_search_tool_allowlist.py @@ -92,7 +92,10 @@ def test_search_tool_allowlist_refuses_other_tools_before_any_upstream_call( assert (refused.status_code, refused.json()) == (403, refusal), refused.text assert wire.drain() == () served: Final = gateway.request("POST", f"/v1/search/{allowed}", _SEARCH_BODY, key=key) - assert (served.status_code, served.json()) == (200, {"object": "search", "results": []}), served.text + assert (served.status_code, served.json()) == ( + 200, + {"object": "search", "results": [], "provider_metadata": {}}, + ), served.text assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/allowed/search")] diff --git a/tests/integration/management/test_search_tool_crud.py b/tests/integration/management/test_search_tool_crud.py index f46da964e14..e97f2727595 100644 --- a/tests/integration/management/test_search_tool_crud.py +++ b/tests/integration/management/test_search_tool_crud.py @@ -113,6 +113,7 @@ def test_search_tool_create_update_delete_is_read_back_and_applied_without_resta "results": [ {"title": target, "url": "https://crud.invalid", "snippet": key, "date": None, "last_updated": None} ], + "provider_metadata": {}, }, response.text read_back(original) From 7a4a255d1d33cb6145546c65b69b444a5f136da2 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:08:11 -0700 Subject: [PATCH 108/124] feat(proxy): lift every earlier billed timed-out retry attempt onto the failed request (#47490) * feat(proxy): lift every earlier billed timed-out retry attempt onto the failed request _billed_earlier_attempts reads the router's previous_models records, keeps the Timeout ones other than the final attempt (matched on deployment_id plus attempted_retries, since the count restarts after a fallback hop and the fallback handler re-logs the exhausting attempt) and prices each at the deployment that served it. post_call_failure_hook lifts the tuple as billed_failed_attempts; nothing writes it yet. previous_models joins the router-reserved metadata fields so a client-planted record never reaches it * fix(proxy): price earlier timed-out attempts from the prompt estimate only The final row may lift a mid-stream recovered usage (prompt plus delivered chunks) or a failed_attempts_usage sum; earlier attempts never produced output, so they are priced from _estimated_prompt_usage, and when failed_attempts_usage already bills them into the final row none is lifted again --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/proxy/utils.py | 153 ++++++++++++++++-- .../unit/proxy/test_litellm_pre_call_utils.py | 6 +- ..._utils_model_creation_and_error_logging.py | 137 ++++++++++++++++ 4 files changed, 282 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index c0222dbcd0b..efc8575feb3 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -467,6 +467,7 @@ _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset( { "attempted_fallbacks", "original_model_group", + "previous_models", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a8bd35644c9..735df38efe6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1031,9 +1031,7 @@ def _failure_usage_to_lift( at the rate the router registered under ``deployment_id`` (``Logging.get_router_model_id``) when the deployment declares one, else the cost map's rate for ``base_model`` (an Azure deployment's ``model_info.base_model``) or the deployment model; any other post-dispatch - failure stays at zero cost. The raw request body backfills - the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a - chat-completions provider) without filling optional_params. Returns the + failure stays at zero cost. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" recovered_usage: Final = model_call_details.get("combined_usage_object") failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) @@ -1047,6 +1045,22 @@ def _failure_usage_to_lift( # The proxy-only error handler stamps this flag on a client disconnect too, even though the # request had already reached the provider, so a billed failure keeps its estimate. return None + estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body) + if estimated_usage is None: + return None + if not provider_billed: + return estimated_usage, 0.0 + return estimated_usage, _estimated_input_cost( + model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model + ) + + +def _estimated_prompt_usage( + model_call_details: Mapping[str, object], request_body: Mapping[str, object] +) -> Usage | None: + """The input-only Usage estimate for the prompt this request sent, or None when the call type has no + estimable input. The raw request body backfills the system prompt when the SDK bridges an endpoint + (e.g. /v1/messages on a chat-completions provider) without filling optional_params.""" if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES: return None optional_params: Final = model_call_details.get("optional_params") @@ -1056,17 +1070,118 @@ def _failure_usage_to_lift( else None ) system_input: Final = dispatched_system or request_body.get("system") or request_body.get("instructions") - estimated_usage: Final = _estimate_dispatched_failure_usage( + return _estimate_dispatched_failure_usage( model=str(model_call_details.get("model") or ""), request_input=model_call_details.get("messages"), system_input=system_input, ) - if estimated_usage is None: - return None - if not provider_billed: - return estimated_usage, 0.0 - return estimated_usage, _estimated_input_cost( - model_call_details, estimated_usage.prompt_tokens, deployment_id=deployment_id, base_model=base_model + + +@dataclass(frozen=True) +class BilledFailedAttempt: + """An earlier retry attempt the provider also billed, written as its own spend row.""" + + attempted_retries: int + deployment_id: str | None + usage: Usage + response_cost: float + + +_NO_RETRY_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _retry_metadata_bucket(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]: + """The request metadata bucket the router writes retry state into, ``litellm_metadata`` when the + route carries one (/messages, /responses) and ``metadata`` otherwise, same order as the deployment id.""" + if litellm_params is None: + return _NO_RETRY_METADATA + for metadata_key in ("litellm_metadata", "metadata"): + metadata = litellm_params.get(metadata_key) + if isinstance(metadata, dict): + return cast("Mapping[str, object]", metadata) # cast-ok: untyped router dict, keys read with .get + return _NO_RETRY_METADATA + + +def _attempt_number(record: Mapping[str, object], index: int) -> int: + """The attempt a retry record describes: its own ``attempted_retries`` count, else its position.""" + recorded: Final = record.get("attempted_retries") + return recorded if isinstance(recorded, int) else index + + +def _retry_attempt_records(litellm_params: Mapping[str, object] | None) -> tuple[Mapping[str, object], ...]: + """The router's ``RetryAttemptRecord`` breadcrumbs, one per failed attempt, read from the same bucket.""" + if litellm_params is None: + return () + for metadata_key in ("litellm_metadata", "metadata"): + metadata = litellm_params.get(metadata_key) + if not isinstance(metadata, dict): + continue + records = cast("Mapping[str, object]", metadata).get("previous_models") # cast-ok: untyped router dict + if isinstance(records, (list, tuple)): + return tuple( + cast("Mapping[str, object]", record) # cast-ok: RetryAttemptRecord TypedDict, read with .get + for record in cast("Sequence[object]", records) # cast-ok: untyped router tuple + if isinstance(record, dict) + ) + return () + + +def _record_deployment_id(record: Mapping[str, object]) -> str | None: + deployment_id: Final = record.get("deployment_id") + return deployment_id if isinstance(deployment_id, str) else None + + +def _attempt_identity(record: Mapping[str, object]) -> tuple[str | None, int | None]: + """What tells one provider call from another across the request: the deployment that served it + and the retry count within its model group. Two records with the same identity describe the + same call (the fallback handler re-logs the attempt that exhausted the previous group).""" + attempted_retries: Final = record.get("attempted_retries") + return _record_deployment_id(record), attempted_retries if isinstance(attempted_retries, int) else None + + +def _timed_out_earlier_records( + litellm_params: Mapping[str, object] | None, final_identity: tuple[str | None, int | None] +) -> tuple[Mapping[str, object], ...]: + """The ``Timeout`` breadcrumbs for calls other than the final attempt, one per provider call.""" + by_identity: Final = { + _attempt_identity(record): record + for record in _retry_attempt_records(litellm_params) + if record.get("exception_type") == litellm.Timeout.__name__ + } + return tuple(record for identity, record in by_identity.items() if identity != final_identity) + + +def _billed_earlier_attempts( + model_call_details: Mapping[str, object], + litellm_params: Mapping[str, object] | None, + estimated_usage: Usage, + base_model: str | None, + deployment_id: str | None, +) -> tuple[BilledFailedAttempt, ...]: + """Every earlier retry attempt the provider also billed. The router records one + ``RetryAttemptRecord`` per failed attempt in ``metadata.previous_models``, the final attempt + included, which the failure row itself bills. The final attempt is the record with the + request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback + hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a + ``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final + attempt, at that deployment's input rate (else the cost map's rate for ``base_model`` or the + model). Only the exception class survives on the record, so an earlier connect timeout is + billed too; any other failure class is not.""" + final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries") + final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None) + return tuple( + BilledFailedAttempt( + attempted_retries=_attempt_number(record, index), + deployment_id=_record_deployment_id(record), + usage=estimated_usage, + response_cost=_estimated_input_cost( + model_call_details, + estimated_usage.prompt_tokens, + deployment_id=_record_deployment_id(record), + base_model=base_model, + ), + ) + for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity)) ) @@ -1265,13 +1380,26 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: return _EMPTY_LIFT _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) _first_handoff: Final = _model_call_details.get("first_api_call_start_time") + _litellm_params: Final = getattr(_logging_obj, "litellm_params", None) + _base_model: Final = get_base_model_from_metadata(model_call_details=_model_call_details) + _deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None _usage_to_lift: Final = _failure_usage_to_lift( model_call_details=_model_call_details, request_body=request_data, dispatched=_first_handoff is not None, provider_billed=provider_billed, - deployment_id=_logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None, - base_model=get_base_model_from_metadata(model_call_details=_model_call_details), + deployment_id=_deployment_id, + base_model=_base_model, + ) + _prompt_estimate: Final = ( + _estimated_prompt_usage(_model_call_details, request_data) + if provider_billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage") + else None + ) + _earlier_attempts: Final = ( + _billed_earlier_attempts(_model_call_details, _litellm_params, _prompt_estimate, _base_model, _deployment_id) + if _prompt_estimate is not None + else () ) _entries: Final = ( ("start_time", _model_call_details.get("start_time")), @@ -1280,6 +1408,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: ("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]), ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)), ("standard_logging_object", _model_call_details.get("standard_logging_object")), + ("billed_failed_attempts", _earlier_attempts or None), ) return MappingProxyType({key: value for key, value in _entries if value is not None}) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 4a97106b3ac..246496ec48f 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -3330,9 +3330,7 @@ def test_x_litellm_trace_id_keeps_the_body_session_beside_it_whatever_the_log_se assert data["litellm_trace_id"] == "req-trace-1" -@pytest.mark.parametrize( - "session_header", ["x-litellm-session-id", "x-claude-code-session-id", "x-session-id"] -) +@pytest.mark.parametrize("session_header", ["x-litellm-session-id", "x-claude-code-session-id", "x-session-id"]) @pytest.mark.parametrize("with_trace_header", [False, True], ids=["session-header-only", "beside-trace-header"]) def test_session_headers_still_beat_the_body_session_with_correlation_logging( monkeypatch: pytest.MonkeyPatch, session_header: str, with_trace_header: bool @@ -8282,6 +8280,7 @@ _PLANTED_STAMPS = { "attempted_fallbacks": 99, "original_model_group": "spoofed-group", "request_retry_count": -100, + "previous_models": [{"exception_type": "Timeout", "deployment_id": "spoofed", "attempted_retries": 101}], "_client_output_ceiling": {"api_base": "https://attacker.example"}, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: 10**9, "client_key": "client_value", @@ -8316,6 +8315,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo assert "original_model_group" not in updated["metadata"] assert "_client_output_ceiling" not in updated["metadata"] assert "request_retry_count" not in updated["metadata"] + assert "previous_models" not in updated["metadata"] assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in updated["metadata"] assert updated["metadata"]["client_key"] == "client_value" diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 77d17741a18..5ab04724bc4 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2135,6 +2135,143 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) +def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(monkeypatch): + """The router leaves one RetryAttemptRecord per failed attempt in metadata.previous_models, the + final attempt included; the earlier Timeout ones are billed (each at the deployment that served + it), a 429 attempt is not, and the final attempt is left to the failure row itself.""" + from litellm.proxy.utils import _billed_earlier_attempts + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit145-b", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.25, "output_cost_per_token": 1.0}, + ) + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + public_rate, _ = litellm.cost_per_token(model=FAILURE_USAGE_MODEL, prompt_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 3, + "previous_models": ( + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-b", "exception_type": "RateLimitError", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 2}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 3}, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-a", + ) + + assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [(0, "dep-lit145-a"), (2, "dep-lit145-b")] + assert attempts[0].response_cost == pytest.approx(public_rate) + assert attempts[0].response_cost > 0 + assert attempts[1].response_cost == pytest.approx(24 * 0.25) + assert all(a.usage is usage for a in attempts) + + +def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_streams_recovered_usage(): + """A final attempt whose stream broke mid-flight lifts its recovered usage (prompt plus the chunks + delivered); the earlier timed-out attempts never produced output, so they are priced from the + input-only estimate and not from that recovered usage.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _estimated_prompt_usage, _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params( + metadata={ + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + {"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0}, + ), + } + ), + ) + recovered = litellm.Usage(prompt_tokens=500, completion_tokens=40, total_tokens=540) + logging_obj.model_call_details["combined_usage_object"] = recovered + logging_obj.model_call_details["response_cost"] = 0.01 + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + estimate = _estimated_prompt_usage(logging_obj.model_call_details, {}) + assert lifted["combined_usage_object"] is recovered + assert [(a.usage.prompt_tokens, a.usage.completion_tokens) for a in lifted["billed_failed_attempts"]] == [ + (estimate.prompt_tokens, 0) + ] + assert estimate.prompt_tokens < recovered.prompt_tokens + + +def test_attempts_already_summed_into_the_final_row_are_not_lifted_again(): + """``Logging.carry_failed_attempt_usage`` moves a cut stream's usage into ``failed_attempts_usage`` + and the final row already bills every entry of it, so no earlier attempt gets a row of its own.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params( + metadata={ + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + {"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0}, + ), + } + ), + ) + logging_obj.model_call_details["failed_attempts_usage"] = ( + (litellm.Usage(prompt_tokens=24, completion_tokens=3, total_tokens=27), 0.001), + ) + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + assert lifted["combined_usage_object"].prompt_tokens == 24 + assert "billed_failed_attempts" not in lifted + + +def test_a_timed_out_attempt_from_the_group_a_fallback_left_is_still_billed(): + """After a fallback hop the router restarts ``attempted_retries`` at 0 for the new model group and + re-logs the attempt that exhausted the previous group, so the final attempt (dep-b, retry 1) must be + told from the previous group's attempt with the same count (dep-a, retry 1), and the re-logged + record must bill once.""" + from litellm.proxy.utils import _billed_earlier_attempts + + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-b"}, + "previous_models": ( + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-a", "exception_type": "Timeout", "attempted_retries": 1}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 0}, + {"deployment_id": "dep-lit145-b", "exception_type": "Timeout", "attempted_retries": 1}, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-b", + ) + + assert [(a.attempted_retries, a.deployment_id) for a in attempts] == [ + (0, "dep-lit145-a"), + (1, "dep-lit145-a"), + (0, "dep-lit145-b"), + ] + assert all(a.response_cost > 0 for a in attempts) + + def _dispatched_failure_logging_obj(model: str, litellm_params: dict) -> object: """A Logging object as the failure hook sees it: litellm_params in the real get_litellm_params() shape (every key present, ``litellm_metadata`` None on chat completions) and a first-handoff From ccd0d4ea493d1c5f9c2f6574261002407687663d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:08:18 -0700 Subject: [PATCH 109/124] fix(proxy): requeue spend logs when a Postgres restart tears down the prisma engine mid-write (#47571) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 6 ++- .../test_proxy_update_spend.py | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 735df38efe6..d6dfd3ea359 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8504,7 +8504,11 @@ MAX_SPEND_LOG_ISOLATION_FAILURES_PER_BATCH: Final = 256 def _is_transient_spend_log_write_error(e: Exception) -> bool: - return PrismaDBExceptionHandler.is_database_transport_error(e) or PrismaDBExceptionHandler.is_deadlock_error(e) + return ( + PrismaDBExceptionHandler.is_database_transport_error(e) + or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) + or PrismaDBExceptionHandler.is_deadlock_error(e) + ) async def _create_spend_logs_with_poison_isolation( diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/unit/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index fc3deb0de18..e3b7bb5b49f 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -952,3 +952,53 @@ async def test_update_spend_logs_requeues_batch_when_postgres_is_out_of_connecti assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1 sleep.assert_not_awaited() assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r0", "r1", "r2", "r3", "e"] + + +async def _create_many_while_postgres_is_restarting(**_: object) -> int: + from prisma.engine.utils import handle_response_errors + + handle_response_errors( + MagicMock(), + [ + { + "error": "Error in connector: Can't reach database server at `127.0.0.1:5432`", + "user_facing_error": {"error_code": "P1001", "message": "Can't reach database server", "meta": None}, + } + ], + ) + return 0 + + +@pytest.mark.asyncio +async def test_update_spend_logs_retries_and_requeues_batch_when_postgres_restart_tears_down_prisma_engine( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """While Postgres restarts the prisma engine answers the write with a P1001 + payload whose ``meta`` is ``None``, which prisma's own response parser trips + over before it can build a typed error. That outage must be retried like + any other, and once retries exhaust the dequeued batch goes back to the head + of the queue so the rows land when Postgres is back. + """ + + async def _fake_sleep(_: float) -> None: + return None + + monkeypatch.setattr(utils_mod.asyncio, "sleep", _fake_sleep) + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_create_many_while_postgres_is_restarting + ) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="c")] + + with pytest.raises(AttributeError): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=2, + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + logs_to_process=[make_spend_log_row(request_id="a"), make_spend_log_row(request_id="b")], + ) + + assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["a", "b", "c"] + assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 3 From 97e5438ffba9a8c4e752686fd4fb040180635572 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:08:29 -0700 Subject: [PATCH 110/124] fix(proxy): fall back to the deployment model when a billed failure row's base_model is not in the cost map (#47568) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 49 ++++++++++++------- ..._utils_model_creation_and_error_logging.py | 30 +++++++++++- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d6dfd3ea359..b01f53c8c6b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1200,6 +1200,25 @@ def _deployment_cost_per_token(deployment_id: str | None, model: str) -> CostPer ) +def _priced_input_cost( + pricing_model: str, + custom_llm_provider: str | None, + prompt_tokens: int, + custom_cost_per_token: CostPerToken | None, +) -> float | None: + try: + estimated_input_cost, _ = litellm.cost_per_token( + model=pricing_model, + custom_llm_provider=custom_llm_provider, + prompt_tokens=prompt_tokens, + custom_cost_per_token=custom_cost_per_token, + ) + except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written + verbose_proxy_logger.debug("failure spend estimate: could not price %s: %s", pricing_model, exc) + return None + return estimated_input_cost + + def _estimated_input_cost( model_call_details: Mapping[str, object], prompt_tokens: int, @@ -1207,23 +1226,19 @@ def _estimated_input_cost( base_model: str | None, ) -> float: """Input price for the estimated prompt: the deployment's registered rate when it has one, else - the cost map's rate for ``base_model`` or the deployment model (an Azure deployment name is not - in the cost map, its ``base_model`` is). An unpriceable model keeps the row at $0 rather than - dropping the failure row.""" - try: - model: Final = str(model_call_details.get("model") or "") - estimated_input_cost, _ = litellm.cost_per_token( - model=base_model or model, - custom_llm_provider=str(model_call_details.get("custom_llm_provider") or "") or None, - prompt_tokens=prompt_tokens, - custom_cost_per_token=_deployment_cost_per_token(deployment_id, model), - ) - except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written - verbose_proxy_logger.debug( - "failure spend estimate: could not price %s: %s", model_call_details.get("model"), exc - ) - return 0.0 - return estimated_input_cost + the cost map's rate for ``base_model``, else the deployment model, the same candidate order + ``completion_cost`` walks (an Azure deployment name is not in the cost map, its ``base_model`` + is; a ``base_model`` the cost map does not know falls through to the deployment model). A model + no candidate can price keeps the row at $0 rather than dropping the failure row.""" + model: Final = str(model_call_details.get("model") or "") + custom_llm_provider: Final = str(model_call_details.get("custom_llm_provider") or "") or None + custom_cost_per_token: Final = _deployment_cost_per_token(deployment_id, model) + pricing_models: Final = (base_model, model) if base_model is not None else (model,) + candidate_costs: Final = ( + _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens, custom_cost_per_token) + for pricing_model in pricing_models + ) + return next((cost for cost in candidate_costs if cost is not None), 0.0) _EMPTY_LIFT: Final = MappingProxyType({}) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 5ab04724bc4..99cd569eb63 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -1174,7 +1174,7 @@ from typing import cast import litellm from litellm.proxy.utils import create_model_info_response from litellm.types.router import DeploymentModelListingInfo -from litellm.types.utils import ModelInfo +from litellm.types.utils import ModelInfo, ModelResponse, Usage def _fake_model_info(**fields: object) -> ModelInfo: @@ -2337,6 +2337,34 @@ def test_an_azure_deployment_is_priced_at_its_base_model_like_the_success_path() assert lifted["response_cost"] == pytest.approx(expected_cost) +def test_an_unmapped_base_model_falls_back_to_the_deployment_model_like_the_success_path(): + """``completion_cost`` tries ``base_model`` first and then the deployment model, so a + ``model_info.base_model`` the cost map does not know still prices a success row at the + deployment model's rate. The failure row must land on the same number instead of $0.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + logging_obj = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params(metadata={"model_info": {"base_model": "not-in-the-cost-map"}}), + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + + lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) + + prompt_tokens = lifted["combined_usage_object"].prompt_tokens + success_cost = litellm.completion_cost( + completion_response=ModelResponse( + model=FAILURE_USAGE_MODEL, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=0) + ), + model=FAILURE_USAGE_MODEL, + custom_llm_provider="openai", + base_model="not-in-the-cost-map", + ) + assert success_cost > 0 + assert lifted["response_cost"] == pytest.approx(success_cost) + + def test_a_litellm_params_key_in_the_request_body_cannot_zero_the_timeout_price(): """``model_call_details`` is flattened with the request's optional_params, so a body key named ``litellm_params`` can land there; the deployment rate must come from the logging From 77e08731eba09b57182283e8bc015c74a449c55b Mon Sep 17 00:00:00 2001 From: Muspi Merol Date: Wed, 21 Oct 2026 00:18:13 +0800 Subject: [PATCH 111/124] test: isolate model cost maps in runtime registration tests (#47559) * test: isolate model cost maps in runtime registration tests * test: restore cost map entries and provider model lists after unit tests * test: restore provider model sets after unit tests and cover the catalog restore * test: snapshot litellm globals before scanning provider model sets --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/unit/conftest.py | 24 ++++++++++++++++++++++++ tests/unit/test_conftest_isolation.py | 20 ++++++++++++++++++++ tests/unit/test_utils.py | 7 ++----- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9028b8461b0..c96e3e2c031 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -307,6 +307,30 @@ def isolate_router_model_cost_state() -> Iterator[None]: litellm.get_model_info.cache_clear() +def _provider_model_sets() -> tuple[set[str], ...]: + module_globals: Final = tuple(vars(litellm).items()) + return tuple(models for name, models in module_globals if name.endswith("_models") and isinstance(models, set)) + + +@pytest.fixture(autouse=True) +def isolate_model_catalog() -> Iterator[None]: + model_cost: Final = litellm.model_cost + model_cost_entries: Final = dict(model_cost) + models_by_provider: Final = litellm.models_by_provider + models_by_provider_entries: Final = dict(models_by_provider) + provider_model_sets: Final = tuple((models, frozenset(models)) for models in _provider_model_sets()) + yield + model_cost.clear() + model_cost.update(model_cost_entries) + models_by_provider.clear() + models_by_provider.update(models_by_provider_entries) + for models, entries in provider_model_sets: + models.clear() + models.update(entries) + litellm_utils_module._invalidate_model_cost_lowercase_map() + litellm.get_model_info.cache_clear() + + @pytest.fixture def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") diff --git a/tests/unit/test_conftest_isolation.py b/tests/unit/test_conftest_isolation.py index 63db0eb5bba..92457119a82 100644 --- a/tests/unit/test_conftest_isolation.py +++ b/tests/unit/test_conftest_isolation.py @@ -88,6 +88,26 @@ def test_register_model_ledger_entry_was_rolled_back(): assert CANARY_MODEL not in litellm_utils_module._runtime_registered_model_cost +def test_register_model_catalog_writes_are_scoped_to_this_test(): + litellm.register_model( + { + CANARY_MODEL: {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001}, + "collection-isolation-ledger": {"input_cost_per_token": 0.5}, + } + ) + assert CANARY_MODEL in litellm.model_cost + assert CANARY_MODEL in litellm.open_ai_chat_completion_models + assert CANARY_MODEL in litellm.models_by_provider["openai"] + assert litellm.model_cost["collection-isolation-ledger"]["input_cost_per_token"] == 0.5 + + +def test_register_model_catalog_writes_were_rolled_back(): + assert CANARY_MODEL not in litellm.model_cost + assert CANARY_MODEL not in litellm.open_ai_chat_completion_models + assert CANARY_MODEL not in litellm.models_by_provider["openai"] + assert litellm.model_cost["collection-isolation-ledger"]["input_cost_per_token"] == 0.001 + + def test_live_router_membership_is_scoped_to_this_test(): _CanaryRouterHolder.router = Router( model_list=[ diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0be3ae05e97..40eb5c31fda 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2358,7 +2358,6 @@ def test_register_model_with_scientific_notation(): _invalidate_model_cost_lowercase_map() -@pytest.mark.usefixtures("isolated_openai_model_sets") @pytest.mark.usefixtures("isolated_openai_model_sets") @respx.mock def test_register_model_url_fetch_uses_single_attempt(monkeypatch): @@ -4803,8 +4802,7 @@ async def test_builtin_string_callback_registers_when_subclass_already_active( assert any(type(cb) is S3Logger for cb in litellm._async_success_callback) -@pytest.mark.usefixtures("isolated_openai_model_sets") -@pytest.mark.usefixtures("isolated_openai_model_sets") +@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") @pytest.mark.parametrize("rate", (0.0, 0.000123)) @pytest.mark.parametrize("updated_rate", (None, 0.0, 0.000456)) def test_reapply_runtime_registrations_replays_register_model_overrides( @@ -4871,8 +4869,7 @@ def test_reapply_runtime_registrations_replays_register_model_overrides( _invalidate_model_cost_lowercase_map() -@pytest.mark.usefixtures("isolated_openai_model_sets") -@pytest.mark.usefixtures("isolated_openai_model_sets") +@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkeypatch): """ Per-request custom pricing describes one call, so it must not be re-asserted From dd22dc9132ba2d32cb809fbcb482c2b76b74d352 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:18:44 -0700 Subject: [PATCH 112/124] test(perf): measure Admin UI journey latency in a browser on the 1TB SpendLogs restore (#47486) * test(perf): measure Admin UI journey latency in a browser on the 1TB restore * test(perf): settle on rendered rows, chart bars and the usage view trigger * test(perf): cover the UI journey summaries in the e2e harness tests * test(perf): fail UI journeys on failed samples and p50/p90 ceilings * test(perf): add Usage, Models and Keys journeys and seed realistic keys, teams, users and models on the disposable restore --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/UI_QUERIES.md | 111 ++--- tests/e2e/perf/setup/ui_journeys.py | 397 ++++++++++++++++++ tests/e2e/perf/setup/ui_queries.py | 227 ---------- tests/e2e/perf/setup/ui_query_cases.py | 206 --------- tests/e2e/perf/setup/ui_query_fixture.py | 72 +++- tests/e2e/perf/setup/ui_query_isolation.py | 4 +- tests/e2e/perf/test_ui_journeys.py | 108 +++++ tests/e2e/perf/test_ui_queries.py | 159 ------- .../perf/setup/test_ui_journeys.py | 144 +++++++ .../e2e_harness/perf/setup/test_ui_queries.py | 257 ------------ tests/e2e_harness/perf/test_upgrade.py | 4 +- 11 files changed, 761 insertions(+), 928 deletions(-) create mode 100644 tests/e2e/perf/setup/ui_journeys.py delete mode 100644 tests/e2e/perf/setup/ui_queries.py delete mode 100644 tests/e2e/perf/setup/ui_query_cases.py create mode 100644 tests/e2e/perf/test_ui_journeys.py delete mode 100644 tests/e2e/perf/test_ui_queries.py create mode 100644 tests/e2e_harness/perf/setup/test_ui_journeys.py delete mode 100644 tests/e2e_harness/perf/setup/test_ui_queries.py diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md index 552f1b953fd..27780537042 100644 --- a/tests/e2e/perf/UI_QUERIES.md +++ b/tests/e2e/perf/UI_QUERIES.md @@ -1,92 +1,55 @@ -# SpendLog UI query performance +# Admin UI journey latency on the 1TB SpendLogs restore -Run `test_ui_queries.py` against a dedicated proxy and a disposable database restored from the metadata-only performance seed. These tests do not call an LLM or measure browser rendering +Run `test_ui_journeys.py` against a dedicated proxy, its Admin UI and a disposable database restored from the metadata-only performance seed. A headless Chromium logs into the dashboard and performs the journeys below, timing each one from the user's click or navigation until the data is on screen. No LLM is called. The test does not look at which routes, SQL or tables served the page, so it stays valid whether the backend gets faster queries or new aggregate tables -The suite measures 42 request shapes across 20 routes used by current dashboard screens. It replays direct HTTP requests against the existing bulk seed, with the seed anchor replacing the wall clock and a UTC browser timezone. It does not run a browser or claim to cover every dashboard request +## Journeys -## UI callers +| name | what the user does | timer stops when | +| --- | --- | --- | +| `logs_open_24h` | opens Logs (default Last 24 Hours) | request rows are rendered and the refresh control is idle | +| `logs_last_7_days` | switches the time range to Last 7 Days | same | +| `logs_next_page` | clicks the next page control | same | +| `logs_filter_key_hash` | filters by the seeded key hash and applies | same | +| `logs_filter_key_alias` | picks the first key alias in the filter drawer and applies | same | +| `logs_filter_team` | picks the first team in the filter drawer and applies | same | +| `logs_filter_model` | picks the first model in the filter drawer and applies | same | +| `usage_open` | opens Usage | summary tiles show dollar values with no skeletons left | +| `usage_last_30_days` | picks Last 30 days in the Usage date picker | same | +| `usage_team_view` | switches the Usage view to Team Usage | same | +| `usage_customer_view` | switches the Usage view to Customer Usage | same | +| `usage_model_activity_30_days` | on the Model Activity tab, picks Last 30 days | same | +| `cache_open` | opens Caching | the Cache Hits tile is visible and the Cache Hits vs API Requests chart has bars | +| `models_open` | opens Models + Endpoints | model rows are rendered and the loading state is gone | +| `keys_open` | opens Virtual Keys | key rows are rendered and the loading state is gone | -All paths below are relative to `ui/litellm-dashboard/src/`. Each case requires an authenticated admin session with access to that screen, and selected resources must exist in the fixture +The mix follows what customers report as slow: the Logs page on `LiteLLM_SpendLogs`, the Usage page on the `LiteLLM_Daily*Spend` rollups (30 day ranges, Customer and Model views), and the Models and Keys pages. -| Cases | UI action and source | -| --- | --- | -| `logs_*` | Logs > Request Logs, change date range, filter, sort or next page. `components/logs/request/useLogFilterLogic.ts` and `RequestLogsFilters.tsx` | -| `usage_*` | Usage overview or entity tab, open key activity, search keys, expand a model group. `app/(dashboard)/usage/_components/components/UsagePageView.tsx`, `EntityUsage/EntityUsage.tsx`, `EntityUsage/entityFetchFns.ts`, `components/UsagePage/components/KeyActivityPanel.tsx` | -| `team_users_*` | Team Usage. `EntityUsage/EntityUsage.tsx` passes selected teams or all available non-dashboard teams to `TeamUserSpendCard.tsx` | -| `cache_activity_*` | Caching > Cache Analytics, optionally filter aliases/models. `app/(dashboard)/hooks/caching/useCacheActivity.ts` | -| `cache_leakage_keys` | Cost Optimization, switch cache leakage to Keys. `app/(dashboard)/cost-optimization/_components/useCacheLeakageKeys.ts` | +## Seeded rows -Request serialization lives in `components/networking.tsx`. Logs use page size 25 and session grouping, including the next-session cursor returned by page one. Individual Logs filters use a 30-day window to exercise larger matching datasets. Their case names end in `_30d` so the measurements do not share history with the old 24-hour filters. The synthetic cursorless page-three request is excluded +The snapshot carries the SpendLogs rows and their Daily* rollups. `setup/ui_query_fixture.py` checks that `LiteLLM_DailyUserSpend` and `LiteLLM_DailyTeamSpend` have rows in the week before the anchor, then fills the tables the snapshot does not have so the Keys, Models and filter journeys see realistic counts: 100 teams (`perf-team-N`), 2,000 users (`perf-user-N`), 5,000 keys whose token hashes and `perf-key-N` aliases match the seeded log rows, and 200 DB models pointed at the perf mock provider. Every insert is `ON CONFLICT DO NOTHING`, so reruns on the same restore are no-ops. Nothing here runs outside the disposable restore -Usage list, search and model drilldown requests do not inject an `api_key` filter. Team activity excludes `litellm-dashboard`. Cache leakage supplies `include_current_utc_day=true`. Dates and filter values are deterministic fixture values, not captured production traffic +Each journey is measured once cold, then `E2E_UI_JOURNEY_REPEATS` more times (default 9) from the same starting state, inside one logged-in tab whose clock is pinned to the seed anchor so relative windows land on seeded rows. The browser timezone is UTC to match the seed -Old Usage is excluded because its expensive-query gate normally hides those calls at this database size. LiteAdmin's agent-selected reports are excluded from this deterministic dashboard set. Missing-token alias recovery is backend functional coverage, not a separate UI action, and its artificial requests and orphan fixtures are removed +## Safety -This mapping is source-verified, not browser-recorded. Recheck the named callers when changing cases. Neither browser capture nor an automatic UI-drift checker is installed by this suite +The seeded source is never the target. `E2E_UI_PERF_DISPOSABLE=1`, `E2E_RESET_SPEND_LOGS=0` and a hostname matching `db-perf-perf-.litellm.svc` are all required, and in-flight SQL is cancelled only on that disposable restore between journeys (see `setup/ui_query_isolation.py`) -## Dataset scope +## Running -Logs, team-user spend and cache activity query the bulk SpendLogs data. User, team, tag and customer usage, plus cache leakage, query daily tables aggregated from that dataset. These are sequential large-dataset queries, not concurrent traffic or saturation tests - -A read-only restore of `db-perf-v1-105-1tb` gave exact request totals from daily aggregates: 9,442,606 across 24 hours, 66,077,435 across seven days and 283,189,684 across 30 days. The retained 30-day key filter matches 1,134,559 requests, user filter 11,316,454, team filter 113,298,119 and model filter 94,395,473. These describe matching requests, not rows scanned or returned by the API, and are documentation rather than performance assertions - -PostgreSQL table-wide estimates (`pg_class.reltuples`) are approximately 2.70 million daily user rows, 2.70 million daily team rows, 5.40 million daily tag rows and 78.27 million daily end-user rows. These are not exact counts within each Usage case's seven-day window - -The individual end-user filter is excluded because it matched only 104 requests in 24 hours and 8,616 across all dates. User/end-user dropdown requests are also excluded because their queries cap the input at 10,000 matching SpendLogs before extracting distinct values. Neither is a broad matching-data workload in this seed - -Excluded because they only had tiny matching fixtures or individual-record lookups: organization/agent usage, guardrail/tool logs, request-ID lookup, payload/session drawers, session/search filters for `perf-ui-*`, the OAuth-token filter, and all prompt-caching request cases. The seed has response-cache hits but no prompt-cache usage or injection metadata, so the prompt-caching cases only matched the 320 added rows. No UI-specific rows are inserted anymore - -| Remaining cases | Count | Variations | -| --- | ---: | --- | -| Logs | 21 | 24h/7d/30d, page 2 with session cursor, 12 individual filters over 30 days, five additional descending sorts | -| User/team/tag/customer usage | 16 | Aggregated totals, keys, key search, model top keys for each | -| Team-user spend | 2 | All 100 seeded teams or `perf-team-0` | -| Cache activity | 2 | All or selected aliases/model | -| Cache leakage keys | 1 | 30-day window | - -## Deployment prerequisites - -Use a fresh proxy against the disposable restore, never the active seed source or a shared test database. Apply the target migrations and wait for readiness first. The source seed's `litellm_perf_seed.checkpoint` must exist and contain rows - -Configure `litellm_settings.service_callback: [prometheus_system]` and expose metrics for every proxy worker/pod. Set `E2E_DATABASE_METRICS_URLS` to their comma-separated URLs, not a load-balanced endpoint that can miss a pod. Absent failure counters count as zero only when the endpoint exposes a positive PostgreSQL request counter. Missing instrumentation stops the run - -No expensive-query override or legacy view creation is needed for these cases. Keep the normal large-database UI restrictions - -The cases use explicit dates from the seed anchor where the UI supplies dates. A missing nonempty fixture result records a contract failure rather than reporting a fast empty query as success - -## Run - -Set `DATABASE_URL` and `LITELLM_MASTER_KEY` securely for this disposable deployment. The URL must name the same database used by the proxy, with hostname `db-perf-perf-.litellm.svc`. The explicit disposable confirmation permits cancellation of leftover queries in this database. Setup reads the existing seed checkpoint without adding UI-specific rows. Destroy the disposable database after the run +The componentized perf stack serves the dashboard behind the same router as the proxy, so `LITELLM_PROXY_URL` covers `/ui`. Set `E2E_UI_BASE_URL` only if the UI lives elsewhere. The admin login uses `E2E_UI_USERNAME` (default `admin`) and `E2E_UI_PASSWORD` (default the master key) ```bash -export LITELLM_PROXY_URL=http://localhost:4000 -export E2E_DATABASE_METRICS_URLS=http://localhost:4000/metrics -export E2E_PERF_TESTS=1 -export E2E_UI_PERF_DISPOSABLE=1 -export E2E_RESET_SPEND_LOGS=0 -export E2E_ARTIFACT_DIR="$PWD/ui-query-results" -mkdir -p "$E2E_ARTIFACT_DIR" -uv run pytest tests/e2e/perf/test_ui_queries.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q +export E2E_PERF_TESTS=1 E2E_UI_PERF_DISPOSABLE=1 E2E_RESET_SPEND_LOGS=0 +export DATABASE_URL=postgresql://...@db-perf-perf-.litellm.svc:5432/litellm +export E2E_DATABASE_METRICS_URLS=http://:4000/metrics +export E2E_ARTIFACT_DIR=/tmp/e2e-artifacts +uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q ``` -Do not enable SpendLog retention or run other traffic on this stack. The `quiet_stack` marker coordinates local pytest callers but cannot stop external traffic. Use one pytest worker for comparable timings +`E2E_UI_JOURNEY_TIMEOUT_SECONDS` (default 120) bounds one journey. A journey that never settles is recorded as a `timeout` sample and a journey whose page rendered empty or threw in the browser as a `ui` sample. The test fails when any sample failed, or when the p50 or p90 across all samples is over the journey's `Ceiling` in `UI_JOURNEY_LIMITS` (`setup/ui_journeys.py`, 10s/20s for Logs journeys, 20s/40s for Usage and Caching, 5s/10s for the Models and Keys tables, generous until the first 1TB run sets real numbers). The artifacts are written before the assertion so a failed run still ships its samples -`E2E_UI_QUERY_REPEATS` defaults to 9, giving ten measured requests including the first, and `E2E_UI_QUERY_TIMEOUT_SECONDS` defaults to 120. Every request is issued once without transport retries. A failed first request stops that case's repeated samples. Cursor prerequisites are recorded separately, and failed prerequisites stop that case's measurements +## Artifacts -## Reading results +Each JUnit testcase `test_ui_journey_latency[]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-.json` holds every sample with its on-screen row count and `ui-isolation-.jsonl` the SQL isolation evidence -Each case writes `ui-query-.json` with its exact method, path, parameters and body, response status counts, elapsed times, row counts, errors and before/after PostgreSQL failure counters. Response bodies and authentication headers are not saved - -The dashboard's `perf_ui_seconds_p50`, `_p90` and `_max` include all successful measured requests, including the first. `perf_ui_samples` is their sample count. p50 is the median and p90 uses nearest rank, the ninth sorted value with ten successes. Failures and cursor prerequisites contribute no latency. Pass percentage is `(perf_ui_requests - perf_ui_failures) / perf_ui_requests * 100`, including cursor prerequisites in the request counts - -The first-request and repeated-request metrics are retained for existing consumers. This is not a cold-cache claim: previous cases and previous runs can warm database buffers or application caches. Restart the disposable proxy between independent runs when comparing independent runs - -Repeated p50 is reported for successful samples, p95 only with at least 20 and p99 only with at least 100. First-request latency is omitted when that request fails. HTTP, JSON, response-shape, nonempty-result, cursor failures are recorded, never asserted. Endpoint failures, database counter increases and latency do not fail CI - -Each JUnit testcase `test_ui_query_performance[]` carries `perf_ui_requests`, `perf_ui_failures`, `perf_ui_repeated_samples`, and nonzero `perf_ui_error_` counts. Codes distinguish `http_400`, `http_500`, `http_503`, other HTTP statuses, `timeout`, `network`, `invalid_json` and `contract`. Successful latency properties are `perf_ui_first_seconds` and `perf_ui_repeated_seconds_p50`, `_p95`, `_p99`, `_max`. A failed cursor prerequisite is counted once and contributes no latency - -Before every case the runner requires three idle database observations and stable failure counters. After each case it cancels leftover SQL, terminates idle transactions and waits for idle and stable counters again. `ui-isolation-.jsonl` retains active SQL, waits, elapsed times, raw samples and counter observations. `perf_ui_database_failures` measures the increase before cleanup, while `perf_ui_cleanup_database_failures` measures the increase observed afterward, including cancellations and potentially delayed request failures. Isolation, instrumentation or setup failures still stop the run rather than publish contaminated measurements. Disk and process caches are not reset between cases - -The Buildkite `ui_queries` lane restores the pinned snapshot independently, waits for migrations and indexes to complete, runs these cases with no retries, uploads JUnit and JSON artifacts, and removes the temporary resources. It follows the nightly regular perf and database-upgrade builds using the same LiteLLM SHA. Shinkiro ingestion is a separate change - -Validate a small restore first. Small-database numbers validate the runner only, not full-size performance. Run the same cases against the completed full-size snapshot before publishing large-database results +The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.py new file mode 100644 index 00000000000..e9d4360d1a4 --- /dev/null +++ b/tests/e2e/perf/setup/ui_journeys.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import math +import re +import statistics +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from itertools import chain +from types import MappingProxyType +from typing import Final + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import Locator, Page, Response, expect, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError +from prometheus_client.parser import text_string_to_metric_families +from pydantic import BaseModel, ConfigDict + +MONEY: Final = re.compile(r"^\$[\d,]+\.\d{2}$") +LOADING_LOGS: Final = "Loading request logs…" +LOADING_MODELS: Final = "Loading models…" +LOADING_KEYS: Final = "Loading keys..." + + +class Sample(BaseModel): + model_config = ConfigDict(frozen=True) + + seconds: float + rows: int = 0 + error: str = "" + + +class Timings(BaseModel): + model_config = ConfigDict(frozen=True) + + first_seconds: float | None + repeated_samples: int + repeated_p50_seconds: float | None + repeated_p95_seconds: float | None + repeated_p99_seconds: float | None + repeated_max_seconds: float | None + + +@dataclass(frozen=True, slots=True) +class Journey: + """One dashboard interaction. ``start`` brings the page to the state the user acts from (untimed), + ``act`` is the click or navigation being measured and ``settled`` blocks until the data the user + was waiting for is on screen, returning the number of rows or tiles that rendered.""" + + name: str + surface: str + start: Callable[[Page, float], None] + act: Callable[[Page], None] + settled: Callable[[Page, float], int] + + +@dataclass(frozen=True, slots=True) +class Dashboard: + base_url: str + username: str + password: str + anchor: datetime + + +def summarize(samples: tuple[Sample, ...]) -> Timings: + assert samples, "No UI journeys were measured" + repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error)) + return Timings( + first_seconds=samples[0].seconds if not samples[0].error else None, + repeated_samples=len(repeated), + repeated_p50_seconds=statistics.median(repeated) if repeated else None, + repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None, + repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None, + repeated_max_seconds=max(repeated) if repeated else None, + ) + + +def error_code(sample: Sample) -> str: + if not sample.error: + return "" + return "timeout" if sample.error == "timeout" else "ui" + + +def ui_junit_properties(samples: tuple[Sample, ...]) -> tuple[tuple[str, float], ...]: + codes: Final = tuple(error_code(sample) for sample in samples if sample.error) + timings: Final = summarize(samples) + successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error)) + latency: Final = ( + ("seconds_p50", statistics.median(successful) if successful else None), + ("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None), + ("seconds_max", max(successful) if successful else None), + ("first_seconds", timings.first_seconds), + ("repeated_seconds_p50", timings.repeated_p50_seconds), + ("repeated_seconds_p95", timings.repeated_p95_seconds), + ("repeated_seconds_p99", timings.repeated_p99_seconds), + ("repeated_seconds_max", timings.repeated_max_seconds), + ) + return ( + ("perf_ui_journeys", len(samples)), + ("perf_ui_failures", len(codes)), + ("perf_ui_samples", len(successful)), + ("perf_ui_repeated_samples", timings.repeated_samples), + *((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))), + *((f"perf_ui_{name}", value) for name, value in latency if value is not None), + ) + + +@dataclass(frozen=True, slots=True) +class Ceiling: + p50_seconds: float + p90_seconds: float + + +LOGS_CEILING: Final = Ceiling(p50_seconds=10, p90_seconds=20) +CHART_CEILING: Final = Ceiling(p50_seconds=20, p90_seconds=40) +TABLE_CEILING: Final = Ceiling(p50_seconds=5, p90_seconds=10) +UI_JOURNEY_LIMITS: Final = MappingProxyType( + { + "logs_open_24h": LOGS_CEILING, + "logs_last_7_days": LOGS_CEILING, + "logs_next_page": LOGS_CEILING, + "logs_filter_key_hash": LOGS_CEILING, + "logs_filter_key_alias": LOGS_CEILING, + "logs_filter_team": LOGS_CEILING, + "logs_filter_model": LOGS_CEILING, + "usage_open": CHART_CEILING, + "usage_last_30_days": CHART_CEILING, + "usage_team_view": CHART_CEILING, + "usage_customer_view": CHART_CEILING, + "usage_model_activity_30_days": CHART_CEILING, + "cache_open": CHART_CEILING, + "models_open": TABLE_CEILING, + "keys_open": TABLE_CEILING, + } +) + + +def assert_within_ceiling(journey_name: str, samples: tuple[Sample, ...]) -> None: + failed: Final = tuple(sample.error for sample in samples if sample.error) + assert not failed, f"{len(failed)} of {len(samples)} {journey_name} samples failed: {failed[0]}" + metrics: Final = dict(ui_junit_properties(samples)) + ceiling: Final = UI_JOURNEY_LIMITS[journey_name] + assert metrics["perf_ui_seconds_p50"] <= ceiling.p50_seconds, ( + f"{journey_name} p50 {metrics['perf_ui_seconds_p50']:.1f}s over {ceiling.p50_seconds}s" + ) + assert metrics["perf_ui_seconds_p90"] <= ceiling.p90_seconds, ( + f"{journey_name} p90 {metrics['perf_ui_seconds_p90']:.1f}s over {ceiling.p90_seconds}s" + ) + + +@contextmanager +def dashboard_page(dashboard: Dashboard, timeout: float) -> Generator[Page]: + """A logged-in admin tab whose clock reads the seed anchor, so relative windows such as + Last 24 Hours land on seeded rows. Live Tail is off so its 15 second refetch cannot land + inside a sample. Playwright keeps a UTC timezone to match the seed.""" + with sync_playwright() as playwright: + browser: Final = playwright.chromium.launch() + context: Final = browser.new_context( + base_url=dashboard.base_url, timezone_id="UTC", viewport={"width": 1600, "height": 1000} + ) + context.set_default_timeout(timeout * 1000) + context.add_init_script('sessionStorage.setItem("isLiveTail", "false")') + page: Final = context.new_page() + page.clock.set_fixed_time(dashboard.anchor) + _login(page, dashboard) + try: + yield page + finally: + context.close() + browser.close() + + +def _login(page: Page, dashboard: Dashboard) -> None: + _ = page.goto("/ui/login") + page.get_by_placeholder("Enter your username").fill(dashboard.username) + page.get_by_placeholder("Enter your password").fill(dashboard.password) + page.get_by_role("button", name="Login", exact=True).click() + page.wait_for_url(lambda url: "/ui" in url and "/login" not in url) + expect(page.get_by_label("LiteLLM home")).to_be_visible() + + +def _is_data_fetch(response: Response) -> bool: + return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url + + +def measure(page: Page, journey: Journey, timeout: float) -> Sample: + """Wall time from the user's action to the data being on screen. Every sample starts from the + same pre-action state so repeats measure the same journey, not a warmed client cache.""" + try: + journey.start(page, timeout) + except (PlaywrightError, AssertionError) as error: + return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300]) + started: Final = time.monotonic() + try: + with page.expect_response(_is_data_fetch, timeout=timeout * 1000): + journey.act(page) + rows: Final = journey.settled(page, timeout) + except PlaywrightTimeoutError: + return Sample(seconds=time.monotonic() - started, error="timeout") + except (PlaywrightError, AssertionError) as error: + return Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]) + return Sample(seconds=time.monotonic() - started, rows=rows) + + +def _visible(page: Page, test_id: str) -> Locator: + return page.get_by_test_id(test_id).filter(visible=True).first + + +def _logs_rows(page: Page) -> Locator: + return page.locator("table").filter(visible=True).first.locator("tbody tr") + + +def _logs_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + rows: Final = _logs_rows(page).filter(has_not_text="No results") + expect(rows.first).to_be_visible(timeout=millis) + expect(_visible(page, "datatable-refresh")).to_be_enabled(timeout=millis) + expect(page.get_by_text(LOADING_LOGS)).to_have_count(0, timeout=millis) + return rows.count() + + +def _no_skeletons(page: Page, timeout: float) -> None: + expect(page.locator('[data-slot="skeleton"]')).to_have_count(0, timeout=timeout * 1000) + + +def _usage_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.get_by_text("Total Spend", exact=True).first).to_be_visible(timeout=millis) + _no_skeletons(page, timeout) + money: Final = page.get_by_text(MONEY) + expect(money.first).to_be_visible(timeout=millis) + return money.count() + + +def _cache_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.get_by_text("Cache Hits", exact=True)).to_be_visible(timeout=millis) + bars: Final = page.locator(".recharts-bar-rectangle") + expect(bars.first).to_be_visible(timeout=millis) + return bars.count() + + +def _table_settled(loading: str) -> Callable[[Page, float], int]: + def settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + rows: Final = page.locator("[data-row-id]") + expect(rows.first).to_be_visible(timeout=millis) + expect(page.get_by_text(loading)).to_have_count(0, timeout=millis) + return rows.count() + + return settled + + +def _open_logs(page: Page, timeout: float) -> None: + _ = page.goto("/ui/logs") + _ = _logs_settled(page, timeout) + + +def _open_usage(page: Page, timeout: float) -> None: + _ = page.goto("/ui/usage") + _ = _usage_settled(page, timeout) + + +def _filters_drawer(page: Page) -> Locator: + _visible(page, "datatable-filters-trigger").click() + drawer: Final = page.get_by_role("dialog", name="Filters") + expect(drawer).to_be_visible() + return drawer + + +def _apply_filters(page: Page) -> None: + page.get_by_role("dialog", name="Filters").get_by_role("button", name="Apply Filters").click() + + +def _pick_first_option(page: Page, combobox: str) -> None: + _filters_drawer(page).get_by_role("combobox", name=combobox).click() + page.get_by_role("option").first.click() + + +def _open_time_range(page: Page, timeout: float) -> None: + _open_logs(page, timeout) + page.get_by_role("button", name="Last 24 Hours").click() + + +def _open_usage_range(page: Page, timeout: float) -> None: + _open_usage(page, timeout) + page.locator('[data-slot="advanced-date-picker-trigger"]').click() + + +def _model_activity_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.locator('[role="tab"][data-active]', has_text="Model Activity")).to_be_visible(timeout=millis) + _no_skeletons(page, timeout) + money: Final = page.get_by_text(MONEY).filter(visible=True) + expect(money.first).to_be_visible(timeout=millis) + return money.count() + + +def _open_model_activity_range(page: Page, timeout: float) -> None: + _open_usage(page, timeout) + page.get_by_role("tab", name="Model Activity").click() + _ = _model_activity_settled(page, timeout) + page.locator('[data-slot="advanced-date-picker-trigger"]').click() + + +def _pick_usage_preset(preset: str) -> Callable[[Page], None]: + def act(page: Page) -> None: + page.locator('[data-slot="advanced-date-picker-preset"]', has_text=preset).click() + + return act + + +def _select_usage_view(page: Page, view: str) -> None: + page.locator('[data-slot="select-trigger"]').filter(has_text="Usage").first.click() + page.get_by_role("option", name=re.compile(rf"^{re.escape(view)}")).click() + + +def _nothing(page: Page, timeout: float) -> None: + return None + + +def _goto(path: str) -> Callable[[Page], None]: + def act(page: Page) -> None: + _ = page.goto(path) + + return act + + +def _logs_filter(name: str, prepare: Callable[[Page], None]) -> Journey: + def start(page: Page, timeout: float) -> None: + _open_logs(page, timeout) + prepare(page) + + return Journey(name, "logs", start, _apply_filters, _logs_settled) + + +def journeys(api_key: str) -> tuple[Journey, ...]: + return ( + Journey("logs_open_24h", "logs", _nothing, _goto("/ui/logs"), _logs_settled), + Journey( + "logs_last_7_days", + "logs", + _open_time_range, + lambda page: page.get_by_role("button", name="Last 7 Days", exact=True).click(), + _logs_settled, + ), + Journey( + "logs_next_page", "logs", _open_logs, lambda page: _visible(page, "pagination-next").click(), _logs_settled + ), + _logs_filter( + "logs_filter_key_hash", + lambda page: _filters_drawer(page).get_by_placeholder("Enter key hash…").fill(api_key), + ), + _logs_filter("logs_filter_key_alias", lambda page: _pick_first_option(page, "Search a key alias")), + _logs_filter("logs_filter_team", lambda page: _pick_first_option(page, "Search or select a team")), + _logs_filter("logs_filter_model", lambda page: _pick_first_option(page, "Search a model")), + Journey("usage_open", "usage", _nothing, _goto("/ui/usage"), _usage_settled), + Journey("usage_last_30_days", "usage", _open_usage_range, _pick_usage_preset("Last 30 days"), _usage_settled), + Journey( + "usage_team_view", "usage", _open_usage, lambda page: _select_usage_view(page, "Team Usage"), _usage_settled + ), + Journey( + "usage_customer_view", + "usage", + _open_usage, + lambda page: _select_usage_view(page, "Customer Usage"), + _usage_settled, + ), + Journey( + "usage_model_activity_30_days", + "usage", + _open_model_activity_range, + _pick_usage_preset("Last 30 days"), + _model_activity_settled, + ), + Journey("cache_open", "caching", _nothing, _goto("/ui/caching"), _cache_settled), + Journey("models_open", "models", _nothing, _goto("/ui/models-and-endpoints"), _table_settled(LOADING_MODELS)), + Journey("keys_open", "keys", _nothing, _goto("/ui/api-keys"), _table_settled(LOADING_KEYS)), + ) + + +def database_failures(text: str) -> float: + families: Final = tuple(text_string_to_metric_families(text)) + required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests")) + missing: Final = required - frozenset(family.name for family in families) + all_samples: Final = tuple(chain.from_iterable(family.samples for family in families)) + has_activity: Final = any( + sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0 + for sample in all_samples + ) + assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}" + names: Final = frozenset(f"{name}_total" for name in required) + values: Final = tuple(sample.value for sample in all_samples if sample.name in names) + assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter" + return sum(values) diff --git a/tests/e2e/perf/setup/ui_queries.py b/tests/e2e/perf/setup/ui_queries.py deleted file mode 100644 index b6945d79896..00000000000 --- a/tests/e2e/perf/setup/ui_queries.py +++ /dev/null @@ -1,227 +0,0 @@ -from __future__ import annotations - -import math -import statistics -import time -from collections.abc import Callable -from dataclasses import dataclass -from itertools import chain -from typing import Final, Literal -from urllib.parse import urlencode - -from e2e_http import ( - NetworkError, - RateLimitedError, - Result, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, - without_retries, -) -from e2e_metadata import step -from prometheus_client.parser import text_string_to_metric_families -from pydantic import BaseModel, ConfigDict, JsonValue, RootModel -from transport import HttpTransport - - -class Query(BaseModel): - model_config = ConfigDict(frozen=True) - - -class Document(RootModel[JsonValue]): - model_config = ConfigDict(frozen=True) - - -@dataclass(frozen=True, slots=True) -class Contract: - collection: tuple[str, ...] = () - row_fields: tuple[str, ...] = () - positive: tuple[str, ...] = () - object_fields: tuple[str, ...] = () - excluded_ids: frozenset[str] = frozenset() - - def count(self, document: Document) -> int: - value: Final = descend(document.root, self.collection) - if self.object_fields: - assert isinstance(value, dict), "Expected a JSON object" - assert set(self.object_fields) <= value.keys(), f"Missing fields {self.object_fields}" - return 1 - assert isinstance(value, list) and value, f"Empty/missing collection {self.collection}, check the fixture" - for row in value: - if self.excluded_ids: - assert isinstance(row, dict) and isinstance(row_id := row.get("request_id"), str), "Missing request_id" - assert row_id not in self.excluded_ids, "Repeated page-one row" - if self.row_fields or self.positive: - assert isinstance(row, dict), "Expected object rows" - assert set(self.row_fields) <= row.keys(), f"Missing row fields {self.row_fields}" - for name in self.positive: - assert any(tuple(_positive(row, name) for row in value)), f"Expected positive {name}" - return len(value) - - -def _positive(row: JsonValue, name: str) -> bool: - assert isinstance(row, dict), "Expected object row" - number: Final = row.get(name) - assert isinstance(number, (int, float)) and not isinstance(number, bool), f"Expected numeric {name}" - assert math.isfinite(number), f"Expected finite {name}" - return number > 0 - - -def descend(value: JsonValue, path: tuple[str, ...]) -> JsonValue: - if not path: - return value - assert isinstance(value, dict) and path[0] in value, f"Missing response path {path}" - return descend(value[path[0]], path[1:]) - - -@dataclass(frozen=True, slots=True) -class UiQueryCase: - name: str - path: str - params: BaseModel - contract: Contract - surface: Literal["current"] = "current" - body: BaseModel | None = None - cursor: Literal["none", "session"] = "none" - - -class Sample(BaseModel): - model_config = ConfigDict(frozen=True) - - seconds: float - status: int - rows: int = 0 - error: str = "" - - -class Timings(BaseModel): - model_config = ConfigDict(frozen=True) - - first_seconds: float | None - repeated_samples: int - repeated_p50_seconds: float | None - repeated_p95_seconds: float | None - repeated_p99_seconds: float | None - repeated_max_seconds: float | None - - -def summarize(samples: tuple[Sample, ...]) -> Timings: - assert samples, "No UI requests were measured" - repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error)) - return Timings( - first_seconds=samples[0].seconds if not samples[0].error else None, - repeated_samples=len(repeated), - repeated_p50_seconds=statistics.median(repeated) if repeated else None, - repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None, - repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None, - repeated_max_seconds=max(repeated) if repeated else None, - ) - - -def error_code(sample: Sample) -> str: - if not sample.error: - return "" - if sample.status > 0 and sample.status != 200: - return f"http_{sample.status}" - if sample.error in ("timeout", "network"): - return sample.error - if sample.error == "Response is not valid JSON": - return "invalid_json" - return "contract" - - -def ui_junit_properties( - samples: tuple[Sample, ...], prerequisite: Sample | None = None -) -> tuple[tuple[str, float], ...]: - requests: Final = (prerequisite, *samples) if prerequisite is not None and not prerequisite.error else samples - codes: Final = tuple(error_code(sample) for sample in requests if sample.error) - timings: Final = summarize(samples) - successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error)) - latency: Final = ( - ("seconds_p50", statistics.median(successful) if successful else None), - ("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None), - ("seconds_max", max(successful) if successful else None), - ("first_seconds", timings.first_seconds), - ("repeated_seconds_p50", timings.repeated_p50_seconds), - ("repeated_seconds_p95", timings.repeated_p95_seconds), - ("repeated_seconds_p99", timings.repeated_p99_seconds), - ("repeated_seconds_max", timings.repeated_max_seconds), - ) - return ( - ("perf_ui_requests", len(requests)), - ("perf_ui_failures", len(codes)), - ("perf_ui_samples", len(successful)), - ("perf_ui_repeated_samples", timings.repeated_samples), - *((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))), - *((f"perf_ui_{name}", value) for name, value in latency if value is not None), - ) - - -def _failure(result: Result[Document], seconds: float) -> Sample: - match result: - case Success(status_code=status): - return Sample(seconds=seconds, status=status, error=f"Unexpected HTTP {status}") - case UnknownApiError(status_code=status): - return Sample(seconds=seconds, status=status, error=f"HTTP {status}") - case UnauthorizedError(): - return Sample(seconds=seconds, status=401, error="Unauthorized") - case RateLimitedError(): - return Sample(seconds=seconds, status=429, error="Rate limited") - case NetworkError(message=message): - return Sample(seconds=seconds, status=-1, error="timeout" if "timed out" in message.lower() else "network") - case ValidationError(): - return Sample(seconds=seconds, status=-1, error="Response is not valid JSON") - - -def measure( - issue: Callable[[], Result[Document]], - contract: Contract, - *, - clock: Callable[[], float] = time.monotonic, -) -> tuple[Sample, Document | None]: - started: Final = clock() - with without_retries(): - result: Final = issue() - seconds: Final = clock() - started - match result: - case Success(status_code=200, data=document): - try: - count: Final = contract.count(document) - except AssertionError as error: - return Sample(seconds=seconds, status=200, error=str(error)), document - return Sample(seconds=seconds, status=200, rows=count), document - case _: - return _failure(result, seconds), None - - -def query_path(case: UiQueryCase) -> str: - query: Final = urlencode(case.params.model_dump(by_alias=True, exclude_none=True), doseq=True) - return f"{case.path}?{query}" if query else case.path - - -@step("Measure UI query {case.name} at {case.path}") -def issue_query(transport: HttpTransport, case: UiQueryCase, *, timeout: float) -> Result[Document]: - if case.body is not None: - return transport.post( - case.path, headers=transport.master, json=case.body, response_type=Document, timeout=timeout - ) - return transport.get( - query_path(case), headers=transport.master, params=Query(), response_type=Document, timeout=timeout - ) - - -def database_failures(text: str) -> float: - families: Final = tuple(text_string_to_metric_families(text)) - required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests")) - missing: Final = required - frozenset(family.name for family in families) - all_samples: Final = tuple(chain.from_iterable(family.samples for family in families)) - has_activity: Final = any( - sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0 - for sample in all_samples - ) - assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}" - names: Final = frozenset(f"{name}_total" for name in required) - values: Final = tuple(sample.value for sample in all_samples if sample.name in names) - assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter" - return sum(values) diff --git a/tests/e2e/perf/setup/ui_query_cases.py b/tests/e2e/perf/setup/ui_query_cases.py deleted file mode 100644 index e9b70aad713..00000000000 --- a/tests/e2e/perf/setup/ui_query_cases.py +++ /dev/null @@ -1,206 +0,0 @@ -from __future__ import annotations - -from dataclasses import replace -from datetime import datetime, timedelta -from itertools import chain -from typing import Final, Literal - -from pydantic import ConfigDict - -from setup.ui_queries import Contract, Document, Query, UiQueryCase, descend - - -class Params(Query): - model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) - - start_date: str | None = None - timezone: str | None = None - offset: int | None = None - model_group: str | None = None - by_model_group: str | None = None - end_date: str | None = None - page: int | None = None - page_size: int | None = None - sort_by: str | None = None - sort_order: str | None = None - group_by_session: Literal["true"] | None = None - session_cursor: str | None = None - api_key: str | None = None - team_id: str | None = None - team_ids: str | None = None - user_id: str | None = None - model: str | None = None - model_id: str | None = None - key_alias: str | None = None - search: str | None = None - status_filter: str | None = None - cache_hit_filter: str | None = None - error_code: str | None = None - error_message: str | None = None - span_type: str | None = None - exclude_internal_health_checks: Literal["true"] | None = None - key_aliases: tuple[str, ...] | None = None - models: tuple[str, ...] | None = None - limit: int | None = None - tags: str | None = None - exclude_team_ids: str | None = None - include_current_utc_day: Literal["true"] | None = None - - -def _timestamp(value: datetime) -> str: - return value.strftime("%Y-%m-%d %H:%M:%S") - - -def next_page(case: UiQueryCase, document: Document) -> UiQueryCase: - params: Final = Params.model_validate(case.params.model_dump()) - rows: Final = descend(document.root, case.contract.collection) - assert isinstance(rows, list), "Missing page-one rows" - ids: Final = frozenset(str(row["request_id"]) for row in rows if isinstance(row, dict)) - contract: Final = replace(case.contract, excluded_ids=ids) - cursor: Final = descend(document.root, ("next_session_cursor",)) - assert isinstance(cursor, str) and cursor, "Missing session cursor" - return replace(case, params=params.model_copy(update={"page": 2, "session_cursor": cursor}), contract=contract) - - -_LOGS: Final = Contract(("data",), ("request_id", "startTime", "metadata")) - - -def log_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - params: Final = Params( - start_date=_timestamp(anchor - timedelta(days=1)), - end_date=_timestamp(anchor), - page=1, - page_size=25, - sort_by="startTime", - sort_order="desc", - group_by_session="true", - ) - base: Final = UiQueryCase("logs_default_24h", "/spend/logs/ui", params, _LOGS) - filtered: Final = params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}) - filters: Final = ( - ("api_key", api_key), - ("team_id", "perf-team-0"), - ("user_id", "perf-user-0"), - ("model", "perf-chat"), - ("model_id", "perf-model-0"), - ("key_alias", "perf-key-0"), - ("status_filter", "failure"), - ("cache_hit_filter", "hit"), - ("error_code", "429"), - ("error_message", "Synthetic"), - ("span_type", "llm"), - ("exclude_internal_health_checks", "true"), - ) - return ( - base, - replace(base, name="logs_cursor_page_2", params=params.model_copy(update={"page": 2}), cursor="session"), - replace( - base, - name="logs_7d", - params=params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=7))}), - ), - replace( - base, - name="logs_30d", - params=params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}), - ), - *( - replace(base, name=f"logs_{name}_30d", params=filtered.model_copy(update={name: value})) - for name, value in filters - ), - *( - replace(base, name=f"logs_sort_{name}", params=params.model_copy(update={"sort_by": name})) - for name in ("spend", "total_tokens", "request_duration_ms", "model", "ttft_ms") - ), - ) - - -def analytics_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat() - ) - cache: Final = UiQueryCase( - "cache_activity_7d", "/global/activity/cache_hits", dates, Contract(("groups",), ("call_type",)) - ) - return ( - UiQueryCase( - "team_users_all_teams", - "/team/spend/by_user", - dates.model_copy(update={"team_ids": ",".join(f"perf-team-{i}" for i in range(100))}), - Contract(("results",), ("team_id", "user_id", "spend"), ("api_requests",)), - ), - UiQueryCase( - "team_users_filtered", - "/team/spend/by_user", - dates.model_copy(update={"team_ids": "perf-team-0"}), - Contract(("results",), ("team_id", "user_id", "spend"), ("api_requests",)), - ), - cache, - replace( - cache, - name="cache_activity_filtered", - params=dates.model_copy(update={"key_aliases": ("perf-key-0", "perf-key-1"), "models": ("perf-chat",)}), - ), - ) - - -def _entity_activity_cases(anchor: datetime, entity: str) -> tuple[UiQueryCase, ...]: - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), - end_date=anchor.date().isoformat(), - timezone="0", - exclude_team_ids="litellm-dashboard" if entity == "team" else None, - ) - routes: Final = ( - ("aggregated", dates, Contract(("results",), ("date", "breakdown"))), - ( - "aggregated/keys", - dates.model_copy(update={"offset": 0, "limit": 50}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ( - "aggregated/search", - dates.model_copy(update={"search": "perf-key"}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ( - "aggregated/model_top_keys", - dates.model_copy(update={"model_group": "perf-chat", "by_model_group": "true"}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ) - return tuple( - UiQueryCase( - f"usage_{entity}_{route.replace('/', '_')}", - f"/{entity}/daily/activity/{route}", - params, - contract, - ) - for route, params, contract in routes - ) - - -def activity_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - groups: Final = tuple(_entity_activity_cases(anchor, entity) for entity in ("user", "team", "tag", "customer")) - return ( - *chain.from_iterable(groups), - UiQueryCase( - "cache_leakage_keys", - "/user/daily/activity/aggregated/cache_leakage_keys", - Params( - start_date=(anchor - timedelta(days=30)).date().isoformat(), - end_date=anchor.date().isoformat(), - timezone="0", - include_current_utc_day="true", - ), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ) - - -def cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - return ( - *log_cases(anchor, api_key), - *analytics_cases(anchor), - *activity_cases(anchor), - ) diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index e3f5236ba47..1cbb5c41ea3 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib import os import time -from datetime import datetime +from datetime import datetime, timedelta from typing import Final import psycopg @@ -20,6 +20,45 @@ class UiFixture(BaseModel): api_key: str +SEEDED_TEAMS: Final = 100 +SEEDED_USERS: Final = 2000 +SEEDED_KEYS: Final = 5000 +SEEDED_MODELS: Final = 200 +DAILY_TABLES: Final = ("LiteLLM_DailyUserSpend", "LiteLLM_DailyTeamSpend") +_SEED_SQL: Final = ( + """ + INSERT INTO "LiteLLM_TeamTable" (team_id, team_alias, members_with_roles, metadata, model_spend, model_max_budget) + SELECT 'perf-team-' || n, 'perf-team-' || n, '[]', '{}', '{}', '{}' FROM generate_series(0, %(teams)s - 1) AS n + ON CONFLICT (team_id) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_UserTable" (user_id, user_alias, user_email, user_role, team_id, metadata, model_spend, model_max_budget) + SELECT 'perf-user-' || n, 'perf-user-' || n, 'perf-user-' || n || '@example.com', 'internal_user', + 'perf-team-' || (n %% %(teams)s), '{}', '{}', '{}' + FROM generate_series(0, %(users)s - 1) AS n + ON CONFLICT (user_id) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_VerificationToken" (token, key_name, key_alias, team_id, user_id, aliases, config, permissions, + metadata, model_spend, model_max_budget, created_at, updated_at) + SELECT md5('perf-key:' || n) || md5('perf-key-tail:' || n), 'sk-...perf' || n, 'perf-key-' || n, + 'perf-team-' || (n %% %(teams)s), 'perf-user-' || (n %% %(users)s), '{}', '{}', '{}', '{}', '{}', '{}', + %(anchor)s - make_interval(days => n %% 365), %(anchor)s + FROM generate_series(0, %(keys)s - 1) AS n + ON CONFLICT (token) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_ProxyModelTable" (model_id, model_name, litellm_params, model_info, created_by, updated_by) + SELECT 'perf-model-' || n, 'perf-chat-' || (n %% 20), + jsonb_build_object('model', 'openai/perf-chat', 'api_key', 'perf-seed', + 'api_base', 'os.environ/E2E_PERF_MOCK_OPENAI_URL'), + jsonb_build_object('id', 'perf-model-' || n, 'db_model', true), 'perf-seed', 'perf-seed' + FROM generate_series(0, %(models)s - 1) AS n + ON CONFLICT (model_id) DO NOTHING + """, +) + + def prepare_fixture() -> UiFixture: validate_target(os.environ["DATABASE_URL"]) _finish_upgrade() @@ -27,6 +66,7 @@ def prepare_fixture() -> UiFixture: _ = connection.execute("SET statement_timeout='30s'") row: Final = connection.execute("SELECT spec->>'anchor', rows FROM litellm_perf_seed.checkpoint").fetchone() assert row is not None and TypeAdapter(int).validate_python(row[1]) > 0, "Missing completed seed rows" + seed_ui_tables(connection, anchor=TypeAdapter(datetime).validate_python(row[0])) fixture: Final = UiFixture( anchor=TypeAdapter(datetime).validate_python(row[0]), api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest() @@ -35,6 +75,36 @@ def prepare_fixture() -> UiFixture: return fixture +def seed_ui_tables(connection: psycopg.Connection, anchor: datetime) -> None: + """Give the Keys, Teams, Users and Models pages realistic row counts on the disposable restore. + + Names follow the SpendLogs seed (perf-key-N aliases, perf-team-N, perf-user-N) so the Logs filters and + Team Usage line up with real spend. The Daily* rollups come from the snapshot itself and are only checked. + """ + window: Final = { + "since": (anchor - timedelta(days=7)).strftime("%Y-%m-%d"), + "until": anchor.strftime("%Y-%m-%d"), + } + for table in DAILY_TABLES: + row = connection.execute(f'SELECT count(*) FROM "{table}" WHERE date BETWEEN %(since)s AND %(until)s', window) + assert _count(row.fetchone()) > 0, f"{table} has no rows in the week before the anchor" + params: Final = { + "teams": SEEDED_TEAMS, + "users": SEEDED_USERS, + "keys": SEEDED_KEYS, + "models": SEEDED_MODELS, + "anchor": anchor, + } + for statement in _SEED_SQL: + _ = connection.execute(statement, params) + connection.commit() + + +def _count(row: tuple[object, ...] | None) -> int: + assert row is not None + return TypeAdapter(int).validate_python(row[0]) + + def _finish_upgrade() -> None: if os.environ.get("E2E_DATABASE_UPGRADE") != "1": return diff --git a/tests/e2e/perf/setup/ui_query_isolation.py b/tests/e2e/perf/setup/ui_query_isolation.py index d7f6d725edc..d892c268fa2 100644 --- a/tests/e2e/perf/setup/ui_query_isolation.py +++ b/tests/e2e/perf/setup/ui_query_isolation.py @@ -62,7 +62,9 @@ def wait_idle(connection: psycopg.Connection[dict[str, object]], evidence: Path, for query in queries: if cancel and query.leader_pid is None: _ = connection.execute( - "SELECT pg_cancel_backend(%s)" if query.state == "active" else "SELECT pg_terminate_backend(%s)", + "SELECT pg_cancel_backend(%s)" + if query.state == "active" + else "SELECT pg_terminate_backend(%s)", (query.pid,), ) else: diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py new file mode 100644 index 00000000000..888a44d2e27 --- /dev/null +++ b/tests/e2e/perf/test_ui_journeys.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import os +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME +from e2e_http import send_text_external, without_retries +from setup.ui_journeys import ( + assert_within_ceiling, + Dashboard, + Journey, + Sample, + dashboard_page, + database_failures, + error_code, + journeys, + measure, + summarize, + ui_junit_properties, +) +from setup.ui_query_fixture import UiFixture, prepare_fixture +from setup.ui_query_isolation import isolated_database, settled_counter +from transport import HttpTransport + +pytestmark = [ + pytest.mark.perf, + pytest.mark.quiet_stack, + pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"), +] + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "journey_name" in metafunc.fixturenames: + metafunc.parametrize("journey_name", [journey.name for journey in journeys("")]) + + +@pytest.fixture(scope="module") +def ui_fixture() -> UiFixture: + return prepare_fixture() + + +def _metrics(transport: HttpTransport) -> float: + urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip()) + try: + assert urls, "Provide a metrics URL for every proxy worker/pod" + readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls) + assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics" + return sum(database_failures(reading.body) for reading in readings) + except Exception as error: + pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2) + + +def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]: + with dashboard_page(dashboard, timeout) as page: + return tuple(measure(page, journey, timeout) for _ in range(repeats + 1)) + + +def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None: + journey: Final = next(journey for journey in journeys(ui_fixture.api_key) if journey.name == journey_name) + transport: Final = HttpTransport(base_url=UI_BASE_URL, master_key=MASTER_KEY) + timeout: Final = float(os.environ.get("E2E_UI_JOURNEY_TIMEOUT_SECONDS", "120")) + repeats: Final = int(os.environ.get("E2E_UI_JOURNEY_REPEATS", "9")) + assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings" + target: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) + assert target.is_dir(), "E2E_ARTIFACT_DIR must exist" + evidence: Final = target / f"ui-isolation-{journey.name}.jsonl" + dashboard: Final = Dashboard( + base_url=UI_BASE_URL, username=UI_USERNAME, password=UI_PASSWORD, anchor=ui_fixture.anchor + ) + with without_retries(): + with isolated_database(journey.name): + before: Final = settled_counter(lambda: _metrics(transport), evidence) + samples: Final = _samples(dashboard, journey, timeout, repeats) + with evidence.open("a", encoding="utf-8") as sample_output: + _ = sample_output.write( + json.dumps({"samples": tuple(sample.model_dump() for sample in samples)}) + "\n" + ) + after_requests: Final = _metrics(transport) + item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped + assert isinstance(item, pytest.Item) + item.user_properties.extend(ui_junit_properties(samples)) + after: Final = settled_counter(lambda: _metrics(transport), evidence) + item.user_properties.extend( + ( + ("perf_ui_database_failures", after_requests - before), + ("perf_ui_cleanup_database_failures", after - after_requests), + ) + ) + artifact: Final = { + "name": journey.name, + "surface": journey.surface, + "measured_at": datetime.now(UTC).isoformat(), + "samples": [sample.model_dump() for sample in samples], + "error_codes": [error_code(sample) for sample in samples], + "timings": summarize(samples).model_dump(), + "timeouts": sum(sample.error == "timeout" for sample in samples), + "failures": sum(bool(sample.error) for sample in samples), + "database_failures_before": before, + "database_failures_after_requests": after_requests, + "database_failures_after_cleanup": after, + } + output: Final = target / f"ui-journey-{journey.name}.json" + _ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8") + assert_within_ceiling(journey.name, samples) diff --git a/tests/e2e/perf/test_ui_queries.py b/tests/e2e/perf/test_ui_queries.py deleted file mode 100644 index 75ac4d3055f..00000000000 --- a/tests/e2e/perf/test_ui_queries.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import json -import os -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from pathlib import Path -from typing import Final - -import pytest -from e2e_config import CONTROL_PLANE_BASE_URL, MASTER_KEY -from e2e_http import send_text_external, without_retries -from setup.ui_queries import ( - Sample, - UiQueryCase, - database_failures, - error_code, - issue_query, - measure, - summarize, - ui_junit_properties, -) -from setup.ui_query_cases import Params, cases, next_page -from setup.ui_query_fixture import UiFixture, prepare_fixture -from setup.ui_query_isolation import isolated_database, settled_counter -from transport import HttpTransport - -pytestmark = [ - pytest.mark.perf, - pytest.mark.quiet_stack, - pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"), -] - - -def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: - if "ui_case_name" not in metafunc.fixturenames: - return - if os.environ.get("E2E_UI_PERF_DISPOSABLE") == "1" and os.environ.get("E2E_RESET_SPEND_LOGS") == "1": - raise pytest.UsageError("UI performance tests cannot run with E2E_RESET_SPEND_LOGS=1") - names: Final = tuple(case.name for case in cases(datetime(2000, 1, 1, tzinfo=UTC), "collection-only")) - metafunc.parametrize("ui_case_name", names, ids=names) - - -@pytest.fixture(scope="module") -def ui_fixture() -> UiFixture: - return prepare_fixture() - - -def _metrics(transport: HttpTransport) -> float: - urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip()) - try: - assert urls, "Provide a metrics URL for every proxy worker/pod" - readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls) - assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics" - return sum(database_failures(reading.body) for reading in readings) - except Exception as error: - pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2) - - -@dataclass(frozen=True, slots=True) -class Prepared: - case: UiQueryCase - prerequisite: Sample | None = None - - -def _page_two(transport: HttpTransport, case: UiQueryCase, timeout: float) -> Prepared: - if case.cursor == "none": - return Prepared(case) - params: Final = Params.model_validate(case.params.model_dump()).model_copy(update={"page": 1}) - first: Final = replace(case, params=params) - sample, document = measure(lambda: issue_query(transport, first, timeout=timeout), case.contract) - if sample.error or document is None: - return Prepared(case, sample) - try: - return Prepared(next_page(case, document), sample) - except AssertionError as error: - return Prepared(case, sample.model_copy(update={"error": str(error)})) - - -def _samples(transport: HttpTransport, case: UiQueryCase, timeout: float, repeats: int) -> tuple[Sample, ...]: - measured, _ = measure(lambda: issue_query(transport, case, timeout=timeout), case.contract) - if measured.error: - return (measured,) - repeated: Final = tuple( - measure(lambda: issue_query(transport, case, timeout=timeout), case.contract)[0] for _ in range(repeats) - ) - return (measured, *repeated) - - -def test_ui_query_performance(ui_case_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None: - case: Final = next(case for case in cases(ui_fixture.anchor, ui_fixture.api_key) if case.name == ui_case_name) - transport: Final = HttpTransport(base_url=CONTROL_PLANE_BASE_URL, master_key=MASTER_KEY) - timeout: Final = float(os.environ.get("E2E_UI_QUERY_TIMEOUT_SECONDS", "120")) - repeats: Final = int(os.environ.get("E2E_UI_QUERY_REPEATS", "9")) - assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings" - target: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) - assert target.is_dir(), "E2E_ARTIFACT_DIR must exist" - evidence: Final = target / f"ui-isolation-{case.name}.jsonl" - with without_retries(): - with isolated_database(case.name): - before: Final = settled_counter(lambda: _metrics(transport), evidence) - paginated: Final = _page_two(transport, case, timeout) - samples: Final = ( - (paginated.prerequisite,) - if paginated.prerequisite is not None and paginated.prerequisite.error - else _samples(transport, paginated.case, timeout, repeats) - ) - with evidence.open("a", encoding="utf-8") as sample_output: - _ = sample_output.write( - json.dumps( - { - "samples": tuple(sample.model_dump() for sample in samples), - "prerequisite": paginated.prerequisite.model_dump() - if paginated.prerequisite is not None - else None, - } - ) - + "\n" - ) - after_requests: Final = _metrics(transport) - item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped - assert isinstance(item, pytest.Item) - item.user_properties.extend(ui_junit_properties(samples, paginated.prerequisite)) - after: Final = settled_counter(lambda: _metrics(transport), evidence) - item.user_properties.extend( - ( - ("perf_ui_database_failures", after_requests - before), - ("perf_ui_cleanup_database_failures", after - after_requests), - ) - ) - timings: Final = summarize(samples) - artifact: Final = { - "name": case.name, - "surface": case.surface, - "path": case.path, - "method": "POST" if case.body is not None else "GET", - "params": paginated.case.params.model_dump(mode="json", by_alias=True, exclude_none=True), - "prerequisite": paginated.prerequisite.model_dump() if paginated.prerequisite is not None else None, - "prerequisite_failed": paginated.prerequisite is not None and bool(paginated.prerequisite.error), - "body": case.body.model_dump(mode="json", exclude_none=True) if case.body is not None else None, - "samples": [sample.model_dump() for sample in samples], - "error_codes": [error_code(sample) for sample in samples], - "timings": timings.model_dump(), - "timeouts": sum(sample.error == "timeout" for sample in samples), - "failures": sum(bool(sample.error) for sample in samples), - "status_counts": { - str(status): sum(sample.status == status for sample in samples) - for status in frozenset(sample.status for sample in samples) - }, - "validation_failures": sum( - bool(sample.error) and (sample.status == 200 or sample.error == "Response is not valid JSON") - for sample in samples - ), - "database_failures_before": before, - "database_failures_after_requests": after_requests, - "database_failures_after_cleanup": after, - } - output: Final = target / f"ui-query-{case.name}.json" - _ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8") diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py new file mode 100644 index 00000000000..7d5b0d531ec --- /dev/null +++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from setup.ui_journeys import ( + UI_JOURNEY_LIMITS, + Sample, + assert_within_ceiling, + database_failures, + journeys, + summarize, + ui_junit_properties, +) + + +def test_first_journey_is_separate_from_repeated_latency() -> None: + samples: Final = (Sample(seconds=80), *(Sample(seconds=i) for i in range(1, 101))) + result: Final = summarize(samples) + assert result.first_seconds == 80 + assert result.repeated_samples == 100 + assert result.repeated_p50_seconds == 50.5 + assert result.repeated_p95_seconds == 95 + assert result.repeated_p99_seconds == 99 + assert result.repeated_max_seconds == 100 + + +def test_failed_repeats_do_not_look_like_fast_successes() -> None: + result: Final = summarize((Sample(seconds=10), Sample(seconds=0.01, error="AssertionError: no rows"))) + assert result.repeated_samples == 0 + assert result.repeated_p50_seconds is None + assert result.repeated_p95_seconds is None + assert result.repeated_p99_seconds is None + assert result.repeated_max_seconds is None + + +def test_small_sample_does_not_report_tail_percentiles() -> None: + result: Final = summarize(tuple(Sample(seconds=1) for _ in range(6))) + assert result.repeated_samples == 5 + assert result.repeated_p50_seconds == 1 + assert result.repeated_p95_seconds is None + assert result.repeated_p99_seconds is None + + +def test_failed_first_journey_emits_error_code_without_latency() -> None: + sample: Final = Sample(seconds=120, error="timeout") + assert summarize((sample,)).first_seconds is None + assert dict(ui_junit_properties((sample,))) == { + "perf_ui_journeys": 1, + "perf_ui_failures": 1, + "perf_ui_samples": 0, + "perf_ui_repeated_samples": 0, + "perf_ui_error_timeout": 1, + } + + +def test_junit_counts_failures_but_only_measures_successful_latency() -> None: + samples: Final = ( + Sample(seconds=2, rows=25), + Sample(seconds=3, rows=25), + Sample(seconds=120, error="timeout"), + Sample(seconds=1, error="AssertionError: Logs table rendered empty"), + Sample(seconds=4, rows=25), + ) + assert dict(ui_junit_properties(samples)) == { + "perf_ui_journeys": 5, + "perf_ui_failures": 2, + "perf_ui_samples": 3, + "perf_ui_repeated_samples": 2, + "perf_ui_error_timeout": 1, + "perf_ui_error_ui": 1, + "perf_ui_seconds_p50": 3, + "perf_ui_seconds_p90": 4, + "perf_ui_seconds_max": 4, + "perf_ui_first_seconds": 2, + "perf_ui_repeated_seconds_p50": 3.5, + "perf_ui_repeated_seconds_max": 4, + } + + +def test_all_journey_percentiles_include_first_and_use_nearest_rank_p90() -> None: + samples: Final = tuple(Sample(seconds=seconds) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5)) + metrics: Final = dict(ui_junit_properties(samples)) + assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == { + "perf_ui_seconds_p50": 5.5, + "perf_ui_seconds_p90": 9, + "perf_ui_seconds_max": 10, + } + assert metrics["perf_ui_samples"] == 10 + + +def test_journeys_have_unique_names_and_cover_every_dashboard_surface() -> None: + names: Final = tuple(journey.name for journey in journeys("key")) + assert len(names) == len(frozenset(names)) + assert frozenset(journey.surface for journey in journeys("key")) == {"logs", "usage", "caching", "models", "keys"} + + +def _metrics(postgres: str, batch: str) -> str: + return ( + "# TYPE litellm_postgres_failed_requests_total counter\n" + f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n' + "# TYPE litellm_batch_write_to_db_failed_requests_total counter\n" + f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n' + ) + + +def test_postgres_failures_include_background_batch_writes() -> None: + assert database_failures(_metrics("0", "0")) == 0 + assert database_failures(_metrics("2", "3")) == 5 + + +def test_lazy_failure_counters_require_evidence_of_database_activity() -> None: + assert ( + database_failures( + "# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n" + ) + == 0 + ) + + +@pytest.mark.parametrize( + "text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0")) +) +def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None: + with pytest.raises(AssertionError): + _ = database_failures(text) + + +def test_every_journey_has_a_latency_ceiling() -> None: + assert frozenset(journey.name for journey in journeys("key")) == frozenset(UI_JOURNEY_LIMITS) + + +def test_failed_sample_fails_the_journey_before_latency_is_judged() -> None: + with pytest.raises(AssertionError, match="1 of 2 logs_open_24h samples failed: timeout"): + assert_within_ceiling("logs_open_24h", (Sample(seconds=0.2, rows=25), Sample(seconds=120, error="timeout"))) + + +def test_slow_journey_fails_its_ceiling() -> None: + fast: Final = tuple(Sample(seconds=1, rows=25) for _ in range(9)) + assert_within_ceiling("logs_open_24h", fast) + with pytest.raises(AssertionError, match=r"logs_open_24h p90 25\.0s over 20s"): + assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25))) + with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"): + assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3))) diff --git a/tests/e2e_harness/perf/setup/test_ui_queries.py b/tests/e2e_harness/perf/setup/test_ui_queries.py deleted file mode 100644 index f24d03c6bba..00000000000 --- a/tests/e2e_harness/perf/setup/test_ui_queries.py +++ /dev/null @@ -1,257 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Final -from urllib.parse import parse_qs, urlsplit - -import pytest -from e2e_http import NetworkError, Result, Success, UnknownApiError -from setup.ui_queries import ( - Contract, - Document, - Sample, - database_failures, - measure, - query_path, - summarize, - ui_junit_properties, -) -from setup.ui_query_cases import Params, cases, next_page - - -def test_query_string_preserves_ui_arrays_and_boolean_strings() -> None: - requests: Final = cases(datetime(2026, 1, 1, tzinfo=UTC), "key") - cache: Final = next(case for case in requests if case.name == "cache_activity_filtered") - assert parse_qs(urlsplit(query_path(cache)).query) == { - "start_date": ["2025-12-25"], - "end_date": ["2026-01-01"], - "key_aliases": ["perf-key-0", "perf-key-1"], - "models": ["perf-chat"], - } - health: Final = next(case for case in requests if case.name == "logs_exclude_internal_health_checks_30d") - query: Final = parse_qs(urlsplit(query_path(health)).query) - assert query["exclude_internal_health_checks"] == ["true"] - assert query["group_by_session"] == ["true"] - - -@pytest.mark.parametrize("entity", ("user", "team", "tag", "customer")) -@pytest.mark.parametrize( - ("route", "extra"), - ( - ("aggregated", {}), - ("aggregated/keys", {"offset": 0, "limit": 50}), - ("aggregated/search", {"search": "perf-key"}), - ("aggregated/model_top_keys", {"model_group": "perf-chat", "by_model_group": "true"}), - ), -) -def test_usage_requests_list_keys_without_a_single_key_filter( - entity: str, route: str, extra: dict[str, str | int] -) -> None: - case: Final = next( - case - for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") - if case.path == f"/{entity}/daily/activity/{route}" - ) - assert case.params.model_dump(exclude_none=True) == { - "start_date": "2025-12-25", - "end_date": "2026-01-01", - "timezone": "0", - **({"exclude_team_ids": "litellm-dashboard"} if entity == "team" else {}), - **extra, - } - - -@pytest.mark.parametrize(("name", "value"), (("api_key", "key"), ("key_alias", "perf-key-0"))) -def test_key_filters_request_30_days_of_logs(name: str, value: str) -> None: - case: Final = next( - case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.name == f"logs_{name}_30d" - ) - assert parse_qs(urlsplit(query_path(case)).query) == { - "start_date": ["2025-12-02 00:00:00"], - "end_date": ["2026-01-01 00:00:00"], - "page_size": ["25"], - "page": ["1"], - "sort_by": ["startTime"], - "sort_order": ["desc"], - "group_by_session": ["true"], - name: [value], - } - - -@pytest.mark.parametrize("invalid", ("not-a-number", True, float("nan"), float("inf"))) -def test_positive_row_does_not_hide_a_malformed_later_row(invalid: str | bool | float) -> None: - with pytest.raises(AssertionError, match=r"Expected (numeric|finite) spend"): - Contract(positive=("spend",)).count(Document([{"spend": 1}, {"spend": invalid}])) - - -def test_session_cursor_is_reused_and_repeated_rows_rejected() -> None: - case: Final = next(case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.cursor == "session") - first: Final = Document.model_validate_json('{"data":[{"request_id":"one"}],"next_session_cursor":"opaque"}') - second: Final = next_page(case, first) - assert Params.model_validate(second.params).session_cursor == "opaque" - assert Params.model_validate(second.params).page == 2 - with pytest.raises(AssertionError, match="Repeated page-one row"): - _ = second.contract.count(first) - - -def test_first_query_is_separate_from_repeated_latency() -> None: - samples: Final = (Sample(seconds=80, status=200), *(Sample(seconds=i, status=200) for i in range(1, 101))) - result: Final = summarize(samples) - assert result.first_seconds == 80 - assert result.repeated_samples == 100 - assert result.repeated_p50_seconds == 50.5 - assert result.repeated_p95_seconds == 95 - assert result.repeated_p99_seconds == 99 - assert result.repeated_max_seconds == 100 - - -def test_failed_repeats_do_not_look_like_fast_successes() -> None: - result: Final = summarize((Sample(seconds=10, status=200), Sample(seconds=0.01, status=503, error="HTTP 503"))) - assert result.repeated_samples == 0 - assert result.repeated_p50_seconds is None - assert result.repeated_p95_seconds is None - assert result.repeated_p99_seconds is None - assert result.repeated_max_seconds is None - - -def test_small_sample_does_not_report_tail_percentiles() -> None: - result: Final = summarize(tuple(Sample(seconds=1, status=200) for _ in range(6))) - assert result.repeated_samples == 5 - assert result.repeated_p50_seconds == 1 - assert result.repeated_p95_seconds is None - assert result.repeated_p99_seconds is None - - -def test_failed_first_request_emits_error_code_without_latency() -> None: - sample: Final = Sample(seconds=30, status=500, error="HTTP 500") - assert summarize((sample,)).first_seconds is None - assert dict(ui_junit_properties((sample,))) == { - "perf_ui_requests": 1, - "perf_ui_failures": 1, - "perf_ui_samples": 0, - "perf_ui_repeated_samples": 0, - "perf_ui_error_http_500": 1, - } - - -def test_junit_counts_failures_but_only_measures_successful_latency() -> None: - samples: Final = ( - Sample(seconds=2, status=200), - Sample(seconds=3, status=200), - Sample(seconds=30, status=-1, error="timeout"), - Sample(seconds=30, status=503, error="HTTP 503"), - Sample(seconds=1, status=200, error="Empty collection"), - ) - assert dict(ui_junit_properties(samples, Sample(seconds=4, status=200))) == { - "perf_ui_requests": 6, - "perf_ui_failures": 3, - "perf_ui_samples": 2, - "perf_ui_seconds_p50": 2.5, - "perf_ui_seconds_p90": 3, - "perf_ui_seconds_max": 3, - "perf_ui_repeated_samples": 1, - "perf_ui_error_timeout": 1, - "perf_ui_error_http_503": 1, - "perf_ui_error_contract": 1, - "perf_ui_first_seconds": 2, - "perf_ui_repeated_seconds_p50": 3, - "perf_ui_repeated_seconds_max": 3, - } - - -def test_failed_cursor_prerequisite_is_counted_once() -> None: - sample: Final = Sample(seconds=30, status=503, error="HTTP 503") - assert dict(ui_junit_properties((sample,), sample)) == dict(ui_junit_properties((sample,))) - - -def test_all_request_percentiles_include_first_and_use_nearest_rank_p90() -> None: - samples: Final = tuple(Sample(seconds=seconds, status=200) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5)) - metrics: Final = dict(ui_junit_properties(samples)) - assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == { - "perf_ui_seconds_p50": 5.5, - "perf_ui_seconds_p90": 9, - "perf_ui_seconds_max": 10, - } - assert metrics["perf_ui_samples"] == 10 - - -@pytest.mark.parametrize("body", ("{}", '{"data": []}', '{"data": [{"spend": 1}]}', '{"data": [null]}')) -def test_empty_or_malformed_rows_fail_even_with_http_200(body: str) -> None: - ticks: Final = iter((10.0, 12.5)) - result, _ = measure( - lambda: Success(status_code=200, data=Document.model_validate_json(body)), - Contract(("data",), ("request_id",)), - clock=lambda: next(ticks), - ) - assert result.status == 200 - assert result.seconds == 2.5 - assert result.rows == 0 - assert result.error - - -def test_valid_response_records_elapsed_seconds_and_rows() -> None: - ticks: Final = iter((5.0, 5.125)) - result, document = measure( - lambda: Success(status_code=200, data=Document.model_validate_json('{"data":[{"request_id":"one"}]}')), - Contract(("data",), ("request_id",)), - clock=lambda: next(ticks), - ) - assert result == Sample(seconds=0.125, status=200, rows=1) - assert document is not None - - -def test_zero_spend_rows_are_allowed_alongside_nonzero_fixture_data() -> None: - contract: Final = Contract(positive=("spend",)) - assert contract.count(Document.model_validate_json('[{"spend":0},{"spend":2}]')) == 2 - with pytest.raises(AssertionError, match="positive spend"): - _ = contract.count(Document.model_validate_json('[{"spend":0}]')) - - -@pytest.mark.parametrize( - "failure,error,status", - ( - (UnknownApiError(status_code=500, body="sensitive error detail"), "HTTP 500", 500), - (NetworkError(message="Read timed out"), "timeout", -1), - (NetworkError(message="connection refused"), "network", -1), - ), -) -def test_failed_requests_are_recorded_without_response_bodies( - failure: Result[Document], - error: str, - status: int, -) -> None: - ticks: Final = iter((1.0, 2.0)) - result, document = measure(lambda: failure, Contract(), clock=lambda: next(ticks)) - assert result == Sample(seconds=1, status=status, error=error) - assert document is None - - -def _metrics(postgres: str, batch: str) -> str: - return ( - "# TYPE litellm_postgres_failed_requests_total counter\n" - f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n' - "# TYPE litellm_batch_write_to_db_failed_requests_total counter\n" - f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n' - ) - - -def test_postgres_failures_include_background_batch_writes() -> None: - assert database_failures(_metrics("0", "0")) == 0 - assert database_failures(_metrics("2", "3")) == 5 - - -def test_lazy_failure_counters_require_evidence_of_database_activity() -> None: - assert ( - database_failures( - "# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n" - ) - == 0 - ) - - -@pytest.mark.parametrize( - "text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0")) -) -def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None: - with pytest.raises(AssertionError): - _ = database_failures(text) diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py index 0904b7a7b5a..89e3a620f48 100644 --- a/tests/e2e_harness/perf/test_upgrade.py +++ b/tests/e2e_harness/perf/test_upgrade.py @@ -308,9 +308,7 @@ def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None: def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples( failures: int, overhead: float, ttft: float, growth: float, expected: tuple[str, ...] ) -> None: - nonstream: Final = tuple( - record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100) - ) + nonstream: Final = tuple(record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100)) stream: Final = record(100).model_copy(update={"stream": True, "first_token_s": ttft}) observed: Final = Observation( records=(*nonstream, stream, *(failed_record(i) for i in range(101, 101 + failures))), From 8d2bf426f59ffbda9aaa49608706aee4acb93395 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:26:38 -0700 Subject: [PATCH 113/124] fix(cost): price base_model deployments past rule-only cost map entries (#47497) * fix(cost): price base_model deployments past rule-only cost map entries and warn when base_model has no rates * fix(router): warn on unpriced base_model from add_deployment too and honour model_info prices * refactor(cost): drop the unpriced base_model startup warning, keep the one-line ladder fix --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 2 +- tests/unit/test_cost_calculator.py | 32 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 20914df8fe2..e8ca60e00af 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1642,7 +1642,7 @@ def completion_cost( *(((model, request_pricing_provider),) if model is not None else ()), ) potential_model_names: Final = ( - raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates) + raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) for idx, (model, pricing_provider) in enumerate(potential_model_names): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index fee4db586cd..2854f9010b2 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -391,6 +391,38 @@ def test_completion_cost_bills_undated_azure_data_zone_base_model_at_its_dated_r ) +def test_completion_cost_prices_rule_only_base_model_from_the_response_model( + _local_model_cost_map: None, + _price_free_gpt_family_generalization: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LIT-9812: a deployment whose base_model is an undated name only a capability rule matches must not + stop the ladder at $0 when the response names a priced model""" + monkeypatch.setattr( + litellm, + "model_cost", + { + "azure/gpt-9.9-2026-01-01": { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": 0.0000111, + "output_cost_per_token": 0.0000222, + } + }, + ) + usage: Final = Usage(prompt_tokens=17, completion_tokens=9, total_tokens=26) + response: Final = ModelResponse(model="gpt-9.9-2026-01-01", choices=[], usage=usage) + + cost: Final = completion_cost( + completion_response=response, + model="azure/my-eu-deployment", + custom_llm_provider="azure", + base_model="azure/eu/gpt-9.9", + ) + + assert cost == pytest.approx(usage.prompt_tokens * 0.0000111 + usage.completion_tokens * 0.0000222) + + def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests( _local_model_cost_map: None, ) -> None: From 9bbf70edb6372dd1f7436562f4bb1dc248af793d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 09:41:22 -0700 Subject: [PATCH 114/124] fix(completion): send prompt_cache_retention to OpenAI instead of silently dropping it (#47480) * fix(completion): send prompt_cache_retention to OpenAI instead of silently dropping it litellm.completion(), Router.completion() and the proxy's /v1/chat/completions accepted prompt_cache_retention and listed it as a supported OpenAI param, but completion() never handed it to get_optional_params, and the leftover-kwargs paths skip known OpenAI params, so it never reached the request body. Make it a named argument of completion() and acompletion() like prompt_cache_key and pass it on. A unit guard sends every OpenAI body param through completion() and fails when a param added to OPENAI_CHAT_COMPLETION_PARAMS has no case proving it reaches the wire, the same hole store and prompt_cache_key fell into in #33184. Live e2e tests capture the body OpenAI receives for the proxy, the SDK and the Router on both openai/gpt-5.5 and openai/responses/gpt-5.5. * chore(completion): type the param-forwarding unit test and keep the basedpyright count flat Every new completion() kwarg adds one unknown-argument error at each call site that splats an untyped dict into it; typing print_verbose's argument and Chat's params removes more than prompt_cache_retention added. * test: freeze the outbound-body model and read captured requests from the mock Addresses review: the e2e body model is frozen, the backend params are a tuple, and the unit test reads the request from the respx route instead of appending to a list. --- litellm/main.py | 13 +- .../coverage_registry/llm_conversational.yaml | 2 + tests/e2e/coverage_registry/schema.py | 1 + .../test_provider_features_e2e.py | 168 +++++++++++++++++- tests/e2e/models.py | 1 + tests/unit/test_main.py | 95 ++++++++++ 6 files changed, 275 insertions(+), 5 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index f9bdf64c667..63375292077 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -400,7 +400,11 @@ class LiteLLM: class Chat: - def __init__(self, params, router_obj: "Router | None"): + def __init__( + self, + params: dict[str, object], # mutable-ok: pops the acompletion flag in place + router_obj: "Router | None", + ): self.params = params if self.params.get("acompletion", False) is True: self.params.pop("acompletion") @@ -480,6 +484,7 @@ async def acompletion( service_tier: str | None = None, store: bool | None = None, prompt_cache_key: str | None = None, + prompt_cache_retention: str | None = None, # set api_base, api_version, api_key base_url: str | None = None, api_version: str | None = None, @@ -657,6 +662,7 @@ async def acompletion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, + "prompt_cache_retention": prompt_cache_retention, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -5356,6 +5362,7 @@ def completion( service_tier: str | None = None, store: bool | None = None, prompt_cache_key: str | None = None, + prompt_cache_retention: str | None = None, # soon to be deprecated params by OpenAI functions: list | None = None, function_call: str | None = None, @@ -5478,6 +5485,7 @@ def completion( service_tier=service_tier, store=store, prompt_cache_key=prompt_cache_key, + prompt_cache_retention=prompt_cache_retention, base_url=base_url, api_version=api_version, api_key=api_key, @@ -5843,6 +5851,7 @@ def completion( "service_tier": service_tier, "store": store, "prompt_cache_key": prompt_cache_key, + "prompt_cache_retention": prompt_cache_retention, "allowed_openai_params": allowed_openai_params, "base_model": base_model, } @@ -9143,7 +9152,7 @@ async def ahealth_check( ####### HELPER FUNCTIONS ################ ## Set verbose to true -> ```litellm.set_verbose = True``` -def print_verbose(print_statement): +def print_verbose(print_statement: object): try: verbose_logger.debug(print_statement) if litellm.set_verbose: diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index c40aeade1c1..e3994e598c6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -13,6 +13,8 @@ - {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"} - {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"} - {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} +- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"} +- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"} - {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"} - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index abef3266aa8..d2ba2c325df 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -81,6 +81,7 @@ LlmCapability = Literal[ "pdf_input", "logprobs", "prompt_cache_1h", + "prompt_cache_24h", "prompt_cache_5m", "response_headers", "service_tier", diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index 9a3de0aba7d..60dfd49638f 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -1,4 +1,4 @@ -"""Live e2e for model-specific request features: service_tier. +"""Live e2e for model-specific request features: service_tier and prompt_cache_retention. Each case asserts the feature took effect, not just a 200. @@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so that cell is OpenAI-only by design. -Prompt caching lives in test_cache_control.py. +prompt_cache_retention is judged on the body OpenAI receives, captured by a live +provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the +request never carried the param, so an echo would pass with the param dropped. + +Prompt caching hits and misses live in test_cache_control.py. """ from __future__ import annotations +import queue +from collections.abc import Mapping +from dataclasses import dataclass, field from typing import Final import pytest +from pydantic import BaseModel -from e2e_config import unique_marker +from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import unwrap from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from passthrough_client import PassthroughClient +from provider_edge import LiveEdge, start_provider_edge +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e SERVICE_TIER = "priority" OPENAI_BACKEND: Final = "openai/gpt-5.5" +OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5" +PROMPT_CACHE_RETENTION: Final = "24h" class TestServiceTier: @@ -74,3 +86,153 @@ class TestServiceTier: f"service_tier not honored: sent {SERVICE_TIER!r}, response reported " f"{response.service_tier!r} ({response})" ) + + +class _OutboundRetention(BaseModel, frozen=True): + prompt_cache_retention: str | None = None + + +@dataclass(frozen=True, slots=True) +class _OpenAIBodyCapture: + """The raw bodies sent to one OpenAI path, seen by a live edge in front of + api.openai.com. Pass ``observe`` as the edge's ``observe_request``.""" + + path: str + _bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue) + + def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None: + if body is not None and url.split("?")[0].endswith(self.path): + self._bodies.put(body) + + def retentions(self) -> tuple[str | None, ...]: + bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize())) + return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies) + + +def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str: + """The proxy may run in a container, so its edge binds where the proxy can + reach it; an in-process SDK call reaches the edge on loopback.""" + edge: Final = ( + start_provider_edge( + LiveEdge(observe_request=capture.observe), + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + ) + if reachable_from_proxy + else start_provider_edge(LiveEdge(observe_request=capture.observe)) + ) + resources.defer(edge.shutdown) + return f"{edge.edge.api_base('openai')}/v1" + + +def _retention_backends() -> tuple[object, ...]: + """gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge + (chat completions in, Responses out) to /v1/responses.""" + return ( + pytest.param( + OPENAI_BACKEND, + "/chat/completions", + marks=( + pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + ), + id="chat_completions", + ), + pytest.param( + OPENAI_RESPONSES_BRIDGE_BACKEND, + "/responses", + marks=( + pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.OPENAI,), + models=(OPENAI_RESPONSES_BRIDGE_BACKEND,), + mode=Mode.NONSTREAM, + ) + ), + ), + id="responses_bridge", + ), + ) + + +class TestPromptCacheRetention: + @pytest.mark.provider_edge_host + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai( + self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True) + model: Final = f"e2e-cache-retention-{unique_marker()}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base) + ) + resources.defer(lambda: proxy.delete_model(model_id)) + + unwrap( + proxy.chat( + resources.key(), + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="reply with one word")], + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ), + ) + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) + + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_sdk_completion_forwards_prompt_cache_retention_to_openai( + self, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + import litellm + + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False) + + litellm.completion( + model=backend, + messages=[{"role": "user", "content": "reply with one word"}], + api_base=api_base, + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) + + @pytest.mark.parametrize(("backend", "openai_path"), _retention_backends()) + def test_router_completion_forwards_prompt_cache_retention_to_openai( + self, resources: ResourceManager, backend: str, openai_path: str + ) -> None: + import litellm + + capture: Final = _OpenAIBodyCapture(openai_path) + api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False) + router: Final = litellm.Router( + model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}] + ) + + router.completion( # pyright: ignore[reportUnknownMemberType] # Router.completion forwards an untyped **kwargs to litellm.completion + model="cache-retention", + messages=[{"role": "user", "content": "reply with one word"}], + prompt_cache_retention=PROMPT_CACHE_RETENTION, + ) + + assert capture.retentions() == (PROMPT_CACHE_RETENTION,), ( + f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f0a8d67b444..afddc7e25d6 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -364,6 +364,7 @@ class ChatBody(BaseModel): thinking: ThinkingParam | None = None service_tier: str | None = None prompt_cache_key: str | None = None + prompt_cache_retention: str | None = None tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 3e5cfeb455e..f5d13aecb6f 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -23,6 +23,7 @@ import respx from openai import APITimeoutError, AsyncOpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion +from pydantic import TypeAdapter import litellm from litellm import ( @@ -3937,6 +3938,100 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" +_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} + +# Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. +_OPENAI_CLIENT_SETTINGS: Final = frozenset( + { + "api_base", + "api_key", + "api_version", + "base_url", + "default_headers", + "deployment_id", + "extra_headers", + "max_retries", + "organization", + "request_timeout", + "timeout", + } +) + +# Accepted by completion() for other providers, never sent to OpenAI's /chat/completions body. +_NOT_OPENAI_BODY_PARAMS: Final = frozenset({"include_server_side_tool_invocations", "thinking"}) + +# param -> (model that accepts it, the call kwargs that carry it) +_OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] = { + "audio": ("openai/gpt-4o", {"audio": {"voice": "alloy", "format": "wav"}}), + "frequency_penalty": ("openai/gpt-4o", {"frequency_penalty": 0.5}), + "function_call": ("openai/gpt-4o", {"function_call": "auto"}), + "functions": ("openai/gpt-4o", {"functions": [_TOOL["function"]]}), + "logit_bias": ("openai/gpt-4o", {"logit_bias": {"1": 1}}), + "logprobs": ("openai/gpt-4o", {"logprobs": True}), + "max_completion_tokens": ("openai/gpt-4o", {"max_completion_tokens": 7}), + "max_tokens": ("openai/gpt-4o", {"max_tokens": 7}), + "modalities": ("openai/gpt-4o", {"modalities": ["text"]}), + "n": ("openai/gpt-4o", {"n": 2}), + "parallel_tool_calls": ("openai/gpt-4o", {"tools": [_TOOL], "parallel_tool_calls": False}), + "prediction": ("openai/gpt-4o", {"prediction": {"type": "content", "content": "x"}}), + "presence_penalty": ("openai/gpt-4o", {"presence_penalty": 0.5}), + "prompt_cache_key": ("openai/gpt-4o", {"prompt_cache_key": "k"}), + "prompt_cache_retention": ("openai/gpt-5.5", {"prompt_cache_retention": "24h"}), + "reasoning_effort": ("openai/gpt-5.5", {"reasoning_effort": "low"}), + "response_format": ("openai/gpt-4o", {"response_format": {"type": "json_object"}}), + "safety_identifier": ("openai/gpt-4o", {"safety_identifier": "s"}), + "seed": ("openai/gpt-4o", {"seed": 3}), + "service_tier": ("openai/gpt-4o", {"service_tier": "priority"}), + "stop": ("openai/gpt-4o", {"stop": ["x"]}), + "store": ("openai/gpt-4o", {"store": False}), + "stream": ("openai/gpt-4o", {"stream": True}), + "stream_options": ("openai/gpt-4o", {"stream": True, "stream_options": {"include_usage": True}}), + "temperature": ("openai/gpt-4o", {"temperature": 0.5}), + "tool_choice": ("openai/gpt-4o", {"tools": [_TOOL], "tool_choice": "auto"}), + "tools": ("openai/gpt-4o", {"tools": [_TOOL]}), + "top_logprobs": ("openai/gpt-4o", {"logprobs": True, "top_logprobs": 2}), + "top_p": ("openai/gpt-4o", {"top_p": 0.5}), + "user": ("openai/gpt-4o", {"user": "u1"}), + "verbosity": ("openai/gpt-5.5", {"verbosity": "low"}), + "web_search_options": ("openai/gpt-4o", {"web_search_options": {"search_context_size": "low"}}), +} + + +def test_every_openai_body_param_has_a_forwarding_case(): + body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( + "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " + "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " + "(#33184) were listed as supported but silently dropped" + ) + + +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.parametrize("param", sorted(_OPENAI_BODY_PARAM_CASES)) +def test_completion_sends_openai_body_param_to_openai( + param: str, respx_mock: respx.MockRouter, openai_api_response: Mapping[str, object] +): + api_base: Final = "http://localhost:12346/v1" + route: Final = respx_mock.post(f"{api_base}/chat/completions").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + model, call_kwargs = _OPENAI_BODY_PARAM_CASES[param] + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hi"}], + api_base=api_base, + api_key="fake_openai_api_key", + **call_kwargs, + ) + + assert route.call_count == 1 + sent: Final = _REQUEST_BODY.validate_json(route.calls.last.request.content) + assert sent.get(param) == call_kwargs[param], f"{param} did not reach the OpenAI request body: {sent}" + + def test_completion_omits_store_and_prompt_cache_key_when_not_passed(): """ When store and prompt_cache_key are not passed, they must not appear in the From 5ab0130abcdf8725dc67afd8e5f30c023589a1f2 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:50:17 -0700 Subject: [PATCH 115/124] test(integration): call _fallback_deployment in the count_tokens wire tests #47363 added (#47593) #47525 renamed _deployment to _fallback_deployment; #47363 was written against the old name and merged after it, leaving three F821 undefined-name errors that fail the python lint job on main Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../providers/test_bedrock_mantle_count_tokens_wire.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index 063b0e46479..25f46be4c5d 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -1683,8 +1683,8 @@ def test_responses_input_tokens_sends_mantle_the_tool_use_invoke_inference_sends wire_server(_mantle(), port=mantle_port) as mantle, gateway.scenario() as scenario, ): - counted_model: Final = _deployment(scenario, runtime.url) - invoked_model: Final = _deployment(scenario, runtime.url, f"invoke/{_OPUS}") + counted_model: Final = _fallback_deployment(scenario, runtime.url) + invoked_model: Final = _fallback_deployment(scenario, runtime.url, f"invoke/{_OPUS}") generated: Final = gateway.request( "POST", "/v1/responses", {**_FORCED_TOOL_CHOICE, "model": invoked_model, "max_output_tokens": 16} ) @@ -1729,7 +1729,7 @@ def test_nova_count_sends_the_tool_specs_converse_inference_sends(counting_proxy path, body, count_field = _NOVA_COUNTS[route] gateway: Final = counting_proxy.gateway with wire_server(_counting_every_model) as runtime, gateway.scenario() as scenario: - model: Final = _deployment(scenario, runtime.url, _NOVA) + model: Final = _fallback_deployment(scenario, runtime.url, _NOVA) generated: Final = gateway.request( "POST", "/v1/chat/completions", From 2c6b280755ba617cd6647c28706abdd581a7dd19 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 09:51:55 -0700 Subject: [PATCH 116/124] chore: lead the PR template with a short plain description and collapse the rest (#47594) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/PULL_REQUEST_TEMPLATE/general.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index 644d1c4a450..7d920b715e8 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -4,6 +4,15 @@ Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or "## Affected release" with nothing under it must not appear in the final description --> +## Description + + + +
+Details + ## TLDR