mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
feat(proxy): general_settings.disable_requester_ip_address_logging keeps caller IP out of spend logs (#45804)
* feat(proxy): general_settings.disable_requester_ip_address_logging keeps caller IP out of spend logs
When the setting is on, spend log rows store requester_ip_address as null in
the column, the metadata and the stored request (which also drops client-IP
forwarding headers), and IP addresses in stored error messages are masked, so
the Logs page shows no caller IP. The setting is read the way the settings API
validates it. Capture is unchanged, so guardrails, logging callbacks and
metrics still receive the IP. Toggleable from the Admin UI General Settings.
Resolves LIT-7442
* fix(proxy): mask IPv6 addresses that end or start with :: across eight groups
The masking pattern allowed seven colons, so valid forms like 1:2:3:4:5:6:7::
and ::1:2:3:4:5:6:7 stayed in stored error text with
disable_requester_ip_address_logging on. Also lists the request metadata walker
in the recursion check, since it only walks the tree the spend log sanitizer
already returned.
* fix(proxy): mask IPs in every stored header value and after labels or underscores
With disable_requester_ip_address_logging on, client IPs carried in headers outside the forwarding list (do-connecting-ip, incap-client-ip, custom names) and addresses written as host:2001:db8::2, peer_2001:db8::2, socket_192.0.2.1 or an unbracketed IPv6 with a port were still stored in spend logs.
* fix(proxy): mask request header IPs before long values are truncated
A header value longer than the spend log string limit kept the part of an IP that fell before the truncation cut. Request headers in metadata and requester_metadata are now masked before the body is truncated, like error text already was. IPv4 followed by a dotted port (192.0.2.1.443) is masked again.
* fix(proxy): mask the IP piece a header truncation cut leaves instead of pre-masking headers
Masking header values before truncation scanned the whole client value,
including text the cut drops, and walked hook-made header cycles. Mask after
truncation instead, then mask the hex and digit run touching each side of the
truncation marker when it is short enough to be part of an IP address. Every
nested headers mapping gets the same treatment.
* docs(proxy): say the requester IP setting masks IPs in other stored header values
* fix(proxy): mask stored error text after truncation and harden the cut-piece pass
Error messages and tracebacks were regex-scanned in full before the DB cap,
which costs seconds on multi-megabyte provider errors. Mask them after the
cap like header values, with the same pass over the run beside each
truncation marker. That pass now handles every marker, skips long hex tokens
so an address beside one is still masked, keeps at most one address length,
and also masks colon-only IPv6 groups.
* fix(proxy): drop the requester IP from the originating request an auto router classifier row stores
* fix(proxy): drop the requester IP from the classifier input an auto router classifier row stores
* refactor(proxy): mask stored IPs with the content filter's ipv4 and ipv6 patterns
The requester IP setting now reuses the content filter guardrail's prebuilt
ipv4 and ipv6 patterns instead of its own address regexes, so spend logs mask
what that guardrail masks. Addresses glued to a label or underscore stay as
the content filter leaves them.
* fix(proxy): mask stored IPs with the spend-log matcher again
The content filter's ipv4 and ipv6 patterns left labelled IPv6 (host:2001:db8::2),
unbracketed IPv6 with a port and IPv4 after an underscore in stored error text and
header values, and masked parts of text like std::io::Error. Restore the
address-validating matcher and its regression cases.
This reverts commit 79537a8aa5.
---------
Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com>
This commit is contained in:
yucheng-berri
GitHub
parent
20da5b949c
commit
1f4ef4b7d1
8 changed files with 1358 additions and 3 deletions
|
|
@ -2792,6 +2792,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
None,
|
||||
description="If True, stores request messages and responses in spend logs. Default is False.",
|
||||
)
|
||||
disable_requester_ip_address_logging: bool | None = Field(
|
||||
None,
|
||||
description="If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.",
|
||||
)
|
||||
spend_logs_metadata_fields: SpendLogsMetadataFields | None = Field(
|
||||
None,
|
||||
description="Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key",
|
||||
|
|
|
|||
|
|
@ -18635,6 +18635,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
|
|||
"pass_through_endpoints": "PydanticModel",
|
||||
"store_model_in_db": "Boolean",
|
||||
"store_prompts_in_spend_logs": "Boolean",
|
||||
"disable_requester_ip_address_logging": "Boolean",
|
||||
"disable_fallbacks_on_per_model_rate_limits": "Boolean",
|
||||
"maximum_spend_logs_retention_period": "String",
|
||||
"maximum_health_check_retention_period": "String",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
|
@ -30,7 +31,11 @@ from litellm.constants import (
|
|||
from litellm.constants import (
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
|
||||
)
|
||||
from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, without_classifier_audit
|
||||
from litellm.litellm_core_utils.classifier_logging import (
|
||||
CLASSIFIER_AUDIT_FIELDS,
|
||||
classifier_audit_fields,
|
||||
without_classifier_audit,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_litellm_metadata_from_kwargs,
|
||||
proxy_stamped_used_client_oauth_token,
|
||||
|
|
@ -770,6 +775,15 @@ def get_logging_payload(
|
|||
if isinstance(cache_write_tokens, int) and cache_write_tokens > 0:
|
||||
additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens
|
||||
clean_metadata["additional_usage_values"] = additional_usage_values
|
||||
drop_requester_ip_address: Final = requester_ip_address_logging_disabled()
|
||||
error_information: Final = clean_metadata["error_information"]
|
||||
if drop_requester_ip_address:
|
||||
clean_metadata["requester_ip_address"] = None
|
||||
if drop_requester_ip_address and isinstance(error_information, Mapping):
|
||||
clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)),
|
||||
)
|
||||
|
||||
if litellm.cache is None:
|
||||
cache_key = "Cache OFF"
|
||||
|
|
@ -831,6 +845,7 @@ def get_logging_payload(
|
|||
else litellm_params
|
||||
),
|
||||
kwargs=kwargs,
|
||||
drop_requester_ip_address=drop_requester_ip_address,
|
||||
),
|
||||
session_id=_get_session_id_for_spend_log(
|
||||
kwargs=kwargs,
|
||||
|
|
@ -1457,7 +1472,12 @@ def sanitize_error_information_for_spend_logs(
|
|||
sanitized[field] = _redact_prompt_leaks_in_error_string(value)
|
||||
|
||||
sanitized = _sanitize_request_body_for_spend_logs_payload(sanitized)
|
||||
return cast(StandardLoggingPayloadErrorInformation, sanitized)
|
||||
return cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(sanitized))
|
||||
if requester_ip_address_logging_disabled()
|
||||
else sanitized,
|
||||
)
|
||||
|
||||
|
||||
_sanitize_error_information_for_spend_logs: Final = sanitize_error_information_for_spend_logs
|
||||
|
|
@ -1580,10 +1600,146 @@ def _placeholder_stored_request_body(
|
|||
)
|
||||
|
||||
|
||||
_CLIENT_IP_HEADERS: Final = frozenset(
|
||||
{
|
||||
"x-forwarded-for",
|
||||
"x-real-ip",
|
||||
"forwarded",
|
||||
"true-client-ip",
|
||||
"cf-connecting-ip",
|
||||
"x-client-ip",
|
||||
"x-cluster-client-ip",
|
||||
"fastly-client-ip",
|
||||
"x-original-forwarded-for",
|
||||
"x-envoy-external-address",
|
||||
"x-azure-clientip",
|
||||
"cloudfront-viewer-address",
|
||||
"fly-client-ip",
|
||||
"cf-connecting-ipv6",
|
||||
"cf-pseudo-ipv4",
|
||||
"x-appengine-user-ip",
|
||||
"x-vercel-forwarded-for",
|
||||
"x-arr-clientip",
|
||||
}
|
||||
)
|
||||
_REQUEST_METADATA_KEYS: Final = ("metadata", "litellm_metadata")
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[object, object])
|
||||
_OBJECT_LIST: Final = TypeAdapter(list[object])
|
||||
_IP_ADDRESS_CHARACTERS: Final = "0123456789abcdefABCDEF.:"
|
||||
_MAX_IP_ADDRESS_LENGTH: Final = len("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")
|
||||
_LONG_HEX_GROUP: Final = re.compile(r"[0-9A-Fa-f]{5,}")
|
||||
_TRUNCATION_MARKER_GROUP: Final = re.compile(f"({_TRUNCATION_MARKER.pattern})")
|
||||
_IPV4_ADDRESS: Final = re.compile(r"(?<![0-9A-Za-z])(?:\d{1,3}\.){3}\d{1,3}(?![0-9A-Za-z])")
|
||||
_IPV6_ADDRESS_CANDIDATE: Final = re.compile(
|
||||
r"(?:(?<![0-9A-Za-z:])|(?<=\w:))(?:[0-9A-Fa-f]{0,4}:){2,9}[0-9A-Fa-f]{0,4}(?![0-9A-Za-z:])"
|
||||
)
|
||||
|
||||
|
||||
def _masked_ipv6_address(candidate: re.Match[str]) -> str:
|
||||
"""Masks the candidate when it, or it without a leading label or a trailing port group, is an IPv6 address."""
|
||||
text: Final = candidate.group(0)
|
||||
unlabelled: Final = text.partition(":")[2]
|
||||
for address in (text, text.rpartition(":")[0], unlabelled, unlabelled.rpartition(":")[0]):
|
||||
try:
|
||||
ipaddress.IPv6Address(address)
|
||||
except ValueError:
|
||||
continue
|
||||
return REDACTED_BY_LITELM_STRING
|
||||
return text
|
||||
|
||||
|
||||
def _without_ip_addresses(text: str) -> str:
|
||||
return _IPV6_ADDRESS_CANDIDATE.sub(_masked_ipv6_address, _IPV4_ADDRESS.sub(REDACTED_BY_LITELM_STRING, text))
|
||||
|
||||
|
||||
def _is_ip_address_piece(piece: str) -> bool:
|
||||
return any(char.isdigit() or char == ":" for char in piece)
|
||||
|
||||
|
||||
def _without_ip_piece_before_cut(text: str) -> str:
|
||||
run: Final = text[len(text.rstrip(_IP_ADDRESS_CHARACTERS)) :]
|
||||
long_groups: Final = list(_LONG_HEX_GROUP.finditer(run))
|
||||
piece: Final = run[long_groups[-1].end() if long_groups else 0 :][-_MAX_IP_ADDRESS_LENGTH:].lstrip(".:")
|
||||
return text[: len(text) - len(piece)] + REDACTED_BY_LITELM_STRING if _is_ip_address_piece(piece) else text
|
||||
|
||||
|
||||
def _without_ip_piece_after_cut(text: str) -> str:
|
||||
run: Final = text[: len(text) - len(text.lstrip(_IP_ADDRESS_CHARACTERS))]
|
||||
long_group: Final = _LONG_HEX_GROUP.search(run)
|
||||
piece: Final = run[: long_group.start() if long_group else len(run)][:_MAX_IP_ADDRESS_LENGTH].rstrip(".:")
|
||||
return REDACTED_BY_LITELM_STRING + text[len(piece) :] if _is_ip_address_piece(piece) else text
|
||||
|
||||
|
||||
def _without_ip_pieces_beside_cuts(text: str, *, after_cut: bool, before_cut: bool) -> str:
|
||||
after: Final = _without_ip_piece_after_cut(text) if after_cut else text
|
||||
return _without_ip_piece_before_cut(after) if before_cut else after
|
||||
|
||||
|
||||
def _masked_ip_text(text: str) -> str:
|
||||
"""The text with every IP address masked, plus the run of short hex, digit, dot and colon groups beside each
|
||||
truncation marker when that run holds a digit or a colon."""
|
||||
parts: Final = _TRUNCATION_MARKER_GROUP.split(_without_ip_addresses(text))
|
||||
last: Final = len(parts) - 1
|
||||
return "".join(
|
||||
part if index % 2 else _without_ip_pieces_beside_cuts(part, after_cut=index > 0, before_cut=index < last)
|
||||
for index, part in enumerate(parts)
|
||||
)
|
||||
|
||||
|
||||
def _error_information_without_ip_addresses(error_information: Mapping[object, object]) -> Mapping[object, object]:
|
||||
"""The error information with every IP address in its error message and traceback masked."""
|
||||
return {
|
||||
key: _masked_ip_text(value) if key in _SCRUBBED_ERROR_TEXT_FIELDS and isinstance(value, str) else value
|
||||
for key, value in error_information.items()
|
||||
}
|
||||
|
||||
|
||||
def _stored_value_without_requester_ip_address(key: object, value: object) -> object:
|
||||
if key == "requester_ip_address":
|
||||
return None
|
||||
if key == "error_information" and isinstance(value, Mapping):
|
||||
return _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(value))
|
||||
return _without_requester_ip_address(value, headers=key == "headers")
|
||||
|
||||
|
||||
def _without_requester_ip_address(value: object, *, headers: bool = False) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
return {
|
||||
key: _masked_ip_text(item)
|
||||
if headers and isinstance(item, str)
|
||||
else _stored_value_without_requester_ip_address(key, item)
|
||||
for key, item in _OBJECT_MAPPING.validate_python(value).items()
|
||||
if not (headers and isinstance(key, str) and key.lower() in _CLIENT_IP_HEADERS)
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_without_requester_ip_address(item) for item in _OBJECT_LIST.validate_python(value)]
|
||||
return value
|
||||
|
||||
|
||||
def _request_metadata_without_requester_ip_address(request_body: Mapping[object, object]) -> Mapping[object, object]:
|
||||
return {
|
||||
key: _without_requester_ip_address(value) if key in _REQUEST_METADATA_KEYS else value
|
||||
for key, value in request_body.items()
|
||||
}
|
||||
|
||||
|
||||
def _stored_request_without_requester_ip_address(request_body: Mapping[object, object]) -> Mapping[object, object]:
|
||||
"""The request body, and the classifier input and originating request a classifier row stores in it, with
|
||||
`requester_ip_address` set to null, client-IP forwarding headers dropped and every IP address in the other header
|
||||
values and in the error text masked inside their metadata."""
|
||||
return {
|
||||
key: _request_metadata_without_requester_ip_address(_OBJECT_MAPPING.validate_python(value))
|
||||
if key in CLASSIFIER_AUDIT_FIELDS and isinstance(value, Mapping)
|
||||
else value
|
||||
for key, value in _request_metadata_without_requester_ip_address(request_body).items()
|
||||
}
|
||||
|
||||
|
||||
def _get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
kwargs: dict | None = None,
|
||||
drop_requester_ip_address: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Only store if should_store_prompts_and_responses_in_spend_logs() is True
|
||||
|
|
@ -1624,7 +1780,12 @@ def _get_proxy_server_request_for_spend_logs_payload(
|
|||
perform_redaction(model_call_details=_request_body, result=None)
|
||||
|
||||
_request_body = _sanitize_request_body_for_spend_logs_payload(_request_body)
|
||||
_request_body_json_str: Final = safe_dumps(_request_body)
|
||||
stored_request_body: Final = _OBJECT_MAPPING.validate_python(_request_body)
|
||||
_request_body_json_str: Final = safe_dumps(
|
||||
_stored_request_without_requester_ip_address(stored_request_body)
|
||||
if drop_requester_ip_address
|
||||
else stored_request_body
|
||||
)
|
||||
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
|
||||
verbose_proxy_logger.info(
|
||||
"Spend Log: request body was truncated before storing in DB. %s",
|
||||
|
|
@ -1749,6 +1910,22 @@ def configured_spend_logs_metadata_fields() -> SpendLogsMetadataFields | None:
|
|||
return None
|
||||
|
||||
|
||||
_OPTIONAL_BOOL_ADAPTER: Final[TypeAdapter[bool | None]] = TypeAdapter(bool | None)
|
||||
|
||||
|
||||
def requester_ip_address_logging_disabled() -> bool:
|
||||
"""True when `general_settings.disable_requester_ip_address_logging` holds a value the settings API accepts as true."""
|
||||
from litellm.proxy.proxy_server import general_settings_view
|
||||
|
||||
try:
|
||||
return (
|
||||
_OPTIONAL_BOOL_ADAPTER.validate_python(general_settings_view().get("disable_requester_ip_address_logging"))
|
||||
is True
|
||||
)
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
|
||||
def spend_log_row_with_retained_metadata(
|
||||
row: Mapping[str, object], fields: SpendLogsMetadataFields | None
|
||||
) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
"scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
|
||||
"_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did.
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
742
tests/integration/spend/test_requester_ip_address_logging.py
Normal file
742
tests/integration/spend/test_requester_ip_address_logging.py
Normal file
|
|
@ -0,0 +1,742 @@
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
list_value,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.database import read_rows, scratch_database
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
FORWARDED_IP: Final = "203.0.113.74"
|
||||
PROXY_HOP: Final = "198.51.100.23"
|
||||
REAL_IP: Final = "203.0.113.75"
|
||||
SETTING: Final = "disable_requester_ip_address_logging"
|
||||
AKTO_KEY: Final = "synthetic-akto-key"
|
||||
AKTO_GUARDRAIL: Final = "akto-requester-ip"
|
||||
PROVIDER_FAILURE: Final = "synthetic-requester-ip-provider-failure"
|
||||
LONG_PROVIDER_FAILURE: Final = "synthetic-requester-ip-long-provider-failure"
|
||||
UPSTREAM_IP: Final = "192.0.2.61"
|
||||
LONG_HEADER_IP: Final = "203.0.113.79"
|
||||
ADDRESS_FORMS_FAILURE: Final = "synthetic-requester-ip-address-forms-failure"
|
||||
UPSTREAM_ADDRESS_FORMS: Final = MappingProxyType(
|
||||
{
|
||||
"2001:db8::62": "connect to host:2001:db8::62 failed",
|
||||
"2001:db8::63": "peer_2001:db8::63",
|
||||
"2001:db8:1:2:3:4:5:64": "upstream 2001:db8:1:2:3:4:5:64:443 refused",
|
||||
"192.0.2.65": "socket_192.0.2.65 closed",
|
||||
}
|
||||
)
|
||||
CLIENT_IP_HEADERS: Final = MappingProxyType(
|
||||
{
|
||||
"203.0.113.76": ("do-connecting-ip", "203.0.113.76"),
|
||||
"203.0.113.77": ("incap-client-ip", "203.0.113.77"),
|
||||
"2001:db8::78": ("x-edge-client", "client=2001:db8::78"),
|
||||
}
|
||||
)
|
||||
CALLBACK_SECRET: Final = "synthetic-callback-sink-secret"
|
||||
_ROW_COLUMNS: Final = (
|
||||
"SELECT s.request_id, s.metadata, s.requester_ip_address, row_to_json(s)::text AS row_text "
|
||||
'FROM "LiteLLM_SpendLogs" s'
|
||||
)
|
||||
|
||||
|
||||
def _config(tmp_path: Path, general_settings: Mapping[str, JsonValue], **sections: JsonValue) -> Path:
|
||||
config: Final = tmp_path / f"requester-ip-{uuid4()}.json"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [],
|
||||
**sections,
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"store_model_in_db": True,
|
||||
"proxy_batch_write_at": 1,
|
||||
"proxy_batch_polling_interval": 1,
|
||||
"use_x_forwarded_for": True,
|
||||
**general_settings,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _caller(scenario: Scenario) -> tuple[str, str, str]:
|
||||
model: Final = scenario.model()
|
||||
user_id: Final = scenario.user(user_role="internal_user")
|
||||
return model, user_id, scenario.key(user_id=user_id, models=[model])
|
||||
|
||||
|
||||
def _chat(proxy: Gateway, model: str, key: str) -> str:
|
||||
response: Final = proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "requester ip"}]},
|
||||
key=key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
return string_value(object_value(response.json())["id"])
|
||||
|
||||
|
||||
def _stored_row(request_id: str, database_url: str | None = None) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(f"{_ROW_COLUMNS} WHERE s.request_id=%s", (request_id,), database_url=database_url),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _served_row(proxy: Gateway, key: str, request_id: str, since: datetime) -> tuple[dict[str, JsonValue], str]:
|
||||
response: Final = proxy.request(
|
||||
"GET",
|
||||
"/spend/logs/ui",
|
||||
key=key,
|
||||
params={
|
||||
"request_id": request_id,
|
||||
"start_date": (since - timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"end_date": (since + timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
served: Final = tuple(
|
||||
object_value(row)
|
||||
for row in list_value(object_value(response.json())["data"])
|
||||
if object_value(row)["request_id"] == request_id
|
||||
)
|
||||
assert len(served) == 1, response.text
|
||||
return served[0], response.text
|
||||
|
||||
|
||||
def _assert_recorded(row: Mapping[str, JsonValue], user_id: str) -> None:
|
||||
metadata: Final = object_value(row["metadata"])
|
||||
assert metadata["user_api_key_user_id"] == user_id, metadata
|
||||
assert row["requester_ip_address"] == FORWARDED_IP, row
|
||||
assert metadata["requester_ip_address"] == FORWARDED_IP, metadata
|
||||
|
||||
|
||||
def _assert_not_recorded(row: Mapping[str, JsonValue], text: str, user_id: str) -> None:
|
||||
metadata: Final = object_value(row["metadata"])
|
||||
assert metadata["user_api_key_user_id"] == user_id, metadata
|
||||
assert row["requester_ip_address"] is None, row
|
||||
assert metadata.get("requester_ip_address") is None, metadata
|
||||
assert FORWARDED_IP not in text, text
|
||||
|
||||
|
||||
def test_allowed_ips_denial_row_masks_the_ip_while_the_caller_still_sees_it(gateway: Gateway, tmp_path: Path) -> None:
|
||||
user_agent: Final = f"requester-ip-allowed-ips-{uuid4()}"
|
||||
config: Final = _config(tmp_path, {SETTING: True, "allowed_ips": ["127.0.0.1"]})
|
||||
with owned_proxy(gateway, tmp_path, {}, config=config) as proxy, proxy.scenario() as scenario:
|
||||
model, _, key = _caller(scenario)
|
||||
response: Final = proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "requester ip"}]},
|
||||
key=key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP, "user-agent": user_agent},
|
||||
)
|
||||
assert response.status_code == 403, response.text
|
||||
assert FORWARDED_IP in response.text, response.text
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(f"{_ROW_COLUMNS} WHERE s.metadata->>'user_agent' = %s", (user_agent,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
denied: Final = rows[0]
|
||||
error_information: Final = object_value(object_value(denied["metadata"])["error_information"])
|
||||
assert object_value(denied["metadata"])["status"] == "failure", denied
|
||||
assert denied["requester_ip_address"] is None, denied
|
||||
assert "IP address REDACTED_BY_LITELM not allowed" in string_value(error_information["error_message"]), denied
|
||||
assert FORWARDED_IP not in string_value(denied["row_text"]), denied
|
||||
|
||||
|
||||
def test_admin_setting_change_applies_to_the_next_request(gateway: Gateway, tmp_path: Path) -> None:
|
||||
since: Final = datetime.now(UTC)
|
||||
with (
|
||||
scratch_database() as database_url,
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"DATABASE_URL": database_url},
|
||||
config=_config(tmp_path, {}),
|
||||
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
||||
) as proxy,
|
||||
proxy.scenario() as scenario,
|
||||
):
|
||||
model, user_id, key = _caller(scenario)
|
||||
before: Final = _chat(proxy, model, key)
|
||||
_assert_recorded(_stored_row(before, database_url), user_id)
|
||||
|
||||
proxy.post(
|
||||
"/config/field/update",
|
||||
{"field_name": SETTING, "field_value": True, "config_type": "general_settings"},
|
||||
)
|
||||
listed: Final = tuple(
|
||||
object_value(field)
|
||||
for field in list_value(
|
||||
proxy.request("GET", "/config/list", params={"config_type": "general_settings"}).json()
|
||||
)
|
||||
if object_value(field)["field_name"] == SETTING
|
||||
)
|
||||
assert len(listed) == 1, listed
|
||||
assert listed[0]["field_type"] == "Boolean" and listed[0]["field_value"] is True, listed
|
||||
|
||||
disabled: Final = _chat(proxy, model, key)
|
||||
stored_disabled: Final = _stored_row(disabled, database_url)
|
||||
served_disabled, served_disabled_text = _served_row(proxy, key, disabled, since)
|
||||
|
||||
proxy.post(
|
||||
"/config/field/update",
|
||||
{"field_name": SETTING, "field_value": False, "config_type": "general_settings"},
|
||||
)
|
||||
enabled: Final = _chat(proxy, model, key)
|
||||
stored_enabled: Final = _stored_row(enabled, database_url)
|
||||
|
||||
_assert_not_recorded(stored_disabled, string_value(stored_disabled["row_text"]), user_id)
|
||||
_assert_not_recorded(served_disabled, served_disabled_text, user_id)
|
||||
_assert_recorded(stored_enabled, user_id)
|
||||
|
||||
|
||||
def _completion_reply(marker: str, streaming: bool) -> Reply:
|
||||
if not streaming:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-" + marker,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
chunks: Final = (
|
||||
{"choices": [{"index": 0, "delta": {"role": "assistant", "content": marker}, "finish_reason": None}]},
|
||||
{"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{"choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}},
|
||||
)
|
||||
frames: Final = tuple(
|
||||
(
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-" + marker,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
**chunk,
|
||||
}
|
||||
)
|
||||
+ "\n\n"
|
||||
).encode()
|
||||
for chunk in chunks
|
||||
)
|
||||
return Reply(chunks=(*frames, b"data: [DONE]\n\n"), content_type="text/event-stream")
|
||||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target.endswith("/v1/models"):
|
||||
return Reply(body=json.dumps({"object": "list", "data": []}).encode())
|
||||
body: Final = object_value(json.loads(request.body))
|
||||
if request.target.endswith("/v1/embeddings"):
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
if request.target.endswith("/v1/messages"):
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"msg_{uuid4().hex}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "requester ip"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
if request.target.endswith("/v1/responses"):
|
||||
identity: Final = uuid4().hex
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"resp_{identity}",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": f"msg_{identity}",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "requester ip", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
assert request.target.endswith("/v1/chat/completions"), request.target
|
||||
if "response_format" in body:
|
||||
return _classifier_verdict()
|
||||
if LONG_PROVIDER_FAILURE.encode() in request.body:
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"message": f"upstream {UPSTREAM_IP} " * 400, "type": "invalid_request_error"}}
|
||||
).encode(),
|
||||
)
|
||||
if ADDRESS_FORMS_FAILURE.encode() in request.body:
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"message": ", ".join(UPSTREAM_ADDRESS_FORMS.values()), "type": "invalid_request_error"}}
|
||||
).encode(),
|
||||
)
|
||||
if PROVIDER_FAILURE.encode() in request.body:
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps({"error": {"message": PROVIDER_FAILURE, "type": "invalid_request_error"}}).encode(),
|
||||
)
|
||||
return _completion_reply(uuid4().hex, body.get("stream") is True)
|
||||
|
||||
|
||||
def _classifier_verdict() -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"chatcmpl-{uuid4().hex}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": json.dumps({"tier": "SIMPLE"})},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _akto_allows(_: Request) -> Reply:
|
||||
return Reply(body=json.dumps({"data": {"guardrailsResult": {"Allowed": True}}}).encode())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: Gateway
|
||||
akto: Wire
|
||||
callbacks: Wire
|
||||
key: str
|
||||
chat_model: str
|
||||
claude_model: str
|
||||
embedding_model: str
|
||||
router_model: str
|
||||
|
||||
def send(self, path: str, body: Mapping[str, JsonValue], forwarded_for: str) -> str:
|
||||
response: Final = self.proxy.request(
|
||||
"POST", path, body, key=self.key, headers={"x-forwarded-for": forwarded_for, "x-real-ip": REAL_IP}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
return response.headers["x-litellm-call-id"]
|
||||
|
||||
def callback_metadata(self, call_id: str) -> dict[str, JsonValue]:
|
||||
received: Final[
|
||||
list[Request]
|
||||
] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches
|
||||
|
||||
def delivered() -> tuple[dict[str, JsonValue], ...]:
|
||||
received.extend(self.callbacks.drain())
|
||||
return tuple(
|
||||
event
|
||||
for batch in received
|
||||
for event in (object_value(item) for item in list_value(json.loads(batch.body)))
|
||||
if event.get("litellm_call_id") == call_id
|
||||
)
|
||||
|
||||
events: Final = eventually(delivered, lambda values: len(values) == 1, seconds=20)
|
||||
return object_value(events[0]["metadata"])
|
||||
|
||||
|
||||
def _row_for_call(call_id: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(f"{_ROW_COLUMNS} WHERE s.litellm_call_id=%s", (call_id,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _auto_router_models(router: str, provider: Wire) -> list[JsonValue]:
|
||||
deployment: Final[JsonValue] = {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": provider.url + "/v1",
|
||||
"api_key": "synthetic-openai-key",
|
||||
}
|
||||
return [
|
||||
{"model_name": f"{router}-judge", "litellm_params": deployment},
|
||||
{"model_name": f"{router}-tier", "litellm_params": deployment},
|
||||
{
|
||||
"model_name": router,
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {"model": f"{router}-judge", "timeout_ms": 20000},
|
||||
"tiers": {tier: f"{router}-tier" for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _rig(stack: ExitStack, gateway: Gateway, root: Path, provider: Wire, disabled: bool) -> Rig:
|
||||
akto: Final = stack.enter_context(wire_server(_akto_allows))
|
||||
callbacks: Final = stack.enter_context(wire_server(lambda _: Reply()))
|
||||
router: Final = f"requester-ip-router-{uuid4().hex[:8]}"
|
||||
config: Final = _config(
|
||||
root,
|
||||
{SETTING: True, "store_prompts_in_spend_logs": True} if disabled else {"store_prompts_in_spend_logs": True},
|
||||
litellm_settings={"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1},
|
||||
guardrails=[
|
||||
{
|
||||
"guardrail_name": AKTO_GUARDRAIL,
|
||||
"litellm_params": {
|
||||
"guardrail": "akto",
|
||||
"mode": "pre_call",
|
||||
"default_on": False,
|
||||
"akto_base_url": akto.url,
|
||||
"akto_api_key": AKTO_KEY,
|
||||
},
|
||||
}
|
||||
],
|
||||
model_list=_auto_router_models(router, provider),
|
||||
)
|
||||
proxy: Final = stack.enter_context(
|
||||
owned_proxy(
|
||||
gateway,
|
||||
root,
|
||||
{
|
||||
"GENERIC_LOGGER_ENDPOINT": callbacks.url,
|
||||
"GENERIC_LOGGER_HEADERS": f"Authorization=Bearer {CALLBACK_SECRET}",
|
||||
},
|
||||
config=config,
|
||||
workers=2,
|
||||
)
|
||||
)
|
||||
scenario: Final = stack.enter_context(proxy.scenario())
|
||||
chat: Final = scenario.model(api_base=provider.url + "/v1", api_key="synthetic-openai-key")
|
||||
claude: Final = scenario.model(
|
||||
model="anthropic/claude-haiku-5-5", api_base=provider.url, api_key="synthetic-anthropic-key"
|
||||
)
|
||||
embedding: Final = scenario.model(
|
||||
model="openai/text-embedding-3-small",
|
||||
api_base=provider.url + "/v1",
|
||||
api_key="synthetic-openai-key",
|
||||
model_info={"mode": "embedding"},
|
||||
)
|
||||
key: Final = scenario.key(models=[chat, claude, embedding, router, f"{router}-judge", f"{router}-tier"])
|
||||
return Rig(proxy, akto, callbacks, key, chat, claude, embedding, router)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rigs(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Mapping[bool, Rig]]:
|
||||
with ExitStack() as stack:
|
||||
gateway: Final = stack.enter_context(gateway_from_environment())
|
||||
provider: Final = stack.enter_context(wire_server(_provider))
|
||||
yield MappingProxyType(
|
||||
{
|
||||
disabled: _rig(stack, gateway, tmp_path_factory.mktemp("requester-ip"), provider, disabled)
|
||||
for disabled in (False, True)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _endpoint_request(rig: Rig, endpoint: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
if endpoint == "embeddings":
|
||||
return "/v1/embeddings", {"model": rig.embedding_model, "input": marker}
|
||||
if endpoint == "responses":
|
||||
return "/v1/responses", {"model": rig.chat_model, "input": marker}
|
||||
if endpoint == "messages":
|
||||
return "/v1/messages", {
|
||||
"model": rig.claude_model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
}
|
||||
return "/v1/chat/completions", {
|
||||
"model": rig.chat_model,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"stream": endpoint == "chat-stream",
|
||||
}
|
||||
|
||||
|
||||
def _ips_in_row(row: Mapping[str, JsonValue]) -> tuple[str, ...]:
|
||||
return tuple(ip for ip in (FORWARDED_IP, PROXY_HOP, REAL_IP) if ip in string_value(row["row_text"]))
|
||||
|
||||
|
||||
DISABLED: Final = pytest.mark.parametrize("disabled", [False, True], ids=["setting-unset", "setting-on"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("forwarded_for", [FORWARDED_IP, f"{FORWARDED_IP}, {PROXY_HOP}"], ids=["single", "chain"])
|
||||
@pytest.mark.parametrize("endpoint", ["chat", "chat-stream", "messages", "responses", "embeddings"])
|
||||
@DISABLED
|
||||
def test_spend_logs_follow_the_setting_on_every_endpoint_while_callbacks_keep_the_ip(
|
||||
rigs: Mapping[bool, Rig], disabled: bool, endpoint: str, forwarded_for: str
|
||||
) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
path, body = _endpoint_request(rig, endpoint, f"requester-ip-{uuid4().hex}")
|
||||
call_id: Final = rig.send(path, body, forwarded_for)
|
||||
|
||||
row: Final = _row_for_call(call_id)
|
||||
callback_metadata: Final = rig.callback_metadata(call_id)
|
||||
|
||||
recorded_ip: Final = None if disabled else forwarded_for
|
||||
assert callback_metadata["requester_ip_address"] == forwarded_for, callback_metadata
|
||||
assert row["requester_ip_address"] == recorded_ip, row
|
||||
assert object_value(row["metadata"])["requester_ip_address"] == recorded_ip, row
|
||||
assert (REAL_IP in _ips_in_row(row)) is not disabled, row
|
||||
assert (_ips_in_row(row) == ()) is disabled, row
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_logs_page_serves_no_ip_when_the_setting_is_on(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
since: Final = datetime.now(UTC)
|
||||
call_id: Final = rig.send(
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": f"requester-ip-{uuid4().hex}"}]},
|
||||
FORWARDED_IP,
|
||||
)
|
||||
request_id: Final = string_value(_row_for_call(call_id)["request_id"])
|
||||
window: Final = {
|
||||
"start_date": (since - timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"end_date": (since + timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
listed: Final = rig.proxy.request("GET", "/spend/logs/ui", params={"request_id": request_id, **window})
|
||||
detail: Final = rig.proxy.request("GET", f"/spend/logs/ui/{request_id}", params=window)
|
||||
|
||||
assert listed.status_code == 200 and detail.status_code == 200, (listed.text, detail.text)
|
||||
assert request_id in listed.text and request_id in detail.text, (listed.text, detail.text)
|
||||
served: Final = listed.text + detail.text
|
||||
assert (FORWARDED_IP in served) is not disabled, served
|
||||
assert (REAL_IP in served) is not disabled, served
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_auth_failure_row_follows_the_setting(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
user_agent: Final = f"requester-ip-auth-failure-{uuid4()}"
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": "requester ip"}]},
|
||||
key=f"sk-requester-ip-{uuid4().hex}",
|
||||
headers={"x-forwarded-for": FORWARDED_IP, "user-agent": user_agent},
|
||||
)
|
||||
assert response.status_code == 401, response.text
|
||||
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(f"{_ROW_COLUMNS} WHERE s.metadata->>'user_agent' = %s", (user_agent,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
assert object_value(rows[0]["metadata"])["status"] == "failure", rows[0]
|
||||
assert rows[0]["requester_ip_address"] == (None if disabled else FORWARDED_IP), rows[0]
|
||||
assert _ips_in_row(rows[0]) == (() if disabled else (FORWARDED_IP,)), rows[0]
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_provider_failure_row_follows_the_setting(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": PROVIDER_FAILURE}]},
|
||||
key=rig.key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP, "x-real-ip": REAL_IP},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
|
||||
row: Final = _row_for_call(response.headers["x-litellm-call-id"])
|
||||
|
||||
assert object_value(row["metadata"])["status"] == "failure", row
|
||||
assert row["requester_ip_address"] == (None if disabled else FORWARDED_IP), row
|
||||
assert _ips_in_row(row) == (() if disabled else (FORWARDED_IP, REAL_IP)), row
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_provider_error_row_masks_addresses_after_labels_and_before_ports(
|
||||
rigs: Mapping[bool, Rig], disabled: bool
|
||||
) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": ADDRESS_FORMS_FAILURE}]},
|
||||
key=rig.key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
|
||||
row_text: Final = string_value(_row_for_call(response.headers["x-litellm-call-id"])["row_text"])
|
||||
|
||||
assert [ip for ip in UPSTREAM_ADDRESS_FORMS if ip in row_text] == ([] if disabled else [*UPSTREAM_ADDRESS_FORMS]), (
|
||||
row_text
|
||||
)
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_client_ip_headers_outside_the_known_list_follow_the_setting(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": f"requester-ip-{uuid4().hex}"}]},
|
||||
key=rig.key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP, **dict(CLIENT_IP_HEADERS.values())},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
row: Final = _row_for_call(response.headers["x-litellm-call-id"])
|
||||
stored_headers: Final = object_value(
|
||||
object_value(object_value(json.loads(string_value(row["row_text"]))["proxy_server_request"])["metadata"])[
|
||||
"headers"
|
||||
]
|
||||
)
|
||||
|
||||
assert [name for name, _ in CLIENT_IP_HEADERS.values() if name in stored_headers] == [
|
||||
name for name, _ in CLIENT_IP_HEADERS.values()
|
||||
], stored_headers
|
||||
assert [ip for ip in CLIENT_IP_HEADERS if ip in string_value(row["row_text"])] == (
|
||||
[] if disabled else [*CLIENT_IP_HEADERS]
|
||||
), row
|
||||
|
||||
|
||||
def test_long_provider_error_row_keeps_no_ip_fragment_at_the_truncation_cut(rigs: Mapping[bool, Rig]) -> None:
|
||||
rig: Final = rigs[True]
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": LONG_PROVIDER_FAILURE}]},
|
||||
key=rig.key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
|
||||
row: Final = _row_for_call(response.headers["x-litellm-call-id"])
|
||||
error_message: Final = string_value(
|
||||
object_value(object_value(row["metadata"])["error_information"])["error_message"]
|
||||
)
|
||||
cut: Final = re.search(r"\.\.\. \(litellm_truncated .*?\) \.\.\.", error_message)
|
||||
|
||||
assert cut is not None, error_message
|
||||
assert re.search(r"[\d.]$", error_message[: cut.start()]) is None, error_message[: cut.start()][-60:]
|
||||
assert re.search(r"^[\d.]", error_message[cut.end() :]) is None, error_message[cut.end() :][:60]
|
||||
assert UPSTREAM_IP not in string_value(row["row_text"]), row
|
||||
|
||||
|
||||
def test_long_header_value_keeps_no_ip_fragment_at_the_truncation_cut(rigs: Mapping[bool, Rig]) -> None:
|
||||
rig: Final = rigs[True]
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": f"requester-ip-{uuid4().hex}"}]},
|
||||
key=rig.key,
|
||||
headers={"x-forwarded-for": FORWARDED_IP, "x-edge-trace": "a" * 710 + f" {LONG_HEADER_IP} " + "b" * 3000},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
row_text: Final = string_value(_row_for_call(response.headers["x-litellm-call-id"])["row_text"])
|
||||
stored_request: Final = object_value(json.loads(row_text)["proxy_server_request"])
|
||||
stored_value: Final = string_value(
|
||||
object_value(object_value(stored_request["metadata"])["headers"])["x-edge-trace"]
|
||||
)
|
||||
cut: Final = re.search(r"\.\.\. \(litellm_truncated .*?\) \.\.\.", stored_value)
|
||||
|
||||
assert cut is not None, stored_value
|
||||
assert re.search(r"[\d.]$", stored_value[: cut.start()]) is None, stored_value[: cut.start()][-60:]
|
||||
assert LONG_HEADER_IP not in row_text, row_text
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_auto_router_classifier_row_follows_the_setting(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
marker: Final = f"requester-ip-classifier-{uuid4().hex}"
|
||||
rig.send(
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.router_model, "messages": [{"role": "user", "content": marker}]},
|
||||
FORWARDED_IP,
|
||||
)
|
||||
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
f"{_ROW_COLUMNS} WHERE s.proxy_server_request::text LIKE %s AND s.proxy_server_request::text LIKE %s",
|
||||
("%originating_request_masked%", f"%{marker}%"),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
stored_request: Final = object_value(json.loads(string_value(rows[0]["row_text"]))["proxy_server_request"])
|
||||
|
||||
assert marker in json.dumps(stored_request["originating_request_masked"]), stored_request
|
||||
assert (FORWARDED_IP in _ips_in_row(rows[0])) is not disabled, rows[0]
|
||||
assert (_ips_in_row(rows[0]) == ()) is disabled, rows[0]
|
||||
|
||||
|
||||
@DISABLED
|
||||
def test_pre_call_guardrail_still_receives_the_ip(rigs: Mapping[bool, Rig], disabled: bool) -> None:
|
||||
rig: Final = rigs[disabled]
|
||||
marker: Final = f"requester-ip-{uuid4().hex}"
|
||||
call_id: Final = rig.send(
|
||||
"/v1/chat/completions",
|
||||
{"model": rig.chat_model, "messages": [{"role": "user", "content": marker}], "guardrails": [AKTO_GUARDRAIL]},
|
||||
f"{FORWARDED_IP}, {PROXY_HOP}",
|
||||
)
|
||||
|
||||
row: Final = _row_for_call(call_id)
|
||||
akto_calls: Final = tuple(
|
||||
object_value(json.loads(call.body)) for call in rig.akto.drain() if marker.encode() in call.body
|
||||
)
|
||||
|
||||
assert [call["ip"] for call in akto_calls] == [FORWARDED_IP], akto_calls
|
||||
assert object_value(row["metadata"])["applied_guardrails"] == [AKTO_GUARDRAIL], row
|
||||
|
|
@ -40,6 +40,8 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
|
|||
configured_spend_logs_metadata_fields,
|
||||
get_logging_payload,
|
||||
get_spend_logs_id,
|
||||
_stored_request_without_requester_ip_address,
|
||||
requester_ip_address_logging_disabled,
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
spend_log_row_with_retained_metadata,
|
||||
)
|
||||
|
|
@ -6470,3 +6472,395 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch: pytest.MonkeyPatch
|
|||
)
|
||||
assert payload_disabled["messages"] == "{}"
|
||||
assert payload_disabled["response"] == "{}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "disabled"),
|
||||
[
|
||||
(True, True),
|
||||
("true", True),
|
||||
("TRUE", True),
|
||||
("yes", True),
|
||||
("on", True),
|
||||
("1", True),
|
||||
(1, True),
|
||||
(False, False),
|
||||
(None, False),
|
||||
("false", False),
|
||||
("no", False),
|
||||
(0, False),
|
||||
("", False),
|
||||
("maybe", False),
|
||||
([1], False),
|
||||
],
|
||||
)
|
||||
def test_requester_ip_address_logging_disabled_reads_the_setting_as_the_settings_api_validates_it(
|
||||
configured: object, disabled: bool
|
||||
) -> None:
|
||||
with patch( # test-quality-ok: general_settings is proxy config, loaded from yaml, not an HTTP boundary
|
||||
"litellm.proxy.proxy_server.general_settings", {"disable_requester_ip_address_logging": configured}
|
||||
):
|
||||
assert requester_ip_address_logging_disabled() is disabled
|
||||
|
||||
|
||||
def _request_ip_metadata() -> dict[str, object]:
|
||||
return {
|
||||
"requester_ip_address": "203.0.113.9",
|
||||
"user_agent": "requester-ip-test",
|
||||
"headers": {"X-Forwarded-For": "203.0.113.9, 10.0.0.1", "x-real-ip": "203.0.113.9", "accept": "*/*"},
|
||||
"requester_metadata": {"headers": {"x-forwarded-for": "203.0.113.9"}, "team": "blue"},
|
||||
"error_information": {
|
||||
"error_code": "403",
|
||||
"error_message": "Access forbidden: IP address 203.0.113.9 not allowed.",
|
||||
"traceback": "HTTPException: client 2001:db8::9 via 198.51.100.7 denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _request_ip_kwargs(request_metadata: dict[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user", "content": "Hello!"}],
|
||||
"litellm_params": {
|
||||
"metadata": request_metadata,
|
||||
"proxy_server_request": {
|
||||
"body": {
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user", "content": "Hello!"}],
|
||||
"metadata": request_metadata,
|
||||
}
|
||||
},
|
||||
},
|
||||
"standard_logging_object": {
|
||||
"messages": [{"role": "user", "content": "Hello!"}],
|
||||
"response": {"role": "assistant", "content": "Hi there!"},
|
||||
"metadata": {"requester_ip_address": "203.0.113.9"},
|
||||
"model_map_information": {"tpm": 1000, "rpm": 1000},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disabled", "recorded_ip", "error_message", "traceback"),
|
||||
[
|
||||
(
|
||||
False,
|
||||
"203.0.113.9",
|
||||
"Access forbidden: IP address 203.0.113.9 not allowed.",
|
||||
"HTTPException: client 2001:db8::9 via 198.51.100.7 denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
|
||||
),
|
||||
(
|
||||
True,
|
||||
None,
|
||||
"Access forbidden: IP address REDACTED_BY_LITELM not allowed.",
|
||||
"HTTPException: client REDACTED_BY_LITELM via REDACTED_BY_LITELM denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
|
||||
),
|
||||
],
|
||||
ids=["setting-off", "setting-on"],
|
||||
)
|
||||
def test_spend_logs_payload_keeps_no_requester_ip_address_when_ip_logging_is_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, disabled: bool, recorded_ip: str | None, error_message: str, traceback: str
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
|
||||
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", disabled)
|
||||
logged_at: Final = datetime.datetime(2026, 1, 15, 12, 0, 0)
|
||||
request_metadata: Final = _request_ip_metadata()
|
||||
|
||||
payload: Final[SpendLogsPayload] = get_logging_payload(
|
||||
kwargs=_request_ip_kwargs(request_metadata),
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-ip", model="gpt-5-mini"),
|
||||
start_time=logged_at,
|
||||
end_time=logged_at,
|
||||
)
|
||||
|
||||
stored_metadata: Final = json.loads(payload["metadata"])
|
||||
stored_request: Final = json.loads(payload["proxy_server_request"] or "{}")
|
||||
stored_errors: Final = (stored_metadata["error_information"], stored_request["metadata"]["error_information"])
|
||||
assert request_metadata == _request_ip_metadata()
|
||||
assert stored_metadata["user_agent"] == "requester-ip-test"
|
||||
assert stored_request["metadata"]["headers"]["accept"] == "*/*"
|
||||
assert stored_request["metadata"]["requester_metadata"]["team"] == "blue"
|
||||
assert payload["requester_ip_address"] == recorded_ip
|
||||
assert stored_metadata["requester_ip_address"] == recorded_ip
|
||||
assert stored_request["metadata"]["requester_ip_address"] == recorded_ip
|
||||
assert [error["error_message"] for error in stored_errors] == [error_message, error_message]
|
||||
assert [error["traceback"] for error in stored_errors] == [traceback, traceback]
|
||||
assert [error["error_code"] for error in stored_errors] == ["403", "403"]
|
||||
assert ("203.0.113.9" in json.dumps(payload, default=str)) is not disabled
|
||||
|
||||
|
||||
def test_stored_request_without_requester_ip_address_drops_or_masks_only_ip_fields_from_request_metadata() -> None:
|
||||
body: Final = {
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"tools": [{"type": "function", "function": {"parameters": {"headers": {"x-real-ip": "keep"}}}}],
|
||||
"error_information": {"error_message": "203.0.113.9"},
|
||||
"litellm_metadata": {
|
||||
"requester_ip_address": "203.0.113.9",
|
||||
"headers": {
|
||||
"Forwarded": "for=203.0.113.9",
|
||||
"CF-Connecting-IP": "203.0.113.9",
|
||||
"X-Envoy-External-Address": "203.0.113.9",
|
||||
"CF-Connecting-IPv6": "2001:db8::9",
|
||||
"CF-Pseudo-IPv4": "203.0.113.9",
|
||||
"X-AppEngine-User-IP": "203.0.113.9",
|
||||
"X-Vercel-Forwarded-For": "203.0.113.9",
|
||||
"X-ARR-ClientIP": "203.0.113.9",
|
||||
"DO-Connecting-IP": "203.0.113.9",
|
||||
"Incap-Client-IP": "2001:db8::9",
|
||||
"x-edge-client": "client=203.0.113.9, peer=[2001:db8::9]:443",
|
||||
"host": "10.0.0.5:4000",
|
||||
"user-agent": "ua",
|
||||
},
|
||||
"forwarded": "not a header",
|
||||
"spend_logs_metadata": [{"headers": {"true-client-ip": "203.0.113.9", "x-client": "host:2001:db8::9"}}],
|
||||
"error_information": {"error_message": "denied 203.0.113.9"},
|
||||
},
|
||||
"originating_request_masked": {
|
||||
"messages": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"metadata": {
|
||||
"requester_ip_address": "203.0.113.9",
|
||||
"headers": {"x-forwarded-for": "203.0.113.9", "host": "10.0.0.5:4000"},
|
||||
},
|
||||
},
|
||||
"classifier_input": {
|
||||
"input": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "curl"},
|
||||
},
|
||||
}
|
||||
|
||||
assert _stored_request_without_requester_ip_address(body) == {
|
||||
"model": "gpt-5-mini",
|
||||
"messages": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"tools": [{"type": "function", "function": {"parameters": {"headers": {"x-real-ip": "keep"}}}}],
|
||||
"error_information": {"error_message": "203.0.113.9"},
|
||||
"litellm_metadata": {
|
||||
"requester_ip_address": None,
|
||||
"headers": {
|
||||
"DO-Connecting-IP": "REDACTED_BY_LITELM",
|
||||
"Incap-Client-IP": "REDACTED_BY_LITELM",
|
||||
"x-edge-client": "client=REDACTED_BY_LITELM, peer=[REDACTED_BY_LITELM]:443",
|
||||
"host": "REDACTED_BY_LITELM:4000",
|
||||
"user-agent": "ua",
|
||||
},
|
||||
"forwarded": "not a header",
|
||||
"spend_logs_metadata": [{"headers": {"x-client": "host:REDACTED_BY_LITELM"}}],
|
||||
"error_information": {"error_message": "denied REDACTED_BY_LITELM"},
|
||||
},
|
||||
"originating_request_masked": {
|
||||
"messages": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"metadata": {"requester_ip_address": None, "headers": {"host": "REDACTED_BY_LITELM:4000"}},
|
||||
},
|
||||
"classifier_input": {
|
||||
"input": [{"role": "user", "content": "203.0.113.9"}],
|
||||
"metadata": {"requester_ip_address": None, "user_agent": "curl"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_field", ["error_message", "traceback"])
|
||||
def test_stored_request_masks_ip_addresses_in_long_error_text_and_at_its_truncation_cut(
|
||||
monkeypatch: pytest.MonkeyPatch, error_field: str
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
|
||||
text: Final = "a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8::9 " + "c" * 644
|
||||
metadata: Final = {"error_information": {"error_code": "500", error_field: text}}
|
||||
|
||||
stored: Final = json.loads(
|
||||
_get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata=metadata,
|
||||
litellm_params={"proxy_server_request": {"body": {"model": "gpt-5-mini", "metadata": metadata}}},
|
||||
drop_requester_ip_address=True,
|
||||
)
|
||||
)
|
||||
|
||||
stored_text: Final = stored["metadata"]["error_information"][error_field]
|
||||
kept_head_and_tail: Final = stored_text[:350] + stored_text[-650:]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_text
|
||||
assert kept_head_and_tail.startswith("a" * 345 + " REDA")
|
||||
assert not any(char.isdigit() for char in kept_head_and_tail)
|
||||
assert metadata["error_information"][error_field] == text
|
||||
|
||||
|
||||
_TRACE_ID: Final = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
_PLANTED_MARKER: Final = (
|
||||
f"x ... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 5 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ... "
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requester_metadata", [False, True], ids=["headers", "requester-metadata-headers"])
|
||||
@pytest.mark.parametrize(
|
||||
"value, stored_head, stored_tail",
|
||||
[
|
||||
(
|
||||
"a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8:0:0:1:0:0:9 " + "c" * 642,
|
||||
"a" * 345 + " REDACTED_BY_LITELM",
|
||||
"REDACTED_BY_LITELM " + "c" * 642,
|
||||
),
|
||||
(
|
||||
"a" * 343 + " ip:203.0.113.9 " + "b" * 2000 + " 192.0.2.1. " + "c" * 647,
|
||||
"a" * 343 + " ip:REDACTED_BY_LITELM",
|
||||
"REDACTED_BY_LITELM. " + "c" * 647,
|
||||
),
|
||||
(
|
||||
"x" * 296 + f" trace={_TRACE_ID}:203.0.113.9 " + "b" * 2000 + f" 203.0.113.9:{_TRACE_ID} " + "c" * 606,
|
||||
"x" * 296 + f" trace={_TRACE_ID}:REDACTED_BY_LITELM",
|
||||
f"REDACTED_BY_LITELM:{_TRACE_ID} " + "c" * 606,
|
||||
),
|
||||
(
|
||||
_PLANTED_MARKER
|
||||
+ "a" * (345 - len(_PLANTED_MARKER))
|
||||
+ " 203.0.113.9 "
|
||||
+ "b" * 2000
|
||||
+ " 2001:db8::dead:beef "
|
||||
+ "c" * 640,
|
||||
_PLANTED_MARKER + "a" * (345 - len(_PLANTED_MARKER)) + " REDACTED_BY_LITELM",
|
||||
"REDACTED_BY_LITELM " + "c" * 640,
|
||||
),
|
||||
("a" * 346 + " cafe:beef " + "b" * 2000 + " deadbeef " + "c" * 646, "a" * 346 + " caf", "eef " + "c" * 646),
|
||||
("0123456789abcdef" * 200, ("0123456789abcdef" * 22)[:350], ("0123456789abcdef" * 41)[-650:]),
|
||||
(
|
||||
"a" * 281 + " " + "ab1." * 17 + "b" * 2000 + " " + "1ba." * 17 + " " + "c" * 581,
|
||||
"a" * 281 + " " + "ab1." * 6 + "REDACTED_BY_LITELM",
|
||||
"REDACTED_BY_LITELMba." + "1ba." * 5 + " " + "c" * 581,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"ipv4-and-ipv6-split",
|
||||
"ipv4-after-a-label-and-before-a-dot",
|
||||
"ipv4-beside-long-hex-tokens",
|
||||
"after-a-planted-marker-and-ipv6-groups-without-digits",
|
||||
"hex-words-without-digits-or-colons",
|
||||
"hex-run-longer-than-an-ip",
|
||||
"short-groups-longer-than-an-ip",
|
||||
],
|
||||
)
|
||||
def test_stored_request_masks_the_piece_of_an_ip_address_a_header_truncation_cut_leaves(
|
||||
monkeypatch: pytest.MonkeyPatch, requester_metadata: bool, value: str, stored_head: str, stored_tail: str
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
|
||||
headers: Final = {"x-edge-trace": value}
|
||||
metadata: Final = {"requester_metadata": {"headers": headers}} if requester_metadata else {"headers": headers}
|
||||
|
||||
stored: Final = json.loads(
|
||||
_get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata=metadata,
|
||||
litellm_params={"proxy_server_request": {"body": {"model": "gpt-5-mini", "metadata": metadata}}},
|
||||
drop_requester_ip_address=True,
|
||||
)
|
||||
)
|
||||
|
||||
stored_metadata: Final = stored["metadata"]["requester_metadata"] if requester_metadata else stored["metadata"]
|
||||
stored_value: Final = stored_metadata["headers"]["x-edge-trace"]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_value
|
||||
assert stored_value.startswith(stored_head + "... (")
|
||||
assert stored_value.endswith(") ..." + stored_tail)
|
||||
assert headers["x-edge-trace"] == value
|
||||
|
||||
|
||||
def test_stored_request_with_ip_logging_disabled_matches_setting_off_for_bodies_without_ip_fields(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
|
||||
loop: Final[dict[str, object]] = {}
|
||||
loop["self"] = (loop,)
|
||||
body: Final[dict[object, object]] = {"model": "gpt-5-mini", 7: "non-string key", "metadata": {"tags": (loop,)}}
|
||||
|
||||
stored: Final = [
|
||||
_get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata={},
|
||||
litellm_params={"proxy_server_request": {"body": body}},
|
||||
drop_requester_ip_address=drop_requester_ip_address,
|
||||
)
|
||||
for drop_requester_ip_address in (False, True)
|
||||
]
|
||||
|
||||
assert json.loads(stored[0])["model"] == "gpt-5-mini"
|
||||
assert stored[1] == stored[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled", [False, True], ids=["setting-off", "setting-on"])
|
||||
def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_truncation_cut(
|
||||
monkeypatch: pytest.MonkeyPatch, disabled: bool
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", disabled)
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
|
||||
text: Final = "a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8::9 " + "c" * 644
|
||||
|
||||
sanitized: Final = sanitize_error_information_for_spend_logs(
|
||||
{"error_code": "500", "error_message": text, "traceback": text}
|
||||
)
|
||||
|
||||
assert sanitized is not None
|
||||
for stored_text in (sanitized["error_message"], sanitized["traceback"]):
|
||||
kept_head_and_tail = stored_text[:350] + stored_text[-650:]
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_text
|
||||
assert any(char.isdigit() for char in kept_head_and_tail) is not disabled
|
||||
assert sanitized["error_code"] == "500"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"]
|
||||
)
|
||||
def test_failure_error_information_masks_every_ip_address_form_when_ip_logging_is_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, address: str
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", True)
|
||||
|
||||
sanitized: Final = sanitize_error_information_for_spend_logs(
|
||||
{
|
||||
"error_code": "403",
|
||||
"error_message": f"IP address {address} not allowed.",
|
||||
"traceback": f"peer [{address}]:443",
|
||||
}
|
||||
)
|
||||
|
||||
assert sanitized == {
|
||||
"error_code": "403",
|
||||
"error_message": "IP address REDACTED_BY_LITELM not allowed.",
|
||||
"traceback": "peer [REDACTED_BY_LITELM]:443",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "stored"),
|
||||
[
|
||||
("connect to host:2001:db8::2 failed", "connect to host:REDACTED_BY_LITELM failed"),
|
||||
("peer_2001:db8::2_tail", "peer_REDACTED_BY_LITELM_tail"),
|
||||
("upstream 1:2:3:4:5:6:7:8:443 refused", "upstream REDACTED_BY_LITELM refused"),
|
||||
("upstream fe80::1:2:3:4:5:6:443 refused", "upstream REDACTED_BY_LITELM refused"),
|
||||
("label cafe:1:2:3:4:5:6:7::", "label REDACTED_BY_LITELM"),
|
||||
("label cafe:1:2:3:4:5:6:7:8:443", "label REDACTED_BY_LITELM"),
|
||||
("socket_192.0.2.65 closed", "socket_REDACTED_BY_LITELM closed"),
|
||||
(
|
||||
"at 2026-10-10T18:20:09 Error::Foo aa:bb:cc:dd:ee:ff 1203.0.113.9 qwen2.5.1.5",
|
||||
"at 2026-10-10T18:20:09 Error::Foo aa:bb:cc:dd:ee:ff 1203.0.113.9 qwen2.5.1.5",
|
||||
),
|
||||
("IP 192.0.2.1.443 > 10.0.0.2.80: Flags [S]", "IP REDACTED_BY_LITELM.443 > REDACTED_BY_LITELM.80: Flags [S]"),
|
||||
],
|
||||
)
|
||||
def test_failure_error_information_masks_addresses_after_labels_and_before_ports(
|
||||
monkeypatch: pytest.MonkeyPatch, text: str, stored: str
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", True)
|
||||
|
||||
assert sanitize_error_information_for_spend_logs({"error_code": "500", "error_message": text}) == {
|
||||
"error_code": "500",
|
||||
"error_message": stored,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -234,6 +234,37 @@ async def test_add_litellm_data_to_request_parses_string_metadata():
|
|||
assert updated_data["metadata"]["generation_name"] == "gen123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("disabled", [False, True])
|
||||
async def test_add_litellm_data_to_request_keeps_requester_ip_address_for_guardrails_when_ip_logging_is_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, disabled: bool
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
|
||||
settings = {"use_x_forwarded_for": True, "disable_requester_ip_address_logging": disabled}
|
||||
monkeypatch.setattr(proxy_server, "general_settings", settings)
|
||||
request_mock = MagicMock(spec=Request)
|
||||
request_mock.url = MagicMock()
|
||||
request_mock.url.path = "/v1/chat/completions"
|
||||
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
|
||||
request_mock.method = "POST"
|
||||
request_mock.query_params = {}
|
||||
request_mock.headers = {"Content-Type": "application/json", "x-forwarded-for": "203.0.113.9"}
|
||||
request_mock.client = MagicMock()
|
||||
request_mock.client.host = "127.0.0.1"
|
||||
|
||||
updated_data = await add_litellm_data_to_request(
|
||||
data={"model": "gpt-4o"},
|
||||
request=request_mock,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings=settings,
|
||||
)
|
||||
|
||||
assert updated_data["metadata"]["requester_ip_address"] == "203.0.113.9"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_otel_service_name_outranks_team_metadata_merge():
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -29791,6 +29791,11 @@ export interface components {
|
|||
* @description If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password login on /login, /v2/login, and /v3/login so SSO is the only way to reach the Admin UI. An admin locked out of the UI can still administer the proxy over the API with the master key; unset this setting and restart the proxy to restore UI username/password login. Default is False.
|
||||
*/
|
||||
disable_password_login_when_sso_enabled?: boolean | null;
|
||||
/**
|
||||
* Disable Requester Ip Address Logging
|
||||
* @description If True, new spend log rows store requester_ip_address as null in the column, the metadata and the stored request. The stored request also drops client-IP forwarding headers such as x-forwarded-for, and IP addresses in its other header values and in stored error messages are masked. Guardrails, metrics and callbacks that read the standard logging payload still receive the IP. Default is False.
|
||||
*/
|
||||
disable_requester_ip_address_logging?: boolean | null;
|
||||
/**
|
||||
* Disable Responses Id Security
|
||||
* @description If True, disables ownership enforcement on Responses API ids. Keys may then retrieve, cancel, delete, and chain from any response id, including ids belonging to another user or team and ids this proxy never issued. WARNING: this removes tenant isolation on /v1/responses
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue