mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 01e53bec4d into 1595adfb2f
This commit is contained in:
commit
ddb4459383
24 changed files with 465 additions and 16 deletions
|
|
@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import CLI_SESSION_KEY_PREFIX
|
||||
from litellm.constants import CLI_SESSION_KEY_PREFIX, USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY
|
||||
from litellm.repositories.table_repositories import (
|
||||
DeletedVerificationTokenRepository,
|
||||
DeprecatedVerificationTokenRepository,
|
||||
|
|
@ -310,6 +310,7 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str,
|
|||
"user_api_key": api_key,
|
||||
"user_api_key_hash": api_key,
|
||||
"user_api_key_team_id": team_id,
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: rows.key is not None,
|
||||
**(
|
||||
{
|
||||
"user_api_key_user_email": getattr(rows.user, "user_email", None),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
-- The team that owns the virtual key a tag row was spent through. "" when the request did not
|
||||
-- come through a virtual key with a team (JWT, custom auth, OAuth2, team-less keys), and on every
|
||||
-- row written before this migration. Not part of the unique key: a team's virtual key already
|
||||
-- maps the row to one team, so older proxies keep upserting on the unchanged key during a rolling
|
||||
-- upgrade. On PostgreSQL 11+ adding a column with a constant default does not rewrite the table.
|
||||
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "team_id" TEXT NOT NULL DEFAULT '';
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
-- CreateIndex (CONCURRENTLY)
|
||||
--
|
||||
-- Serves team-scoped tag reads: WHERE team_id = ANY(...) AND date BETWEEN ... grouped by tag.
|
||||
--
|
||||
-- Disclaimer:
|
||||
-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
|
||||
-- single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
|
||||
-- - If the build is interrupted, Postgres may leave an INVALID index that must be dropped and
|
||||
-- recreated.
|
||||
-- - Do not edit this file after it has been applied to any database: Prisma checksums
|
||||
-- migrations; add a new migration instead.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_DailyTagSpend_team_id_date_tag_idx" ON "LiteLLM_DailyTagSpend"("team_id", "date", "tag");
|
||||
|
|
@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend {
|
|||
id String @id @default(uuid())
|
||||
request_id String?
|
||||
tag String?
|
||||
team_id String @default("")
|
||||
date String
|
||||
api_key String
|
||||
model String?
|
||||
|
|
@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend {
|
|||
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
|
||||
@@index([date])
|
||||
@@index([tag, date])
|
||||
@@index([team_id, date, tag])
|
||||
@@index([api_key])
|
||||
@@index([model])
|
||||
@@index([mcp_namespaced_tool_name])
|
||||
|
|
|
|||
|
|
@ -1693,6 +1693,7 @@ ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted
|
|||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: Final = "user_api_key_is_virtual_key"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.constants import (
|
|||
PROVIDER_REQUEST_ID_HEADERS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REDACTED_BY_LITELLM,
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
|
|
@ -328,9 +329,10 @@ _in_memory_loggers: Final[list[CustomLogger]] = []
|
|||
_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
|
||||
("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment")
|
||||
)
|
||||
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
|
||||
frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
|
||||
)
|
||||
_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = frozenset(StandardLoggingMetadata.__annotations__.keys()) - {
|
||||
*_STANDARD_LOGGING_METADATA_RESOLVED_KEYS,
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY,
|
||||
}
|
||||
|
||||
|
||||
def _get_provider_request_id(original_exception: Exception) -> str | None:
|
||||
|
|
@ -8064,7 +8066,10 @@ def get_standard_logging_metadata(
|
|||
)
|
||||
if isinstance(metadata, dict):
|
||||
# Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
|
||||
for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}:
|
||||
for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {
|
||||
"is_silent_experiment",
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY,
|
||||
}:
|
||||
clean_metadata[key] = metadata[key]
|
||||
|
||||
if metadata.get("user_api_key") is not None:
|
||||
|
|
|
|||
|
|
@ -5270,6 +5270,7 @@ class DailyEndUserSpendTransaction(BaseDailySpendTransaction):
|
|||
class DailyTagSpendTransaction(BaseDailySpendTransaction):
|
||||
request_id: str | None
|
||||
tag: str
|
||||
team_id: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
class DailyAgentSpendTransaction(BaseDailySpendTransaction):
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ BatchStatus: TypeAlias = Literal[
|
|||
TERMINAL_BATCH_STATUSES: Final[frozenset[str]] = frozenset({"completed", "failed", "cancelled", "expired"})
|
||||
_STOP_STATUSES: Final[frozenset[str]] = TERMINAL_BATCH_STATUSES | frozenset({"cancelling"})
|
||||
_BATCH_ENDPOINT_ADAPTER: Final[TypeAdapter[BatchEndpoint]] = TypeAdapter(BatchEndpoint)
|
||||
_CLIENT_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_CANCEL_POLL_SECONDS: Final = 1.0
|
||||
_HEARTBEAT_SECONDS: Final = 30.0
|
||||
_STALE_AFTER_SECONDS: Final = 180.0
|
||||
|
|
@ -350,6 +351,19 @@ def _error_body(error: Exception) -> _ErrorBody:
|
|||
return body
|
||||
|
||||
|
||||
def _without_client_key_metadata(body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
litellm_metadata: Final = body.get("litellm_metadata")
|
||||
if not isinstance(litellm_metadata, Mapping):
|
||||
return body
|
||||
client_metadata: Final = _CLIENT_METADATA_ADAPTER.validate_python(litellm_metadata)
|
||||
return {
|
||||
**body,
|
||||
"litellm_metadata": {
|
||||
key: value for key, value in client_metadata.items() if not key.startswith("user_api_key_")
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _line_response(outcome: RowOutcome | ExpiredRow) -> _ResultResponse | None:
|
||||
if isinstance(outcome, ExpiredRow):
|
||||
return None
|
||||
|
|
@ -645,7 +659,12 @@ class LiteLLMExecutedBatchRunner:
|
|||
|
||||
async def _dispatch(self, run: _BatchRun, line: BatchInputLine) -> Mapping[str, object]:
|
||||
params: Final = MappingProxyType(
|
||||
{**line.body, "model": run.model, "metadata": self._row_metadata(run), "disable_fallbacks": True}
|
||||
{
|
||||
**_without_client_key_metadata(line.body),
|
||||
"model": run.model,
|
||||
"metadata": self._row_metadata(run),
|
||||
"disable_fallbacks": True,
|
||||
}
|
||||
)
|
||||
return _dump(await self._router_call(run.endpoint)(**params))
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ class DailySpendTable:
|
|||
name: str
|
||||
entity_id_column: str
|
||||
carries_request_id: bool = False
|
||||
carries_team_id: bool = False
|
||||
|
||||
|
||||
DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingProxyType(
|
||||
|
|
@ -41,7 +42,9 @@ DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingP
|
|||
"org": DailySpendTable(name="LiteLLM_DailyOrganizationSpend", entity_id_column="organization_id"),
|
||||
"end_user": DailySpendTable(name="LiteLLM_DailyEndUserSpend", entity_id_column="end_user_id"),
|
||||
"agent": DailySpendTable(name="LiteLLM_DailyAgentSpend", entity_id_column="agent_id"),
|
||||
"tag": DailySpendTable(name="LiteLLM_DailyTagSpend", entity_id_column="tag", carries_request_id=True),
|
||||
"tag": DailySpendTable(
|
||||
name="LiteLLM_DailyTagSpend", entity_id_column="tag", carries_request_id=True, carries_team_id=True
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -168,6 +171,7 @@ def _row_params(
|
|||
*(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS),
|
||||
*(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS),
|
||||
*((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()),
|
||||
*((_as_text(transaction.get("team_id")),) if table.carries_team_id else ()),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -180,6 +184,7 @@ def _insert_columns(table: DailySpendTable) -> tuple[str, ...]:
|
|||
*_COUNTER_COLUMNS,
|
||||
*_SPEND_COLUMNS,
|
||||
*(("request_id",) if table.carries_request_id else ()),
|
||||
*(("team_id",) if table.carries_team_id else ()),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -210,11 +215,16 @@ def build_bulk_upsert(
|
|||
if table.carries_request_id
|
||||
else ""
|
||||
)
|
||||
team_id_update: Final = (
|
||||
f', "team_id" = COALESCE(NULLIF(EXCLUDED."team_id", \'\'), {quoted_table}."team_id")'
|
||||
if table.carries_team_id
|
||||
else ""
|
||||
)
|
||||
sql: Final = (
|
||||
f'INSERT INTO {quoted_table} ({_quoted(columns)}, "updated_at")\n'
|
||||
f"VALUES {rows}\n"
|
||||
f"ON CONFLICT ({_quoted((table.entity_id_column, *_KEY_COLUMNS))}) DO UPDATE SET\n"
|
||||
f" {increments}{request_id_update},\n"
|
||||
f" {increments}{request_id_update}{team_id_update},\n"
|
||||
f" \"updated_at\" = (NOW() AT TIME ZONE 'UTC')"
|
||||
)
|
||||
return sql, tuple(value for key, transaction in batch for value in _row_params(table, key, transaction))
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.constants import (
|
|||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
|
||||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line
|
||||
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
|
||||
|
|
@ -351,6 +352,20 @@ def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadat
|
|||
return parsed
|
||||
|
||||
|
||||
def _virtual_key_team_id(kwargs: Mapping[str, object]) -> str:
|
||||
"""The team of the virtual key that authenticated the request, or "" for JWT, custom auth, OAuth2,
|
||||
UI sessions, the master key and team-less keys, whose api_key does not pin a tag row to one team."""
|
||||
params: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs.get("litellm_params") or {})
|
||||
buckets: Final = tuple(
|
||||
_SPEND_METADATA_ADAPTER.validate_python(params.get(bucket) or {}) for bucket in ("litellm_metadata", "metadata")
|
||||
)
|
||||
marked: Final = next(
|
||||
(bucket for bucket in buckets if bucket.get(USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY) is True), None
|
||||
)
|
||||
team_id: Final = marked.get("user_api_key_team_id") if marked is not None else None
|
||||
return team_id if isinstance(team_id, str) else ""
|
||||
|
||||
|
||||
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
|
||||
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
|
||||
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
|
||||
|
|
@ -721,6 +736,7 @@ class DBSpendUpdateWriter:
|
|||
litellm_proxy_budget_name=litellm_proxy_budget_name,
|
||||
payload=payload,
|
||||
request_model_access_groups=get_request_model_access_groups(kwargs),
|
||||
virtual_key_team_id=_virtual_key_team_id(_SPEND_METADATA_ADAPTER.validate_python(kwargs or {})),
|
||||
),
|
||||
task_name="spend rollups",
|
||||
)
|
||||
|
|
@ -1145,6 +1161,7 @@ class DBSpendUpdateWriter:
|
|||
payload: SpendLogsPayload,
|
||||
request_model_access_groups: Sequence[str] = (),
|
||||
project_id: str | None = None,
|
||||
virtual_key_team_id: str = "",
|
||||
):
|
||||
"""
|
||||
Runs every spend-update helper for one request. Each helper goes through
|
||||
|
|
@ -1244,7 +1261,9 @@ class DBSpendUpdateWriter:
|
|||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_tag_transaction",
|
||||
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
self.add_spend_log_transaction_to_daily_tag_transaction(
|
||||
payload=payload_copy, prisma_client=prisma_client, virtual_key_team_id=virtual_key_team_id
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
|
||||
|
|
@ -3079,6 +3098,7 @@ class DBSpendUpdateWriter:
|
|||
self,
|
||||
payload: SpendLogsPayload,
|
||||
prisma_client: PrismaClient | None = None,
|
||||
virtual_key_team_id: str = "",
|
||||
) -> None:
|
||||
if prisma_client is None:
|
||||
verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.")
|
||||
|
|
@ -3100,7 +3120,7 @@ class DBSpendUpdateWriter:
|
|||
endpoint_str = base_daily_transaction.get("endpoint") or ""
|
||||
daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}"
|
||||
daily_transaction = DailyTagSpendTransaction(
|
||||
tag=tag, **base_daily_transaction, request_id=payload["request_id"]
|
||||
tag=tag, team_id=virtual_key_team_id, **base_daily_transaction, request_id=payload["request_id"]
|
||||
)
|
||||
|
||||
await self.daily_tag_spend_update_queue.add_update(update={daily_transaction_key: daily_transaction})
|
||||
|
|
|
|||
|
|
@ -2082,6 +2082,7 @@ class LiteLLMProxyRequestSetup:
|
|||
user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None
|
||||
),
|
||||
user_api_key_auth_metadata=auth_metadata,
|
||||
user_api_key_is_virtual_key=LiteLLMProxyRequestSetup.authenticated_with_virtual_key(user_api_key_dict),
|
||||
)
|
||||
return user_api_key_logged_metadata
|
||||
|
||||
|
|
@ -2093,6 +2094,12 @@ class LiteLLMProxyRequestSetup:
|
|||
project_tags: Final = (user_api_key_dict.project_metadata or {}).get("tags") or ()
|
||||
return tuple(dict.fromkeys((*key_tags, *team_tags, *project_tags)))
|
||||
|
||||
@staticmethod
|
||||
def authenticated_with_virtual_key(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
"""A serializable copy of the unforgeable via_virtual_key marker, carried in the key's logged
|
||||
metadata so spend paths that drop the auth object still know the request came through a virtual key."""
|
||||
return user_api_key_dict.via_virtual_key and not user_api_key_dict.is_session_token
|
||||
|
||||
@staticmethod
|
||||
def add_user_api_key_auth_to_request_metadata(
|
||||
data: dict,
|
||||
|
|
|
|||
|
|
@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend {
|
|||
id String @id @default(uuid())
|
||||
request_id String?
|
||||
tag String?
|
||||
team_id String @default("")
|
||||
date String
|
||||
api_key String
|
||||
model String?
|
||||
|
|
@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend {
|
|||
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
|
||||
@@index([date])
|
||||
@@index([tag, date])
|
||||
@@index([team_id, date, tag])
|
||||
@@index([api_key])
|
||||
@@index([model])
|
||||
@@index([mcp_namespaced_tool_name])
|
||||
|
|
|
|||
|
|
@ -3074,6 +3074,7 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict):
|
|||
user_api_key_end_user_id: str | None
|
||||
user_api_key_request_route: str | None
|
||||
user_api_key_auth_metadata: dict[str, str] | None
|
||||
user_api_key_is_virtual_key: ReadOnly[NotRequired[bool]]
|
||||
|
||||
|
||||
class StandardLoggingMCPToolCall(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend {
|
|||
id String @id @default(uuid())
|
||||
request_id String?
|
||||
tag String?
|
||||
team_id String @default("")
|
||||
date String
|
||||
api_key String
|
||||
model String?
|
||||
|
|
@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend {
|
|||
@@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
|
||||
@@index([date])
|
||||
@@index([tag, date])
|
||||
@@index([team_id, date, tag])
|
||||
@@index([api_key])
|
||||
@@index([model])
|
||||
@@index([mcp_namespaced_tool_name])
|
||||
|
|
|
|||
112
tests/integration/spend/test_team_tag_spend_attribution.py
Normal file
112
tests/integration/spend/test_team_tag_spend_attribution.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import math
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway, Scenario, eventually
|
||||
from integration._support.database import read_rows, write_rows
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
_SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class _Reply(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
usage: _Usage
|
||||
|
||||
|
||||
class _TagRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
team_id: str
|
||||
spend: float
|
||||
api_requests: int
|
||||
|
||||
|
||||
_TAG_ROWS: Final = TypeAdapter(tuple[_TagRow, ...])
|
||||
|
||||
|
||||
def _send_tagged(gateway: Gateway, *, key: str, model: str, tag: str, calls: int) -> None:
|
||||
for _ in range(calls):
|
||||
reply = gateway.post(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"tagged {uuid.uuid4().hex}"}],
|
||||
"metadata": {"tags": [tag]},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
assert _Reply.model_validate(reply).usage.total_tokens == 40, reply
|
||||
|
||||
|
||||
def _tag_rows(tag: str, *, requests: int) -> tuple[_TagRow, ...]:
|
||||
return eventually(
|
||||
lambda: _TAG_ROWS.validate_python(
|
||||
read_rows(
|
||||
'SELECT team_id, spend, api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag = %s ORDER BY team_id',
|
||||
(tag,),
|
||||
)
|
||||
),
|
||||
lambda rows: sum(row.api_requests for row in rows) == requests,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
|
||||
def _model(scenario: Scenario) -> str:
|
||||
return scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
|
||||
|
||||
def test_a_shared_tag_is_split_by_the_team_of_each_virtual_key(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = _model(scenario)
|
||||
tag: Final = f"cost-center:{uuid.uuid4().hex}"
|
||||
teams: Final = tuple(scenario.team(models=[model]) for _ in range(2))
|
||||
calls: Final = {teams[0]: 1, teams[1]: 2, "": 1}
|
||||
for team, count in calls.items():
|
||||
key = scenario.key(team_id=team, models=[model]) if team else scenario.key(models=[model])
|
||||
_send_tagged(gateway, key=key, model=model, tag=tag, calls=count)
|
||||
|
||||
rows: Final = _tag_rows(tag, requests=sum(calls.values()))
|
||||
|
||||
assert sorted((row.team_id, row.api_requests) for row in rows) == sorted(calls.items()), rows
|
||||
for row in rows:
|
||||
assert math.isclose(row.spend, calls[row.team_id] * _SPEND_PER_CALL), rows
|
||||
|
||||
|
||||
def test_a_key_moved_between_teams_keeps_one_row_under_its_latest_team(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = _model(scenario)
|
||||
tag: Final = f"moved-key:{uuid.uuid4().hex}"
|
||||
teams: Final = tuple(scenario.team(models=[model]) for _ in range(2))
|
||||
key: Final = scenario.key(team_id=teams[0], models=[model])
|
||||
_send_tagged(gateway, key=key, model=model, tag=tag, calls=1)
|
||||
_tag_rows(tag, requests=1)
|
||||
gateway.post("/key/update", {"key": key, "team_id": teams[1]})
|
||||
_send_tagged(gateway, key=key, model=model, tag=tag, calls=2)
|
||||
|
||||
rows: Final = _tag_rows(tag, requests=3)
|
||||
|
||||
assert [(row.team_id, row.api_requests) for row in rows] == [(teams[1], 3)], rows
|
||||
|
||||
|
||||
def test_a_row_an_older_proxy_wrote_without_a_team_gets_the_team_on_the_next_write(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = _model(scenario)
|
||||
tag: Final = f"rolling-upgrade:{uuid.uuid4().hex}"
|
||||
team: Final = scenario.team(models=[model])
|
||||
key: Final = scenario.key(team_id=team, models=[model])
|
||||
_send_tagged(gateway, key=key, model=model, tag=tag, calls=1)
|
||||
_tag_rows(tag, requests=1)
|
||||
write_rows("UPDATE \"LiteLLM_DailyTagSpend\" SET team_id = '' WHERE tag = %s", (tag,))
|
||||
_send_tagged(gateway, key=key, model=model, tag=tag, calls=1)
|
||||
|
||||
rows: Final = _tag_rows(tag, requests=2)
|
||||
|
||||
assert [(row.team_id, row.api_requests) for row in rows] == [(team, 2)], rows
|
||||
|
|
@ -5737,6 +5737,22 @@ def test_get_standard_logging_metadata_merges_recorded_applied_guardrails():
|
|||
assert result["applied_guardrails"] == ["guard-a", "blocker"]
|
||||
|
||||
|
||||
def test_logging_metadata_leaves_out_the_virtual_key_flag():
|
||||
from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup, get_standard_logging_metadata
|
||||
|
||||
metadata: Final = {USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True, "user_api_key_team_id": "team-a"}
|
||||
|
||||
payload_metadata: Final = StandardLoggingPayloadSetup.get_standard_logging_metadata(
|
||||
metadata=metadata, litellm_params={}
|
||||
)
|
||||
standalone_metadata: Final = get_standard_logging_metadata(metadata)
|
||||
|
||||
assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in payload_metadata
|
||||
assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in standalone_metadata
|
||||
assert payload_metadata["user_api_key_team_id"] == standalone_metadata["user_api_key_team_id"] == "team-a"
|
||||
|
||||
|
||||
def test_function_setup_metadata_takes_precedence_over_litellm_metadata():
|
||||
"""
|
||||
Test that when BOTH metadata and litellm_metadata are present (e.g., user sets
|
||||
|
|
|
|||
|
|
@ -660,7 +660,9 @@ async def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_sc
|
|||
assert (
|
||||
await resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1", FakeFilesApiProbe(True, [])) == expected
|
||||
)
|
||||
router.registry.return_value.credentials.assert_called_once_with(BATCH_MODEL, form="with_provider", team_id="team-1")
|
||||
router.registry.return_value.credentials.assert_called_once_with(
|
||||
BATCH_MODEL, form="with_provider", team_id="team-1"
|
||||
)
|
||||
|
||||
|
||||
async def test_create_stores_a_validating_batch_and_completes_it_in_the_background() -> None:
|
||||
|
|
@ -711,6 +713,37 @@ async def test_create_dispatches_each_row_with_the_batch_model_and_the_key_metad
|
|||
assert metadata["user_api_key_user_email"] == "user@example.com"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("via_virtual_key", "expected"), [(True, "team-1"), (False, "")])
|
||||
async def test_each_row_keeps_the_virtual_key_team_for_tag_spend(via_virtual_key: bool, expected: str) -> None:
|
||||
from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id
|
||||
|
||||
user = UserAPIKeyAuth(api_key="sk-batch-key", team_id="team-1").model_copy(
|
||||
update={"via_virtual_key": via_virtual_key}
|
||||
)
|
||||
harness = make_runner(user=user)
|
||||
await harness.create_and_finish()
|
||||
|
||||
teams = [
|
||||
_virtual_key_team_id({"litellm_params": {"metadata": call.kwargs["metadata"]}})
|
||||
for call in harness.router.acompletion.await_args_list
|
||||
]
|
||||
assert teams == [expected] * 2
|
||||
|
||||
|
||||
async def test_a_row_cannot_claim_another_team_through_its_own_litellm_metadata() -> None:
|
||||
from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id
|
||||
|
||||
forged = {"user_api_key_is_virtual_key": True, "user_api_key_team_id": "team-other", "custom_field": "kept"}
|
||||
user = UserAPIKeyAuth(api_key="sk-batch-key", team_id="team-1").model_copy(update={"via_virtual_key": False})
|
||||
harness = make_runner(content=jsonl(chat_row("row-1", "hi 1", litellm_metadata=forged)), user=user)
|
||||
await harness.create_and_finish()
|
||||
|
||||
(call,) = harness.router.acompletion.await_args_list
|
||||
params = {"metadata": call.kwargs["metadata"], "litellm_metadata": call.kwargs["litellm_metadata"]}
|
||||
assert _virtual_key_team_id({"litellm_params": params}) == ""
|
||||
assert call.kwargs["litellm_metadata"] == {"custom_field": "kept"}
|
||||
|
||||
|
||||
async def test_create_dispatches_each_row_with_the_key_team_and_project_tags_as_inherited_tags() -> None:
|
||||
user = UserAPIKeyAuth(
|
||||
api_key="sk-batch-key",
|
||||
|
|
|
|||
|
|
@ -2486,6 +2486,21 @@ class TestBatchCostAttribution:
|
|||
assert metadata["user_api_key_team_alias"] == "Team Alpha"
|
||||
assert metadata["tags"] == ["env:prod"]
|
||||
|
||||
@pytest.mark.parametrize(("key_row_exists", "expected"), [(True, "team-alpha"), (False, "")])
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_spend_team_needs_the_batch_to_come_from_a_virtual_key(self, key_row_exists, expected):
|
||||
"""Tag spend is attributed to a team only when the creating key is a real virtual key row,
|
||||
not a JWT, a CLI session or a deleted key."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id
|
||||
|
||||
instance = self._instance(key_row=SimpleNamespace(key_alias="prod-key") if key_row_exists else None)
|
||||
|
||||
metadata = await instance.batch_metadata(self._job(), "batch-1")
|
||||
|
||||
assert _virtual_key_team_id({"litellm_params": {"metadata": metadata}}) == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_tolerates_legacy_row_without_columns(self):
|
||||
"""Rows created before the columns existed carry only created_by/team_id and must
|
||||
|
|
|
|||
|
|
@ -88,10 +88,10 @@ def test_one_statement_carries_every_row_in_the_batch():
|
|||
|
||||
assert sql.count("INSERT INTO") == 1
|
||||
assert len(re.findall(r"ON CONFLICT", sql)) == 1
|
||||
# 25 bound columns per row plus the inlined updated_at, so the row count is what
|
||||
# 26 bound columns per row plus the inlined updated_at, so the row count is what
|
||||
# separates one multi-row statement from a hundred single-row ones.
|
||||
assert len(params) == 100 * 25
|
||||
assert "$2500::text" in sql
|
||||
assert len(params) == 100 * 26
|
||||
assert "$2600::text" in sql
|
||||
assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,14 +20,20 @@ from redis.exceptions import DataError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
|
||||
from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem
|
||||
from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS, USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY
|
||||
from litellm.proxy._types import (
|
||||
DailyTagSpendTransaction,
|
||||
Litellm_EntityType,
|
||||
SpendLogsPayload,
|
||||
SpendUpdateQueueItem,
|
||||
)
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_TEAM_ADVISORY_LOCK_SQL,
|
||||
_TEAM_MEMBER_SPEND_SQL,
|
||||
DBSpendUpdateWriter,
|
||||
_spend_tables_left_to_send,
|
||||
_SpendTableName,
|
||||
_virtual_key_team_id,
|
||||
)
|
||||
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue
|
||||
from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer
|
||||
|
|
@ -1338,6 +1344,45 @@ async def test_batch_database_updates_queues_org_member_spend_for_the_request_us
|
|||
assert transactions["org_member_list_transactions"] == {"organization_id::org1::user_id::u1": 0.1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_database_updates_puts_the_virtual_key_team_on_tag_rows():
|
||||
db_writer: Final = DBSpendUpdateWriter()
|
||||
prisma: Final = MagicMock()
|
||||
prisma.get_request_status = MagicMock(return_value="success")
|
||||
await db_writer._batch_database_updates(
|
||||
response_cost=0.01,
|
||||
user_id="u1",
|
||||
hashed_token="a" * 64,
|
||||
team_id="team-a",
|
||||
org_id=None,
|
||||
end_user_id=None,
|
||||
prisma_client=prisma,
|
||||
litellm_proxy_budget_name=None,
|
||||
payload=cast(
|
||||
SpendLogsPayload,
|
||||
{
|
||||
"request_id": "req-1",
|
||||
"request_tags": '["cost-center:cc-42"]',
|
||||
"team_id": "team-a",
|
||||
"user": "u1",
|
||||
"startTime": "2026-10-29T00:00:00",
|
||||
"api_key": "a" * 64,
|
||||
"model": "gpt-4o",
|
||||
"custom_llm_provider": "openai",
|
||||
"model_group": "gpt-4o",
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"spend": 0.01,
|
||||
"metadata": '{"usage_object": {}}',
|
||||
},
|
||||
),
|
||||
virtual_key_team_id="team-a",
|
||||
)
|
||||
queued: Final = await db_writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
|
||||
assert [cast(DailyTagSpendTransaction, txn).get("team_id") for txn in queued.values()] == ["team-a"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_project_spend_is_persisted_to_project_table_and_project_cache_is_evicted():
|
||||
db_writer: Final = DBSpendUpdateWriter()
|
||||
|
|
@ -1494,6 +1539,72 @@ async def test_add_spend_log_transaction_to_daily_tag_transaction_with_request_i
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("bucket", "marker", "team_id", "expected"),
|
||||
[
|
||||
("metadata", True, "team-a", "team-a"),
|
||||
("litellm_metadata", True, "team-a", "team-a"),
|
||||
("metadata", True, None, ""),
|
||||
("metadata", False, "team-a", ""),
|
||||
("metadata", "true", "team-a", ""),
|
||||
("metadata", None, "team-a", ""),
|
||||
],
|
||||
ids=[
|
||||
"virtual-key",
|
||||
"virtual-key-litellm-metadata",
|
||||
"virtual-key-without-team",
|
||||
"jwt-or-custom-auth",
|
||||
"non-boolean-marker",
|
||||
"no-marker",
|
||||
],
|
||||
)
|
||||
def test_tag_team_comes_only_from_a_request_marked_as_a_virtual_key(
|
||||
bucket: str, marker: object, team_id: str | None, expected: str
|
||||
) -> None:
|
||||
metadata: Final = {
|
||||
"user_api_key_team_id": team_id,
|
||||
**({USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: marker} if marker is not None else {}),
|
||||
}
|
||||
|
||||
assert _virtual_key_team_id({"litellm_params": {bucket: metadata}}) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_tag_row_of_a_request_carries_its_virtual_key_team() -> None:
|
||||
writer: Final = DBSpendUpdateWriter()
|
||||
prisma: Final = MagicMock()
|
||||
prisma.get_request_status = MagicMock(return_value="success")
|
||||
payload: Final = cast(
|
||||
SpendLogsPayload,
|
||||
{
|
||||
"request_id": "req-1",
|
||||
"request_tags": '["cost-center:cc-42", "prod"]',
|
||||
"team_id": "team-from-jwt",
|
||||
"user": "user-1",
|
||||
"startTime": "2026-10-29T00:00:00",
|
||||
"api_key": "a" * 64,
|
||||
"model": "gpt-4o",
|
||||
"custom_llm_provider": "openai",
|
||||
"model_group": "gpt-4o",
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"spend": 0.01,
|
||||
"metadata": '{"usage_object": {}}',
|
||||
},
|
||||
)
|
||||
|
||||
await writer.add_spend_log_transaction_to_daily_tag_transaction(
|
||||
payload=payload, prisma_client=prisma, virtual_key_team_id="team-a"
|
||||
)
|
||||
queued: Final = await writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
tag_rows: Final = tuple(cast(DailyTagSpendTransaction, txn) for txn in queued.values())
|
||||
|
||||
assert sorted((txn["tag"], txn.get("team_id")) for txn in tag_rows) == [
|
||||
("cost-center:cc-42", "team-a"),
|
||||
("prod", "team-a"),
|
||||
], queued
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_spend_log_transaction_to_daily_org_transaction_injects_org_id_and_queues_update():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -8156,6 +8156,19 @@ def _passthrough_kwargs_for_reservation(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("via_virtual_key", "expected"), [(True, "team-a"), (False, "")])
|
||||
def test_pass_through_tag_spend_keeps_the_virtual_key_team(via_virtual_key: bool, expected: str) -> None:
|
||||
from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id
|
||||
|
||||
auth: Final = UserAPIKeyAuth(api_key="a" * 64, team_id="team-a").model_copy(
|
||||
update={"via_virtual_key": via_virtual_key}
|
||||
)
|
||||
|
||||
kwargs: Final = _passthrough_kwargs_for_reservation(auth)
|
||||
|
||||
assert _virtual_key_team_id(kwargs) == expected
|
||||
|
||||
|
||||
async def _track_cost_for_passthrough_kwargs(kwargs: dict) -> AsyncMock:
|
||||
from datetime import datetime
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id
|
||||
from litellm.proxy.spend_tracking.spend_event import (
|
||||
CACHE_OFF_KEY,
|
||||
SpendEventBuildError,
|
||||
|
|
@ -151,6 +153,21 @@ def test_round_trip_preserves_identity_usage_reservation_and_tools():
|
|||
assert "complete_streaming_response" not in args.kwargs
|
||||
|
||||
|
||||
def test_round_trip_keeps_the_virtual_key_team_for_tag_spend():
|
||||
base: Final = _success_kwargs()
|
||||
kwargs: Final = {
|
||||
**base,
|
||||
"litellm_params": {
|
||||
**base["litellm_params"],
|
||||
"metadata": {**base["litellm_params"]["metadata"], USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True},
|
||||
},
|
||||
}
|
||||
event: Final = decode_spend_event(_build(kwargs, _response()))
|
||||
assert not isinstance(event, SpendEventDecodeError)
|
||||
|
||||
assert _virtual_key_team_id(spend_event_callback_args(event).kwargs) == "team-1"
|
||||
|
||||
|
||||
def test_streaming_event_reconstructs_complete_streaming_response():
|
||||
kwargs: Final = {**_success_kwargs(), "stream": True, "complete_streaming_response": _response()}
|
||||
event: Final = decode_spend_event(_build(kwargs, _response()))
|
||||
|
|
|
|||
|
|
@ -597,6 +597,46 @@ async def test_add_litellm_data_to_request_strips_all_user_api_key_prefix_keys()
|
|||
assert attacker_leaks == [], f"Unexpected leaked keys: {attacker_leaks}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("auth_fields", "expected"),
|
||||
[
|
||||
({"via_virtual_key": True}, True),
|
||||
({}, False),
|
||||
({"via_virtual_key": True, "is_session_token": True}, False),
|
||||
],
|
||||
ids=["virtual-key", "jwt-or-custom-auth", "ui-session"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_the_proxy_marks_a_request_as_made_with_a_virtual_key(
|
||||
auth_fields: dict[str, bool], expected: bool
|
||||
) -> None:
|
||||
from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
|
||||
request_mock = MagicMock(spec=Request)
|
||||
request_mock.url = MagicMock()
|
||||
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
|
||||
request_mock.method = "POST"
|
||||
request_mock.query_params = {}
|
||||
request_mock.headers = {"Content-Type": "application/json"}
|
||||
request_mock.client = MagicMock()
|
||||
request_mock.client.host = "127.0.0.1"
|
||||
forged: Final = {USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True, "user_api_key_team_id": "victim-team"}
|
||||
|
||||
updated: Final = await add_litellm_data_to_request(
|
||||
data={"model": "gpt-3.5-turbo", "metadata": {**forged}, "litellm_metadata": {**forged}},
|
||||
request=request_mock,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="a" * 64, team_id="team-a").model_copy(update=auth_fields),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
assert updated["metadata"][USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY] is expected
|
||||
assert updated["metadata"]["user_api_key_team_id"] == "team-a"
|
||||
assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in (updated.get("litellm_metadata") or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_string_metadata_does_not_crash():
|
||||
"""Regression: pre-strip code that pre-populated data['metadata'][k]=v
|
||||
|
|
|
|||
|
|
@ -600,6 +600,7 @@ def test_foward_litellm_user_info_to_backend_llm_call():
|
|||
"x-litellm-user_api_key_hash": "test_api_key",
|
||||
"x-litellm-user_api_key_spend": "0.0",
|
||||
"x-litellm-user_api_key_auth_metadata": "{}",
|
||||
"x-litellm-user_api_key_is_virtual_key": "False",
|
||||
}
|
||||
|
||||
assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue