diff --git a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py index a4a40a7a3d7..b0300e1653c 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger -from litellm.constants import CLI_SESSION_KEY_PREFIX +from litellm.constants import CLI_SESSION_KEY_PREFIX, USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY from litellm.repositories.table_repositories import ( DeletedVerificationTokenRepository, DeprecatedVerificationTokenRepository, @@ -310,6 +310,7 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, "user_api_key": api_key, "user_api_key_hash": api_key, "user_api_key_team_id": team_id, + USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: rows.key is not None, **( { "user_api_key_user_email": getattr(rows.user, "user_email", None), diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_daily_tag_spend_add_team_id/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_daily_tag_spend_add_team_id/migration.sql new file mode 100644 index 00000000000..0e15d9745b5 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_daily_tag_spend_add_team_id/migration.sql @@ -0,0 +1,8 @@ +SET LOCAL lock_timeout = '10s'; + +-- The team that owns the virtual key a tag row was spent through. "" when the request did not +-- come through a virtual key with a team (JWT, custom auth, OAuth2, team-less keys), and on every +-- row written before this migration. Not part of the unique key: a team's virtual key already +-- maps the row to one team, so older proxies keep upserting on the unchanged key during a rolling +-- upgrade. On PostgreSQL 11+ adding a column with a constant default does not rewrite the table. +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "team_id" TEXT NOT NULL DEFAULT ''; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000001_daily_tag_spend_team_id_date_tag_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000001_daily_tag_spend_team_id_date_tag_index/migration.sql new file mode 100644 index 00000000000..474d5311305 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000001_daily_tag_spend_team_id_date_tag_index/migration.sql @@ -0,0 +1,12 @@ +-- CreateIndex (CONCURRENTLY) +-- +-- Serves team-scoped tag reads: WHERE team_id = ANY(...) AND date BETWEEN ... grouped by tag. +-- +-- Disclaimer: +-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a +-- single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction. +-- - If the build is interrupted, Postgres may leave an INVALID index that must be dropped and +-- recreated. +-- - Do not edit this file after it has been applied to any database: Prisma checksums +-- migrations; add a new migration instead. +CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_DailyTagSpend_team_id_date_tag_idx" ON "LiteLLM_DailyTagSpend"("team_id", "date", "tag"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 179b741fe7c..a02d2e46bd8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend { id String @id @default(uuid()) request_id String? tag String? + team_id String @default("") date String api_key String model String? @@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend { @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag, date]) + @@index([team_id, date, tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) diff --git a/litellm/constants.py b/litellm/constants.py index e21751c7fd1..8e101950cf3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1693,6 +1693,7 @@ ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id" BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" +USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: Final = "user_api_key_is_virtual_key" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3d4c29c5712..83bc85042da 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,7 @@ from litellm.constants import ( PROVIDER_REQUEST_ID_HEADERS, REALTIME_METERED_USAGE_KEY, REDACTED_BY_LITELLM, + USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY, ) from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, @@ -328,9 +329,10 @@ _in_memory_loggers: Final[list[CustomLogger]] = [] _STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY, "is_silent_experiment") ) -_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( - frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS -) +_STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = frozenset(StandardLoggingMetadata.__annotations__.keys()) - { + *_STANDARD_LOGGING_METADATA_RESOLVED_KEYS, + USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY, +} def _get_provider_request_id(original_exception: Exception) -> str | None: @@ -8064,7 +8066,10 @@ def get_standard_logging_metadata( ) if isinstance(metadata, dict): # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields - for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - {"is_silent_experiment"}: + for key in (metadata.keys() & StandardLoggingMetadata.__annotations__.keys()) - { + "is_silent_experiment", + USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY, + }: clean_metadata[key] = metadata[key] if metadata.get("user_api_key") is not None: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7b7bfc55ade..bf4772af26d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5270,6 +5270,7 @@ class DailyEndUserSpendTransaction(BaseDailySpendTransaction): class DailyTagSpendTransaction(BaseDailySpendTransaction): request_id: str | None tag: str + team_id: ReadOnly[NotRequired[str]] class DailyAgentSpendTransaction(BaseDailySpendTransaction): diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 4d99dc5db34..496d2e55a67 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -49,6 +49,7 @@ BatchStatus: TypeAlias = Literal[ TERMINAL_BATCH_STATUSES: Final[frozenset[str]] = frozenset({"completed", "failed", "cancelled", "expired"}) _STOP_STATUSES: Final[frozenset[str]] = TERMINAL_BATCH_STATUSES | frozenset({"cancelling"}) _BATCH_ENDPOINT_ADAPTER: Final[TypeAdapter[BatchEndpoint]] = TypeAdapter(BatchEndpoint) +_CLIENT_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CANCEL_POLL_SECONDS: Final = 1.0 _HEARTBEAT_SECONDS: Final = 30.0 _STALE_AFTER_SECONDS: Final = 180.0 @@ -350,6 +351,19 @@ def _error_body(error: Exception) -> _ErrorBody: return body +def _without_client_key_metadata(body: Mapping[str, object]) -> Mapping[str, object]: + litellm_metadata: Final = body.get("litellm_metadata") + if not isinstance(litellm_metadata, Mapping): + return body + client_metadata: Final = _CLIENT_METADATA_ADAPTER.validate_python(litellm_metadata) + return { + **body, + "litellm_metadata": { + key: value for key, value in client_metadata.items() if not key.startswith("user_api_key_") + }, + } + + def _line_response(outcome: RowOutcome | ExpiredRow) -> _ResultResponse | None: if isinstance(outcome, ExpiredRow): return None @@ -645,7 +659,12 @@ class LiteLLMExecutedBatchRunner: async def _dispatch(self, run: _BatchRun, line: BatchInputLine) -> Mapping[str, object]: params: Final = MappingProxyType( - {**line.body, "model": run.model, "metadata": self._row_metadata(run), "disable_fallbacks": True} + { + **_without_client_key_metadata(line.body), + "model": run.model, + "metadata": self._row_metadata(run), + "disable_fallbacks": True, + } ) return _dump(await self._router_call(run.endpoint)(**params)) diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..2c20c521fb5 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -32,6 +32,7 @@ class DailySpendTable: name: str entity_id_column: str carries_request_id: bool = False + carries_team_id: bool = False DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingProxyType( @@ -41,7 +42,9 @@ DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingP "org": DailySpendTable(name="LiteLLM_DailyOrganizationSpend", entity_id_column="organization_id"), "end_user": DailySpendTable(name="LiteLLM_DailyEndUserSpend", entity_id_column="end_user_id"), "agent": DailySpendTable(name="LiteLLM_DailyAgentSpend", entity_id_column="agent_id"), - "tag": DailySpendTable(name="LiteLLM_DailyTagSpend", entity_id_column="tag", carries_request_id=True), + "tag": DailySpendTable( + name="LiteLLM_DailyTagSpend", entity_id_column="tag", carries_request_id=True, carries_team_id=True + ), } ) @@ -168,6 +171,7 @@ def _row_params( *(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS), *(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS), *((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()), + *((_as_text(transaction.get("team_id")),) if table.carries_team_id else ()), ) @@ -180,6 +184,7 @@ def _insert_columns(table: DailySpendTable) -> tuple[str, ...]: *_COUNTER_COLUMNS, *_SPEND_COLUMNS, *(("request_id",) if table.carries_request_id else ()), + *(("team_id",) if table.carries_team_id else ()), ) @@ -210,11 +215,16 @@ def build_bulk_upsert( if table.carries_request_id else "" ) + team_id_update: Final = ( + f', "team_id" = COALESCE(NULLIF(EXCLUDED."team_id", \'\'), {quoted_table}."team_id")' + if table.carries_team_id + else "" + ) sql: Final = ( f'INSERT INTO {quoted_table} ({_quoted(columns)}, "updated_at")\n' f"VALUES {rows}\n" f"ON CONFLICT ({_quoted((table.entity_id_column, *_KEY_COLUMNS))}) DO UPDATE SET\n" - f" {increments}{request_id_update},\n" + f" {increments}{request_id_update}{team_id_update},\n" f" \"updated_at\" = (NOW() AT TIME ZONE 'UTC')" ) return sql, tuple(value for key, transaction in batch for value in _row_params(table, key, transaction)) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index d9fb236632b..a95675c79f0 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -33,6 +33,7 @@ from litellm.constants import ( DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, + USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY, ) from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups @@ -351,6 +352,20 @@ def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadat return parsed +def _virtual_key_team_id(kwargs: Mapping[str, object]) -> str: + """The team of the virtual key that authenticated the request, or "" for JWT, custom auth, OAuth2, + UI sessions, the master key and team-less keys, whose api_key does not pin a tag row to one team.""" + params: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs.get("litellm_params") or {}) + buckets: Final = tuple( + _SPEND_METADATA_ADAPTER.validate_python(params.get(bucket) or {}) for bucket in ("litellm_metadata", "metadata") + ) + marked: Final = next( + (bucket for bucket in buckets if bucket.get(USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY) is True), None + ) + team_id: Final = marked.get("user_api_key_team_id") if marked is not None else None + return team_id if isinstance(team_id, str) else "" + + def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: """An earlier billed retry attempt's own spend row, not a request the caller sent.""" return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True @@ -721,6 +736,7 @@ class DBSpendUpdateWriter: litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, request_model_access_groups=get_request_model_access_groups(kwargs), + virtual_key_team_id=_virtual_key_team_id(_SPEND_METADATA_ADAPTER.validate_python(kwargs or {})), ), task_name="spend rollups", ) @@ -1145,6 +1161,7 @@ class DBSpendUpdateWriter: payload: SpendLogsPayload, request_model_access_groups: Sequence[str] = (), project_id: str | None = None, + virtual_key_team_id: str = "", ): """ Runs every spend-update helper for one request. Each helper goes through @@ -1244,7 +1261,9 @@ class DBSpendUpdateWriter: ) await self._run_spend_update_step( "add_spend_log_transaction_to_daily_tag_transaction", - self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client), + self.add_spend_log_transaction_to_daily_tag_transaction( + payload=payload_copy, prisma_client=prisma_client, virtual_key_team_id=virtual_key_team_id + ), payload=payload_copy, ) await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client) @@ -3079,6 +3098,7 @@ class DBSpendUpdateWriter: self, payload: SpendLogsPayload, prisma_client: PrismaClient | None = None, + virtual_key_team_id: str = "", ) -> None: if prisma_client is None: verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.") @@ -3100,7 +3120,7 @@ class DBSpendUpdateWriter: endpoint_str = base_daily_transaction.get("endpoint") or "" daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}" daily_transaction = DailyTagSpendTransaction( - tag=tag, **base_daily_transaction, request_id=payload["request_id"] + tag=tag, team_id=virtual_key_team_id, **base_daily_transaction, request_id=payload["request_id"] ) await self.daily_tag_spend_update_queue.add_update(update={daily_transaction_key: daily_transaction}) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4444d6d25f9..98d87d3c034 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2082,6 +2082,7 @@ class LiteLLMProxyRequestSetup: user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None ), user_api_key_auth_metadata=auth_metadata, + user_api_key_is_virtual_key=LiteLLMProxyRequestSetup.authenticated_with_virtual_key(user_api_key_dict), ) return user_api_key_logged_metadata @@ -2093,6 +2094,12 @@ class LiteLLMProxyRequestSetup: project_tags: Final = (user_api_key_dict.project_metadata or {}).get("tags") or () return tuple(dict.fromkeys((*key_tags, *team_tags, *project_tags))) + @staticmethod + def authenticated_with_virtual_key(user_api_key_dict: UserAPIKeyAuth) -> bool: + """A serializable copy of the unforgeable via_virtual_key marker, carried in the key's logged + metadata so spend paths that drop the auth object still know the request came through a virtual key.""" + return user_api_key_dict.via_virtual_key and not user_api_key_dict.is_session_token + @staticmethod def add_user_api_key_auth_to_request_metadata( data: dict, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 179b741fe7c..a02d2e46bd8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend { id String @id @default(uuid()) request_id String? tag String? + team_id String @default("") date String api_key String model String? @@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend { @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag, date]) + @@index([team_id, date, tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f4763acaeda..2f12b628349 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3074,6 +3074,7 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict): user_api_key_end_user_id: str | None user_api_key_request_route: str | None user_api_key_auth_metadata: dict[str, str] | None + user_api_key_is_virtual_key: ReadOnly[NotRequired[bool]] class StandardLoggingMCPToolCall(TypedDict, total=False): diff --git a/schema.prisma b/schema.prisma index 179b741fe7c..a02d2e46bd8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1086,6 +1086,7 @@ model LiteLLM_DailyTagSpend { id String @id @default(uuid()) request_id String? tag String? + team_id String @default("") date String api_key String model String? @@ -1114,6 +1115,7 @@ model LiteLLM_DailyTagSpend { @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint]) @@index([date]) @@index([tag, date]) + @@index([team_id, date, tag]) @@index([api_key]) @@index([model]) @@index([mcp_namespaced_tool_name]) diff --git a/tests/integration/spend/test_team_tag_spend_attribution.py b/tests/integration/spend/test_team_tag_spend_attribution.py new file mode 100644 index 00000000000..558735f282e --- /dev/null +++ b/tests/integration/spend/test_team_tag_spend_attribution.py @@ -0,0 +1,112 @@ +import math +import uuid +from typing import Final + +from integration._support.client import Gateway, Scenario, eventually +from integration._support.database import read_rows, write_rows +from pydantic import BaseModel, ConfigDict, TypeAdapter + +_SPEND_PER_CALL: Final = 20 * 0.001 + 20 * 0.002 + + +class _Usage(BaseModel): + model_config = ConfigDict(frozen=True) + + total_tokens: int + + +class _Reply(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: _Usage + + +class _TagRow(BaseModel): + model_config = ConfigDict(frozen=True) + + team_id: str + spend: float + api_requests: int + + +_TAG_ROWS: Final = TypeAdapter(tuple[_TagRow, ...]) + + +def _send_tagged(gateway: Gateway, *, key: str, model: str, tag: str, calls: int) -> None: + for _ in range(calls): + reply = gateway.post( + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tagged {uuid.uuid4().hex}"}], + "metadata": {"tags": [tag]}, + }, + key=key, + ) + assert _Reply.model_validate(reply).usage.total_tokens == 40, reply + + +def _tag_rows(tag: str, *, requests: int) -> tuple[_TagRow, ...]: + return eventually( + lambda: _TAG_ROWS.validate_python( + read_rows( + 'SELECT team_id, spend, api_requests FROM "LiteLLM_DailyTagSpend" WHERE tag = %s ORDER BY team_id', + (tag,), + ) + ), + lambda rows: sum(row.api_requests for row in rows) == requests, + seconds=70, + ) + + +def _model(scenario: Scenario) -> str: + return scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + + +def test_a_shared_tag_is_split_by_the_team_of_each_virtual_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = _model(scenario) + tag: Final = f"cost-center:{uuid.uuid4().hex}" + teams: Final = tuple(scenario.team(models=[model]) for _ in range(2)) + calls: Final = {teams[0]: 1, teams[1]: 2, "": 1} + for team, count in calls.items(): + key = scenario.key(team_id=team, models=[model]) if team else scenario.key(models=[model]) + _send_tagged(gateway, key=key, model=model, tag=tag, calls=count) + + rows: Final = _tag_rows(tag, requests=sum(calls.values())) + + assert sorted((row.team_id, row.api_requests) for row in rows) == sorted(calls.items()), rows + for row in rows: + assert math.isclose(row.spend, calls[row.team_id] * _SPEND_PER_CALL), rows + + +def test_a_key_moved_between_teams_keeps_one_row_under_its_latest_team(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = _model(scenario) + tag: Final = f"moved-key:{uuid.uuid4().hex}" + teams: Final = tuple(scenario.team(models=[model]) for _ in range(2)) + key: Final = scenario.key(team_id=teams[0], models=[model]) + _send_tagged(gateway, key=key, model=model, tag=tag, calls=1) + _tag_rows(tag, requests=1) + gateway.post("/key/update", {"key": key, "team_id": teams[1]}) + _send_tagged(gateway, key=key, model=model, tag=tag, calls=2) + + rows: Final = _tag_rows(tag, requests=3) + + assert [(row.team_id, row.api_requests) for row in rows] == [(teams[1], 3)], rows + + +def test_a_row_an_older_proxy_wrote_without_a_team_gets_the_team_on_the_next_write(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = _model(scenario) + tag: Final = f"rolling-upgrade:{uuid.uuid4().hex}" + team: Final = scenario.team(models=[model]) + key: Final = scenario.key(team_id=team, models=[model]) + _send_tagged(gateway, key=key, model=model, tag=tag, calls=1) + _tag_rows(tag, requests=1) + write_rows("UPDATE \"LiteLLM_DailyTagSpend\" SET team_id = '' WHERE tag = %s", (tag,)) + _send_tagged(gateway, key=key, model=model, tag=tag, calls=1) + + rows: Final = _tag_rows(tag, requests=2) + + assert [(row.team_id, row.api_requests) for row in rows] == [(team, 2)], rows diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e952127a32c..1c23673a158 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -5737,6 +5737,22 @@ def test_get_standard_logging_metadata_merges_recorded_applied_guardrails(): assert result["applied_guardrails"] == ["guard-a", "blocker"] +def test_logging_metadata_leaves_out_the_virtual_key_flag(): + from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup, get_standard_logging_metadata + + metadata: Final = {USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True, "user_api_key_team_id": "team-a"} + + payload_metadata: Final = StandardLoggingPayloadSetup.get_standard_logging_metadata( + metadata=metadata, litellm_params={} + ) + standalone_metadata: Final = get_standard_logging_metadata(metadata) + + assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in payload_metadata + assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in standalone_metadata + assert payload_metadata["user_api_key_team_id"] == standalone_metadata["user_api_key_team_id"] == "team-a" + + def test_function_setup_metadata_takes_precedence_over_litellm_metadata(): """ Test that when BOTH metadata and litellm_metadata are present (e.g., user sets diff --git a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py index 93292a3a864..0272cbb3399 100644 --- a/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py +++ b/tests/unit/proxy/batches_endpoints/test_litellm_executed_batches.py @@ -660,7 +660,9 @@ async def test_resolve_litellm_executed_provider_asks_the_router_for_the_team_sc assert ( await resolve_litellm_executed_provider(router, BATCH_MODEL, "team-1", FakeFilesApiProbe(True, [])) == expected ) - router.registry.return_value.credentials.assert_called_once_with(BATCH_MODEL, form="with_provider", team_id="team-1") + router.registry.return_value.credentials.assert_called_once_with( + BATCH_MODEL, form="with_provider", team_id="team-1" + ) async def test_create_stores_a_validating_batch_and_completes_it_in_the_background() -> None: @@ -711,6 +713,37 @@ async def test_create_dispatches_each_row_with_the_batch_model_and_the_key_metad assert metadata["user_api_key_user_email"] == "user@example.com" +@pytest.mark.parametrize(("via_virtual_key", "expected"), [(True, "team-1"), (False, "")]) +async def test_each_row_keeps_the_virtual_key_team_for_tag_spend(via_virtual_key: bool, expected: str) -> None: + from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id + + user = UserAPIKeyAuth(api_key="sk-batch-key", team_id="team-1").model_copy( + update={"via_virtual_key": via_virtual_key} + ) + harness = make_runner(user=user) + await harness.create_and_finish() + + teams = [ + _virtual_key_team_id({"litellm_params": {"metadata": call.kwargs["metadata"]}}) + for call in harness.router.acompletion.await_args_list + ] + assert teams == [expected] * 2 + + +async def test_a_row_cannot_claim_another_team_through_its_own_litellm_metadata() -> None: + from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id + + forged = {"user_api_key_is_virtual_key": True, "user_api_key_team_id": "team-other", "custom_field": "kept"} + user = UserAPIKeyAuth(api_key="sk-batch-key", team_id="team-1").model_copy(update={"via_virtual_key": False}) + harness = make_runner(content=jsonl(chat_row("row-1", "hi 1", litellm_metadata=forged)), user=user) + await harness.create_and_finish() + + (call,) = harness.router.acompletion.await_args_list + params = {"metadata": call.kwargs["metadata"], "litellm_metadata": call.kwargs["litellm_metadata"]} + assert _virtual_key_team_id({"litellm_params": params}) == "" + assert call.kwargs["litellm_metadata"] == {"custom_field": "kept"} + + async def test_create_dispatches_each_row_with_the_key_team_and_project_tags_as_inherited_tags() -> None: user = UserAPIKeyAuth( api_key="sk-batch-key", diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 5b37cee88c5..faec25d68cd 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -2486,6 +2486,21 @@ class TestBatchCostAttribution: assert metadata["user_api_key_team_alias"] == "Team Alpha" assert metadata["tags"] == ["env:prod"] + @pytest.mark.parametrize(("key_row_exists", "expected"), [(True, "team-alpha"), (False, "")]) + @pytest.mark.asyncio + async def test_tag_spend_team_needs_the_batch_to_come_from_a_virtual_key(self, key_row_exists, expected): + """Tag spend is attributed to a team only when the creating key is a real virtual key row, + not a JWT, a CLI session or a deleted key.""" + from types import SimpleNamespace + + from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id + + instance = self._instance(key_row=SimpleNamespace(key_alias="prod-key") if key_row_exists else None) + + metadata = await instance.batch_metadata(self._job(), "batch-1") + + assert _virtual_key_team_id({"litellm_params": {"metadata": metadata}}) == expected + @pytest.mark.asyncio async def test_metadata_tolerates_legacy_row_without_columns(self): """Rows created before the columns existed carry only created_by/team_id and must diff --git a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py index 5bce9137666..1f80e184516 100644 --- a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py @@ -88,10 +88,10 @@ def test_one_statement_carries_every_row_in_the_batch(): assert sql.count("INSERT INTO") == 1 assert len(re.findall(r"ON CONFLICT", sql)) == 1 - # 25 bound columns per row plus the inlined updated_at, so the row count is what + # 26 bound columns per row plus the inlined updated_at, so the row count is what # separates one multi-row statement from a hundred single-row ones. - assert len(params) == 100 * 25 - assert "$2500::text" in sql + assert len(params) == 100 * 26 + assert "$2600::text" in sql assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1 diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 298b153f621..29ad4979a1b 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -20,14 +20,20 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes -from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS -from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem +from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS, USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY +from litellm.proxy._types import ( + DailyTagSpendTransaction, + Litellm_EntityType, + SpendLogsPayload, + SpendUpdateQueueItem, +) from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, _TEAM_MEMBER_SPEND_SQL, DBSpendUpdateWriter, _spend_tables_left_to_send, _SpendTableName, + _virtual_key_team_id, ) from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer @@ -1338,6 +1344,45 @@ async def test_batch_database_updates_queues_org_member_spend_for_the_request_us assert transactions["org_member_list_transactions"] == {"organization_id::org1::user_id::u1": 0.1} +@pytest.mark.asyncio +async def test_batch_database_updates_puts_the_virtual_key_team_on_tag_rows(): + db_writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + await db_writer._batch_database_updates( + response_cost=0.01, + user_id="u1", + hashed_token="a" * 64, + team_id="team-a", + org_id=None, + end_user_id=None, + prisma_client=prisma, + litellm_proxy_budget_name=None, + payload=cast( + SpendLogsPayload, + { + "request_id": "req-1", + "request_tags": '["cost-center:cc-42"]', + "team_id": "team-a", + "user": "u1", + "startTime": "2026-10-29T00:00:00", + "api_key": "a" * 64, + "model": "gpt-4o", + "custom_llm_provider": "openai", + "model_group": "gpt-4o", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.01, + "metadata": '{"usage_object": {}}', + }, + ), + virtual_key_team_id="team-a", + ) + queued: Final = await db_writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + + assert [cast(DailyTagSpendTransaction, txn).get("team_id") for txn in queued.values()] == ["team-a"] + + @pytest.mark.asyncio async def test_project_spend_is_persisted_to_project_table_and_project_cache_is_evicted(): db_writer: Final = DBSpendUpdateWriter() @@ -1494,6 +1539,72 @@ async def test_add_spend_log_transaction_to_daily_tag_transaction_with_request_i ) +@pytest.mark.parametrize( + ("bucket", "marker", "team_id", "expected"), + [ + ("metadata", True, "team-a", "team-a"), + ("litellm_metadata", True, "team-a", "team-a"), + ("metadata", True, None, ""), + ("metadata", False, "team-a", ""), + ("metadata", "true", "team-a", ""), + ("metadata", None, "team-a", ""), + ], + ids=[ + "virtual-key", + "virtual-key-litellm-metadata", + "virtual-key-without-team", + "jwt-or-custom-auth", + "non-boolean-marker", + "no-marker", + ], +) +def test_tag_team_comes_only_from_a_request_marked_as_a_virtual_key( + bucket: str, marker: object, team_id: str | None, expected: str +) -> None: + metadata: Final = { + "user_api_key_team_id": team_id, + **({USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: marker} if marker is not None else {}), + } + + assert _virtual_key_team_id({"litellm_params": {bucket: metadata}}) == expected + + +@pytest.mark.asyncio +async def test_every_tag_row_of_a_request_carries_its_virtual_key_team() -> None: + writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + payload: Final = cast( + SpendLogsPayload, + { + "request_id": "req-1", + "request_tags": '["cost-center:cc-42", "prod"]', + "team_id": "team-from-jwt", + "user": "user-1", + "startTime": "2026-10-29T00:00:00", + "api_key": "a" * 64, + "model": "gpt-4o", + "custom_llm_provider": "openai", + "model_group": "gpt-4o", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.01, + "metadata": '{"usage_object": {}}', + }, + ) + + await writer.add_spend_log_transaction_to_daily_tag_transaction( + payload=payload, prisma_client=prisma, virtual_key_team_id="team-a" + ) + queued: Final = await writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + tag_rows: Final = tuple(cast(DailyTagSpendTransaction, txn) for txn in queued.values()) + + assert sorted((txn["tag"], txn.get("team_id")) for txn in tag_rows) == [ + ("cost-center:cc-42", "team-a"), + ("prod", "team-a"), + ], queued + + @pytest.mark.asyncio async def test_add_spend_log_transaction_to_daily_org_transaction_injects_org_id_and_queues_update(): """ diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 4fe358c38a3..1a35bb72ebd 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -8156,6 +8156,19 @@ def _passthrough_kwargs_for_reservation( ) +@pytest.mark.parametrize(("via_virtual_key", "expected"), [(True, "team-a"), (False, "")]) +def test_pass_through_tag_spend_keeps_the_virtual_key_team(via_virtual_key: bool, expected: str) -> None: + from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id + + auth: Final = UserAPIKeyAuth(api_key="a" * 64, team_id="team-a").model_copy( + update={"via_virtual_key": via_virtual_key} + ) + + kwargs: Final = _passthrough_kwargs_for_reservation(auth) + + assert _virtual_key_team_id(kwargs) == expected + + async def _track_cost_for_passthrough_kwargs(kwargs: dict) -> AsyncMock: from datetime import datetime diff --git a/tests/unit/proxy/spend_tracking/test_spend_event.py b/tests/unit/proxy/spend_tracking/test_spend_event.py index ff449235582..e65d834e41d 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_event.py +++ b/tests/unit/proxy/spend_tracking/test_spend_event.py @@ -6,7 +6,9 @@ import pytest import litellm from litellm.caching.caching import Cache +from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.db.db_spend_update_writer import _virtual_key_team_id from litellm.proxy.spend_tracking.spend_event import ( CACHE_OFF_KEY, SpendEventBuildError, @@ -151,6 +153,21 @@ def test_round_trip_preserves_identity_usage_reservation_and_tools(): assert "complete_streaming_response" not in args.kwargs +def test_round_trip_keeps_the_virtual_key_team_for_tag_spend(): + base: Final = _success_kwargs() + kwargs: Final = { + **base, + "litellm_params": { + **base["litellm_params"], + "metadata": {**base["litellm_params"]["metadata"], USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True}, + }, + } + event: Final = decode_spend_event(_build(kwargs, _response())) + assert not isinstance(event, SpendEventDecodeError) + + assert _virtual_key_team_id(spend_event_callback_args(event).kwargs) == "team-1" + + def test_streaming_event_reconstructs_complete_streaming_response(): kwargs: Final = {**_success_kwargs(), "stream": True, "complete_streaming_response": _response()} event: Final = decode_spend_event(_build(kwargs, _response())) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 91c87c63067..dbb06c53c7c 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -597,6 +597,46 @@ async def test_add_litellm_data_to_request_strips_all_user_api_key_prefix_keys() assert attacker_leaks == [], f"Unexpected leaked keys: {attacker_leaks}" +@pytest.mark.parametrize( + ("auth_fields", "expected"), + [ + ({"via_virtual_key": True}, True), + ({}, False), + ({"via_virtual_key": True, "is_session_token": True}, False), + ], + ids=["virtual-key", "jwt-or-custom-auth", "ui-session"], +) +@pytest.mark.asyncio +async def test_only_the_proxy_marks_a_request_as_made_with_a_virtual_key( + auth_fields: dict[str, bool], expected: bool +) -> None: + from litellm.constants import USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + forged: Final = {USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY: True, "user_api_key_team_id": "victim-team"} + + updated: Final = await add_litellm_data_to_request( + data={"model": "gpt-3.5-turbo", "metadata": {**forged}, "litellm_metadata": {**forged}}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="a" * 64, team_id="team-a").model_copy(update=auth_fields), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["metadata"][USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY] is expected + assert updated["metadata"]["user_api_key_team_id"] == "team-a" + assert USER_API_KEY_IS_VIRTUAL_KEY_METADATA_KEY not in (updated.get("litellm_metadata") or {}) + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_string_metadata_does_not_crash(): """Regression: pre-strip code that pre-populated data['metadata'][k]=v diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index 1b9cdd003ab..5d48226191f 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -600,6 +600,7 @@ def test_foward_litellm_user_info_to_backend_llm_call(): "x-litellm-user_api_key_hash": "test_api_key", "x-litellm-user_api_key_spend": "0.0", "x-litellm-user_api_key_auth_metadata": "{}", + "x-litellm-user_api_key_is_virtual_key": "False", } assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True)