From 29e13cc9dfa1f4ff951a1e23fdbef8e67b5cde8a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:48:49 -0700 Subject: [PATCH 1/5] fix(managed_files): keep the upload the managed id encodes when two target models share a deployment --- .../proxy/hooks/managed_files.py | 106 +++++++++++----- ...test_vector_store_file_list_managed_ids.py | 71 +++++++++++ .../proxy/test_managed_files_hook.py | 113 ++++++++++++++++++ 3 files changed, 263 insertions(+), 27 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 416444664a3..3f27290c1f2 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -6,6 +6,7 @@ import base64 import json import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta from itertools import chain from types import MappingProxyType @@ -388,6 +389,76 @@ class _RouterFileCallKwargs(TypedDict, total=False): custom_llm_provider: ReadOnly[str | None] +_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str]) +_FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"}) +_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"}) + + +@dataclass(frozen=True, slots=True) +class _ProviderUpload: + model_id: str + provider_file_id: str + + +def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, ...]: + file_hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic file metadata + dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR) + ) + model_file_id_mapping: Final = _MODEL_FILE_ID_MAPPING.validate_python( + file_hidden_params.get("model_file_id_mapping") or {} + ) + return tuple( + _ProviderUpload(model_id=model_id, provider_file_id=provider_file_id) + for model_id, provider_file_id in model_file_id_mapping.items() + ) + + +def _is_first_upload_to_its_deployment(uploads: Sequence[_ProviderUpload], index: int) -> bool: + return all(earlier.model_id != uploads[index].model_id for earlier in uploads[:index]) + + +async def _delete_provider_file( + llm_router: Router, + model_id: str, + provider_file_id: str, + request_data: Mapping[str, object], +) -> None: + credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id=model_id) + delete_data: Final = { + **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}, + **( + {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} + if credentials is not None + else {} + ), + } + await llm_router.afile_delete( + model=model_id, + file_id=provider_file_id, + **cast(_RouterFileCallKwargs, delete_data), + ) + + +async def _delete_redundant_upload( + llm_router: Router, + upload: _ProviderUpload, + create_file_request: CreateFileRequest, +) -> None: + try: + await _delete_provider_file( + llm_router, + upload.model_id, + upload.provider_file_id, + {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}, + ) + except Exception as error: + verbose_logger.warning( + f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} " + f"at deployment model_id={sanitize_for_log(upload.model_id)}: " + f"{type(error).__name__} {sanitize_for_log(error)}" + ) + + def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase: return prisma_client.db @@ -1989,16 +2060,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): target_model_names_list=target_model_names_list, ) - ## STORE MODEL MAPPINGS IN DB - model_mappings: Dict[str, str] = {} - - for file_object in responses: - file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata - dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR) - ) - model_file_id_mapping = file_hidden_params.get("model_file_id_mapping") - if model_file_id_mapping and isinstance(model_file_id_mapping, dict): - model_mappings.update(model_file_id_mapping) + uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses))) + kept_uploads: Final = tuple( + upload for index, upload in enumerate(uploads) if _is_first_upload_to_its_deployment(uploads, index) + ) + for redundant_upload in (upload for upload in uploads if upload not in kept_uploads): + await _delete_redundant_upload(llm_router, redundant_upload, create_file_request) + model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads} await self.store_unified_file_id( file_id=response.id, @@ -2646,24 +2714,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): specific_model_file_id_mapping: Final = model_file_id_mapping.get(file_id) if not specific_model_file_id_mapping: return - filtered_data: Final = { - k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials") - } for model_id, model_file_id in specific_model_file_id_mapping.items(): - credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id) - delete_data = { - **filtered_data, - **( - {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} - if credentials is not None - else {} - ), - } - await llm_router.afile_delete( - model=model_id, - file_id=model_file_id, - **cast(_RouterFileCallKwargs, delete_data), - ) + await _delete_provider_file(llm_router, model_id, model_file_id, data) async def afile_content( self, diff --git a/tests/integration/management/test_vector_store_file_list_managed_ids.py b/tests/integration/management/test_vector_store_file_list_managed_ids.py index 16d33ee83af..ebf4b2aa33c 100644 --- a/tests/integration/management/test_vector_store_file_list_managed_ids.py +++ b/tests/integration/management/test_vector_store_file_list_managed_ids.py @@ -9,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass from functools import partial +from itertools import count from pathlib import Path from queue import SimpleQueue from types import MappingProxyType @@ -1042,6 +1043,76 @@ def test_multi_model_upload_maps_only_the_provider_id_the_managed_id_carries(gat assert second_ids == ((managed_a,) if carried == second.file_id("a.txt") else (second.file_id("a.txt"),)) +def _team_wildcard(scenario: Scenario, pattern: str, api_base: str, bearer: str, team: str) -> str: + identity: Final = f"wildcard-{uuid.uuid4().hex}" + scenario.gateway.post( + "/model/new", + { + "model_name": pattern, + "litellm_params": {"model": "openai/*", "api_key": bearer, "api_base": api_base}, + "model_info": {"id": identity, "team_id": team}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, identity) + eventually(lambda: _every_worker_has_deployment(scenario.gateway, identity), lambda served: served, seconds=90) + return identity + + +def test_two_targets_served_by_one_team_wildcard_keep_one_upload_that_delete_removes(gateway: Gateway) -> None: + store: Final = "vs_" + uuid.uuid4().hex + bearer: Final = "provider-key-" + uuid.uuid4().hex[:8] + prefix: Final = f"wc-{uuid.uuid4().hex}" + upload_numbers: Final = count(1) + first_upload: Final = f"file-{prefix}-1" + second_upload: Final = f"file-{prefix}-2" + served: Final = _provider(store, _constant_listing(store, first_upload)) + + def respond(request: Request) -> Reply: + if (request.method, urlsplit(request.target).path) != ("POST", "/v1/files"): + return served(request) + _assert_provider_upload(request, bearer, "a.txt") + return _json_reply(_file_object(f"file-{prefix}-{next(upload_numbers)}")) + + with gateway.scenario() as scenario, wire_server(respond) as wire: + team: Final = scenario.team(models=[]) + _team_wildcard(scenario, f"{prefix}/*", f"{wire.url}/v1", bearer, team) + key: Final = scenario.key(team_id=team, user_id=scenario.member(team)) + wire.drain() + + managed: Final = _upload(gateway, key, f"{prefix}/gpt-one,{prefix}/gpt-two", "a.txt") + assert _carried_provider_file_id(managed) == first_upload, _decoded(managed) + assert [row["flat_model_file_ids"] for row in read_rows(MANAGED_FILE_ROW, (managed,))] == [[first_upload]] + create_requests: Final = wire.drain() + assert [(request.method, request.target) for request in create_requests] == [ + ("POST", "/v1/files"), + ("POST", "/v1/files"), + ("DELETE", f"/v1/files/{second_upload}"), + ], create_requests + assert create_requests[2].headers["authorization"] == f"Bearer {bearer}", create_requests[2].headers + + attached: Final = gateway.request( + "POST", f"/v1/vector_stores/{store}/files", {"file_id": managed}, key=key, params={"model": f"{prefix}/*"} + ) + assert attached.status_code == 200, attached.text + listed: Final = gateway.request( + "GET", f"/v1/vector_stores/{store}/files", key=key, params={"model": f"{prefix}/*"} + ) + assert _ids(_listed(listed)) == (managed,), listed.text + attach_bodies: Final = [ + JSON_OBJECT.validate_json(request.body) for request in wire.drain() if request.method == "POST" + ] + assert attach_bodies == [{"file_id": first_upload}], attach_bodies + + deleted: Final = gateway.request("DELETE", f"/v1/files/{managed}", key=key) + assert deleted.status_code == 200, deleted.text + assert read_rows(MANAGED_FILE_ROW, (managed,)) == [] + delete_requests: Final = wire.drain() + assert [(request.method, request.target) for request in delete_requests] == [ + ("DELETE", f"/v1/files/{first_upload}") + ], delete_requests + assert delete_requests[0].headers["authorization"] == f"Bearer {bearer}", delete_requests[0].headers + + def test_deleting_the_managed_file_makes_its_listing_raw_again(gateway: Gateway) -> None: with _rig(gateway, "a.txt") as rig: member: Final = _member(rig.scenario, rig.model) diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index f9f4e17c760..11a2a229dd9 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -2161,3 +2161,116 @@ async def test_store_unified_file_id_caches_the_storage_location_the_db_row_gets assert (cached.storage_backend, cached.storage_url) == ("litellm_db", "litellm_db://content-row-1") create_data = file_table.upsert.await_args.kwargs["data"]["create"] assert (create_data["storage_backend"], create_data["storage_url"]) == ("litellm_db", "litellm_db://content-row-1") + + + + +class _FakeFilesRouter: + """Uploads each target model's file to every deployment it routes to, one new provider id per upload.""" + + def __init__(self, deployments_by_model, delete_error=None): + self.deployments_by_model = deployments_by_model + self.delete_error = delete_error + self.upload_count = 0 + self.deletes = () + + async def acreate_file(self, model, file, purpose, litellm_metadata): + deployment_ids: Final = self.deployments_by_model[model] + provider_file_ids: Final = tuple( + f"file-upload-{self.upload_count + offset + 1}" for offset in range(len(deployment_ids)) + ) + self.upload_count += len(deployment_ids) + response: Final = _make_file_object(provider_file_ids[0]).model_copy(update={"purpose": purpose}) + response._hidden_params = { + "model_id": deployment_ids[0], + "custom_llm_provider": "openai", + "model_file_id_mapping": dict(zip(deployment_ids, provider_file_ids)), + } + return response + + def get_deployment_credentials_with_provider(self, model_id): + return {"custom_llm_provider": "openai", "api_key": f"key-of-{model_id}"} + + async def afile_delete(self, model, file_id, litellm_metadata, _litellm_internal_model_credentials): + self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) + if self.delete_error is not None: + raise self.delete_error + + +_REQUEST_METADATA: Final = MappingProxyType({"user_api_key_team_id": "test-team"}) + + +async def _create_managed_file(router, target_model_names): + managed_files, mock_prisma = _make_real_managed_files_instance() + response: Final = await managed_files.acreate_file( + create_file_request={ + "file": ("notes.txt", b"the launch code is ORCHID-42\n", "text/plain"), + "purpose": "user_data", + "litellm_metadata": _REQUEST_METADATA, + }, + llm_router=router, + target_model_names_list=target_model_names, + litellm_parent_otel_span=None, + user_api_key_dict=_make_team_member_api_key_dict(), + ) + stored: Final = mock_prisma.db.litellm_managedfiletable.upsert.await_args.kwargs["data"]["create"] + decoded: Final = base64.urlsafe_b64decode(response.id + "=" * (-len(response.id) % 4)).decode() + carried: Final = ( + managed_files.get_model_id_from_unified_file_id(decoded), + managed_files.get_output_file_id_from_unified_file_id(decoded), + ) + return json.loads(stored["model_mappings"]), stored["flat_model_file_ids"], carried + + +@pytest.mark.asyncio +async def test_acreate_file_keeps_the_upload_the_managed_id_carries_when_two_targets_share_a_deployment(): + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}) + + model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b"]) + + assert carried == ("wildcard-dep", "file-upload-1") + assert model_mappings == {"wildcard-dep": "file-upload-1"} + assert flat_ids == ["file-upload-1"] + assert router.deletes == ( + ( + "wildcard-dep", + "file-upload-2", + _REQUEST_METADATA, + {"custom_llm_provider": "openai", "api_key": "key-of-wildcard-dep"}, + ), + ) + assert isinstance(router.deletes[0][3], MappingProxyType) + + +@pytest.mark.asyncio +async def test_acreate_file_deletes_only_the_repeat_upload_and_keeps_every_deployment_reached_once(): + router: Final = _FakeFilesRouter({"gpt-a": ("dep-1", "dep-2"), "gpt-b": ("dep-2",), "gpt-c": ("dep-3",)}) + + model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b", "gpt-c"]) + + assert carried == ("dep-1", "file-upload-1") + assert list(model_mappings.items()) == [ + ("dep-1", "file-upload-1"), + ("dep-2", "file-upload-2"), + ("dep-3", "file-upload-4"), + ] + assert flat_ids == ["file-upload-1", "file-upload-2", "file-upload-4"] + assert [delete[:2] for delete in router.deletes] == [("dep-2", "file-upload-3")] + + +@pytest.mark.asyncio +async def test_acreate_file_still_returns_the_managed_file_when_deleting_the_repeat_upload_fails(caplog): + router: Final = _FakeFilesRouter( + {"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}, + delete_error=RuntimeError("provider unavailable"), + ) + + with caplog.at_level(logging.WARNING): + model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b"]) + + assert carried == ("wildcard-dep", "file-upload-1") + assert model_mappings == {"wildcard-dep": "file-upload-1"} + assert flat_ids == ["file-upload-1"] + assert [delete[:2] for delete in router.deletes] == [("wildcard-dep", "file-upload-2")] + assert "file-upload-2" in caplog.text and "wildcard-dep" in caplog.text + assert "provider unavailable" in caplog.text From e483d9efeab5d203340274f5c70abfca5730719d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:56:03 -0700 Subject: [PATCH 2/5] fix(proxy): skip cost tracking for a file delete routed to a deployment --- litellm/proxy/hooks/proxy_track_cost_callback.py | 8 +++++--- .../proxy/hooks/test_proxy_track_cost_callback.py | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c11b59a809a..1c847d1ac5e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -124,6 +124,8 @@ _UNATTRIBUTED_TRACKABLE_CALL_TYPES: Final[frozenset[str]] = frozenset( } ) +_UNBILLED_CALL_TYPES: Final[frozenset[str]] = frozenset({"_arealtime", CallTypes.afile_delete.value}) + # Both spellings, because call_type reaches the callback as str(...) of either the # enum member or its value. _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -730,9 +732,9 @@ class _ProxyDBLogger(CustomLogger): ) return await _release_budget_reservation(budget_reservation=budget_reservation) - # Non-model call types (health checks, afile_delete) have no model or standard_logging_object. - # Use .get() for "stream" to avoid KeyError on health checks. - if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"): + # Health checks and file deletes have no standard_logging_object, and a file delete routed + # to a deployment still carries its model. Use .get() for "stream" to avoid KeyError on health checks. + if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") in _UNBILLED_CALL_TYPES): verbose_proxy_logger.warning( "Cost tracking - skipping, no standard_logging_object for call_type=%s", kwargs.get("call_type", "unknown"), diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 79193d37b04..b4ec4aacb3b 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1442,11 +1442,20 @@ class _FailedTrackingAlerts: @pytest.mark.asyncio -@pytest.mark.parametrize(("call_type", "alerted"), [("_aresponses_websocket", True), ("_arealtime", False)]) -async def test_track_cost_callback_alerts_on_a_websocket_turn_with_no_logging_object(call_type: str, alerted: bool): +@pytest.mark.parametrize( + ("call_type", "model", "alerted"), + [ + ("_aresponses_websocket", "gpt-5-mini", True), + ("_arealtime", "gpt-5-mini", False), + ("afile_delete", "openai/*", False), + ], +) +async def test_track_cost_callback_alerts_only_on_a_billable_call_with_no_logging_object( + call_type: str, model: str, alerted: bool +): kwargs: Final = { "call_type": call_type, - "model": "gpt-5-mini", + "model": model, "litellm_call_id": "turn-call-id", "litellm_params": {}, "stream": False, From 63e2067e43f799cbad3daa925e7e91f0097d38b8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:57:25 -0700 Subject: [PATCH 3/5] fix(managed_files): delete repeat uploads after the upload answers Run the repeat-copy deletes together in a background task so a slow provider delete no longer holds the upload's response, log the unbilled file-delete cost skip at debug, and type the new test helpers --- .../proxy/hooks/managed_files.py | 43 ++++++++++--- .../proxy/hooks/proxy_track_cost_callback.py | 11 ++-- .../proxy/test_managed_files_hook.py | 64 ++++++++++++++++--- .../hooks/test_proxy_track_cost_callback.py | 22 +++++++ 4 files changed, 118 insertions(+), 22 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index cffe835ebe6..d615f5adbad 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -35,7 +35,7 @@ from typing_extensions import ReadOnly, Unpack import litellm from litellm import Router, verbose_logger -from litellm._internal_context import with_service_target +from litellm._internal_context import post_response_phase, with_service_target from litellm._uuid import uuid from litellm.caching.caching import DualCache from litellm.constants import ( @@ -446,18 +446,16 @@ async def _delete_provider_file( ) +_REDUNDANT_UPLOAD_DELETES: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep cleanup deletes alive + + async def _delete_redundant_upload( llm_router: Router, upload: _ProviderUpload, - create_file_request: CreateFileRequest, + delete_request: Mapping[str, object], ) -> None: try: - await _delete_provider_file( - llm_router, - upload.model_id, - upload.provider_file_id, - {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS}, - ) + await _delete_provider_file(llm_router, upload.model_id, upload.provider_file_id, delete_request) except Exception as error: verbose_logger.warning( f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} " @@ -466,6 +464,30 @@ async def _delete_redundant_upload( ) +async def _delete_redundant_uploads( + llm_router: Router, + uploads: Sequence[_ProviderUpload], + delete_request: Mapping[str, object], +) -> None: + await asyncio.gather(*(_delete_redundant_upload(llm_router, upload, delete_request) for upload in uploads)) + + +def _start_deleting_redundant_uploads( + llm_router: Router, + uploads: Sequence[_ProviderUpload], + create_file_request: CreateFileRequest, +) -> None: + if not uploads: + return + delete_request: Final = MappingProxyType( + {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS} + ) + with post_response_phase(): + task: Final = asyncio.create_task(_delete_redundant_uploads(llm_router, uploads, delete_request)) + _REDUNDANT_UPLOAD_DELETES.add(task) + task.add_done_callback(_REDUNDANT_UPLOAD_DELETES.discard) + + def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase: return prisma_client.db @@ -2109,8 +2131,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): kept_uploads: Final = tuple( upload for index, upload in enumerate(uploads) if _is_first_upload_to_its_deployment(uploads, index) ) - for redundant_upload in (upload for upload in uploads if upload not in kept_uploads): - await _delete_redundant_upload(llm_router, redundant_upload, create_file_request) + _start_deleting_redundant_uploads( + llm_router, tuple(upload for upload in uploads if upload not in kept_uploads), create_file_request + ) model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads} await self.store_unified_file_id( diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index b37381b1b51..535bb0d8ef3 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -128,6 +128,12 @@ _UNATTRIBUTED_TRACKABLE_CALL_TYPES: Final[frozenset[str]] = frozenset( _UNBILLED_CALL_TYPES: Final[frozenset[str]] = frozenset({"_arealtime", CallTypes.afile_delete.value}) + +def _log_untracked_call_skip(call_type: object) -> None: + log: Final = verbose_proxy_logger.debug if call_type in _UNBILLED_CALL_TYPES else verbose_proxy_logger.warning + log("Cost tracking - skipping, no standard_logging_object for call_type=%s", call_type) + + # Both spellings, because call_type reaches the callback as str(...) of either the # enum member or its value. _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -920,10 +926,7 @@ class _ProxyDBLogger(CustomLogger): # Health checks and file deletes have no standard_logging_object, and a file delete routed # to a deployment still carries its model. Use .get() for "stream" to avoid KeyError on health checks. if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") in _UNBILLED_CALL_TYPES): - verbose_proxy_logger.warning( - "Cost tracking - skipping, no standard_logging_object for call_type=%s", - kwargs.get("call_type", "unknown"), - ) + _log_untracked_call_skip(kwargs.get("call_type", "unknown")) return if kwargs.get("stream") is not True or ( kwargs.get("stream") is True and "complete_streaming_response" in kwargs diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index b1b2a44b83d..a9293e7a964 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -9,6 +9,7 @@ import asyncio import base64 import json import logging +from collections.abc import Callable, Mapping from types import MappingProxyType import pytest @@ -2273,13 +2274,21 @@ async def test_batch_create_hook_passes_the_router_deployment_as_creating_model_ class _FakeFilesRouter: """Uploads each target model's file to every deployment it routes to, one new provider id per upload.""" - def __init__(self, deployments_by_model, delete_error=None): + def __init__( + self, + deployments_by_model: Mapping[str, tuple[str, ...]], + delete_error: Exception | None = None, + delete_gate: asyncio.Event | None = None, + ) -> None: self.deployments_by_model = deployments_by_model self.delete_error = delete_error + self.delete_gate = delete_gate self.upload_count = 0 - self.deletes = () + self.deletes: tuple[tuple[str, str, Mapping[str, object], Mapping[str, str]], ...] = () - async def acreate_file(self, model, file, purpose, litellm_metadata): + async def acreate_file( + self, model: str, file: tuple[str, bytes, str], purpose: str, litellm_metadata: Mapping[str, object] + ) -> OpenAIFileObject: deployment_ids: Final = self.deployments_by_model[model] provider_file_ids: Final = tuple( f"file-upload-{self.upload_count + offset + 1}" for offset in range(len(deployment_ids)) @@ -2293,14 +2302,22 @@ class _FakeFilesRouter: } return response - def registry(self): + def registry(self) -> "_FakeFilesRouter": return self - def credentials(self, model_id, form): + def credentials(self, model_id: str, form: str) -> dict[str, str]: assert form == "with_provider" return {"custom_llm_provider": "openai", "api_key": f"key-of-{model_id}"} - async def afile_delete(self, model, file_id, litellm_metadata, _litellm_internal_model_credentials): + async def afile_delete( + self, + model: str, + file_id: str, + litellm_metadata: Mapping[str, object], + _litellm_internal_model_credentials: Mapping[str, str], + ) -> None: + if self.delete_gate is not None: + await self.delete_gate.wait() self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) if self.delete_error is not None: raise self.delete_error @@ -2309,7 +2326,17 @@ class _FakeFilesRouter: _REQUEST_METADATA: Final = MappingProxyType({"user_api_key_team_id": "test-team"}) -async def _create_managed_file(router, target_model_names): +async def _eventually(condition: Callable[[], bool]) -> None: + async def _poll() -> None: + while not condition(): + await asyncio.sleep(0) + + await asyncio.wait_for(_poll(), timeout=1) + + +async def _create_managed_file( + router: _FakeFilesRouter, target_model_names: list[str] +) -> tuple[dict[str, str], list[str], tuple[str, str]]: managed_files, mock_prisma = _make_real_managed_files_instance() response: Final = await managed_files.acreate_file( create_file_request={ @@ -2336,6 +2363,7 @@ async def test_acreate_file_keeps_the_upload_the_managed_id_carries_when_two_tar router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}) model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b"]) + await _eventually(lambda: len(router.deletes) == 1) assert carried == ("wildcard-dep", "file-upload-1") assert model_mappings == {"wildcard-dep": "file-upload-1"} @@ -2356,6 +2384,7 @@ async def test_acreate_file_deletes_only_the_repeat_upload_and_keeps_every_deplo router: Final = _FakeFilesRouter({"gpt-a": ("dep-1", "dep-2"), "gpt-b": ("dep-2",), "gpt-c": ("dep-3",)}) model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b", "gpt-c"]) + await _eventually(lambda: len(router.deletes) == 1) assert carried == ("dep-1", "file-upload-1") assert list(model_mappings.items()) == [ @@ -2368,7 +2397,7 @@ async def test_acreate_file_deletes_only_the_repeat_upload_and_keeps_every_deplo @pytest.mark.asyncio -async def test_acreate_file_still_returns_the_managed_file_when_deleting_the_repeat_upload_fails(caplog): +async def test_acreate_file_still_returns_the_managed_file_when_deleting_the_repeat_upload_fails(caplog: pytest.LogCaptureFixture): router: Final = _FakeFilesRouter( {"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}, delete_error=RuntimeError("provider unavailable"), @@ -2376,6 +2405,7 @@ async def test_acreate_file_still_returns_the_managed_file_when_deleting_the_rep with caplog.at_level(logging.WARNING): model_mappings, flat_ids, carried = await _create_managed_file(router, ["gpt-a", "gpt-b"]) + await _eventually(lambda: "provider unavailable" in caplog.text) assert carried == ("wildcard-dep", "file-upload-1") assert model_mappings == {"wildcard-dep": "file-upload-1"} @@ -2383,3 +2413,21 @@ async def test_acreate_file_still_returns_the_managed_file_when_deleting_the_rep assert [delete[:2] for delete in router.deletes] == [("wildcard-dep", "file-upload-2")] assert "file-upload-2" in caplog.text and "wildcard-dep" in caplog.text assert "provider unavailable" in caplog.text + + +@pytest.mark.asyncio +async def test_acreate_file_answers_before_deleting_the_repeat_upload_finishes(): + delete_gate: Final = asyncio.Event() + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}, delete_gate=delete_gate) + + model_mappings, flat_ids, carried = await asyncio.wait_for( + _create_managed_file(router, ["gpt-a", "gpt-b"]), timeout=1 + ) + + assert carried == ("wildcard-dep", "file-upload-1") + assert model_mappings == {"wildcard-dep": "file-upload-1"} + assert flat_ids == ["file-upload-1"] + assert router.deletes == () + delete_gate.set() + await _eventually(lambda: len(router.deletes) == 1) + assert router.deletes[0][:2] == ("wildcard-dep", "file-upload-2") diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 41da5452f7f..4df85cc17c5 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1527,6 +1527,28 @@ async def test_track_cost_callback_alerts_only_on_a_billable_call_with_no_loggin assert any(f"call_type: {call_type}" in message for message in alerts.messages) is alerted +@pytest.mark.asyncio +async def test_track_cost_callback_logs_no_warning_for_a_file_delete_routed_to_a_deployment(caplog: pytest.LogCaptureFixture): + kwargs: Final = { + "call_type": "afile_delete", + "model": "openai/*", + "litellm_call_id": "delete-call-id", + "litellm_params": {}, + "stream": False, + } + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", _FailedTrackingAlerts()): + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await ProxyDBLogger()._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] + + @pytest.mark.asyncio async def test_track_cost_callback_defers_in_progress_background_interaction(): # test-quality-ok: writing no spend row and raising no alert is the whole observable contract of the deferral path """ From 505c713913aae47ed2e9eab82c4ce392594f534d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:48:34 -0700 Subject: [PATCH 4/5] fix(proxy): run managed file repeat-upload cleanup as a drained internal call Delete repeat uploads after the record is saved, through the shared background task list the shutdown drain waits on, and as an internal call so logging, spend tracking and alerts skip it. Leave litellm_proxy deployments on the old keep-last behavior, since their per-target copies are distinct files. Revert the cost tracker skip change, which the internal call makes unnecessary. --- .../proxy/hooks/managed_files.py | 47 +++++++++++++------ .../proxy/hooks/proxy_track_cost_callback.py | 19 +++----- ...test_vector_store_file_list_managed_ids.py | 1 + .../proxy/test_managed_files_hook.py | 47 ++++++++++++++++++- .../hooks/test_proxy_track_cost_callback.py | 37 ++------------- 5 files changed, 89 insertions(+), 62 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index d615f5adbad..1932bfc5bc3 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -35,7 +35,7 @@ from typing_extensions import ReadOnly, Unpack import litellm from litellm import Router, verbose_logger -from litellm._internal_context import post_response_phase, with_service_target +from litellm._internal_context import internal_sub_call, post_response_phase, with_service_target from litellm._uuid import uuid from litellm.caching.caching import DualCache from litellm.constants import ( @@ -96,6 +96,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_metadata, ) from litellm.repositories.managed_file_repository import ManagedFileRepository +from litellm.responses.streaming_iterator import spawn_logging_task from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] AllMessageValues, @@ -420,8 +421,15 @@ def _provider_uploads(file_object: OpenAIFileObject) -> tuple[_ProviderUpload, . ) -def _is_first_upload_to_its_deployment(uploads: Sequence[_ProviderUpload], index: int) -> bool: - return all(earlier.model_id != uploads[index].model_id for earlier in uploads[:index]) +def _keeps_one_upload_per_target(llm_router: Router, model_id: str) -> bool: + credentials: Final = _provider_file_retrieve_credentials(llm_router=llm_router, model_id=model_id) + return (credentials or {}).get("custom_llm_provider") == LlmProviders.LITELLM_PROXY.value + + +def _is_kept_upload(uploads: Sequence[_ProviderUpload], index: int, per_target_model_ids: frozenset[str]) -> bool: + upload: Final = uploads[index] + other_uploads: Final = uploads[index + 1 :] if upload.model_id in per_target_model_ids else uploads[:index] + return all(other.model_id != upload.model_id for other in other_uploads) async def _delete_provider_file( @@ -446,9 +454,6 @@ async def _delete_provider_file( ) -_REDUNDANT_UPLOAD_DELETES: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep cleanup deletes alive - - async def _delete_redundant_upload( llm_router: Router, upload: _ProviderUpload, @@ -469,7 +474,8 @@ async def _delete_redundant_uploads( uploads: Sequence[_ProviderUpload], delete_request: Mapping[str, object], ) -> None: - await asyncio.gather(*(_delete_redundant_upload(llm_router, upload, delete_request) for upload in uploads)) + with internal_sub_call(): + await asyncio.gather(*(_delete_redundant_upload(llm_router, upload, delete_request) for upload in uploads)) def _start_deleting_redundant_uploads( @@ -483,9 +489,11 @@ def _start_deleting_redundant_uploads( {k: v for k, v in create_file_request.items() if k not in _FILE_UPLOAD_ONLY_FIELDS} ) with post_response_phase(): - task: Final = asyncio.create_task(_delete_redundant_uploads(llm_router, uploads, delete_request)) - _REDUNDANT_UPLOAD_DELETES.add(task) - task.add_done_callback(_REDUNDANT_UPLOAD_DELETES.discard) + spawn_logging_task( + asyncio.get_running_loop(), + _delete_redundant_uploads(llm_router, uploads, delete_request), + task_name="managed file repeat upload cleanup", + ) def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase: @@ -2128,11 +2136,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) uploads: Final = tuple(chain.from_iterable(map(_provider_uploads, responses))) - kept_uploads: Final = tuple( - upload for index, upload in enumerate(uploads) if _is_first_upload_to_its_deployment(uploads, index) + per_target_model_ids: Final = frozenset( + model_id + for model_id in {upload.model_id for upload in uploads} + if _keeps_one_upload_per_target(llm_router, model_id) ) - _start_deleting_redundant_uploads( - llm_router, tuple(upload for upload in uploads if upload not in kept_uploads), create_file_request + kept_uploads: Final = tuple( + upload for index, upload in enumerate(uploads) if _is_kept_upload(uploads, index, per_target_model_ids) ) model_mappings: Final = {upload.model_id: upload.provider_file_id for upload in kept_uploads} @@ -2143,6 +2153,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_mappings=model_mappings, user_api_key_dict=user_api_key_dict, ) + _start_deleting_redundant_uploads( + llm_router, + tuple( + upload + for upload in uploads + if upload not in kept_uploads and upload.model_id not in per_target_model_ids + ), + create_file_request, + ) # Emit Prometheus metrics for managed file creation prom_logger = self._get_prometheus_logger() diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 535bb0d8ef3..2951bc02d8c 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -126,14 +126,6 @@ _UNATTRIBUTED_TRACKABLE_CALL_TYPES: Final[frozenset[str]] = frozenset( } ) -_UNBILLED_CALL_TYPES: Final[frozenset[str]] = frozenset({"_arealtime", CallTypes.afile_delete.value}) - - -def _log_untracked_call_skip(call_type: object) -> None: - log: Final = verbose_proxy_logger.debug if call_type in _UNBILLED_CALL_TYPES else verbose_proxy_logger.warning - log("Cost tracking - skipping, no standard_logging_object for call_type=%s", call_type) - - # Both spellings, because call_type reaches the callback as str(...) of either the # enum member or its value. _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -923,10 +915,13 @@ class _ProxyDBLogger(CustomLogger): ) return await _release_budget_reservation(budget_reservation=budget_reservation) - # Health checks and file deletes have no standard_logging_object, and a file delete routed - # to a deployment still carries its model. Use .get() for "stream" to avoid KeyError on health checks. - if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") in _UNBILLED_CALL_TYPES): - _log_untracked_call_skip(kwargs.get("call_type", "unknown")) + # Non-model call types (health checks, afile_delete) have no model or standard_logging_object. + # Use .get() for "stream" to avoid KeyError on health checks. + if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"): + verbose_proxy_logger.warning( + "Cost tracking - skipping, no standard_logging_object for call_type=%s", + kwargs.get("call_type", "unknown"), + ) return if kwargs.get("stream") is not True or ( kwargs.get("stream") is True and "complete_streaming_response" in kwargs diff --git a/tests/integration/management/test_vector_store_file_list_managed_ids.py b/tests/integration/management/test_vector_store_file_list_managed_ids.py index ebf4b2aa33c..5ef709d7365 100644 --- a/tests/integration/management/test_vector_store_file_list_managed_ids.py +++ b/tests/integration/management/test_vector_store_file_list_managed_ids.py @@ -1082,6 +1082,7 @@ def test_two_targets_served_by_one_team_wildcard_keep_one_upload_that_delete_rem managed: Final = _upload(gateway, key, f"{prefix}/gpt-one,{prefix}/gpt-two", "a.txt") assert _carried_provider_file_id(managed) == first_upload, _decoded(managed) assert [row["flat_model_file_ids"] for row in read_rows(MANAGED_FILE_ROW, (managed,))] == [[first_upload]] + eventually(lambda: wire.received.qsize(), lambda received: received >= 3) create_requests: Final = wire.drain() assert [(request.method, request.target) for request in create_requests] == [ ("POST", "/v1/files"), diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index a9293e7a964..e2cbd0a8cfc 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -16,9 +16,11 @@ import pytest from typing import Final, Optional from unittest.mock import AsyncMock, MagicMock, patch +from litellm._internal_context import is_internal_call from litellm.llms.base_llm.files.transformation import FileListModelScope from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import BATCH_CREATE_HIDDEN_PARAM +from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.llms.openai import FileListPage, OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -2279,12 +2281,15 @@ class _FakeFilesRouter: deployments_by_model: Mapping[str, tuple[str, ...]], delete_error: Exception | None = None, delete_gate: asyncio.Event | None = None, + provider: str = "openai", ) -> None: self.deployments_by_model = deployments_by_model self.delete_error = delete_error self.delete_gate = delete_gate + self.provider = provider self.upload_count = 0 self.deletes: tuple[tuple[str, str, Mapping[str, object], Mapping[str, str]], ...] = () + self.deletes_marked_internal: tuple[bool, ...] = () async def acreate_file( self, model: str, file: tuple[str, bytes, str], purpose: str, litellm_metadata: Mapping[str, object] @@ -2297,7 +2302,7 @@ class _FakeFilesRouter: response: Final = _make_file_object(provider_file_ids[0]).model_copy(update={"purpose": purpose}) response._hidden_params = { "model_id": deployment_ids[0], - "custom_llm_provider": "openai", + "custom_llm_provider": self.provider, "model_file_id_mapping": dict(zip(deployment_ids, provider_file_ids)), } return response @@ -2307,7 +2312,7 @@ class _FakeFilesRouter: def credentials(self, model_id: str, form: str) -> dict[str, str]: assert form == "with_provider" - return {"custom_llm_provider": "openai", "api_key": f"key-of-{model_id}"} + return {"custom_llm_provider": self.provider, "api_key": f"key-of-{model_id}"} async def afile_delete( self, @@ -2319,6 +2324,7 @@ class _FakeFilesRouter: if self.delete_gate is not None: await self.delete_gate.wait() self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) + self.deletes_marked_internal = (*self.deletes_marked_internal, is_internal_call.get()) if self.delete_error is not None: raise self.delete_error @@ -2431,3 +2437,40 @@ async def test_acreate_file_answers_before_deleting_the_repeat_upload_finishes() delete_gate.set() await _eventually(lambda: len(router.deletes) == 1) assert router.deletes[0][:2] == ("wildcard-dep", "file-upload-2") + + +@pytest.mark.asyncio +async def test_shutdown_drain_waits_for_the_repeat_upload_delete(): + delete_gate: Final = asyncio.Event() + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}, delete_gate=delete_gate) + + await _create_managed_file(router, ["gpt-a", "gpt-b"]) + asyncio.get_running_loop().call_later(0.05, delete_gate.set) + await drain_logging_tasks(timeout=1) + + assert [delete[:2] for delete in router.deletes] == [("wildcard-dep", "file-upload-2")] + + +@pytest.mark.asyncio +async def test_repeat_upload_delete_runs_as_an_internal_call_the_client_is_not_billed_or_alerted_for(): + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}) + + await _create_managed_file(router, ["gpt-a", "gpt-b"]) + await drain_logging_tasks(timeout=1) + + assert router.deletes_marked_internal == (True,) + assert is_internal_call.get() is False + + +@pytest.mark.asyncio +async def test_litellm_proxy_wildcard_keeps_its_last_per_target_upload_and_deletes_nothing(): + router: Final = _FakeFilesRouter( + {"gpt-a": ("proxy-wildcard-dep",), "gpt-b": ("proxy-wildcard-dep",)}, provider="litellm_proxy" + ) + + model_mappings, flat_ids, _ = await _create_managed_file(router, ["gpt-a", "gpt-b"]) + await drain_logging_tasks(timeout=1) + + assert model_mappings == {"proxy-wildcard-dep": "file-upload-2"} + assert flat_ids == ["file-upload-2"] + assert router.deletes == () diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 4df85cc17c5..e482b6552a4 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1495,20 +1495,11 @@ class _FailedTrackingAlerts: @pytest.mark.asyncio -@pytest.mark.parametrize( - ("call_type", "model", "alerted"), - [ - ("_aresponses_websocket", "gpt-5-mini", True), - ("_arealtime", "gpt-5-mini", False), - ("afile_delete", "openai/*", False), - ], -) -async def test_track_cost_callback_alerts_only_on_a_billable_call_with_no_logging_object( - call_type: str, model: str, alerted: bool -): +@pytest.mark.parametrize(("call_type", "alerted"), [("_aresponses_websocket", True), ("_arealtime", False)]) +async def test_track_cost_callback_alerts_on_a_websocket_turn_with_no_logging_object(call_type: str, alerted: bool): kwargs: Final = { "call_type": call_type, - "model": model, + "model": "gpt-5-mini", "litellm_call_id": "turn-call-id", "litellm_params": {}, "stream": False, @@ -1527,28 +1518,6 @@ async def test_track_cost_callback_alerts_only_on_a_billable_call_with_no_loggin assert any(f"call_type: {call_type}" in message for message in alerts.messages) is alerted -@pytest.mark.asyncio -async def test_track_cost_callback_logs_no_warning_for_a_file_delete_routed_to_a_deployment(caplog: pytest.LogCaptureFixture): - kwargs: Final = { - "call_type": "afile_delete", - "model": "openai/*", - "litellm_call_id": "delete-call-id", - "litellm_params": {}, - "stream": False, - } - - with patch("litellm.proxy.proxy_server.proxy_logging_obj", _FailedTrackingAlerts()): - with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): - await ProxyDBLogger()._PROXY_track_cost_callback( - kwargs=kwargs, - completion_response=None, - start_time=datetime.now(), - end_time=datetime.now(), - ) - - assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] - - @pytest.mark.asyncio async def test_track_cost_callback_defers_in_progress_background_interaction(): # test-quality-ok: writing no spend row and raising no alert is the whole observable contract of the deferral path """ From 81038f82c1f5211f84e269f6800c9be30bd3adad Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:31:30 -0700 Subject: [PATCH 5/5] fix(managed_files): delete the copy a managed id names on DELETE and isolate cleanup metadata --- .../proxy/hooks/managed_files.py | 24 ++++- .../proxy/test_managed_files_hook.py | 101 ++++++++++++++++-- 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 1932bfc5bc3..897556e0c13 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -3,6 +3,7 @@ import asyncio import base64 +import copy import json import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence @@ -454,13 +455,19 @@ async def _delete_provider_file( ) +def _with_own_nested_dicts(request_data: Mapping[str, object]) -> MappingProxyType[str, object]: + return MappingProxyType({k: copy.copy(v) if isinstance(v, dict) else v for k, v in request_data.items()}) + + async def _delete_redundant_upload( llm_router: Router, upload: _ProviderUpload, delete_request: Mapping[str, object], ) -> None: try: - await _delete_provider_file(llm_router, upload.model_id, upload.provider_file_id, delete_request) + await _delete_provider_file( + llm_router, upload.model_id, upload.provider_file_id, _with_own_nested_dicts(delete_request) + ) except Exception as error: verbose_logger.warning( f"Could not delete the redundant upload provider_file_id={sanitize_for_log(upload.provider_file_id)} " @@ -2804,6 +2811,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return for model_id, model_file_id in specific_model_file_id_mapping.items(): await _delete_provider_file(llm_router, model_id, model_file_id, data) + unrecorded_upload: Final = self._carried_upload_missing_from(file_id, specific_model_file_id_mapping) + if unrecorded_upload is not None: + await _delete_redundant_upload(llm_router, unrecorded_upload, data) + + def _carried_upload_missing_from( + self, unified_file_id: str, model_file_ids: Mapping[str, str] + ) -> _ProviderUpload | None: + decoded: Final = _is_base64_encoded_unified_file_id(unified_file_id) + if not decoded or "llm_output_file_model_id," not in decoded: + return None + carried: Final = _ProviderUpload( + model_id=self.get_model_id_from_unified_file_id(decoded), + provider_file_id=self.get_output_file_id_from_unified_file_id(decoded), + ) + return carried if model_file_ids.get(carried.model_id) not in (None, carried.provider_file_id) else None async def afile_content( self, diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index e2cbd0a8cfc..1d222555527 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -13,7 +13,7 @@ from collections.abc import Callable, Mapping from types import MappingProxyType import pytest -from typing import Final, Optional +from typing import TYPE_CHECKING, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch from litellm._internal_context import is_internal_call @@ -24,6 +24,9 @@ from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.llms.openai import FileListPage, OpenAIFileObject from litellm.types.utils import LiteLLMBatch +if TYPE_CHECKING: + from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles + def _make_file_object(file_id: str = "file-output-abc") -> OpenAIFileObject: return OpenAIFileObject( @@ -2274,17 +2277,17 @@ async def test_batch_create_hook_passes_the_router_deployment_as_creating_model_ class _FakeFilesRouter: - """Uploads each target model's file to every deployment it routes to, one new provider id per upload.""" - def __init__( self, deployments_by_model: Mapping[str, tuple[str, ...]], delete_error: Exception | None = None, delete_gate: asyncio.Event | None = None, provider: str = "openai", + gone_file_ids: frozenset[str] = frozenset(), ) -> None: self.deployments_by_model = deployments_by_model self.delete_error = delete_error + self.gone_file_ids = gone_file_ids self.delete_gate = delete_gate self.provider = provider self.upload_count = 0 @@ -2323,8 +2326,12 @@ class _FakeFilesRouter: ) -> None: if self.delete_gate is not None: await self.delete_gate.wait() + if isinstance(litellm_metadata, dict): + litellm_metadata.update({"model_info": {"id": model}}) self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) self.deletes_marked_internal = (*self.deletes_marked_internal, is_internal_call.get()) + if file_id in self.gone_file_ids: + raise RuntimeError(f"No such File object: {file_id}") if self.delete_error is not None: raise self.delete_error @@ -2340,21 +2347,30 @@ async def _eventually(condition: Callable[[], bool]) -> None: await asyncio.wait_for(_poll(), timeout=1) -async def _create_managed_file( - router: _FakeFilesRouter, target_model_names: list[str] -) -> tuple[dict[str, str], list[str], tuple[str, str]]: - managed_files, mock_prisma = _make_real_managed_files_instance() - response: Final = await managed_files.acreate_file( +async def _upload( + managed_files: "PROXY_LiteLLMManagedFiles", + router: _FakeFilesRouter, + target_model_names: list[str], + litellm_metadata: Mapping[str, object] = _REQUEST_METADATA, +) -> OpenAIFileObject: + return await managed_files.acreate_file( create_file_request={ "file": ("notes.txt", b"the launch code is ORCHID-42\n", "text/plain"), "purpose": "user_data", - "litellm_metadata": _REQUEST_METADATA, + "litellm_metadata": litellm_metadata, }, llm_router=router, target_model_names_list=target_model_names, litellm_parent_otel_span=None, user_api_key_dict=_make_team_member_api_key_dict(), ) + + +async def _create_managed_file( + router: _FakeFilesRouter, target_model_names: list[str] +) -> tuple[dict[str, str], list[str], tuple[str, str]]: + managed_files, mock_prisma = _make_real_managed_files_instance() + response: Final = await _upload(managed_files, router, target_model_names) stored: Final = mock_prisma.db.litellm_managedfiletable.upsert.await_args.kwargs["data"]["create"] decoded: Final = base64.urlsafe_b64decode(response.id + "=" * (-len(response.id) % 4)).decode() carried: Final = ( @@ -2474,3 +2490,70 @@ async def test_litellm_proxy_wildcard_keeps_its_last_per_target_upload_and_delet assert model_mappings == {"proxy-wildcard-dep": "file-upload-2"} assert flat_ids == ["file-upload-2"] assert router.deletes == () + + +async def _upload_then_delete(router: _FakeFilesRouter, target_model_names: list[str]) -> tuple[str, ...]: + managed_files, mock_prisma = _make_real_managed_files_instance() + response: Final = await _upload(managed_files, router, target_model_names) + await drain_logging_tasks(timeout=1) + deletes_during_upload: Final = len(router.deletes) + managed_files.internal_usage_cache.async_get_cache = AsyncMock( + return_value=managed_files.internal_usage_cache.async_set_cache.await_args.kwargs["value"] + ) + mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=MagicMock()) + mock_prisma.db.litellm_managedfiletable.delete = AsyncMock() + + deleted: Final = await managed_files.afile_delete( + file_id=response.id, litellm_parent_otel_span=None, llm_router=router, litellm_metadata=_REQUEST_METADATA + ) + + assert deleted.deleted is True + mock_prisma.db.litellm_managedfiletable.delete.assert_awaited_once_with(where={"unified_file_id": response.id}) + return tuple(delete[1] for delete in router.deletes[deletes_during_upload:]) + + +@pytest.mark.asyncio +async def test_deleting_a_litellm_proxy_wildcard_file_removes_the_upload_its_id_carries_too(): + router: Final = _FakeFilesRouter( + {"gpt-a": ("proxy-wildcard-dep",), "gpt-b": ("proxy-wildcard-dep",)}, provider="litellm_proxy" + ) + + assert await _upload_then_delete(router, ["gpt-a", "gpt-b"]) == ("file-upload-2", "file-upload-1") + + +@pytest.mark.asyncio +async def test_deleting_a_wildcard_file_removes_its_kept_upload_once(): + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}) + + assert await _upload_then_delete(router, ["gpt-a", "gpt-b"]) == ("file-upload-1",) + + +@pytest.mark.asyncio +async def test_deleting_a_file_still_succeeds_when_the_upload_its_id_carries_is_already_gone( + caplog: pytest.LogCaptureFixture, +): + router: Final = _FakeFilesRouter( + {"gpt-a": ("proxy-wildcard-dep",), "gpt-b": ("proxy-wildcard-dep",)}, + provider="litellm_proxy", + gone_file_ids=frozenset({"file-upload-1"}), + ) + + with caplog.at_level(logging.WARNING): + assert await _upload_then_delete(router, ["gpt-a", "gpt-b"]) == ("file-upload-2", "file-upload-1") + + assert "file-upload-1" in caplog.text and "No such File object" in caplog.text + + +@pytest.mark.asyncio +async def test_repeat_upload_delete_leaves_the_upload_request_metadata_unchanged(): + router: Final = _FakeFilesRouter({"gpt-a": ("wildcard-dep",), "gpt-b": ("wildcard-dep",)}) + upload_metadata: Final = {"user_api_key_team_id": "test-team"} + managed_files, _ = _make_real_managed_files_instance() + + await _upload(managed_files, router, ["gpt-a", "gpt-b"], upload_metadata) + await drain_logging_tasks(timeout=1) + + assert upload_metadata == {"user_api_key_team_id": "test-team"} + assert [delete[2] for delete in router.deletes] == [ + {"user_api_key_team_id": "test-team", "model_info": {"id": "wildcard-dep"}} + ]