Merge pull request #38117 from BerriAI/litellm_lit5902_managed_files_e2e

test(e2e): pin require_managed_files enforcement behind a marker-gated stack phase

tests/e2e/batches/COVERAGE.md

@@ -81,6 +81,21 @@ File delete asserts `object=="file"` and `deleted==True`.
 | `capabilities.py` | the provider x scenario matrix + per-provider /model/new params + id-shape classifiers + per-provider raw-id assertion |
 | `conftest.py` | session-scoped batch deployment registration and teardown |
 | `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial, per-backend content download, failure paths, second-hop routing, terminal state + cost |
+| `test_managed_files_enforcement_e2e.py` | require_managed_files enforcement pins; deselected unless `E2E_MANAGED_FILES_STACK` is set (see below) |
+
+## require_managed_files enforcement (separate stack phase)
+
+`litellm_settings.require_managed_files` is a boot-time module global with no per-key
+or runtime override, and turning it on 400s every upload that lacks
+`target_model_names`, including the files_settings-routed `provider_fallback`
+scenario above. So its pins cannot share a proxy with the rest of this suite:
+`test_managed_files_enforcement_e2e.py` carries the `managed_files` marker, is
+deselected unless `E2E_MANAGED_FILES_STACK` is set (the same pattern as the `weekly`
+marker), and the PR gate runs it in a sequential phase after the main suite, against
+the same ephemeral stack redeployed with the flag on. The pins: upload without
+`target_model_names` is a 400, upload carrying a `model` param is a 400, a raw
+provider file id on retrieve is a 400, and another user's managed unified id is a
+403 while the owning user still retrieves it.
 
 ## Failure paths
 

tests/e2e/batches/conftest.py

@@ -12,12 +12,14 @@ the proxy config.
 
 from __future__ import annotations
 
+import os
 from typing import Iterator
 
 import pytest
 
 from batch_client import BatchClient, build_client
 from capabilities import PROVIDERS
+from e2e_config import MANAGED_FILES_OPT_IN_ENV
 from e2e_http import NoBody
 from proxy_client import ProxyClient
 
@@ -29,6 +31,22 @@ def pytest_configure(config: pytest.Config) -> None:
     )
 
 
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    if os.environ.get(MANAGED_FILES_OPT_IN_ENV):
+        return
+    deselected = [
+        item for item in items if item.get_closest_marker("managed_files") is not None
+    ]
+    if not deselected:
+        return
+    config.hook.pytest_deselected(items=deselected)
+    items[:] = [
+        item for item in items if item.get_closest_marker("managed_files") is None
+    ]
+
+
 @pytest.fixture(scope="session")
 def client(proxy: ProxyClient) -> BatchClient:
     return build_client(proxy)

tests/e2e/batches/test_managed_files_enforcement_e2e.py

@@ -0,0 +1,118 @@
+"""Live e2e pins for litellm_settings.require_managed_files enforcement.
+
+require_managed_files is a boot-time module global, so these tests need a proxy
+whose config enables it. The main ephemeral stack can never run with it on: the
+flag would 400 every files_settings-routed upload in the rest of the suite. The
+PR gate instead reconfigures the same stack sequentially after the main run and
+executes only this file with E2E_MANAGED_FILES_STACK set; without that env every
+test here is deselected (see conftest.py, mirroring the weekly marker).
+
+Pins: an upload without target_model_names is rejected 400, an upload that also
+carries a model param is rejected 400, a raw provider file id is rejected 400 on
+retrieve, and another user's managed unified file id is denied 403 while the
+owning user still retrieves it.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Iterator
+
+import pytest
+
+from batch_client import BatchClient, FileObject
+from capabilities import batch_model_name, is_managed_id, openai_batch_params
+from e2e_config import unique_marker
+from e2e_http import FileUploadForm, Result, UnknownApiError, unwrap
+from lifecycle import ResourceManager
+
+pytestmark = [pytest.mark.e2e, pytest.mark.managed_files]
+
+UPLOAD_ROW = "llm.files.openai.require_managed_files_upload.nonstream.works"
+ISOLATION_ROW = "llm.files.openai.require_managed_files_isolation.nonstream.works"
+
+
+def batch_jsonl(model: str) -> bytes:
+    line = {
+        "custom_id": "req-1",
+        "method": "POST",
+        "url": "/v1/chat/completions",
+        "body": {
+            "model": model,
+            "messages": [{"role": "user", "content": "ping"}],
+            "max_tokens": 8,
+        },
+    }
+    return (json.dumps(line) + "\n").encode()
+
+
+def expect_api_error(result: Result[FileObject], status: int, needle: str) -> None:
+    match result:
+        case UnknownApiError(status_code=code, body=body) if code == status:
+            assert needle in body, f"expected {needle!r} in HTTP {status} body: {body[:300]}"
+        case _:
+            raise AssertionError(f"expected HTTP {status} containing {needle!r}, got: {result}")
+
+
+@pytest.fixture(scope="module")
+def managed_model(client: BatchClient) -> Iterator[str]:
+    model_name = batch_model_name("managed-files-openai")
+    model_id = client.create_model(model_name, openai_batch_params())
+    yield model_name
+    client.delete_model(model_id)
+
+
+@pytest.mark.covers(UPLOAD_ROW)
+def test_upload_without_target_model_names_rejected(
+    client: BatchClient, scoped_key: str, managed_model: str
+) -> None:
+    result = client.upload_file(
+        content=batch_jsonl(managed_model),
+        form=FileUploadForm(purpose="batch"),
+        key=scoped_key,
+    )
+    expect_api_error(result, 400, "target_model_names is required")
+
+
+@pytest.mark.covers(UPLOAD_ROW)
+def test_upload_with_model_param_rejected(
+    client: BatchClient, scoped_key: str, managed_model: str
+) -> None:
+    result = client.upload_file(
+        content=batch_jsonl(managed_model),
+        form=FileUploadForm(purpose="batch", target_model_names=managed_model),
+        model=managed_model,
+        key=scoped_key,
+    )
+    expect_api_error(result, 400, "model is not allowed")
+
+
+@pytest.mark.covers(ISOLATION_ROW)
+def test_raw_provider_file_id_rejected(client: BatchClient, scoped_key: str) -> None:
+    result = client.retrieve_file("file-e2e-raw-provider-id", key=scoped_key)
+    expect_api_error(result, 400, "Raw provider file ids cannot be used")
+
+
+@pytest.mark.covers(ISOLATION_ROW)
+def test_cross_user_managed_id_denied_owner_allowed(
+    client: BatchClient, resources: ResourceManager, managed_model: str
+) -> None:
+    run = unique_marker()
+    owner_key = resources.key(user_id=f"managed-files-owner-{run}")
+    other_key = resources.key(user_id=f"managed-files-other-{run}")
+
+    uploaded = unwrap(
+        client.upload_file(
+            content=batch_jsonl(managed_model),
+            form=FileUploadForm(purpose="batch", target_model_names=managed_model),
+            key=owner_key,
+        )
+    )
+    resources.defer(lambda: client.delete_file(uploaded.id, key=owner_key))
+    assert is_managed_id(uploaded.id), f"expected a managed unified file id, got {uploaded.id}"
+
+    denied = client.retrieve_file(uploaded.id, key=other_key)
+    expect_api_error(denied, 403, "does not have access to this managed file")
+
+    retrieved = unwrap(client.retrieve_file(uploaded.id, key=owner_key))
+    assert retrieved.id == uploaded.id

tests/e2e/conftest.py

@@ -51,6 +51,10 @@ def pytest_configure(config: pytest.Config) -> None:
         "markers",
         "weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set",
     )
+    config.addinivalue_line(
+        "markers",
+        "managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set",
+    )
 
 
 def pytest_sessionstart(session: pytest.Session) -> None:

tests/e2e/coverage_registry/llm_nonconversational.yaml

@@ -45,6 +45,8 @@
 - {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"}
 - {id: llm.files.gemini.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Gemini Files API upload via proxy"}
 - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"}
+- {id: llm.files.openai.require_managed_files_upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_managed_files_enforcement_e2e.py / LIT-5902", rationale: "With require_managed_files enabled, an upload without target_model_names and an upload carrying a model param are both rejected 400; runs only in the sequential managed-files stack phase (E2E_MANAGED_FILES_STACK)"}
+- {id: llm.files.openai.require_managed_files_isolation.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_managed_files_enforcement_e2e.py / LIT-5902", rationale: "With require_managed_files enabled, a raw provider file id is rejected 400 and another user's managed unified id is denied 403 while the owner still retrieves it; runs only in the managed-files stack phase"}
 - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"}
 - {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"}
 - {id: llm.files.azure_openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on an Azure unified file returns the uploaded JSONL bytes verbatim"}

tests/e2e/e2e_config.py

@@ -133,6 +133,7 @@ LOAD_MAX_SERIAL_LATENCY_SECONDS = float(os.environ.get("E2E_LOAD_MAX_SERIAL_LATE
 LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY_EFFICIENCY", "0.8"))
 
 WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
+MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
 ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
 ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
 ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))

tests/e2e/pytest.ini

@@ -7,3 +7,4 @@ markers =
     e2e: live test that requires a running proxy and real provider keys
     load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites
     weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set
+    managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set