mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): require opt-in for project forwarding
This commit is contained in:
parent
f13aaad25a
commit
3725fa91d0
6 changed files with 216 additions and 48 deletions
|
|
@ -2932,6 +2932,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
None,
|
||||
description="If True, forwards client headers (e.g. Authorization) to the LLM API. Required for Claude Code with Max subscription.",
|
||||
)
|
||||
forward_openai_project_id: bool = Field(
|
||||
False,
|
||||
description="Allow client OpenAI-Project headers and project query/body fields on file uploads and batch creation. Configured deployment projects take precedence. Explicit extra_headers retain their existing behavior.",
|
||||
)
|
||||
mcp_required_fields: list[str] | None = Field(
|
||||
None,
|
||||
description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).",
|
||||
|
|
|
|||
|
|
@ -183,7 +183,23 @@ async def create_batch(
|
|||
or get_custom_llm_provider_from_request_headers(request=request)
|
||||
)
|
||||
custom_llm_provider: Final = requested_provider or "openai"
|
||||
_create_batch_data: Final = LiteLLMBatchCreateRequest(**data)
|
||||
request_fields: Final = cast( # cast-ok: parsed batch payload is a string-keyed mapping
|
||||
Mapping[str, object], data
|
||||
)
|
||||
_create_batch_data: Final = cast( # cast-ok: preserve the parsed batch fields without runtime filtering
|
||||
LiteLLMBatchCreateRequest,
|
||||
dict(request_fields), # mutable-ok: request copy receives provider credentials before dispatch
|
||||
)
|
||||
forward_project: Final = (
|
||||
cast( # cast-ok: resolved server configuration mapping
|
||||
Mapping[str, object], general_settings
|
||||
).get("forward_openai_project_id")
|
||||
is True
|
||||
)
|
||||
header_project: Final = request.headers.get("openai-project") if forward_project else None
|
||||
query_project: Final = request.query_params.get("project") if forward_project else None
|
||||
requested_project: Final = _create_batch_data.get("project")
|
||||
body_project: Final = requested_project if forward_project and isinstance(requested_project, str) else None
|
||||
|
||||
# Apply team-level batch output expiry enforcement
|
||||
team_metadata: Final = user_api_key_dict.team_metadata or {}
|
||||
|
|
@ -232,6 +248,8 @@ async def create_batch(
|
|||
|
||||
original_file_id: Final = get_original_file_id(input_file_id)
|
||||
_create_batch_data["input_file_id"] = original_file_id
|
||||
if credentials["custom_llm_provider"] == "openai":
|
||||
_create_batch_data.pop("project", None)
|
||||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
credentials=credentials,
|
||||
|
|
@ -242,8 +260,9 @@ async def create_batch(
|
|||
with_openai_project_header(
|
||||
_create_batch_data,
|
||||
cast(str, credentials["custom_llm_provider"]), # cast-ok: router credentials identify the provider
|
||||
request.headers.get("openai-project"),
|
||||
request.query_params.get("project"),
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -289,13 +308,16 @@ async def create_batch(
|
|||
detail={"error": "LLM Router not initialized. Ensure models added to proxy."},
|
||||
)
|
||||
|
||||
if custom_llm_provider == "openai":
|
||||
_create_batch_data.pop("project", None)
|
||||
loadbalanced_batch_data: Final = cast( # cast-ok: conversion preserves batch request fields
|
||||
LiteLLMBatchCreateRequest,
|
||||
with_openai_project_header(
|
||||
_create_batch_data,
|
||||
custom_llm_provider,
|
||||
request.headers.get("openai-project"),
|
||||
request.query_params.get("project"),
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
response = await llm_router.acreate_batch(**loadbalanced_batch_data)
|
||||
|
|
@ -323,13 +345,16 @@ async def create_batch(
|
|||
)
|
||||
|
||||
_create_batch_data.update(disable_fallbacks=True) # pyright: ignore[reportCallIssue] # router flag
|
||||
if custom_llm_provider == "openai":
|
||||
_create_batch_data.pop("project", None)
|
||||
unified_batch_data: Final = cast( # cast-ok: conversion preserves batch request fields
|
||||
LiteLLMBatchCreateRequest,
|
||||
with_openai_project_header(
|
||||
_create_batch_data,
|
||||
custom_llm_provider,
|
||||
request.headers.get("openai-project"),
|
||||
request.query_params.get("project"),
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
response = await llm_router.acreate_batch(**unified_batch_data)
|
||||
|
|
@ -350,6 +375,8 @@ async def create_batch(
|
|||
operation_context="batch creation",
|
||||
)
|
||||
|
||||
if credentials["custom_llm_provider"] == "openai":
|
||||
_create_batch_data.pop("project", None)
|
||||
prepare_data_with_credentials(
|
||||
data=_create_batch_data,
|
||||
credentials=credentials,
|
||||
|
|
@ -362,8 +389,9 @@ async def create_batch(
|
|||
cast(
|
||||
str, credentials["custom_llm_provider"]
|
||||
), # cast-ok: router credentials identify the provider
|
||||
request.headers.get("openai-project"),
|
||||
request.query_params.get("project"),
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -378,6 +406,8 @@ async def create_batch(
|
|||
verbose_proxy_logger.debug("Created batch using model: %s", model_param)
|
||||
else:
|
||||
# SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
|
||||
if custom_llm_provider == "openai":
|
||||
_create_batch_data.pop("project", None)
|
||||
apply_team_provider_credentials(
|
||||
data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime
|
||||
llm_router=llm_router,
|
||||
|
|
@ -389,8 +419,9 @@ async def create_batch(
|
|||
with_openai_project_header(
|
||||
_create_batch_data,
|
||||
cast(str, custom_llm_provider), # cast-ok: request provider selection is a string
|
||||
request.headers.get("openai-project"),
|
||||
request.query_params.get("project"),
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
_raise_not_found_when_openai_fallback_unservable(
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ async def route_create_file(
|
|||
target_storage: str | None = "default",
|
||||
request: Request | None = None,
|
||||
request_project: str | None = None,
|
||||
forward_request_project: bool = False,
|
||||
) -> OpenAIFileObject:
|
||||
"""
|
||||
Route file creation request to the appropriate provider.
|
||||
|
|
@ -247,6 +248,14 @@ async def route_create_file(
|
|||
5. Else -> use custom_llm_provider with files_settings
|
||||
"""
|
||||
|
||||
header_project: Final = (
|
||||
request.headers.get("openai-project") if forward_request_project and request is not None else None
|
||||
)
|
||||
query_project: Final = (
|
||||
request.query_params.get("project") if forward_request_project and request is not None else None
|
||||
)
|
||||
body_project: Final = request_project if forward_request_project else None
|
||||
|
||||
# Handle custom storage backend
|
||||
if target_storage and target_storage != "default":
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -291,9 +300,9 @@ async def route_create_file(
|
|||
with_openai_project_header(
|
||||
_create_file_request,
|
||||
cast(str, credentials["custom_llm_provider"]), # cast-ok: router credentials identify the provider
|
||||
request.headers.get("openai-project") if request is not None else None,
|
||||
request.query_params.get("project") if request is not None else None,
|
||||
body_project=request_project,
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -342,9 +351,9 @@ async def route_create_file(
|
|||
with_openai_project_header(
|
||||
_create_file_request,
|
||||
custom_llm_provider,
|
||||
request.headers.get("openai-project") if request is not None else None,
|
||||
request.query_params.get("project") if request is not None else None,
|
||||
body_project=request_project,
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
# Managed files internally calls llm_router.acreate_file() which includes loadbalancing
|
||||
|
|
@ -362,9 +371,9 @@ async def route_create_file(
|
|||
with_openai_project_header(
|
||||
_create_file_request,
|
||||
custom_llm_provider,
|
||||
request.headers.get("openai-project") if request is not None else None,
|
||||
request.query_params.get("project") if request is not None else None,
|
||||
body_project=request_project,
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
response = await _deprecated_loadbalanced_create_file(
|
||||
|
|
@ -390,9 +399,9 @@ async def route_create_file(
|
|||
with_openai_project_header(
|
||||
_create_file_request,
|
||||
custom_llm_provider,
|
||||
request.headers.get("openai-project") if request is not None else None,
|
||||
request.query_params.get("project") if request is not None else None,
|
||||
body_project=request_project,
|
||||
header_project,
|
||||
query_project,
|
||||
body_project=body_project,
|
||||
),
|
||||
)
|
||||
# for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch
|
||||
|
|
@ -726,6 +735,10 @@ async def create_file(
|
|||
target_storage=target_storage,
|
||||
request=request,
|
||||
request_project=project_in_body if isinstance(project_in_body, str) else None,
|
||||
forward_request_project=cast( # cast-ok: resolved server configuration mapping
|
||||
Mapping[str, object], general_settings
|
||||
).get("forward_openai_project_id")
|
||||
is True,
|
||||
)
|
||||
|
||||
if response is None:
|
||||
|
|
|
|||
|
|
@ -512,6 +512,7 @@ class CreateBatchRequest(TypedDict, total=False):
|
|||
|
||||
class LiteLLMBatchCreateRequest(CreateBatchRequest, total=False):
|
||||
model: str
|
||||
project: str # writable-ok: proxy removes caller aliases before merging provider credentials
|
||||
|
||||
|
||||
class RetrieveBatchRequest(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -554,6 +554,9 @@ async def test_create__unified_file_id_single_model_disables_cross_model_fallbac
|
|||
with (
|
||||
patch.object(endpoints, "_is_base64_encoded_unified_file_id", return_value="unified-xyz"),
|
||||
patch.object(endpoints, "get_models_from_unified_file_id", return_value=["gpt-4o-mini"]),
|
||||
patch.object( # test-quality-ok: TQ008 configures server opt-in without replacing request processing
|
||||
proxy_server, "general_settings", {"forward_openai_project_id": True}
|
||||
),
|
||||
):
|
||||
resp = await call_create(harness, headers={"openai-project": "proj-request"})
|
||||
|
||||
|
|
@ -802,7 +805,12 @@ async def test_create__loadbalancing_routes_to_router(harness):
|
|||
},
|
||||
)
|
||||
harness.is_known_model.return_value = True
|
||||
with patch.object(litellm, "enable_loadbalancing_on_batch_endpoints", True):
|
||||
with (
|
||||
patch.object(litellm, "enable_loadbalancing_on_batch_endpoints", True),
|
||||
patch.object( # test-quality-ok: TQ008 configures server opt-in without replacing request processing
|
||||
proxy_server, "general_settings", {"forward_openai_project_id": True}
|
||||
),
|
||||
):
|
||||
await call_create(harness, headers={"openai-project": "proj-request"})
|
||||
|
||||
harness.is_known_model.assert_called_once_with(model="lb-model", llm_router=harness.router)
|
||||
|
|
@ -994,27 +1002,62 @@ async def test_create__uses_acreate_batch_route_type(harness, openai_env_creds):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__forwards_openai_project_header(harness, openai_env_creds):
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": "file-plain",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
},
|
||||
)
|
||||
|
||||
await call_create(harness, headers={"openai-project": "proj-request"})
|
||||
|
||||
assert harness.acreate_kwargs()["extra_headers"] == {"OpenAI-Project": "proj-request"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__model_project_overrides_request_header(harness):
|
||||
@pytest.mark.parametrize("forward_project", [None, False, True])
|
||||
@pytest.mark.parametrize("source", ["header", "query", "body"])
|
||||
@pytest.mark.parametrize("route", ["direct", "model", "encoded", "loadbalanced", "unified"])
|
||||
async def test_create__project_alias_forwarding_requires_opt_in(harness, openai_env_creds, forward_project, source, route):
|
||||
harness.creds_resolver.side_effect = lambda *, model_id: {
|
||||
"custom_llm_provider": "openai",
|
||||
"api_key": "sk-test-openai",
|
||||
"project": "proj-config",
|
||||
}
|
||||
input_file_id = {
|
||||
"encoded": encode_file_id_with_model("file-plain", "openai-model"),
|
||||
"unified": base64.urlsafe_b64encode(
|
||||
b"litellm_proxy:application/octet-stream;unified_id,input-uuid;target_model_names,openai-model"
|
||||
).decode(),
|
||||
}.get(route, "file-plain")
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": input_file_id,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
**({"model": "openai-model"} if route in ("model", "loadbalanced") else {}),
|
||||
**({"project": "proj-request"} if source == "body" else {}),
|
||||
},
|
||||
)
|
||||
harness.is_known_model.return_value = route == "loadbalanced"
|
||||
with (
|
||||
patch.object( # test-quality-ok: TQ008 parameterizes server opt-in without replacing request processing
|
||||
proxy_server,
|
||||
"general_settings",
|
||||
{} if forward_project is None else {"forward_openai_project_id": forward_project},
|
||||
),
|
||||
patch.object( # test-quality-ok: TQ008 selects configured routing mode while real routing logic executes
|
||||
litellm, "enable_loadbalancing_on_batch_endpoints", route == "loadbalanced"
|
||||
),
|
||||
):
|
||||
await call_create(
|
||||
harness,
|
||||
headers={"openai-project": "proj-request"} if source == "header" else None,
|
||||
query={"project": "proj-request"} if source == "query" else None,
|
||||
)
|
||||
|
||||
kwargs = harness.router_kwargs() if route in ("loadbalanced", "unified") else harness.acreate_kwargs()
|
||||
assert kwargs.get("extra_headers", {}) == ({"OpenAI-Project": "proj-request"} if forward_project is True else {})
|
||||
assert "project" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("forward_project", [None, False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"configured_project", [{"project": "proj-config"}, {"extra_headers": {"openai-project": "proj-config"}}]
|
||||
)
|
||||
async def test_create__model_project_overrides_request_aliases(harness, forward_project, configured_project):
|
||||
harness.creds_resolver.side_effect = lambda *, model_id: {
|
||||
"custom_llm_provider": "openai",
|
||||
"api_key": "sk-test-openai",
|
||||
**configured_project,
|
||||
}
|
||||
set_body(
|
||||
harness,
|
||||
|
|
@ -1023,16 +1066,68 @@ async def test_create__model_project_overrides_request_header(harness):
|
|||
"model": "openai-model",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"project": "proj-body",
|
||||
},
|
||||
)
|
||||
|
||||
await call_create(harness, headers={"openai-project": "proj-request"})
|
||||
with patch.object( # test-quality-ok: TQ008 parameterizes server opt-in without replacing request processing
|
||||
proxy_server,
|
||||
"general_settings",
|
||||
{} if forward_project is None else {"forward_openai_project_id": forward_project},
|
||||
):
|
||||
await call_create(harness, headers={"openai-project": "proj-header"}, query={"project": "proj-query"})
|
||||
|
||||
kwargs = harness.acreate_kwargs()
|
||||
assert kwargs["extra_headers"] == {"OpenAI-Project": "proj-config"}
|
||||
assert "project" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("forward_project", [None, False, True])
|
||||
async def test_create__explicit_extra_headers_unchanged_by_project_alias_opt_in(harness, openai_env_creds, forward_project):
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": "file-plain",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"extra_headers": {"OpenAI-Project": "proj-explicit", "X-Caller": "preserved"},
|
||||
"project": "proj-body",
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object( # test-quality-ok: TQ008 parameterizes server opt-in without replacing request processing
|
||||
proxy_server,
|
||||
"general_settings",
|
||||
{} if forward_project is None else {"forward_openai_project_id": forward_project},
|
||||
):
|
||||
await call_create(harness, headers={"openai-project": "proj-header"}, query={"project": "proj-query"})
|
||||
|
||||
kwargs = harness.acreate_kwargs()
|
||||
assert kwargs["extra_headers"] == {"OpenAI-Project": "proj-explicit", "X-Caller": "preserved"}
|
||||
assert "project" not in kwargs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create__non_openai_body_project_is_preserved(harness):
|
||||
set_body(
|
||||
harness,
|
||||
{
|
||||
"input_file_id": AZURE_FILE_ID,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"project": "azure-project",
|
||||
},
|
||||
)
|
||||
|
||||
await call_create(harness)
|
||||
|
||||
kwargs = harness.acreate_kwargs()
|
||||
assert kwargs["custom_llm_provider"] == "azure"
|
||||
assert kwargs["project"] == "azure-project"
|
||||
assert "extra_headers" not in kwargs
|
||||
|
||||
|
||||
def install_managed_files_hook(harness: Harness) -> AsyncMock:
|
||||
prisma_client = AsyncMock()
|
||||
managed_files = _PROXY_LiteLLMManagedFiles(MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client)
|
||||
|
|
|
|||
|
|
@ -5238,11 +5238,15 @@ def test_get_file_content_keeps_the_status_of_a_rejection_raised_inside_the_rout
|
|||
({"project": "proj-form"}, {"openai-project": "proj-config", "X-Deployment": "keep"}, "proj-config"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("forward_project", [None, False, True])
|
||||
@pytest.mark.parametrize("query_project", [False, True])
|
||||
def test_upload_forwards_openai_project(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
form_fields: dict[str, str],
|
||||
configured_headers: dict[str, str],
|
||||
expected_project: str,
|
||||
forward_project: bool | None,
|
||||
query_project: bool,
|
||||
) -> None:
|
||||
router = Router(
|
||||
model_list=[
|
||||
|
|
@ -5260,6 +5264,11 @@ def test_upload_forwards_openai_project(
|
|||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "master_key", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"general_settings",
|
||||
{"forward_openai_project_id": forward_project} if forward_project is not None else {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
files_endpoints, "files_config", [{"custom_llm_provider": "openai", "extra_headers": configured_headers}]
|
||||
)
|
||||
|
|
@ -5290,22 +5299,36 @@ def test_upload_forwards_openai_project(
|
|||
"/v1/files",
|
||||
files={"file": ("batch.jsonl", VALID_BATCH_LINE, "application/jsonl")},
|
||||
data={"purpose": "batch", **form_fields},
|
||||
headers={"Authorization": "Bearer test-key", "OpenAI-Project": "proj-request"},
|
||||
headers={
|
||||
"Authorization": "Bearer test-key",
|
||||
**({} if query_project else {"OpenAI-Project": "proj-request"}),
|
||||
},
|
||||
params={"project": "proj-query"} if query_project else {},
|
||||
)
|
||||
finally:
|
||||
proxy_server.app.dependency_overrides.pop(proxy_server.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs["extra_headers"] == {
|
||||
selected_project: Final = (
|
||||
"proj-config"
|
||||
if configured_headers
|
||||
else ("proj-query" if query_project and expected_project == "proj-request" else expected_project)
|
||||
if forward_project
|
||||
else None
|
||||
)
|
||||
assert captured_kwargs.get("extra_headers", {}) == {
|
||||
**{key: value for key, value in configured_headers.items() if key.lower() != "openai-project"},
|
||||
"OpenAI-Project": expected_project,
|
||||
**({"OpenAI-Project": selected_project} if selected_project else {}),
|
||||
}
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("managed", [False, True])
|
||||
async def test_routed_upload_forwards_openai_project(monkeypatch: pytest.MonkeyPatch, managed: bool) -> None:
|
||||
@pytest.mark.parametrize("forward_project", [False, True])
|
||||
async def test_routed_upload_forwards_openai_project(
|
||||
monkeypatch: pytest.MonkeyPatch, managed: bool, forward_project: bool
|
||||
) -> None:
|
||||
request: Final = Request({"type": "http", "headers": [(b"openai-project", b"proj-request")], "query_string": b""})
|
||||
file_request: Final = CreateFileRequest(file=("batch.jsonl", VALID_BATCH_LINE), purpose="batch")
|
||||
file_response: Final = OpenAIFileObject(
|
||||
|
|
@ -5336,6 +5359,7 @@ async def test_routed_upload_forwards_openai_project(monkeypatch: pytest.MonkeyP
|
|||
router_model="gpt-4o",
|
||||
custom_llm_provider="openai",
|
||||
request=request,
|
||||
forward_request_project=forward_project,
|
||||
)
|
||||
|
||||
sent: Final = (
|
||||
|
|
@ -5343,4 +5367,4 @@ async def test_routed_upload_forwards_openai_project(monkeypatch: pytest.MonkeyP
|
|||
if managed
|
||||
else router.acreate_file.call_args.kwargs
|
||||
)
|
||||
assert sent["extra_headers"] == {"OpenAI-Project": "proj-request"}
|
||||
assert sent.get("extra_headers", {}) == ({"OpenAI-Project": "proj-request"} if forward_project else {})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue