From 2899c97601ffed73f64465923e5a94ef2c296c17 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sun, 15 Nov 2026 11:28:31 -0800 Subject: [PATCH] fix(proxy): run the /fal_ai passthrough price gate on the path sent upstream (#51671) * fix(proxy): run the /fal_ai passthrough price gate on the path sent upstream The gate checked the raw request path for /requests/ and priced the raw path, so a path that collapses to a submit URL upstream (a /requests/.. dot segment, or an encoded ? or # tail) skipped the gate and reached fal on the proxy's key * test(proxy): use an endpoint with no catalog row in the fal passthrough gate test * fix(proxy): price the fal passthrough gate on the encoded upstream path * fix(proxy): strip the fal queue base prefix in the form it is sent upstream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_passthrough_endpoints.py | 15 +++- .../providers/test_fal_ai_passthrough_wire.py | 42 ++++++++++ .../test_llm_pass_through_endpoints.py | 79 +++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index f7ecda99d9d..90e6ff369e7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -598,6 +598,13 @@ def _fal_target(endpoint: str) -> httpx.URL: ) +def _fal_upstream_endpoint(target: httpx.URL) -> str: + base_url: Final = httpx.URL(fal_ai_queue_base()) + base_path: Final = base_url.copy_with(path=base_url.path).raw_path.partition(b"?")[0].decode("ascii").strip("/") + upstream_path: Final = target.raw_path.partition(b"?")[0].decode("ascii").lstrip("/") + return upstream_path.removeprefix(f"{base_path}/") if base_path else upstream_path + + @router.api_route( "/fal_ai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -619,10 +626,14 @@ async def fal_ai_proxy_route( status_code=401, detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured", ) - if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await read_request_body(request)) is None: + upstream_endpoint: Final = _fal_upstream_endpoint(updated_url) + if ( + "/requests/" not in upstream_endpoint + and fal_ai_passthrough_cost(upstream_endpoint, await read_request_body(request)) is None + ): raise HTTPException( status_code=400, - detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai", + detail=f"fal_ai/{upstream_endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai", ) endpoint_func: Final = create_pass_through_route( endpoint=endpoint, diff --git a/tests/integration/providers/test_fal_ai_passthrough_wire.py b/tests/integration/providers/test_fal_ai_passthrough_wire.py index b97bd1f2c1c..f8f8126ec5d 100644 --- a/tests/integration/providers/test_fal_ai_passthrough_wire.py +++ b/tests/integration/providers/test_fal_ai_passthrough_wire.py @@ -191,3 +191,45 @@ def test_fal_queue_submit_prices_string_resolution_like_the_integer(gateway: Gat ("POST", f"/{_MODEL}"), ] assert [json.loads(request.body) for request in forwarded] == [numeric_body, string_body] + + +@pytest.mark.parametrize( + "path", + ( + "/fal_ai/synthetic-vendor/unpriced-endpoint", + "/fal_ai/synthetic-vendor/x/requests/%2e%2e/%2e%2e/unpriced-endpoint", + "/fal_ai/synthetic-vendor/unpriced-endpoint%3Fa=/requests/x", + "/fal_ai/synthetic-vendor/unpriced-endpoint%23/requests/x", + ), +) +def test_fal_queue_submit_to_unpriced_endpoint_is_rejected_however_the_path_is_spelled( + gateway: Gateway, tmp_path: Path, path: str +) -> None: + def respond(request: Request) -> Reply: + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + submit: Final = candidate.request("POST", path, {"prompt": "a red boat"}) + assert submit.status_code == 400, submit.text + assert "pricing" in submit.text + priced: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", _REQUEST_BODY) + assert priced.status_code == 200, priced.text + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_MODEL}")] diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index da7578e71bc..103ee8ca9a5 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -10775,6 +10775,85 @@ class TestFalAIPassthroughRoute: assert "no pricing entry" in response.text assert not route.calls + @pytest.mark.parametrize( + "path", + ( + "/fal_ai/acme/x/requests/%2e%2e/%2e%2e/unit-billed-model", + "/fal_ai/acme/unit-billed-model/requests/%2e%2e", + "/fal_ai/acme/unit-billed-model%3Fa=/requests/x", + "/fal_ai/acme/unit-billed-model%23/requests/x", + ), + ) + def test_submit_gate_prices_the_upstream_path_not_the_spelling_of_the_request_path( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, path: str + ) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/acme/unit-billed-model", + {"litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_billable_unit": 1.0}, + ) + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://queue.fal.run/acme/unit-billed-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + response: Final = client.post(path, json={"prompt": "a red boat"}) + + assert response.status_code == 400, response.text + assert "no pricing entry" in response.text + assert not route.calls + + def test_submit_gate_rejects_an_encoded_spelling_that_stays_encoded_upstream(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post(url__regex=r"https://queue\.fal\.run/.*").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + plain: Final = client.post( + "/fal_ai/fal-ai/trellis-2", json={"image_url": "https://example.com/in.png", "resolution": 1536} + ) + encoded: Final = client.post( + "/fal_ai/fal-ai/trellis%2525252D2", json={"image_url": "https://example.com/in.png", "resolution": 1536} + ) + + assert plain.status_code == 200, plain.text + assert encoded.status_code == 400, encoded.text + assert [str(call.request.url) for call in route.calls] == ["https://queue.fal.run/fal-ai/trellis-2"] + + @pytest.mark.parametrize( + ("queue_base", "sent_base"), + ( + ("https://queue.example/base", "https://queue.example/base"), + ("https://queue.example/fal%2Dqueue", "https://queue.example/fal-queue"), + ("https://queue.example/fal%252Dqueue", "https://queue.example/fal%2Dqueue"), + ), + ) + def test_submit_gate_strips_the_queue_base_path_before_pricing( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, queue_base: str, sent_base: str + ) -> None: + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", queue_base) + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/acme/unit-billed-model", + {"litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_billable_unit": 1.0}, + ) + with respx.mock(assert_all_called=False) as upstream: + priced_route: Final = upstream.post(f"{sent_base}/fal-ai/trellis-2").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + unit_billed_route: Final = upstream.post(f"{sent_base}/acme/unit-billed-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-2"}) + ) + priced: Final = client.post( + "/fal_ai/fal-ai/trellis-2", json={"image_url": "https://example.com/in.png", "resolution": 1536} + ) + unit_billed: Final = client.post( + "/fal_ai/acme/x/requests/%2e%2e/%2e%2e/unit-billed-model", json={"prompt": "a red boat"} + ) + + assert priced.status_code == 200, priced.text + assert unit_billed.status_code == 400, unit_billed.text + assert len(priced_route.calls) == 1 + assert not unit_billed_route.calls + def test_submit_gate_prices_the_request_body_not_an_empty_one( self, client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: