From 0fd1c191ca6e8f814de09b082a545e15274c9c5e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:59:58 +0000 Subject: [PATCH] feat(fal_ai): add queue-only /fal_ai pass-through route with spend tracking (#42360) --- gateway/routes/allowlist.py | 1 + litellm/llms/fal_ai/cost_calculator.py | 17 ++ ...odel_prices_and_context_window_backup.json | 21 ++ litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 217 ++++++++++++++++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 51 ++++ .../fal_ai_passthrough_logging_handler.py | 71 ++++++ .../pass_through_endpoints/success_handler.py | 13 ++ litellm/types/utils.py | 6 + model_prices_and_context_window.json | 21 ++ model_prices_and_context_window.schema.json | 12 + tests/integration/contracts.json | 3 + .../providers/test_fal_ai_passthrough_wire.py | 86 +++++++ .../llms/fal_ai/test_cost_calculator.py | 31 ++- ...test_fal_ai_passthrough_logging_handler.py | 132 +++++++++++ .../test_llm_pass_through_endpoints.py | 115 ++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 188 +++++++++++++++ 18 files changed, 986 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py create mode 100644 tests/integration/providers/test_fal_ai_passthrough_wire.py create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index aac36fe3f04..01ba9da3364 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -96,6 +96,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/assemblyai/", "/eu.assemblyai/", "/deepgram/", + "/fal_ai/", "/langfuse/", "/vllm/", "/mistral/", diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index 7ab5e055e71..31f0995bf9f 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -1,3 +1,4 @@ +import os from collections.abc import Mapping from math import ceil from types import MappingProxyType @@ -25,6 +26,12 @@ FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( _OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +FAL_AI_QUEUE_DEFAULT_BASE: Final[str] = "https://queue.fal.run" + + +def fal_ai_queue_base() -> str: + return os.getenv("FAL_AI_QUEUE_API_BASE") or FAL_AI_QUEUE_DEFAULT_BASE + def _keyed_size(optional_params: Mapping[str, object]) -> str | None: image_size: Final = optional_params.get("image_size") @@ -128,6 +135,16 @@ def _entry(key: str) -> Mapping[str, object] | None: return _OBJECT_MAP.validate_python(raw_entry) +def fal_ai_passthrough_cost(model: str, request_body: Mapping[str, object]) -> float | None: + entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{model}") + if entry is None: + return None + resolution: Final = request_body.get("resolution") + keyed_cost: Final = entry.get(f"output_cost_per_image_{resolution}") if isinstance(resolution, int) else None + cost: Final = keyed_cost if isinstance(keyed_cost, (int, float)) else entry.get("output_cost_per_image") + return float(cost) if isinstance(cost, (int, float)) else None + + def cost_calculator( model: str, image_response: object, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f2b10c436ef..70c54b91cbd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24966,6 +24966,27 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, "fal_ai/fal-ai/flux-lora-depth": { "litellm_provider": "fal_ai", "metadata": { diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 007a7aa5c29..c53f7625550 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -203,6 +203,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/cursor/", "/deepgram/", "/eu.assemblyai/", + "/fal_ai/", "/gemini/", "/gigachat/", "/milvus/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 12041c69785..3985feff08a 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -18518,6 +18518,223 @@ ] } }, + "/fal_ai/{endpoint}": { + "delete": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/gemini/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 126b0105ca1..2a6b15e4097 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -500,6 +500,7 @@ class LiteLLMRoutes(enum.Enum): "/watsonx", "/nvidia_nim", "/deepgram", + "/fal_ai", ] ######################################################### diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0a4bc31ec2e..74caa1050bb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,6 +52,7 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_requested_model, deepgram_listen_websocket_target, ) +from litellm.llms.fal_ai.cost_calculator import fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse @@ -421,6 +422,56 @@ async def cohere_proxy_route( return received_value +def _fal_target(endpoint: str) -> httpx.URL: + base_target_url: Final = fal_ai_queue_base() + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + return base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + + +@router.api_route( + "/fal_ai/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["Fal AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def fal_ai_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + updated_url: Final = _fal_target(endpoint) + fal_ai_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="fal_ai", + region_name=None, + ) + if fal_ai_api_key is None: + raise HTTPException( + status_code=401, + detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured", + ) + if "/requests/" not in endpoint: + priced_model: Final = f"fal_ai/{endpoint}" + if priced_model not in (litellm.model_cost or {}): + raise HTTPException( + status_code=400, + detail=f"{priced_model} has no pricing entry; only priced Fal endpoints can be submitted through /fal_ai", + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ + "Authorization": f"Key {fal_ai_api_key}" + }, # mutable-ok: pass-through request headers require a mutable mapping + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/vllm/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..3d1fad90e03 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py @@ -0,0 +1,71 @@ +from collections.abc import Mapping, Sequence +from typing import Final +from urllib.parse import urlparse + +import httpx + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import ImageObject, ImageResponse + +FAL_AI_PROVIDER: Final[str] = litellm.LlmProviders.FAL_AI.value + + +def _url_parts(value: object) -> tuple[Mapping[str, object], ...]: + if isinstance(value, Mapping): + return (value,) if isinstance(value.get("url"), str) else () + if isinstance(value, Sequence) and not isinstance(value, str): + return tuple(item for item in value if isinstance(item, Mapping) and isinstance(item.get("url"), str)) + return () + + +class FalAIPassthroughLoggingHandler: + @staticmethod + def is_fal_ai_route(url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == FAL_AI_PROVIDER + + def fal_ai_passthrough_handler( + self, + response_body: Mapping[str, object], + request_body: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + url_route: str, + kwargs: Mapping[str, object], + ) -> PassThroughEndpointLoggingTypedDict: + base_path: Final = httpx.URL(fal_ai_queue_base()).path.strip("/") + raw_path: Final = urlparse(url_route).path.strip("/") + upstream_path: Final = raw_path.removeprefix(f"{base_path}/") if base_path else raw_path + model: Final = upstream_path.partition("/requests/")[0] + is_submit: Final = "/requests/" not in upstream_path + response: Final = ImageResponse( + data=tuple( + ImageObject(url=url) + for value in response_body.values() + for part in _url_parts(value) + if isinstance((url := part.get("url")), str) + ) + ) + response_cost: Final = fal_ai_passthrough_cost(model, request_body) if is_submit else None + response._hidden_params["response_cost"] = response_cost # pyright: ignore[reportPrivateUsage] # the logger reads a precomputed cost off the response's hidden params + logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = FAL_AI_PROVIDER # rebind-ok: same shared logging object + logging_obj.model_call_details["response_cost"] = response_cost # rebind-ok: same shared logging object + verbose_proxy_logger.debug( + "Fal AI passthrough cost tracking: model %s, cost %s", + model, + response_cost, + ) + logging_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": response, + "kwargs": { + **kwargs, + "model": model, + "custom_llm_provider": FAL_AI_PROVIDER, + "response_cost": response_cost, + }, + } + return logging_result diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 2b40cfaa221..d1e4da2e47c 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -29,6 +29,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import ( DeepgramListenPassthroughLoggingHandler, ) +from .llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) @@ -370,6 +373,16 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider): + fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}), + request_body=request_body, + logging_obj=logging_obj, + url_route=url_route, + kwargs=kwargs, + ) + standard_logging_response_object = fal_ai_handler_result["result"] # rebind-ok: elif-chain + kwargs = fal_ai_handler_result["kwargs"] # rebind-ok: elif-chain contract return_dict["standard_logging_response_object"] = standard_logging_response_object return_dict["kwargs"] = kwargs diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 35d967fc1d5..2e8c869b89d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -356,6 +356,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_second_768p: ReadOnly[float | None] output_cost_per_second_2k: ReadOnly[float | None] output_cost_per_second_4k: ReadOnly[float | None] + output_cost_per_image_512: ReadOnly[float | None] + output_cost_per_image_1024: ReadOnly[float | None] + output_cost_per_image_1536: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_page_batches: ReadOnly[float | None] ocr_cost_per_credit: float | None # for OCR models priced by credit @@ -3682,6 +3685,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_second_768p: float | None = None output_cost_per_second_2k: float | None = None output_cost_per_second_4k: float | None = None + output_cost_per_image_512: float | None = None + output_cost_per_image_1024: float | None = None + output_cost_per_image_1536: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f2b10c436ef..70c54b91cbd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24966,6 +24966,27 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, "fal_ai/fal-ai/flux-lora-depth": { "litellm_provider": "fal_ai", "metadata": { diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 0509516ac32..f3a4e614f59 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -586,6 +586,18 @@ "type": "number", "minimum": 0 }, + "output_cost_per_image_1024": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_1536": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_512": { + "type": "number", + "minimum": 0 + }, "output_cost_per_image_token": { "type": "number", "minimum": 0 diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 6629b1fa1f4..e1b5940935f 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -181,6 +181,9 @@ "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" ], diff --git a/tests/integration/providers/test_fal_ai_passthrough_wire.py b/tests/integration/providers/test_fal_ai_passthrough_wire.py new file mode 100644 index 00000000000..f103135124e --- /dev/null +++ b/tests/integration/providers/test_fal_ai_passthrough_wire.py @@ -0,0 +1,86 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +_MODEL: Final = "fal-ai/trellis-2" +_REQUEST_BODY: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} +_UPSTREAM_BODY: Final = { + "model_glb": { + "url": "https://fal.media/model.glb", + "content_type": "model/gltf-binary", + "file_name": "model.glb", + "file_size": 123, + } +} +_EXPECTED_SPEND: Final = 0.35 + + +@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not") +def test_fal_queue_submit_charges_and_polls_pass_through_free(gateway: Gateway, tmp_path) -> None: + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_MODEL}" + assert json.loads(request.body) == _REQUEST_BODY + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + if request.target == f"/{_MODEL}/requests/req-1/status": + return Reply(body=json.dumps({"status": "COMPLETED"}).encode()) + assert request.target == f"/{_MODEL}/requests/req-1" + return Reply(body=json.dumps(_UPSTREAM_BODY).encode()) + + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + submit: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", _REQUEST_BODY) + assert submit.status_code == 200, submit.text + assert json.loads(submit.content) == {"request_id": "req-1", "status": "IN_QUEUE"} + status_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1/status") + assert status_response.status_code == 200, status_response.text + assert json.loads(status_response.content) == {"status": "COMPLETED"} + result_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1") + assert result_response.status_code == 200, result_response.text + assert json.loads(result_response.content) == _UPSTREAM_BODY + submit_spend: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (submit.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(submit_spend[0]["spend"]) == pytest.approx(_EXPECTED_SPEND) + poll_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=ANY(%s)', + ([status_response.headers["x-litellm-call-id"], result_response.headers["x-litellm-call-id"]],), + ), + lambda values: len(values) == 2, + seconds=70, + ) + assert sorted(float(row["spend"]) for row in poll_rows) == [0.0, 0.0] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", f"/{_MODEL}"), + ("GET", f"/{_MODEL}/requests/req-1/status"), + ("GET", f"/{_MODEL}/requests/req-1"), + ] diff --git a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py index 71c93112635..56dcba04b5c 100644 --- a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py +++ b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py @@ -4,7 +4,7 @@ import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils -from litellm.llms.fal_ai.cost_calculator import cost_calculator +from litellm.llms.fal_ai.cost_calculator import cost_calculator, fal_ai_passthrough_cost from litellm.types.utils import ImageObject, ImageResponse @pytest.fixture(autouse=True) @@ -174,3 +174,32 @@ def test_image_edit_call_type_routes_to_fal_keyed_pricing(): call_type="aimage_edit", ) assert cost == litellm.model_cost[f"fal_ai/medium/1024-x-1024/{model}"]["output_cost_per_image"] > 0 + + +def test_passthrough_trellis_charges_flat_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis", {}) + == litellm.model_cost["fal_ai/fal-ai/trellis"]["output_cost_per_image"] + > 0 + ) + + +@pytest.mark.parametrize("resolution", [512, 1024, 1536]) +def test_passthrough_trellis_2_resolution_picks_keyed_tier(resolution): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"resolution": resolution}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"][f"output_cost_per_image_{resolution}"] + > 0 + ) + + +def test_passthrough_trellis_2_without_resolution_falls_back_to_default_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"image_url": "https://a"}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image"] + > 0 + ) + + +def test_passthrough_unknown_model_returns_none(): + assert fal_ai_passthrough_cost("fal-ai/no-such-model", {"resolution": 512}) is None diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..1c945b9110b --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py @@ -0,0 +1,132 @@ +"""Fal AI pass-through: upstream URL to model extraction and resolution-keyed spend tracking.""" + +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) +from litellm.types.utils import ImageResponse + +pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") + +UPSTREAM_URL: Final = "https://queue.fal.run/fal-ai/trellis-2" + + +def _logging_obj(call_id: str = "call-fal") -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "passthrough"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id=call_id, + function_id="passthrough", + ) + + +def test_is_fal_ai_route_matches_only_the_fal_ai_provider(): + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "fal_ai") is True + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "deepgram") is False + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, None) is False + + +def test_handler_extracts_model_urls_and_resolution_keyed_cost(): + upstream_body: Final = { + "model_glb": {"url": "https://fal.media/model.glb", "content_type": "model/gltf-binary"}, + "images": [{"url": "https://fal.media/preview.png"}], + "timings": {"inference": 1.2}, + } + logging_obj: Final = _logging_obj() + expected_cost: Final = litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=upstream_body, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=logging_obj, + url_route=UPSTREAM_URL, + kwargs={"litellm_params": {"metadata": {}}}, + ) + + result = handler_result["result"] + assert isinstance(result, ImageResponse) + assert [image.url for image in result.data or ()] == [ + "https://fal.media/model.glb", + "https://fal.media/preview.png", + ] + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["custom_llm_provider"] == "fal_ai" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["litellm_params"] == {"metadata": {}} + assert logging_obj.model == "fal-ai/trellis-2" + assert logging_obj.model_call_details["model"] == "fal-ai/trellis-2" + assert logging_obj.model_call_details["custom_llm_provider"] == "fal_ai" + assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) + + +def test_handler_charges_nothing_and_names_the_model_for_queue_status_and_result_polls(): + for upstream_url in ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status", + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1", + ): + logging_obj: Final = _logging_obj() + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=logging_obj, + url_route=upstream_url, + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] is None + assert logging_obj.model_call_details["response_cost"] is None + + +def test_handler_charges_for_queue_submit(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/trellis-2", + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_strips_queue_base_path_prefix(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://gw.example/fal/queue") + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://gw.example/fal/queue/fal-ai/trellis-2", + kwargs={}, + ) + + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_without_url_values_returns_empty_image_response_and_no_cost(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/no-such-model", + kwargs={}, + ) + + assert isinstance(handler_result["result"], ImageResponse) + assert not handler_result["result"].data + assert handler_result["kwargs"]["response_cost"] is None + assert handler_result["kwargs"]["model"] == "fal-ai/no-such-model" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index db3b15c29a8..dcc3bd2b690 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -29,6 +29,7 @@ from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, + _fal_target, _join_url_paths, _proxy_general_settings, anthropic_proxy_route, @@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, is_azure_ai_search_service_level_index_create, @@ -7117,6 +7119,119 @@ class TestTypeSafePassthroughRoute: ) +class TestFalAIPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_submit_forwards_body_and_key_scheme_to_queue_fal_run(self, client: TestClient) -> None: + body: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/trellis-2").mock( + return_value=httpx.Response(200, json={"request_id": "req-1", "status": "IN_QUEUE"}) + ) + response = client.post("/fal_ai/fal-ai/trellis-2", json=body) + + assert response.status_code == 200, response.text + assert response.json() == {"request_id": "req-1", "status": "IN_QUEUE"} + sent = route.calls.last.request + assert sent.headers["authorization"] == "Key fal-test-key" + assert json.loads(sent.content or b"{}") == body + + def test_status_get_forwards_to_queue_fal_run(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.get("https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status").mock( + return_value=httpx.Response(200, json={"status": "COMPLETED"}) + ) + response = client.get("/fal_ai/fal-ai/trellis-2/requests/req-1/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "COMPLETED"} + assert route.calls.last.request.headers["authorization"] == "Key fal-test-key" + + def test_honours_fal_ai_queue_api_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + request.json = AsyncMock(return_value={}) + + result = asyncio.run( + fal_ai_proxy_route( + endpoint="fal-ai/trellis", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + ) + + assert result == {"ok": True} + create_route.assert_called_once_with( + endpoint="fal-ai/trellis", + target="https://queue.example/base/fal-ai/trellis", + custom_headers={"Authorization": "Key fal-test-key"}, + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + + def test_submit_to_unpriced_endpoint_returns_400_without_upstream_call(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/unpriced-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + response = client.post("/fal_ai/fal-ai/unpriced-model", json={"image_url": "https://example.com/in.png"}) + + assert response.status_code == 400, response.text + assert "no pricing entry" in response.text + assert not route.calls + + def test_status_get_on_unpriced_endpoint_forwards(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://queue.fal.run/fal-ai/unpriced-model/requests/req-9/status").mock( + return_value=httpx.Response(200, json={"status": "IN_PROGRESS"}) + ) + response = client.get("/fal_ai/fal-ai/unpriced-model/requests/req-9/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "IN_PROGRESS"} + + def test_missing_fal_key_returns_401(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_API_KEY", raising=False) + response = client.post("/fal_ai/fal-ai/trellis", json={}) + assert response.status_code == 401 + + +class TestFalTargetSelection: + def test_endpoint_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.fal.run/fal-ai/trellis-2" + + def test_status_path_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2/requests/req-1/status")) == ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status" + ) + + def test_queue_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.example/base/fal-ai/trellis-2" + + class TestOpenRouterPassthroughRoute: @staticmethod def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 931dc3363fc..bd0de11e9fb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4872,6 +4872,27 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/fal_ai/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Fal Ai Proxy Route */ + get: operations["fal_ai_proxy_route_fal_ai__endpoint__get"]; + /** Fal Ai Proxy Route */ + put: operations["fal_ai_proxy_route_fal_ai__endpoint__put"]; + /** Fal Ai Proxy Route */ + post: operations["fal_ai_proxy_route_fal_ai__endpoint__post"]; + /** Fal Ai Proxy Route */ + delete: operations["fal_ai_proxy_route_fal_ai__endpoint__delete"]; + options?: never; + head?: never; + /** Fal Ai Proxy Route */ + patch: operations["fal_ai_proxy_route_fal_ai__endpoint__patch"]; + trace?: never; + }; "/fallback": { parameters: { query?: never; @@ -31258,6 +31279,12 @@ export interface components { output_cost_per_character_above_128k_tokens?: number | null; /** Output Cost Per Image */ output_cost_per_image?: number | null; + /** Output Cost Per Image 1024 */ + output_cost_per_image_1024?: number | null; + /** Output Cost Per Image 1536 */ + output_cost_per_image_1536?: number | null; + /** Output Cost Per Image 512 */ + output_cost_per_image_512?: number | null; /** Output Cost Per Image Token */ output_cost_per_image_token?: number | null; /** Output Cost Per Pixel */ @@ -42087,6 +42114,12 @@ export interface components { output_cost_per_character_above_128k_tokens?: number | null; /** Output Cost Per Image */ output_cost_per_image?: number | null; + /** Output Cost Per Image 1024 */ + output_cost_per_image_1024?: number | null; + /** Output Cost Per Image 1536 */ + output_cost_per_image_1536?: number | null; + /** Output Cost Per Image 512 */ + output_cost_per_image_512?: number | null; /** Output Cost Per Image Token */ output_cost_per_image_token?: number | null; /** Output Cost Per Pixel */ @@ -49377,6 +49410,161 @@ export interface operations { }; }; }; + fal_ai_proxy_route_fal_ai__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; create_fallback_fallback_post: { parameters: { query?: never;