scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

feat(fal_ai): add queue-only /fal_ai pass-through route with spend tracking (#42360)

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-09-22 02:59:58 +00:00 • committed by GitHub
parent 2f76aa1b1b
commit 0fd1c191ca
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 986 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
gateway/routes/allowlist.py
View file

@ -96,6 +96,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/assemblyai/",
"/eu.assemblyai/",
"/deepgram/",
"/fal_ai/",
"/langfuse/",
"/vllm/",
"/mistral/",

17
litellm/llms/fal_ai/cost_calculator.py
View file

@ -1,3 +1,4 @@
import os
from collections.abc import Mapping
from math import ceil
from types import MappingProxyType
@ -25,6 +26,12 @@ FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType(
_OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
FAL_AI_QUEUE_DEFAULT_BASE: Final[str] = "https://queue.fal.run"
def fal_ai_queue_base() -> str:
return os.getenv("FAL_AI_QUEUE_API_BASE") or FAL_AI_QUEUE_DEFAULT_BASE
def _keyed_size(optional_params: Mapping[str, object]) -> str | None:
image_size: Final = optional_params.get("image_size")
@ -128,6 +135,16 @@ def _entry(key: str) -> Mapping[str, object] | None:
return _OBJECT_MAP.validate_python(raw_entry)
def fal_ai_passthrough_cost(model: str, request_body: Mapping[str, object]) -> float | None:
entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{model}")
if entry is None:
return None
resolution: Final = request_body.get("resolution")
keyed_cost: Final = entry.get(f"output_cost_per_image_{resolution}") if isinstance(resolution, int) else None
cost: Final = keyed_cost if isinstance(keyed_cost, (int, float)) else entry.get("output_cost_per_image")
return float(cost) if isinstance(cost, (int, float)) else None
def cost_calculator(
model: str,
image_response: object,

21
litellm/model_prices_and_context_window_backup.json
View file

@ -24966,6 +24966,27 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/trellis": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.02,
"source": "https://fal.ai/models/fal-ai/trellis",
"metadata": {
"comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route"
}
},
"fal_ai/fal-ai/trellis-2": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.3,
"output_cost_per_image_512": 0.25,
"output_cost_per_image_1024": 0.3,
"output_cost_per_image_1536": 0.35,
"source": "https://fal.ai/models/fal-ai/trellis-2",
"metadata": {
"comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route"
}
},
"fal_ai/fal-ai/flux-lora-depth": {
"litellm_provider": "fal_ai",
"metadata": {

1
litellm/proxy/_lazy_features.py
View file

@ -203,6 +203,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/cursor/",
"/deepgram/",
"/eu.assemblyai/",
"/fal_ai/",
"/gemini/",
"/gigachat/",
"/milvus/",

217
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -18518,6 +18518,223 @@
]
}
},
"/fal_ai/{endpoint}": {
"delete": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Fal Ai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"get": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Fal Ai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"patch": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__patch",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Fal Ai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"post": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Fal Ai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"put": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__put",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Fal Ai Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/gemini/{endpoint}": {
"delete": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)",

1
litellm/proxy/_types.py
View file

@ -500,6 +500,7 @@ class LiteLLMRoutes(enum.Enum):
"/watsonx",
"/nvidia_nim",
"/deepgram",
"/fal_ai",
]
#########################################################

51
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -52,6 +52,7 @@ from litellm.llms.deepgram.common_utils import (
deepgram_listen_requested_model,
deepgram_listen_websocket_target,
)
from litellm.llms.fal_ai.cost_calculator import fal_ai_queue_base
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.passthrough.main import AsyncPassthroughStreamingResponse
@ -421,6 +422,56 @@ async def cohere_proxy_route(
return received_value
def _fal_target(endpoint: str) -> httpx.URL:
base_target_url: Final = fal_ai_queue_base()
encoded_endpoint: Final = httpx.URL(endpoint).path
normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
base_url: Final = httpx.URL(base_target_url)
return base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
)
@router.api_route(
"/fal_ai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
tags=["Fal AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
)
async def fal_ai_proxy_route(
endpoint: str,
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
updated_url: Final = _fal_target(endpoint)
fal_ai_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider="fal_ai",
region_name=None,
)
if fal_ai_api_key is None:
raise HTTPException(
status_code=401,
detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured",
)
if "/requests/" not in endpoint:
priced_model: Final = f"fal_ai/{endpoint}"
if priced_model not in (litellm.model_cost or {}):
raise HTTPException(
status_code=400,
detail=f"{priced_model} has no pricing entry; only priced Fal endpoints can be submitted through /fal_ai",
)
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers={
"Authorization": f"Key {fal_ai_api_key}"
}, # mutable-ok: pass-through request headers require a mutable mapping
custom_llm_provider="fal_ai",
is_streaming_request=False,
)
return await endpoint_func(request, fastapi_response, user_api_key_dict)
@router.api_route(
"/vllm/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

71
litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,71 @@
from collections.abc import Mapping, Sequence
from typing import Final
from urllib.parse import urlparse
import httpx
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.utils import ImageObject, ImageResponse
FAL_AI_PROVIDER: Final[str] = litellm.LlmProviders.FAL_AI.value
def _url_parts(value: object) -> tuple[Mapping[str, object], ...]:
if isinstance(value, Mapping):
return (value,) if isinstance(value.get("url"), str) else ()
if isinstance(value, Sequence) and not isinstance(value, str):
return tuple(item for item in value if isinstance(item, Mapping) and isinstance(item.get("url"), str))
return ()
class FalAIPassthroughLoggingHandler:
@staticmethod
def is_fal_ai_route(url_route: str, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == FAL_AI_PROVIDER
def fal_ai_passthrough_handler(
self,
response_body: Mapping[str, object],
request_body: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
url_route: str,
kwargs: Mapping[str, object],
) -> PassThroughEndpointLoggingTypedDict:
base_path: Final = httpx.URL(fal_ai_queue_base()).path.strip("/")
raw_path: Final = urlparse(url_route).path.strip("/")
upstream_path: Final = raw_path.removeprefix(f"{base_path}/") if base_path else raw_path
model: Final = upstream_path.partition("/requests/")[0]
is_submit: Final = "/requests/" not in upstream_path
response: Final = ImageResponse(
data=tuple(
ImageObject(url=url)
for value in response_body.values()
for part in _url_parts(value)
if isinstance((url := part.get("url")), str)
)
)
response_cost: Final = fal_ai_passthrough_cost(model, request_body) if is_submit else None
response._hidden_params["response_cost"] = response_cost # pyright: ignore[reportPrivateUsage] # the logger reads a precomputed cost off the response's hidden params
logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object
logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object
logging_obj.model_call_details["custom_llm_provider"] = FAL_AI_PROVIDER # rebind-ok: same shared logging object
logging_obj.model_call_details["response_cost"] = response_cost # rebind-ok: same shared logging object
verbose_proxy_logger.debug(
"Fal AI passthrough cost tracking: model %s, cost %s",
model,
response_cost,
)
logging_result: Final[PassThroughEndpointLoggingTypedDict] = {
"result": response,
"kwargs": {
**kwargs,
"model": model,
"custom_llm_provider": FAL_AI_PROVIDER,
"response_cost": response_cost,
},
}
return logging_result

13
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -29,6 +29,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import (
from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import (
DeepgramListenPassthroughLoggingHandler,
)
from .llm_provider_handlers.fal_ai_passthrough_logging_handler import (
FalAIPassthroughLoggingHandler,
)
from .llm_provider_handlers.gemini_passthrough_logging_handler import (
GeminiPassthroughLoggingHandler,
)
@ -370,6 +373,16 @@ class PassThroughEndpointLogging:
)
standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain
kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider):
fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}),
request_body=request_body,
logging_obj=logging_obj,
url_route=url_route,
kwargs=kwargs,
)
standard_logging_response_object = fal_ai_handler_result["result"] # rebind-ok: elif-chain
kwargs = fal_ai_handler_result["kwargs"] # rebind-ok: elif-chain contract
return_dict["standard_logging_response_object"] = standard_logging_response_object
return_dict["kwargs"] = kwargs

6
litellm/types/utils.py
View file

@ -356,6 +356,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_second_768p: ReadOnly[float | None]
output_cost_per_second_2k: ReadOnly[float | None]
output_cost_per_second_4k: ReadOnly[float | None]
output_cost_per_image_512: ReadOnly[float | None]
output_cost_per_image_1024: ReadOnly[float | None]
output_cost_per_image_1536: ReadOnly[float | None]
ocr_cost_per_page: float | None # for OCR models
ocr_cost_per_page_batches: ReadOnly[float | None]
ocr_cost_per_credit: float | None # for OCR models priced by credit
@ -3682,6 +3685,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_second_768p: float | None = None
output_cost_per_second_2k: float | None = None
output_cost_per_second_4k: float | None = None
output_cost_per_image_512: float | None = None
output_cost_per_image_1024: float | None = None
output_cost_per_image_1536: float | None = None
input_cost_per_pixel: float | None = None
output_cost_per_pixel: float | None = None

21
model_prices_and_context_window.json
View file

@ -24966,6 +24966,27 @@
"/v1/images/generations"
]
},
"fal_ai/fal-ai/trellis": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.02,
"source": "https://fal.ai/models/fal-ai/trellis",
"metadata": {
"comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route"
}
},
"fal_ai/fal-ai/trellis-2": {
"litellm_provider": "fal_ai",
"mode": "image_generation",
"output_cost_per_image": 0.3,
"output_cost_per_image_512": 0.25,
"output_cost_per_image_1024": 0.3,
"output_cost_per_image_1536": 0.35,
"source": "https://fal.ai/models/fal-ai/trellis-2",
"metadata": {
"comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route"
}
},
"fal_ai/fal-ai/flux-lora-depth": {
"litellm_provider": "fal_ai",
"metadata": {

12
model_prices_and_context_window.schema.json
View file

@ -586,6 +586,18 @@
"type": "number",
"minimum": 0
},
"output_cost_per_image_1024": {
"type": "number",
"minimum": 0
},
"output_cost_per_image_1536": {
"type": "number",
"minimum": 0
},
"output_cost_per_image_512": {
"type": "number",
"minimum": 0
},
"output_cost_per_image_token": {
"type": "number",
"minimum": 0

3
tests/integration/contracts.json
View file

@ -181,6 +181,9 @@
"tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [
"other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing"
],
"tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [
"other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not"
],
"tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [
"other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing"
],

86
tests/integration/providers/test_fal_ai_passthrough_wire.py Normal file
View file

@ -0,0 +1,86 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway, eventually
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
_MODEL: Final = "fal-ai/trellis-2"
_REQUEST_BODY: Final = {"image_url": "https://example.com/in.png", "resolution": 1536}
_UPSTREAM_BODY: Final = {
"model_glb": {
"url": "https://fal.media/model.glb",
"content_type": "model/gltf-binary",
"file_name": "model.glb",
"file_size": 123,
}
}
_EXPECTED_SPEND: Final = 0.35
@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not")
def test_fal_queue_submit_charges_and_polls_pass_through_free(gateway: Gateway, tmp_path) -> None:
def respond(request: Request) -> Reply:
assert request.headers["authorization"] == "Key synthetic-fal-key"
if request.method == "POST":
assert request.target == f"/{_MODEL}"
assert json.loads(request.body) == _REQUEST_BODY
return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode())
if request.target == f"/{_MODEL}/requests/req-1/status":
return Reply(body=json.dumps({"status": "COMPLETED"}).encode())
assert request.target == f"/{_MODEL}/requests/req-1"
return Reply(body=json.dumps(_UPSTREAM_BODY).encode())
config: Final = tmp_path / "proxy_config.yaml"
config.write_text(
"model_list: []\n"
"general_settings:\n"
" master_key: os.environ/LITELLM_MASTER_KEY\n"
" database_url: os.environ/DATABASE_URL\n"
" store_model_in_db: true\n"
" disable_spend_logs: false\n"
" proxy_batch_write_at: 1\n"
"router_settings:\n"
" disable_cooldowns: true\n"
)
with wire_server(respond) as wire:
with owned_proxy(
gateway,
tmp_path,
{"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"},
config=config,
) as candidate:
submit: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", _REQUEST_BODY)
assert submit.status_code == 200, submit.text
assert json.loads(submit.content) == {"request_id": "req-1", "status": "IN_QUEUE"}
status_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1/status")
assert status_response.status_code == 200, status_response.text
assert json.loads(status_response.content) == {"status": "COMPLETED"}
result_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1")
assert result_response.status_code == 200, result_response.text
assert json.loads(result_response.content) == _UPSTREAM_BODY
submit_spend: Final = eventually(
lambda: read_rows(
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(submit.headers["x-litellm-call-id"],),
),
lambda values: len(values) == 1,
seconds=70,
)
assert float(submit_spend[0]["spend"]) == pytest.approx(_EXPECTED_SPEND)
poll_rows: Final = eventually(
lambda: read_rows(
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=ANY(%s)',
([status_response.headers["x-litellm-call-id"], result_response.headers["x-litellm-call-id"]],),
),
lambda values: len(values) == 2,
seconds=70,
)
assert sorted(float(row["spend"]) for row in poll_rows) == [0.0, 0.0]
assert [(request.method, request.target) for request in wire.drain()] == [
("POST", f"/{_MODEL}"),
("GET", f"/{_MODEL}/requests/req-1/status"),
("GET", f"/{_MODEL}/requests/req-1"),
]

31
tests/test_litellm/llms/fal_ai/test_cost_calculator.py
View file

@ -4,7 +4,7 @@ import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
from litellm.llms.fal_ai.cost_calculator import cost_calculator
from litellm.llms.fal_ai.cost_calculator import cost_calculator, fal_ai_passthrough_cost
from litellm.types.utils import ImageObject, ImageResponse
@pytest.fixture(autouse=True)
@ -174,3 +174,32 @@ def test_image_edit_call_type_routes_to_fal_keyed_pricing():
call_type="aimage_edit",
)
assert cost == litellm.model_cost[f"fal_ai/medium/1024-x-1024/{model}"]["output_cost_per_image"] > 0
def test_passthrough_trellis_charges_flat_rate():
assert (
fal_ai_passthrough_cost("fal-ai/trellis", {})
== litellm.model_cost["fal_ai/fal-ai/trellis"]["output_cost_per_image"]
> 0
)
@pytest.mark.parametrize("resolution", [512, 1024, 1536])
def test_passthrough_trellis_2_resolution_picks_keyed_tier(resolution):
assert (
fal_ai_passthrough_cost("fal-ai/trellis-2", {"resolution": resolution})
== litellm.model_cost["fal_ai/fal-ai/trellis-2"][f"output_cost_per_image_{resolution}"]
> 0
)
def test_passthrough_trellis_2_without_resolution_falls_back_to_default_rate():
assert (
fal_ai_passthrough_cost("fal-ai/trellis-2", {"image_url": "https://a"})
== litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image"]
> 0
)
def test_passthrough_unknown_model_returns_none():
assert fal_ai_passthrough_cost("fal-ai/no-such-model", {"resolution": 512}) is None

132
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,132 @@
"""Fal AI pass-through: upstream URL to model extraction and resolution-keyed spend tracking."""
from datetime import datetime
from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.fal_ai_passthrough_logging_handler import (
FalAIPassthroughLoggingHandler,
)
from litellm.types.utils import ImageResponse
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
UPSTREAM_URL: Final = "https://queue.fal.run/fal-ai/trellis-2"
def _logging_obj(call_id: str = "call-fal") -> LiteLLMLoggingObj:
return LiteLLMLoggingObj(
model="unknown",
messages=[{"role": "user", "content": "passthrough"}],
stream=False,
call_type="pass_through_endpoint",
start_time=datetime.now(),
litellm_call_id=call_id,
function_id="passthrough",
)
def test_is_fal_ai_route_matches_only_the_fal_ai_provider():
assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "fal_ai") is True
assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "deepgram") is False
assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, None) is False
def test_handler_extracts_model_urls_and_resolution_keyed_cost():
upstream_body: Final = {
"model_glb": {"url": "https://fal.media/model.glb", "content_type": "model/gltf-binary"},
"images": [{"url": "https://fal.media/preview.png"}],
"timings": {"inference": 1.2},
}
logging_obj: Final = _logging_obj()
expected_cost: Final = litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"]
handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body=upstream_body,
request_body={"image_url": "https://example.com/in.png", "resolution": 1536},
logging_obj=logging_obj,
url_route=UPSTREAM_URL,
kwargs={"litellm_params": {"metadata": {}}},
)
result = handler_result["result"]
assert isinstance(result, ImageResponse)
assert [image.url for image in result.data or ()] == [
"https://fal.media/model.glb",
"https://fal.media/preview.png",
]
assert result._hidden_params["response_cost"] == pytest.approx(expected_cost)
assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2"
assert handler_result["kwargs"]["custom_llm_provider"] == "fal_ai"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
assert handler_result["kwargs"]["litellm_params"] == {"metadata": {}}
assert logging_obj.model == "fal-ai/trellis-2"
assert logging_obj.model_call_details["model"] == "fal-ai/trellis-2"
assert logging_obj.model_call_details["custom_llm_provider"] == "fal_ai"
assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost)
def test_handler_charges_nothing_and_names_the_model_for_queue_status_and_result_polls():
for upstream_url in (
"https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status",
"https://queue.fal.run/fal-ai/trellis-2/requests/req-1",
):
logging_obj: Final = _logging_obj()
handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body={"status": "COMPLETED"},
request_body={},
logging_obj=logging_obj,
url_route=upstream_url,
kwargs={},
)
assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2"
assert handler_result["kwargs"]["response_cost"] is None
assert logging_obj.model_call_details["response_cost"] is None
def test_handler_charges_for_queue_submit():
handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body={"request_id": "req-1", "status": "IN_QUEUE"},
request_body={"image_url": "https://example.com/in.png", "resolution": 1536},
logging_obj=_logging_obj(),
url_route="https://queue.fal.run/fal-ai/trellis-2",
kwargs={},
)
assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(
litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"]
)
def test_handler_strips_queue_base_path_prefix(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://gw.example/fal/queue")
handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body={"request_id": "req-1", "status": "IN_QUEUE"},
request_body={"image_url": "https://example.com/in.png", "resolution": 1536},
logging_obj=_logging_obj(),
url_route="https://gw.example/fal/queue/fal-ai/trellis-2",
kwargs={},
)
assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(
litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"]
)
def test_handler_without_url_values_returns_empty_image_response_and_no_cost():
handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
response_body={"status": "COMPLETED"},
request_body={},
logging_obj=_logging_obj(),
url_route="https://queue.fal.run/fal-ai/no-such-model",
kwargs={},
)
assert isinstance(handler_result["result"], ImageResponse)
assert not handler_result["result"].data
assert handler_result["kwargs"]["response_cost"] is None
assert handler_result["kwargs"]["model"] == "fal-ai/no-such-model"

115
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -29,6 +29,7 @@ from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
BaseOpenAIPassThroughHandler,
RouteChecks,
_fal_target,
_join_url_paths,
_proxy_general_settings,
anthropic_proxy_route,
@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
bedrock_proxy_route,
create_pass_through_route,
cursor_proxy_route,
fal_ai_proxy_route,
get_azure_ai_search_index_from_endpoint,
get_vertex_base_url,
is_azure_ai_search_service_level_index_create,
@ -7117,6 +7119,119 @@ class TestTypeSafePassthroughRoute:
)
class TestFalAIPassthroughRoute:
@pytest.fixture
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key")
monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
yield TestClient(app)
def test_submit_forwards_body_and_key_scheme_to_queue_fal_run(self, client: TestClient) -> None:
body: Final = {"image_url": "https://example.com/in.png", "resolution": 1536}
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post("https://queue.fal.run/fal-ai/trellis-2").mock(
return_value=httpx.Response(200, json={"request_id": "req-1", "status": "IN_QUEUE"})
)
response = client.post("/fal_ai/fal-ai/trellis-2", json=body)
assert response.status_code == 200, response.text
assert response.json() == {"request_id": "req-1", "status": "IN_QUEUE"}
sent = route.calls.last.request
assert sent.headers["authorization"] == "Key fal-test-key"
assert json.loads(sent.content or b"{}") == body
def test_status_get_forwards_to_queue_fal_run(self, client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
route = upstream.get("https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status").mock(
return_value=httpx.Response(200, json={"status": "COMPLETED"})
)
response = client.get("/fal_ai/fal-ai/trellis-2/requests/req-1/status")
assert response.status_code == 200, response.text
assert response.json() == {"status": "COMPLETED"}
assert route.calls.last.request.headers["authorization"] == "Key fal-test-key"
def test_honours_fal_ai_queue_api_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key")
monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base")
endpoint_func = AsyncMock(return_value={"ok": True})
create_route = Mock(return_value=endpoint_func)
monkeypatch.setattr(
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
create_route,
)
request = MagicMock(spec=Request)
request.method = "POST"
request.query_params = {}
request.json = AsyncMock(return_value={})
result = asyncio.run(
fal_ai_proxy_route(
endpoint="fal-ai/trellis",
request=request,
fastapi_response=MagicMock(spec=Response),
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
)
)
assert result == {"ok": True}
create_route.assert_called_once_with(
endpoint="fal-ai/trellis",
target="https://queue.example/base/fal-ai/trellis",
custom_headers={"Authorization": "Key fal-test-key"},
custom_llm_provider="fal_ai",
is_streaming_request=False,
)
def test_submit_to_unpriced_endpoint_returns_400_without_upstream_call(self, client: TestClient) -> None:
with respx.mock(assert_all_called=False) as upstream:
route = upstream.post("https://queue.fal.run/fal-ai/unpriced-model").mock(
return_value=httpx.Response(200, json={"request_id": "req-1"})
)
response = client.post("/fal_ai/fal-ai/unpriced-model", json={"image_url": "https://example.com/in.png"})
assert response.status_code == 400, response.text
assert "no pricing entry" in response.text
assert not route.calls
def test_status_get_on_unpriced_endpoint_forwards(self, client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
upstream.get("https://queue.fal.run/fal-ai/unpriced-model/requests/req-9/status").mock(
return_value=httpx.Response(200, json={"status": "IN_PROGRESS"})
)
response = client.get("/fal_ai/fal-ai/unpriced-model/requests/req-9/status")
assert response.status_code == 200, response.text
assert response.json() == {"status": "IN_PROGRESS"}
def test_missing_fal_key_returns_401(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("FAL_AI_API_KEY", raising=False)
response = client.post("/fal_ai/fal-ai/trellis", json={})
assert response.status_code == 401
class TestFalTargetSelection:
def test_endpoint_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False)
assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.fal.run/fal-ai/trellis-2"
def test_status_path_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False)
assert str(_fal_target("fal-ai/trellis-2/requests/req-1/status")) == (
"https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status"
)
def test_queue_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base")
assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.example/base/fal-ai/trellis-2"
class TestOpenRouterPassthroughRoute:
@staticmethod
def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock:

188
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -4872,6 +4872,27 @@ export interface paths {
patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"];
trace?: never;
};
"/fal_ai/{endpoint}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Fal Ai Proxy Route */
get: operations["fal_ai_proxy_route_fal_ai__endpoint__get"];
/** Fal Ai Proxy Route */
put: operations["fal_ai_proxy_route_fal_ai__endpoint__put"];
/** Fal Ai Proxy Route */
post: operations["fal_ai_proxy_route_fal_ai__endpoint__post"];
/** Fal Ai Proxy Route */
delete: operations["fal_ai_proxy_route_fal_ai__endpoint__delete"];
options?: never;
head?: never;
/** Fal Ai Proxy Route */
patch: operations["fal_ai_proxy_route_fal_ai__endpoint__patch"];
trace?: never;
};
"/fallback": {
parameters: {
query?: never;
@ -31258,6 +31279,12 @@ export interface components {
output_cost_per_character_above_128k_tokens?: number | null;
/** Output Cost Per Image */
output_cost_per_image?: number | null;
/** Output Cost Per Image 1024 */
output_cost_per_image_1024?: number | null;
/** Output Cost Per Image 1536 */
output_cost_per_image_1536?: number | null;
/** Output Cost Per Image 512 */
output_cost_per_image_512?: number | null;
/** Output Cost Per Image Token */
output_cost_per_image_token?: number | null;
/** Output Cost Per Pixel */
@ -42087,6 +42114,12 @@ export interface components {
output_cost_per_character_above_128k_tokens?: number | null;
/** Output Cost Per Image */
output_cost_per_image?: number | null;
/** Output Cost Per Image 1024 */
output_cost_per_image_1024?: number | null;
/** Output Cost Per Image 1536 */
output_cost_per_image_1536?: number | null;
/** Output Cost Per Image 512 */
output_cost_per_image_512?: number | null;
/** Output Cost Per Image Token */
output_cost_per_image_token?: number | null;
/** Output Cost Per Pixel */
@ -49377,6 +49410,161 @@ export interface operations {
};
};
};
fal_ai_proxy_route_fal_ai__endpoint__get: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
fal_ai_proxy_route_fal_ai__endpoint__put: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
fal_ai_proxy_route_fal_ai__endpoint__post: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
fal_ai_proxy_route_fal_ai__endpoint__delete: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
fal_ai_proxy_route_fal_ai__endpoint__patch: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
create_fallback_fallback_post: {
parameters: {
query?: never;