mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(integration): fal image generation and edit wire contracts
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry
parent
342bde7a8d
commit
7510697355
3 changed files with 224 additions and 0 deletions
|
|
@ -23728,6 +23728,16 @@
|
|||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"fal_ai/high/1536-x-1024/openai/gpt-image-2.5/flare/text-to-image": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.04116,
|
||||
"source": "https://fal.ai/models/openai/gpt-image-2.5/flare/text-to-image",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"fal_ai/high/1920-x-1080/openai/gpt-image-2.5/flare/text-to-image": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "image_generation",
|
||||
|
|
|
|||
|
|
@ -166,6 +166,15 @@
|
|||
"tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": [
|
||||
"other.provider_wire.fal_ai.video_queue_create_status_and_content_download"
|
||||
],
|
||||
"tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [
|
||||
"other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing"
|
||||
],
|
||||
"tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [
|
||||
"other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing"
|
||||
],
|
||||
"tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [
|
||||
"other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing"
|
||||
],
|
||||
"tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [
|
||||
"mcp.call_tool.saved_headers.reach_actual_transport"
|
||||
],
|
||||
|
|
|
|||
205
tests/integration/providers/test_fal_ai_image_wire.py
Normal file
205
tests/integration/providers/test_fal_ai_image_wire.py
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_GPT_IMAGE_MODEL: Final = "openai/gpt-image-2.5/flare/text-to-image"
|
||||
_FLUX_MODEL: Final = "fal-ai/flux/dev"
|
||||
_EDIT_MODEL: Final = "openai/gpt-image-2.5/flare/edit"
|
||||
_PNG_BYTES: Final = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
|
||||
b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff"
|
||||
b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
_PROMPT: Final = "a red circle on a blue background"
|
||||
_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
||||
|
||||
def _catalog_cost(key: str) -> float:
|
||||
cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes())
|
||||
cost_value: Final = cost_map[key]["output_cost_per_image"]
|
||||
assert isinstance(cost_value, (int, float))
|
||||
return float(cost_value)
|
||||
|
||||
|
||||
def _image_response(urls: tuple[str, ...], prompt: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"url": url,
|
||||
"content_type": "image/png",
|
||||
"file_name": url.rsplit("/", 1)[-1],
|
||||
"file_size": 123456,
|
||||
"width": 1024,
|
||||
"height": 768,
|
||||
}
|
||||
for url in urls
|
||||
],
|
||||
"timings": {"inference": 2.1},
|
||||
"seed": 1234567,
|
||||
"has_nsfw_concepts": [False],
|
||||
"prompt": prompt,
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _response_data(response: httpx.Response) -> list[JsonValue]:
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
data: Final = payload["data"]
|
||||
assert isinstance(data, list)
|
||||
return data
|
||||
|
||||
|
||||
def _image_urls(response: httpx.Response) -> tuple[str, ...]:
|
||||
data: Final = _response_data(response)
|
||||
values: Final = tuple(
|
||||
image["url"] for image in data if isinstance(image, dict) and isinstance(image.get("url"), str)
|
||||
)
|
||||
assert len(values) == len(data)
|
||||
return tuple(value for value in values if isinstance(value, str))
|
||||
|
||||
|
||||
def _response_cost(response: httpx.Response) -> tuple[float, str]:
|
||||
headers: Final = response.headers
|
||||
if "x-litellm-response-cost" in headers:
|
||||
return float(headers["x-litellm-response-cost"]), "x-litellm-response-cost"
|
||||
call_id: Final = headers["x-litellm-call-id"]
|
||||
assert isinstance(call_id, str)
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
spend: Final = rows[0]["spend"]
|
||||
assert isinstance(spend, (int, float, str))
|
||||
return float(spend), "LiteLLM_SpendLogs.spend"
|
||||
|
||||
|
||||
def _approx(value: float) -> object:
|
||||
return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing")
|
||||
def test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.headers["authorization"] == "Key synthetic-fal-key"
|
||||
assert request.target == "/openai/gpt-image-2.5/flare/text-to-image"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
if body.get("quality") == "high":
|
||||
assert body == {"prompt": _PROMPT, "quality": "high", "image_size": {"width": 1536, "height": 1024}}
|
||||
return Reply(body=_image_response((f"{wire_url}/files/high.png",), _PROMPT))
|
||||
assert body == {"prompt": _PROMPT, "quality": "low"}
|
||||
return Reply(body=_image_response((f"{wire_url}/files/low.png",), _PROMPT))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
wire_url: Final = wire.url
|
||||
model: Final = scenario.model(
|
||||
model=f"fal_ai/{_GPT_IMAGE_MODEL}", api_base=wire.url, api_key="synthetic-fal-key"
|
||||
)
|
||||
high_response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/images/generations",
|
||||
{"model": model, "prompt": _PROMPT, "quality": "high", "size": "1536x1024"},
|
||||
)
|
||||
assert high_response.status_code == 200, high_response.text
|
||||
assert _image_urls(high_response) == (f"{wire.url}/files/high.png",)
|
||||
high_cost, high_cost_path = _response_cost(high_response)
|
||||
assert high_cost == _approx(_catalog_cost("fal_ai/high/1536-x-1024/openai/gpt-image-2.5/flare/text-to-image"))
|
||||
|
||||
low_response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/images/generations",
|
||||
{"model": model, "prompt": _PROMPT, "quality": "low"},
|
||||
)
|
||||
assert low_response.status_code == 200, low_response.text
|
||||
assert _image_urls(low_response) == (f"{wire.url}/files/low.png",)
|
||||
low_cost, low_cost_path = _response_cost(low_response)
|
||||
assert low_cost == _approx(_catalog_cost("fal_ai/low/1024-x-768/openai/gpt-image-2.5/flare/text-to-image"))
|
||||
assert high_cost != low_cost
|
||||
assert high_cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend")
|
||||
assert low_cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend")
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/openai/gpt-image-2.5/flare/text-to-image"),
|
||||
("POST", "/openai/gpt-image-2.5/flare/text-to-image"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing")
|
||||
def test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.headers["authorization"] == "Key synthetic-fal-key"
|
||||
assert request.target == "/fal-ai/flux/dev"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"prompt": _PROMPT,
|
||||
"num_images": 2,
|
||||
"image_size": "square_hd",
|
||||
}
|
||||
return Reply(
|
||||
body=_image_response(
|
||||
(f"{wire_url}/files/flux-1.png", f"{wire_url}/files/flux-2.png"),
|
||||
_PROMPT,
|
||||
)
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
wire_url: Final = wire.url
|
||||
model: Final = scenario.model(model=f"fal_ai/{_FLUX_MODEL}", api_base=wire.url, api_key="synthetic-fal-key")
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/images/generations",
|
||||
{"model": model, "prompt": _PROMPT, "n": 2, "size": "1024x1024"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _image_urls(response) == (
|
||||
f"{wire.url}/files/flux-1.png",
|
||||
f"{wire.url}/files/flux-2.png",
|
||||
)
|
||||
cost, cost_path = _response_cost(response)
|
||||
assert cost == _approx(2 * _catalog_cost("fal_ai/fal-ai/flux/dev"))
|
||||
assert cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend")
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/fal-ai/flux/dev")]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing")
|
||||
def test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.headers["authorization"] == "Key synthetic-fal-key"
|
||||
assert request.target == "/openai/gpt-image-2.5/flare/edit"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"prompt": _PROMPT,
|
||||
"image_urls": ["data:image/png;base64," + base64.b64encode(_PNG_BYTES).decode()],
|
||||
"quality": "low",
|
||||
}
|
||||
return Reply(body=_image_response((f"{wire_url}/files/edit.png",), _PROMPT))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
wire_url: Final = wire.url
|
||||
model: Final = scenario.model(model=f"fal_ai/{_EDIT_MODEL}", api_base=wire.url, api_key="synthetic-fal-key")
|
||||
response: Final = gateway.client.post(
|
||||
"/v1/images/edits",
|
||||
data={"model": model, "prompt": _PROMPT, "quality": "low"},
|
||||
files={"image": ("red_circle.png", _PNG_BYTES, "image/png")},
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _image_urls(response) == (f"{wire.url}/files/edit.png",)
|
||||
cost, cost_path = _response_cost(response)
|
||||
assert cost == _approx(_catalog_cost("fal_ai/low/1024-x-768/openai/gpt-image-2.5/flare/edit"))
|
||||
assert cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend")
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/openai/gpt-image-2.5/flare/edit")
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue