From 75106973550121d04166617380b5245ccfb5690d Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 21 Sep 2026 17:39:53 +0000 Subject: [PATCH] test(integration): fal image generation and edit wire contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- model_prices_and_context_window.json | 10 + tests/integration/contracts.json | 9 + .../providers/test_fal_ai_image_wire.py | 205 ++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 tests/integration/providers/test_fal_ai_image_wire.py diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b5d0b75e5bc..4ba02ecea09 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23728,6 +23728,16 @@ ], "supports_vision": true }, + "fal_ai/high/1536-x-1024/openai/gpt-image-2.5/flare/text-to-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.04116, + "source": "https://fal.ai/models/openai/gpt-image-2.5/flare/text-to-image", + "supported_endpoints": [ + "/v1/images/generations" + ], + "supports_vision": true + }, "fal_ai/high/1920-x-1080/openai/gpt-image-2.5/flare/text-to-image": { "litellm_provider": "fal_ai", "mode": "image_generation", diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 712ff928a48..cd7e84f81b6 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -166,6 +166,15 @@ "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": [ "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ + "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ + "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ + "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" + ], "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [ "mcp.call_tool.saved_headers.reach_actual_transport" ], diff --git a/tests/integration/providers/test_fal_ai_image_wire.py b/tests/integration/providers/test_fal_ai_image_wire.py new file mode 100644 index 00000000000..61b476e89b7 --- /dev/null +++ b/tests/integration/providers/test_fal_ai_image_wire.py @@ -0,0 +1,205 @@ +import base64 +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_GPT_IMAGE_MODEL: Final = "openai/gpt-image-2.5/flare/text-to-image" +_FLUX_MODEL: Final = "fal-ai/flux/dev" +_EDIT_MODEL: Final = "openai/gpt-image-2.5/flare/edit" +_PNG_BYTES: Final = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00" + b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff" + b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82" +) +_PROMPT: Final = "a red circle on a blue background" +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _catalog_cost(key: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[key]["output_cost_per_image"] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _image_response(urls: tuple[str, ...], prompt: str) -> bytes: + return json.dumps( + { + "images": [ + { + "url": url, + "content_type": "image/png", + "file_name": url.rsplit("/", 1)[-1], + "file_size": 123456, + "width": 1024, + "height": 768, + } + for url in urls + ], + "timings": {"inference": 2.1}, + "seed": 1234567, + "has_nsfw_concepts": [False], + "prompt": prompt, + } + ).encode() + + +def _response_data(response: httpx.Response) -> list[JsonValue]: + payload: Final = _JSON_OBJECT.validate_json(response.content) + data: Final = payload["data"] + assert isinstance(data, list) + return data + + +def _image_urls(response: httpx.Response) -> tuple[str, ...]: + data: Final = _response_data(response) + values: Final = tuple( + image["url"] for image in data if isinstance(image, dict) and isinstance(image.get("url"), str) + ) + assert len(values) == len(data) + return tuple(value for value in values if isinstance(value, str)) + + +def _response_cost(response: httpx.Response) -> tuple[float, str]: + headers: Final = response.headers + if "x-litellm-response-cost" in headers: + return float(headers["x-litellm-response-cost"]), "x-litellm-response-cost" + call_id: Final = headers["x-litellm-call-id"] + assert isinstance(call_id, str) + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + spend: Final = rows[0]["spend"] + assert isinstance(spend, (int, float, str)) + return float(spend), "LiteLLM_SpendLogs.spend" + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +@pytest.mark.covers("other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing") +def test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/text-to-image" + body: Final = _JSON_OBJECT.validate_json(request.body) + if body.get("quality") == "high": + assert body == {"prompt": _PROMPT, "quality": "high", "image_size": {"width": 1536, "height": 1024}} + return Reply(body=_image_response((f"{wire_url}/files/high.png",), _PROMPT)) + assert body == {"prompt": _PROMPT, "quality": "low"} + return Reply(body=_image_response((f"{wire_url}/files/low.png",), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model=f"fal_ai/{_GPT_IMAGE_MODEL}", api_base=wire.url, api_key="synthetic-fal-key" + ) + high_response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "quality": "high", "size": "1536x1024"}, + ) + assert high_response.status_code == 200, high_response.text + assert _image_urls(high_response) == (f"{wire.url}/files/high.png",) + high_cost, high_cost_path = _response_cost(high_response) + assert high_cost == _approx(_catalog_cost("fal_ai/high/1536-x-1024/openai/gpt-image-2.5/flare/text-to-image")) + + low_response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "quality": "low"}, + ) + assert low_response.status_code == 200, low_response.text + assert _image_urls(low_response) == (f"{wire.url}/files/low.png",) + low_cost, low_cost_path = _response_cost(low_response) + assert low_cost == _approx(_catalog_cost("fal_ai/low/1024-x-768/openai/gpt-image-2.5/flare/text-to-image")) + assert high_cost != low_cost + assert high_cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend") + assert low_cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend") + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/text-to-image"), + ("POST", "/openai/gpt-image-2.5/flare/text-to-image"), + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing") +def test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/fal-ai/flux/dev" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "num_images": 2, + "image_size": "square_hd", + } + return Reply( + body=_image_response( + (f"{wire_url}/files/flux-1.png", f"{wire_url}/files/flux-2.png"), + _PROMPT, + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model(model=f"fal_ai/{_FLUX_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "n": 2, "size": "1024x1024"}, + ) + assert response.status_code == 200, response.text + assert _image_urls(response) == ( + f"{wire.url}/files/flux-1.png", + f"{wire.url}/files/flux-2.png", + ) + cost, cost_path = _response_cost(response) + assert cost == _approx(2 * _catalog_cost("fal_ai/fal-ai/flux/dev")) + assert cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend") + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/fal-ai/flux/dev")] + + +@pytest.mark.covers("other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing") +def test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/edit" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "image_urls": ["data:image/png;base64," + base64.b64encode(_PNG_BYTES).decode()], + "quality": "low", + } + return Reply(body=_image_response((f"{wire_url}/files/edit.png",), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model(model=f"fal_ai/{_EDIT_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT, "quality": "low"}, + files={"image": ("red_circle.png", _PNG_BYTES, "image/png")}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + assert response.status_code == 200, response.text + assert _image_urls(response) == (f"{wire.url}/files/edit.png",) + cost, cost_path = _response_cost(response) + assert cost == _approx(_catalog_cost("fal_ai/low/1024-x-768/openai/gpt-image-2.5/flare/edit")) + assert cost_path in ("x-litellm-response-cost", "LiteLLM_SpendLogs.spend") + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/edit") + ]