From a11fdf7728ceca2d2d30613999b2290e20cda284 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 22 Sep 2026 06:16:09 +0000 Subject: [PATCH] test(e2e): collect the responses callback cost test without importing litellm The Buildkite e2e image ships only tests/e2e, so the test module cannot import the CustomLogger helper that pulls in litellm. Split the capture row into callback_cost_record.py and import that from the test. Also pin that a Fal result poll with hidden response_cost=None stays unbilled through the new fallthrough. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/logging/callback_cost_capture.py | 12 +----- tests/e2e/logging/callback_cost_record.py | 18 +++++++++ ...test_custom_callback_responses_cost_e2e.py | 2 +- .../test_litellm_logging.py | 37 +++++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/logging/callback_cost_record.py diff --git a/tests/e2e/logging/callback_cost_capture.py b/tests/e2e/logging/callback_cost_capture.py index efc1a8edfbb..e835b96317f 100644 --- a/tests/e2e/logging/callback_cost_capture.py +++ b/tests/e2e/logging/callback_cost_capture.py @@ -16,17 +16,7 @@ from typing import Final from pydantic import BaseModel, ConfigDict from litellm.integrations.custom_logger import CustomLogger - -CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" - - -class CapturedCost(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - - request_id: str - call_type: str | None - hook: str - response_cost: float | None +from tests.e2e.logging.callback_cost_record import CAPTURE_FILE_ENV, CapturedCost class _StandardLoggingCostFields(BaseModel): diff --git a/tests/e2e/logging/callback_cost_record.py b/tests/e2e/logging/callback_cost_record.py new file mode 100644 index 00000000000..35394fe1108 --- /dev/null +++ b/tests/e2e/logging/callback_cost_record.py @@ -0,0 +1,18 @@ +"""Capture row shared by callback_cost_capture (proxy side) and the test; must stay importable without litellm.""" + +from __future__ import annotations + +from typing import Final + +from pydantic import BaseModel, ConfigDict + +CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" + + +class CapturedCost(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + request_id: str + call_type: str | None + hook: str + response_cost: float | None diff --git a/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py b/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py index 13024e7ed31..db40db9ff51 100644 --- a/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py +++ b/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py @@ -16,7 +16,7 @@ import time from pathlib import Path import pytest -from callback_cost_capture import CAPTURE_FILE_ENV, CapturedCost +from callback_cost_record import CAPTURE_FILE_ENV, CapturedCost from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from lifecycle import ResourceManager from logging_client import LoggingClient, first_ok diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 527a786fbc3..2d58f0038d8 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3896,6 +3896,43 @@ def test_process_hidden_params_explicit_zero_cost_overrides_earlier_cost(): assert slo.get("response_cost") == 0.0 +def test_process_hidden_params_none_cost_on_fal_poll_stays_unbilled(): + """A Fal result poll hands back image URLs with hidden response_cost=None; the fallthrough must not price them.""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, + ) + + logging_obj = LiteLLMLoggingObj( + model="fal-ai/flux-pro/v1.1", + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-fal-poll-unbilled", + function_id="test-fal-poll-unbilled", + ) + logging_obj.model_call_details["litellm_params"] = {"model": "fal-ai/flux-pro/v1.1"} + logging_obj.optional_params = {} + + poll = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"images": [{"url": "https://example.invalid/out.png"}]}, + request_body={"prompt": "a cat"}, + logging_obj=logging_obj, + url_route="https://queue.fal.run/fal-ai/flux-pro/v1.1/requests/req-1", + kwargs={}, + ) + assert poll["result"]._hidden_params["response_cost"] is None + + logging_obj._process_hidden_params_and_response_cost(poll["result"], datetime.now(), datetime.now()) + + assert not logging_obj.model_call_details.get("response_cost") + slo = logging_obj.model_call_details.get("standard_logging_object") or {} + assert not slo.get("response_cost") + + def test_function_setup_litellm_metadata_populates_metadata(): """ Test that function_setup() properly handles litellm_metadata (used by /v1/messages,