mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
f6882246d4
* ci: run the unit_selection.sh shard files on every event instead of only fork pull requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rename fork-flag to unit-flag now that it applies on every event * test: move tests/test_litellm root and small trees into tests/unit Pure renames, no content changes. Follow-up commits in this PR fix references, merge the three files that already existed in tests/unit, keep live-provider tests in tests/test_litellm and wire CI. * test: carry tests/test_litellm conftest isolation into tests/unit Callback lists, routing fallbacks, cached HTTP clients, logger state, AWS, proxy-URL and keychain env, and session-end client cleanup now reset for unit tests too. The environment isolation owns its MonkeyPatch so a test's own monkeypatch is undone before the model-cost teardown runs. * test: merge, split and prune the moved root and small-tree tests Merge batches/test_batch_utils.py and the chat_completions and messages dispatch tests into the files that already existed in tests/unit. Keep the live Gemini interactions tests, the async image-fetch format test and the OpenAI embedding scorer test in tests/test_litellm since they need real network or keys. Put test_router.py under tests/unit/test_router so the existing package no longer shadows it. Delete eight tests the audit found superseded by stronger ones kept in this move. * ci: run the moved root and small-tree tests under their legacy flags Add the misc and responses-caching-types flags to unit_selection.sh and CircleCI, extend enterprise-routing and mcp-integration, and point the legacy GHA shards, Makefile, redis-compat workflow, merge smoke manifest and change classifier at the new paths. * test: make the new tests/unit directories packages tests/unit/test_package_layout.py requires every directory to carry an __init__.py, and without one the moved and retained test_litellm_responses_bridge.py modules collide on import. * test: scope the unit socket block to tests/unit in shared sessions The GHA shards collect the legacy test-path and the unit selection in one pytest session. The unit conftest's loopback-only block leaked into legacy modules that reach the network at import. The legacy conftest now lifts the restriction at collect and setup time, and the unit conftest re-applies it when collecting its own modules. * test: give the shard-script tests their own GITHUB_OUTPUT They only passed where the runner set it. The CircleCI unit job's env allowlist drops it, so the script's redirect failed there. * test: point the router and module-deletion checks at tests/unit router_code_coverage and code_qa_check_tests only searched tests/test_litellm, so the moved router tests no longer counted. The two silent-experiment tests the audit deleted were the only direct callers of those methods; they are replaced with tests that assert the forwarded shadow request and the recursion guard. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
122 lines
3.6 KiB
Python
122 lines
3.6 KiB
Python
"""
|
|
Tests for OpenAI gpt-image-1 cost calculator
|
|
|
|
This tests the fix for GitHub issue #13847:
|
|
https://github.com/BerriAI/litellm/issues/13847
|
|
|
|
gpt-image-1 uses token-based pricing:
|
|
- Text Input: $5.00/1M tokens
|
|
- Image Input: $10.00/1M tokens
|
|
- Image Output: $40.00/1M tokens
|
|
"""
|
|
|
|
import pytest
|
|
|
|
import litellm
|
|
from litellm.types.utils import (
|
|
ImageObject,
|
|
ImageResponse,
|
|
ImageUsage,
|
|
ImageUsageInputTokensDetails,
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _use_local_model_cost_map(monkeypatch):
|
|
original_model_cost = litellm.model_cost
|
|
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
|
litellm.model_cost = litellm.get_model_cost_map(url="")
|
|
litellm.get_model_info.cache_clear()
|
|
try:
|
|
yield
|
|
finally:
|
|
litellm.model_cost = original_model_cost
|
|
litellm.get_model_info.cache_clear()
|
|
|
|
|
|
class TestGPTImageCostCalculator:
|
|
"""Test the OpenAI gpt-image cost calculator"""
|
|
|
|
def test_gpt_image_1_cost_no_usage(self):
|
|
"""Test that cost returns 0 when no usage data is available"""
|
|
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
|
|
|
|
image_response = ImageResponse(
|
|
created=1234567890,
|
|
data=[ImageObject(url="http://example.com/image.jpg")],
|
|
)
|
|
|
|
cost = cost_calculator(
|
|
model="gpt-image-1",
|
|
image_response=image_response,
|
|
custom_llm_provider="openai",
|
|
)
|
|
|
|
assert cost == 0.0
|
|
|
|
@pytest.mark.parametrize(
|
|
"usage",
|
|
[
|
|
None,
|
|
ImageUsage(
|
|
input_tokens=0,
|
|
input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
|
|
output_tokens=0,
|
|
total_tokens=0,
|
|
),
|
|
],
|
|
)
|
|
def test_gpt_image_1_bills_deployment_output_cost_per_image_without_usage_tokens(
|
|
self, usage: ImageUsage | None
|
|
) -> None:
|
|
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
|
|
|
|
image_response = ImageResponse(
|
|
created=1234567890,
|
|
data=[ImageObject(url="http://example.com/one.jpg"), ImageObject(url="http://example.com/two.jpg")],
|
|
usage=usage,
|
|
)
|
|
|
|
cost = cost_calculator(
|
|
model="gpt-image-1",
|
|
image_response=image_response,
|
|
custom_llm_provider="openai",
|
|
model_info={"output_cost_per_image": 0.05},
|
|
)
|
|
|
|
assert cost == pytest.approx(0.10)
|
|
|
|
|
|
class TestGPTImageCostRouting:
|
|
"""Test that gpt-image models are properly routed to the token-based calculator"""
|
|
|
|
|
|
|
|
class TestGPTImage15OutputImageTokens:
|
|
"""
|
|
Test for GitHub issue #19508:
|
|
Image usage calculation does not include image tokens in gpt-image-1.5
|
|
|
|
gpt-image-1.5 returns output_tokens_details with separate image_tokens and text_tokens,
|
|
and these must be correctly included in cost calculation.
|
|
"""
|
|
|
|
|
|
class TestCompletionCostIntegration:
|
|
"""Test the full completion_cost integration for gpt-image-1"""
|
|
|
|
|
|
class TestGPTImage2OutputImageTokensNoBreakdown:
|
|
"""
|
|
Regression test: the OpenAI Images endpoints (/v1/images/generations and
|
|
/v1/images/edits) return usage with NO output token breakdown — litellm's
|
|
ImageUsage has no ``output_tokens_details`` field. Before the fix, the
|
|
generated-image OUTPUT tokens were priced at the text rate
|
|
(``output_cost_per_token`` = $10/1M for gpt-image-2) instead of the image rate
|
|
(``output_cost_per_image_token`` = $30/1M), a ~3x undercount on the dominant
|
|
cost component.
|
|
"""
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|