diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 65eaf14471d..4f25e6c1096 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -1021,6 +1021,11 @@ router_settings: | UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication | USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption | USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments. +| VANTAGE_API_KEY | API key for Vantage cost-import integration +| VANTAGE_BASE_URL | Base URL for Vantage API. Default is `https://api.vantage.sh` +| VANTAGE_EXPORT_FREQUENCY | Export frequency for Vantage — `hourly` (default), `daily`, or `interval` +| VANTAGE_EXPORT_INTERVAL_SECONDS | Interval in seconds when VANTAGE_EXPORT_FREQUENCY is `interval` +| VANTAGE_INTEGRATION_TOKEN | Vantage integration token for the cost-import endpoint | WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration | WANDB_HOST | Host URL for Weights & Biases (W&B) service | WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration diff --git a/litellm/integrations/focus/destinations/vantage_destination.py b/litellm/integrations/focus/destinations/vantage_destination.py index 9e6028900f9..e5c04365efb 100644 --- a/litellm/integrations/focus/destinations/vantage_destination.py +++ b/litellm/integrations/focus/destinations/vantage_destination.py @@ -9,6 +9,11 @@ from typing import Any, Optional import httpx from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + httpxSpecialProvider, +) from .base import FocusDestination, FocusTimeWindow @@ -131,45 +136,37 @@ class FocusVantageDestination(FocusDestination): # rejection (e.g. InvoiceIssuerName, ProviderName, PublisherName). content = _strip_unsupported_columns(content) - # Reuse a single HTTP client for the entire deliver() call - async with httpx.AsyncClient(timeout=60.0) as client: - # Check both size and row-count limits before single-shot upload - lines = content.split(b"\n") - data_line_count = sum(1 for line in lines[1:] if line.strip()) - within_limits = ( - len(content) <= VANTAGE_MAX_BYTES_PER_UPLOAD - and data_line_count <= VANTAGE_MAX_ROWS_PER_UPLOAD - ) - if within_limits: - await self._upload_csv(client, content, filename) - return + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback, + ) - # Otherwise split into batches respecting both limits - await self._upload_batched(client, content, filename) + # Check both size and row-count limits before single-shot upload + lines = content.split(b"\n") + data_line_count = sum(1 for line in lines[1:] if line.strip()) + within_limits = ( + len(content) <= VANTAGE_MAX_BYTES_PER_UPLOAD + and data_line_count <= VANTAGE_MAX_ROWS_PER_UPLOAD + ) + if within_limits: + await self._upload_csv(client, content, filename) + return + + # Otherwise split into batches respecting both limits + await self._upload_batched(client, content, filename) async def _upload_csv( - self, client: httpx.AsyncClient, csv_bytes: bytes, filename: str + self, client: AsyncHTTPHandler, csv_bytes: bytes, filename: str ) -> None: url = f"{self.base_url}/v2/integrations/" f"{self.integration_token}/costs.csv" headers = { "Authorization": f"Bearer {self.api_key}", } - response = await client.post( + await client.post( url, headers=headers, files={"csv": (filename, csv_bytes, "text/csv")}, ) - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - verbose_logger.error( - "Vantage destination: upload failed for %s — %s — response body: %s", - filename, - e, - response.text, - ) - raise verbose_logger.debug( "Vantage destination: uploaded %d bytes (%s)", @@ -178,7 +175,7 @@ class FocusVantageDestination(FocusDestination): ) async def _upload_batched( - self, client: httpx.AsyncClient, csv_bytes: bytes, filename: str + self, client: AsyncHTTPHandler, csv_bytes: bytes, filename: str ) -> None: """Split the CSV into batches and upload each. @@ -217,7 +214,7 @@ class FocusVantageDestination(FocusDestination): async def _upload_size_limited( self, - client: httpx.AsyncClient, + client: AsyncHTTPHandler, header: bytes, data_lines: list[bytes], filename: str, diff --git a/litellm/main.py b/litellm/main.py index 3b1f5dc96f5..a9a1c38e3ff 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1613,7 +1613,7 @@ def completion( # type: ignore # noqa: PLR0915 ) ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map - model_info, model = responses_api_bridge_check( + responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py index 417a7335a8a..b3be5729e57 100644 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ b/tests/local_testing/test_anthropic_prompt_caching.py @@ -151,6 +151,7 @@ async def test_litellm_anthropic_prompt_caching_tools(): }, "required": ["location"], }, + "type": "custom", } ], "max_tokens": 64000, diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py index 8e8033d885f..4b72eb7a6c7 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py +++ b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py @@ -110,8 +110,8 @@ async def test_claude_agent_sdk_streaming(litellm_proxy_config, model_name, mode # Note: Very short responses might come in 1 chunk, so we just verify we got content assert len(received_chunks) > 0, f"No chunks received from {model_name}" - # Verify response contains expected content (case insensitive) - assert "hello" in full_response.lower(), f"Response doesn't contain expected greeting: {full_response}" + # Verify response is non-empty (don't assert on specific LLM content — it's non-deterministic) + assert len(full_response.strip()) > 0, f"Empty response received from {model_name}" print(f"✅ Test passed for {model_name}") diff --git a/tests/test_litellm/integrations/focus/test_vantage_destination.py b/tests/test_litellm/integrations/focus/test_vantage_destination.py index c3fa9bda63d..10f72399193 100644 --- a/tests/test_litellm/integrations/focus/test_vantage_destination.py +++ b/tests/test_litellm/integrations/focus/test_vantage_destination.py @@ -4,7 +4,7 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone from typing import Any, Dict, List -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -15,6 +15,8 @@ from litellm.integrations.focus.destinations.vantage_destination import ( VANTAGE_MAX_ROWS_PER_UPLOAD, ) +MOCK_TARGET = "litellm.integrations.focus.destinations.vantage_destination.get_async_httpx_client" + def _window(freq: str = "hourly", hour: int = 5) -> FocusTimeWindow: start = datetime(2024, 1, 2, hour, tzinfo=timezone.utc) @@ -72,17 +74,14 @@ async def test_should_skip_empty_content(): @pytest.mark.asyncio async def test_should_upload_csv_to_correct_url(): dest = FocusVantageDestination(prefix="exports", config=_config()) - captured: Dict[str, Any] = {} mock_response = AsyncMock() mock_response.raise_for_status = lambda: None - mock_client = AsyncMock() + mock_client = MagicMock() mock_client.post = AsyncMock(return_value=mock_response) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) - with patch("litellm.integrations.focus.destinations.vantage_destination.httpx.AsyncClient", return_value=mock_client): + with patch(MOCK_TARGET, return_value=mock_client): await dest.deliver( content=b"header\nrow1\n", time_window=_window(), @@ -100,8 +99,9 @@ async def test_should_upload_csv_to_correct_url(): async def test_should_batch_large_content(): dest = FocusVantageDestination(prefix="exports", config=_config()) - # Create content larger than 2 MB - header = b"col1,col2,col3" + # Create content larger than 2 MB — use supported column names so + # _strip_unsupported_columns does not remove them. + header = b"ChargeCategory,ChargePeriodStart,BilledCost" row = b"a" * 100 + b"," + b"b" * 100 + b"," + b"c" * 100 num_rows = (VANTAGE_MAX_BYTES_PER_UPLOAD // len(row)) + 100 large_content = header + b"\n" + b"\n".join([row] * num_rows) + b"\n" @@ -113,19 +113,17 @@ async def test_should_batch_large_content(): mock_response = AsyncMock() mock_response.raise_for_status = lambda: None - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client = MagicMock() async def capture_post(url, **kwargs): files = kwargs.get("files", {}) - if "file" in files: - upload_calls.append(files["file"][1]) + if "csv" in files: + upload_calls.append(files["csv"][1]) return mock_response mock_client.post = capture_post - with patch("litellm.integrations.focus.destinations.vantage_destination.httpx.AsyncClient", return_value=mock_client): + with patch(MOCK_TARGET, return_value=mock_client): await dest.deliver( content=large_content, time_window=_window(), @@ -144,7 +142,7 @@ async def test_should_batch_by_row_count(): """Verify batching triggers when row count exceeds 10K even if under 2 MB.""" dest = FocusVantageDestination(prefix="exports", config=_config()) - header = b"col1" + header = b"ChargeCategory" # Short rows so total size stays well under 2 MB row = b"x" num_rows = VANTAGE_MAX_ROWS_PER_UPLOAD + 500 @@ -159,19 +157,17 @@ async def test_should_batch_by_row_count(): mock_response = AsyncMock() mock_response.raise_for_status = lambda: None - mock_client = AsyncMock() - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client = MagicMock() async def capture_post(url, **kwargs): files = kwargs.get("files", {}) - if "file" in files: - upload_calls.append(files["file"][1]) + if "csv" in files: + upload_calls.append(files["csv"][1]) return mock_response mock_client.post = capture_post - with patch("litellm.integrations.focus.destinations.vantage_destination.httpx.AsyncClient", return_value=mock_client): + with patch(MOCK_TARGET, return_value=mock_client): await dest.deliver( content=content, time_window=_window(),