From d3a0b0d45b814494e64edf82cd13b0683cd4aba8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:51:45 -0700 Subject: [PATCH] fix(voyage): let caller params win for contextual auto-chunking and drop duplicate cost map entries A flat list[str] sent to voyage-context-4 is treated as independent inputs and forwarded flat with enable_auto_chunking=True, chunk_size=32000, and input_type=document unless the caller already set input_type=query. Caller-supplied params now override the defaults instead of being clobbered. The voyage-4 family and voyage-context-4 cost map entries already exist on litellm_internal_staging, and voyage-4-nano is not served by the Voyage API, so those additions and their pricing test are dropped. --- .../embedding/transformation_contextual.py | 69 ++------- ...odel_prices_and_context_window_backup.json | 40 ------ model_prices_and_context_window.json | 40 ------ tests/llm_translation/test_voyage_ai.py | 131 ------------------ .../test_voyage_contextual_embedding.py | 51 ++++++- 5 files changed, 59 insertions(+), 272 deletions(-) diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index a9a4a01f8e3..870de8756bb 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -3,6 +3,7 @@ This module is used to transform the request and response for the Voyage context This would be used for all the contextualized embeddings models in Voyage. """ +from collections.abc import Mapping from typing import Final import httpx @@ -100,7 +101,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): "Authorization": f"Bearer {api_key}", } - AUTO_CHUNK_SIZE = 32000 + AUTO_CHUNK_SIZE: Final = 32000 def transform_embedding_request( self, @@ -109,71 +110,27 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, headers: dict, ) -> dict: - inputs, extra_params = self._transform_contextual_inputs(input, optional_params) return { - "inputs": inputs, + "inputs": [input] if isinstance(input, str) else input, "model": model, + **self._auto_chunk_params(input, optional_params), **optional_params, - **extra_params, } @classmethod - def _transform_contextual_inputs( + def _auto_chunk_params( cls, - input: AllEmbeddingInputValues | list[list[str]], # mutable-ok: union with AllEmbeddingInputValues - optional_params: dict, # mutable-ok: matches public API - ) -> tuple[list[str] | list[list[str]], dict]: # mutable-ok: returned to caller who owns it - """ - Normalize ``input`` for Voyage's contextualized embeddings API and - return ``(inputs, extra_params)`` where ``extra_params`` carries any - request fields (e.g. auto-chunking) needed for the chosen shape. - - The API contract (verified against the live endpoint) is: - - - A flat ``list[str]`` is only accepted with ``input_type="query"`` or - with ``enable_auto_chunking=True`` (which itself requires - ``input_type="document"``). - - A ``list[list[str]]`` (each inner list = one document's chunks) is - always accepted. - - So we prefer to send a flat ``list[str]`` and let the API auto-chunk, - instead of pre-wrapping into ``list[list[str]]``: - - - ``str`` -> ``[str]`` + ``enable_auto_chunking`` (input_type=document) - - flat ``list[str]`` + ``input_type="query"`` -> kept flat, as-is - - flat ``list[str]`` otherwise -> kept flat + ``enable_auto_chunking`` - (input_type=document) - - ``list[list[str]]`` -> passed through unchanged - - Reference: https://docs.voyageai.com/reference/contextualized-embeddings-api - """ - if isinstance(input, str): - if optional_params.get("input_type") == "query": - return [input], {} # mutable-ok: fresh list returned to caller - return [input], cls._auto_chunk_params(optional_params) # mutable-ok: fresh list returned to caller - - if all(isinstance(i, str) for i in input): - if optional_params.get("input_type") == "query": - return input, {} # pyright: ignore[reportReturnType] # narrowed to list[str] by all(isinstance) check - return input, cls._auto_chunk_params(optional_params) # pyright: ignore[reportReturnType] # narrowed to list[str] - - return input, {} # pyright: ignore[reportReturnType] # list[list[str]] branch - - @classmethod - def _auto_chunk_params(cls, optional_params: dict) -> dict: # mutable-ok: matches public API - """ - Params required to send a flat ``list[str]`` to the contextualized API. - - ``enable_auto_chunking=True`` requires ``input_type="document"``, so set - it unless the caller already provided an ``input_type``. - """ - params: dict[str, object] = { # mutable-ok: building return value + input: AllEmbeddingInputValues | list[list[str]], + optional_params: Mapping[str, object], + ) -> Mapping[str, object]: + is_flat: Final = isinstance(input, str) or all(isinstance(item, str) for item in input) + if not is_flat or optional_params.get("input_type") == "query": + return {} + return { "enable_auto_chunking": True, "chunk_size": cls.AUTO_CHUNK_SIZE, + "input_type": "document", } - if not optional_params.get("input_type"): - params["input_type"] = "document" - return params def transform_embedding_response( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fb1cef7ff58..12c3c5e2be9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -39676,38 +39676,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "voyage/voyage-4": { - "input_cost_per_token": 6e-08, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-large": { - "input_cost_per_token": 1.2e-07, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-lite": { - "input_cost_per_token": 2e-08, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-nano": { - "input_cost_per_token": 0.0, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -39732,14 +39700,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "voyage/voyage-context-4": { - "input_cost_per_token": 1.2e-07, - "litellm_provider": "voyage", - "max_input_tokens": 120000, - "max_tokens": 120000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fb1cef7ff58..12c3c5e2be9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -39676,38 +39676,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "voyage/voyage-4": { - "input_cost_per_token": 6e-08, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-large": { - "input_cost_per_token": 1.2e-07, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-lite": { - "input_cost_per_token": 2e-08, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, - "voyage/voyage-4-nano": { - "input_cost_per_token": 0.0, - "litellm_provider": "voyage", - "max_input_tokens": 32000, - "max_tokens": 32000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "voyage/voyage-code-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", @@ -39732,14 +39700,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "voyage/voyage-context-4": { - "input_cost_per_token": 1.2e-07, - "litellm_provider": "voyage", - "max_input_tokens": 120000, - "max_tokens": 120000, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "voyage/voyage-finance-2": { "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index e6c73240a0e..438bc4f3507 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -72,26 +72,6 @@ class TestVoyageAI(BaseLLMEmbeddingTest): assert response.usage.total_tokens > 0 -@pytest.mark.parametrize( - "model, expected_input_cost", - [ - ("voyage/voyage-4", 6e-08), - ("voyage/voyage-4-large", 1.2e-07), - ("voyage/voyage-4-lite", 2e-08), - ("voyage/voyage-4-nano", 0.0), - ("voyage/voyage-context-4", 1.2e-07), - ], -) -def test_voyage_4_family_pricing_registered(model, expected_input_cost): - """The voyage-4 family and voyage-context-4 must be in the cost map.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - info = litellm.get_model_info(model=model) - assert info["litellm_provider"] == "voyage" - assert info["mode"] == "embedding" - assert info["input_cost_per_token"] == expected_input_cost - - def test_voyage_ai_embedding_extra_params(): """Test Voyage AI embedding with extra parameters""" try: @@ -220,117 +200,6 @@ class TestVoyageContextualEmbeddings: assert transformed["model"] == "voyage-context-3" assert transformed["encoding_format"] == "float" - def test_contextual_flat_list_str_is_auto_chunked(self): - """Flat list[str] without input_type stays flat and is auto-chunked. - - The API accepts a flat list[str] only with input_type="query" or with - enable_auto_chunking=True (which requires input_type="document"), so we - keep the list flat and let the API auto-chunk it. - """ - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - transformed = config.transform_embedding_request( - "voyage-context-4", ["Hello", "world"], {}, {} - ) - - assert transformed["inputs"] == ["Hello", "world"] - assert transformed["model"] == "voyage-context-4" - assert transformed["enable_auto_chunking"] is True - assert transformed["chunk_size"] == 32000 - assert transformed["input_type"] == "document" - - def test_contextual_flat_list_str_query_stays_flat(self): - """Flat list[str] with input_type='query' is sent as-is (list[str]).""" - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - transformed = config.transform_embedding_request( - "voyage-context-4", - ["Hello", "world"], - {"input_type": "query"}, - {}, - ) - - assert transformed["inputs"] == ["Hello", "world"] - assert transformed["input_type"] == "query" - # A query list is accepted as-is, no auto-chunking needed. - assert "enable_auto_chunking" not in transformed - - def test_contextual_flat_list_str_document_input_type_preserved(self): - """Explicit input_type='document' is preserved while auto-chunking.""" - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - transformed = config.transform_embedding_request( - "voyage-context-4", - ["Hello", "world"], - {"input_type": "document"}, - {}, - ) - - assert transformed["inputs"] == ["Hello", "world"] - assert transformed["input_type"] == "document" - assert transformed["enable_auto_chunking"] is True - assert transformed["chunk_size"] == 32000 - - def test_contextual_single_string_is_auto_chunked(self): - """A single string becomes a one-element flat list and is auto-chunked.""" - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - transformed = config.transform_embedding_request( - "voyage-context-4", "Hello", {}, {} - ) - - assert transformed["inputs"] == ["Hello"] - assert transformed["enable_auto_chunking"] is True - assert transformed["chunk_size"] == 32000 - assert transformed["input_type"] == "document" - - def test_contextual_single_string_query_no_auto_chunk(self): - """A single string with input_type='query' is not auto-chunked.""" - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - transformed = config.transform_embedding_request( - "voyage-context-4", "Hello", {"input_type": "query"}, {} - ) - - assert transformed["inputs"] == ["Hello"] - assert transformed["input_type"] == "query" - assert "enable_auto_chunking" not in transformed - - def test_contextual_nested_input_passthrough(self): - """Already-nested list[list[str]] input is passed through unchanged.""" - from litellm.llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, - ) - - config = VoyageContextualEmbeddingConfig() - - nested = [["Hello", "world"], ["Test"]] - transformed = config.transform_embedding_request( - "voyage-context-4", nested, {}, {} - ) - - assert transformed["inputs"] == nested - def test_contextual_embedding_response_transformation(self): """Test response transformation for contextual embeddings""" from litellm.llms.voyage.embedding.transformation_contextual import ( diff --git a/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py index 865608e286f..d3e912ce6af 100644 --- a/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py +++ b/tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py @@ -74,15 +74,11 @@ class TestVoyageContextualEmbeddings: assert headers == {"Authorization": "Bearer test-key"} def test_validate_environment_secret_fallback(self, monkeypatch): - import litellm.llms.voyage.embedding.transformation_contextual as module from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig, ) - def fake_get_secret(name): - return "secret-key" if name == "VOYAGE_API_KEY" else None - - monkeypatch.setattr(module, "get_secret_str", fake_get_secret) + monkeypatch.setenv("VOYAGE_API_KEY", "secret-key") config = VoyageContextualEmbeddingConfig() headers = config.validate_environment( {}, "voyage-context-4", [], {}, {}, api_key=None @@ -142,6 +138,51 @@ class TestVoyageContextualEmbeddings: assert transformed["input_type"] == "document" assert transformed["enable_auto_chunking"] is True + def test_flat_list_str_caller_chunk_params_win(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", + ["Hello", "world"], + {"input_type": "document", "chunk_size": 512, "chunk_overlap": 32}, + {}, + ) + assert transformed["enable_auto_chunking"] is True + assert transformed["chunk_size"] == 512 + assert transformed["chunk_overlap"] == 32 + assert transformed["input_type"] == "document" + + def test_flat_list_str_caller_can_disable_auto_chunking(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", ["Hello"], {"enable_auto_chunking": False}, {} + ) + assert transformed["enable_auto_chunking"] is False + assert transformed["input_type"] == "document" + + def test_nested_list_keeps_caller_params(self): + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + + config = VoyageContextualEmbeddingConfig() + transformed = config.transform_embedding_request( + "voyage-context-4", [["Hello", "world"]], {"input_type": "document", "output_dimension": 512}, {} + ) + assert transformed == { + "inputs": [["Hello", "world"]], + "model": "voyage-context-4", + "input_type": "document", + "output_dimension": 512, + } + def test_single_string_auto_chunked(self): from litellm.llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig,