diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index df6036ff8d7..e278a796146 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -19,6 +19,7 @@ class Request: target: str headers: Mapping[str, str] body: bytes + header_lines: tuple[tuple[str, str], ...] = () class AbortedBody(Exception): @@ -136,7 +137,7 @@ def wire_server( self.close_connection = True disconnected.put(self.path) return - request: Final = Request(self.command, self.path, headers, body) + request: Final = Request(self.command, self.path, headers, body, tuple(self.headers.items())) received.put(request) try: reply = respond(request) diff --git a/tests/integration/providers/test_api_base_trailing_slash_wire.py b/tests/integration/providers/test_api_base_trailing_slash_wire.py index e914199e825..01b68f52032 100644 --- a/tests/integration/providers/test_api_base_trailing_slash_wire.py +++ b/tests/integration/providers/test_api_base_trailing_slash_wire.py @@ -42,7 +42,7 @@ _MESSAGES_ROWS: Final = ( Row("ollama_chat/llama3.1", "/ol", "/api/chat"), ) _EMBEDDING_ROWS: Final = ( - Row("ollama/nomic-embed-text", "/ol", "/api/embed", keyed=False), + Row("ollama/nomic-embed-text", "/ol", "/api/embed"), Row("jina_ai/jina-embeddings-v3", "/jn/v1", "/embeddings"), Row("voyage/voyage-3", "/vy/v1", "/embeddings"), Row("voyage/voyage-multimodal-3", "/vy/v1", "/multimodalembeddings"), @@ -65,6 +65,7 @@ _FULL_ROUTE_EMBEDDING_ROWS: Final = ( Row("jina_ai/jina-embeddings-v3", "/jn/v1/embeddings", ""), Row("voyage/voyage-3", "/vy/v1/embeddings", ""), Row("perplexity/pplx-embed-v1-0.6b", "/px/v1/embeddings", ""), + Row("ollama/nomic-embed-text", "/ol/api/embed", ""), ) diff --git a/tests/integration/providers/test_ollama_embedding_wire.py b/tests/integration/providers/test_ollama_embedding_wire.py new file mode 100644 index 00000000000..edf587d84a4 --- /dev/null +++ b/tests/integration/providers/test_ollama_embedding_wire.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +import threading +import time +import uuid +from collections.abc import Mapping +from typing import Final + +import httpx +import pytest +from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value +from integration._support.timing import assert_cut_short +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +_MODEL: Final = "nomic-embed-text" +_DEPLOYMENT_KEY: Final = "synthetic-ollama-deployment-key" +_CALLER_TOKEN: Final = "Bearer synthetic-deployment-header-token" +_DELAY_SECONDS: Final = 5.0 + + +def _vector(index: int) -> list[float]: + return [0.5, float(index)] + + +def _embeddings(request: Request) -> Reply: + body: Final = JSON_OBJECT.validate_json(request.body) + count: Final = len(body["input"]) if isinstance(body["input"], list) else 1 + return Reply( + body=json.dumps({"embeddings": [_vector(index) for index in range(count)], "prompt_eval_count": 3}).encode() + ) + + +def _vectors(response: httpx.Response) -> list[JsonValue]: + data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + return [object_value(row)["embedding"] for row in data] + + +def _only(wire: Wire) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/api/embed")], [ + (request.method, request.target) for request in received + ] + return received[0] + + +def _values_of(request: Request, name: str) -> list[str]: + return [value for header, value in request.header_lines if header.lower() == name.lower()] + + +def _texts(count: int) -> list[str]: + return [f"ollama proxy embedding {uuid.uuid4().hex}" for _ in range(count)] + + +@pytest.mark.parametrize( + ("route", "batched"), (("/v1/embeddings", False), ("/embeddings", True)), ids=("v1-string", "alias-batch") +) +@pytest.mark.parametrize( + ("credentials", "authorization"), + ( + ({"api_key": _DEPLOYMENT_KEY}, f"Bearer {_DEPLOYMENT_KEY}"), + ( + {"api_key": _DEPLOYMENT_KEY, "extra_headers": {"authorization": _CALLER_TOKEN, "X-Embed-Tenant": "t-1"}}, + _CALLER_TOKEN, + ), + ), + ids=("api-key", "lowercase-header-overrides-key"), +) +def test_regression_proxy_ollama_embedding_sends_deployment_credentials( + gateway: Gateway, route: str, batched: bool, credentials: Mapping[str, JsonValue], authorization: str +) -> None: + texts: Final = _texts(2 if batched else 1) + with wire_server(_embeddings) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url, **credentials) + response: Final = gateway.request("POST", route, {"model": model, "input": texts if batched else texts[0]}) + request: Final = _only(wire) + assert response.status_code == 200, response.text + assert _vectors(response) == [_vector(i) for i in range(len(texts))] + assert _values_of(request, "authorization") == [authorization], request.header_lines + assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body + expected_tenant: Final = ["t-1"] if "extra_headers" in credentials else [] + assert _values_of(request, "x-embed-tenant") == expected_tenant, request.header_lines + + +@pytest.mark.parametrize("caller_timeout", ({}, {"timeout": 0}), ids=("omitted", "zero")) +def test_regression_proxy_ollama_deployment_timeout_cuts_a_held_read( + gateway: Gateway, caller_timeout: Mapping[str, JsonValue] +) -> None: + release: Final = threading.Event() + + def respond(request: Request) -> Reply: + release.wait(timeout=_DELAY_SECONDS) + return _embeddings(request) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url, timeout=0.5) + started: Final = time.monotonic() + try: + response: Final = gateway.request( + "POST", "/v1/embeddings", {"model": model, "input": _texts(1)[0], **caller_timeout} + ) + finally: + elapsed: Final = time.monotonic() - started + release.set() + _only(wire) + assert response.status_code == 408, response.text + assert_cut_short(elapsed, injected_delay=_DELAY_SECONDS) + + +def test_guard_proxy_keyless_ollama_embedding_sends_no_authorization(gateway: Gateway) -> None: + texts: Final = _texts(2) + with wire_server(_embeddings) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=f"{wire.url}/") + response: Final = gateway.request("POST", "/v1/embeddings", {"model": model, "input": texts}) + request: Final = _only(wire) + assert response.status_code == 200, response.text + assert _vectors(response) == [_vector(0), _vector(1)] + assert _values_of(request, "authorization") == [], request.header_lines + assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body + + +def test_guard_proxy_ollama_upstream_error_reaches_the_caller(gateway: Gateway) -> None: + marker: Final = f"scripted-bad-request-{uuid.uuid4().hex}" + + def respond(_: Request) -> Reply: + return Reply(status=400, body=json.dumps({"error": marker}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url) + response: Final = gateway.request("POST", "/v1/embeddings", {"model": model, "input": _texts(1)[0]}) + _only(wire) + assert response.status_code == 400, response.text + assert marker in response.text, response.text diff --git a/tests/integration/sdk/test_ollama_embedding_wire.py b/tests/integration/sdk/test_ollama_embedding_wire.py new file mode 100644 index 00000000000..21b803b8728 --- /dev/null +++ b/tests/integration/sdk/test_ollama_embedding_wire.py @@ -0,0 +1,218 @@ +import json +import threading +import time +import uuid +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from typing import Final, Literal + +import httpx +import pytest +from integration._support.client import JSON_OBJECT +from integration._support.timing import assert_cut_short +from integration._support.wire import Reply, Request, Wire, wire_server + +import litellm +from litellm.types.utils import EmbeddingResponse + +_MODEL: Final = "nomic-embed-text" +_EXPLICIT_KEY: Final = "synthetic-ollama-explicit-key" +_ENV_KEY: Final = "synthetic-ollama-env-key" +_GLOBAL_KEY: Final = "synthetic-global-key-never-sent" +_CALLER_TOKEN: Final = "Bearer synthetic-caller-token" +_DELAY_SECONDS: Final = 5.0 +_SHORT_DELAY_SECONDS: Final = 1.0 + +Mode = Literal["sync", "async"] +_MODES: Final[tuple[Mode, ...]] = ("sync", "async") + + +def _vector(index: int) -> list[float]: + return [0.25, float(index)] + + +def _embeddings(request: Request) -> Reply: + body: Final = JSON_OBJECT.validate_json(request.body) + count: Final = len(body["input"]) if isinstance(body["input"], list) else 1 + return Reply( + body=json.dumps({"embeddings": [_vector(index) for index in range(count)], "prompt_eval_count": 3}).encode() + ) + + +def _held(release: threading.Event, seconds: float) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + release.wait(timeout=seconds) + return _embeddings(request) + + return respond + + +@pytest.fixture(autouse=True) +def _no_ambient_ollama_settings(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OLLAMA_API_KEY", raising=False) + monkeypatch.delenv("OLLAMA_API_BASE", raising=False) + + +@contextmanager +def _peer(respond: Callable[[Request], Reply] = _embeddings) -> Iterator[Wire]: + with wire_server(respond) as wire: + yield wire + + +async def _call(mode: Mode, **kwargs: object) -> EmbeddingResponse: + if mode == "sync": + return litellm.embedding(model=f"ollama/{_MODEL}", **kwargs) + return await litellm.aembedding(model=f"ollama/{_MODEL}", **kwargs) + + +async def _seconds_until_timeout(mode: Mode, **kwargs: object) -> float: + started: Final = time.monotonic() + with pytest.raises(litellm.Timeout): + await _call(mode, **kwargs) + return time.monotonic() - started + + +def _only(wire: Wire) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/api/embed")], [ + (request.method, request.target) for request in received + ] + return received[0] + + +def _values_of(request: Request, name: str) -> list[str]: + return [value for header, value in request.header_lines if header.lower() == name.lower()] + + +def _text() -> str: + return f"ollama embedding {uuid.uuid4().hex}" + + +@pytest.mark.parametrize("mode", _MODES) +@pytest.mark.parametrize( + ("explicit", "env", "expected"), + ( + (_EXPLICIT_KEY, None, _EXPLICIT_KEY), + (None, _ENV_KEY, _ENV_KEY), + (_EXPLICIT_KEY, _ENV_KEY, _EXPLICIT_KEY), + ), + ids=("explicit-key", "env-key", "explicit-beats-env"), +) +async def test_regression_embedding_sends_the_resolved_ollama_key_as_one_bearer_header( + monkeypatch: pytest.MonkeyPatch, mode: Mode, explicit: str | None, env: str | None, expected: str +) -> None: + if env is not None: + monkeypatch.setenv("OLLAMA_API_KEY", env) + texts: Final = [_text(), _text()] + with _peer() as wire: + response: Final = await _call(mode, input=texts, api_base=wire.url, api_key=explicit) + request: Final = _only(wire) + assert [row["embedding"] for row in response.data] == [_vector(0), _vector(1)], response + assert _values_of(request, "authorization") == [f"Bearer {expected}"], request.header_lines + assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body + + +@pytest.mark.parametrize("mode", _MODES) +@pytest.mark.parametrize("auth_name", ("Authorization", "authorization"), ids=("canonical", "lowercase")) +@pytest.mark.parametrize("header_kwarg", ("extra_headers", "headers")) +async def test_regression_caller_headers_reach_ollama_and_override_the_key_once( + mode: Mode, auth_name: str, header_kwarg: str +) -> None: + caller: Final[dict[str, str]] = {auth_name: _CALLER_TOKEN, "X-Embed-Tenant": "tenant-a"} + snapshot: Final = dict(caller) + text: Final = _text() + with _peer() as wire: + response: Final = await _call( + mode, input=text, api_base=wire.url, api_key=_EXPLICIT_KEY, **{header_kwarg: caller} + ) + request: Final = _only(wire) + assert [row["embedding"] for row in response.data] == [_vector(0)], response + assert _values_of(request, "authorization") == [_CALLER_TOKEN], request.header_lines + assert _values_of(request, "x-embed-tenant") == ["tenant-a"], request.header_lines + assert caller == snapshot, "the caller's header map was changed by the call" + assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": [text]}, request.body + + +@pytest.mark.parametrize("mode", _MODES) +async def test_regression_extra_headers_win_over_headers_and_both_reach_ollama(mode: Mode) -> None: + extra: Final[dict[str, str]] = {"X-Embed-Tenant": "from-extra-headers"} + extra_snapshot: Final = dict(extra) + with _peer() as wire: + await _call( + mode, + input=_text(), + api_base=wire.url, + headers={"X-Embed-Tenant": "from-headers", "X-Embed-Trace": "trace-1"}, + extra_headers=extra, + ) + request: Final = _only(wire) + assert _values_of(request, "x-embed-tenant") == ["from-extra-headers"], request.header_lines + assert _values_of(request, "x-embed-trace") == ["trace-1"], request.header_lines + assert extra == extra_snapshot, "the caller's extra_headers map was changed by the call" + + +@pytest.mark.parametrize("mode", _MODES) +@pytest.mark.parametrize("timeout", (0.5, httpx.Timeout(0.5)), ids=("float", "httpx-timeout")) +async def test_regression_short_timeout_cuts_a_held_ollama_read(mode: Mode, timeout: float | httpx.Timeout) -> None: + release: Final = threading.Event() + with _peer(_held(release, _DELAY_SECONDS)) as wire: + try: + elapsed: Final = await _seconds_until_timeout(mode, input=_text(), api_base=wire.url, timeout=timeout) + finally: + release.set() + _only(wire) + assert_cut_short(elapsed, injected_delay=_DELAY_SECONDS) + + +@pytest.mark.parametrize("mode", _MODES) +@pytest.mark.parametrize("timeout_kwargs", ({}, {"timeout": None}), ids=("omitted", "none")) +async def test_guard_default_timeout_does_not_cut_a_slow_ollama_reply( + mode: Mode, timeout_kwargs: Mapping[str, None] +) -> None: + release: Final = threading.Event() + with _peer(_held(release, _SHORT_DELAY_SECONDS)) as wire: + response: Final = await _call(mode, input=_text(), api_base=wire.url, **timeout_kwargs) + _only(wire) + assert [row["embedding"] for row in response.data] == [_vector(0)], response + + +@pytest.mark.parametrize("mode", _MODES) +async def test_guard_no_ollama_key_sends_no_authorization_even_with_global_keys( + monkeypatch: pytest.MonkeyPatch, mode: Mode +) -> None: + monkeypatch.setattr(litellm, "api_key", _GLOBAL_KEY) + monkeypatch.setattr(litellm, "ollama_key", _GLOBAL_KEY, raising=False) + monkeypatch.setattr(litellm, "openai_key", _GLOBAL_KEY) + texts: Final = [_text(), _text()] + with _peer() as wire: + response: Final = await _call(mode, input=texts, api_base=f"{wire.url}/") + request: Final = _only(wire) + assert [row["embedding"] for row in response.data] == [_vector(0), _vector(1)], response + assert _values_of(request, "authorization") == [], request.header_lines + assert not any(_GLOBAL_KEY in value for _, value in request.header_lines), request.header_lines + assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body + + +@pytest.mark.parametrize("mode", _MODES) +async def test_guard_non_text_input_is_rejected_before_ollama_is_called(mode: Mode) -> None: + valid: Final = _text() + with _peer() as wire: + with pytest.raises(litellm.BadRequestError): + await _call(mode, input=[[1, 2, 3]], api_base=wire.url, api_key=_EXPLICIT_KEY) + await _call(mode, input=valid, api_base=wire.url) + received: Final = wire.drain() + assert [JSON_OBJECT.validate_json(request.body)["input"] for request in received] == [[valid]] + + +@pytest.mark.parametrize("mode", _MODES) +async def test_guard_ollama_error_reaches_the_caller_as_its_status(mode: Mode) -> None: + marker: Final = f"scripted-unauthorized-{uuid.uuid4().hex}" + + def respond(_: Request) -> Reply: + return Reply(status=401, body=json.dumps({"error": marker}).encode()) + + with _peer(respond) as wire: + with pytest.raises(litellm.AuthenticationError) as caught: + await _call(mode, input=_text(), api_base=wire.url) + _only(wire) + assert marker in str(caught.value), caught.value