mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(integration): cover ollama embedding key, header and timeout forwarding
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
b9e3587721
commit
37c626718d
4 changed files with 356 additions and 2 deletions
|
|
@ -19,6 +19,7 @@ class Request:
|
|||
target: str
|
||||
headers: Mapping[str, str]
|
||||
body: bytes
|
||||
header_lines: tuple[tuple[str, str], ...] = ()
|
||||
|
||||
|
||||
class AbortedBody(Exception):
|
||||
|
|
@ -136,7 +137,7 @@ def wire_server(
|
|||
self.close_connection = True
|
||||
disconnected.put(self.path)
|
||||
return
|
||||
request: Final = Request(self.command, self.path, headers, body)
|
||||
request: Final = Request(self.command, self.path, headers, body, tuple(self.headers.items()))
|
||||
received.put(request)
|
||||
try:
|
||||
reply = respond(request)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ _MESSAGES_ROWS: Final = (
|
|||
Row("ollama_chat/llama3.1", "/ol", "/api/chat"),
|
||||
)
|
||||
_EMBEDDING_ROWS: Final = (
|
||||
Row("ollama/nomic-embed-text", "/ol", "/api/embed", keyed=False),
|
||||
Row("ollama/nomic-embed-text", "/ol", "/api/embed"),
|
||||
Row("jina_ai/jina-embeddings-v3", "/jn/v1", "/embeddings"),
|
||||
Row("voyage/voyage-3", "/vy/v1", "/embeddings"),
|
||||
Row("voyage/voyage-multimodal-3", "/vy/v1", "/multimodalembeddings"),
|
||||
|
|
@ -65,6 +65,7 @@ _FULL_ROUTE_EMBEDDING_ROWS: Final = (
|
|||
Row("jina_ai/jina-embeddings-v3", "/jn/v1/embeddings", ""),
|
||||
Row("voyage/voyage-3", "/vy/v1/embeddings", ""),
|
||||
Row("perplexity/pplx-embed-v1-0.6b", "/px/v1/embeddings", ""),
|
||||
Row("ollama/nomic-embed-text", "/ol/api/embed", ""),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
134
tests/integration/providers/test_ollama_embedding_wire.py
Normal file
134
tests/integration/providers/test_ollama_embedding_wire.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value
|
||||
from integration._support.timing import assert_cut_short
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
_MODEL: Final = "nomic-embed-text"
|
||||
_DEPLOYMENT_KEY: Final = "synthetic-ollama-deployment-key"
|
||||
_CALLER_TOKEN: Final = "Bearer synthetic-deployment-header-token"
|
||||
_DELAY_SECONDS: Final = 5.0
|
||||
|
||||
|
||||
def _vector(index: int) -> list[float]:
|
||||
return [0.5, float(index)]
|
||||
|
||||
|
||||
def _embeddings(request: Request) -> Reply:
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
count: Final = len(body["input"]) if isinstance(body["input"], list) else 1
|
||||
return Reply(
|
||||
body=json.dumps({"embeddings": [_vector(index) for index in range(count)], "prompt_eval_count": 3}).encode()
|
||||
)
|
||||
|
||||
|
||||
def _vectors(response: httpx.Response) -> list[JsonValue]:
|
||||
data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"])
|
||||
return [object_value(row)["embedding"] for row in data]
|
||||
|
||||
|
||||
def _only(wire: Wire) -> Request:
|
||||
received: Final = wire.drain()
|
||||
assert [(request.method, request.target) for request in received] == [("POST", "/api/embed")], [
|
||||
(request.method, request.target) for request in received
|
||||
]
|
||||
return received[0]
|
||||
|
||||
|
||||
def _values_of(request: Request, name: str) -> list[str]:
|
||||
return [value for header, value in request.header_lines if header.lower() == name.lower()]
|
||||
|
||||
|
||||
def _texts(count: int) -> list[str]:
|
||||
return [f"ollama proxy embedding {uuid.uuid4().hex}" for _ in range(count)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "batched"), (("/v1/embeddings", False), ("/embeddings", True)), ids=("v1-string", "alias-batch")
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("credentials", "authorization"),
|
||||
(
|
||||
({"api_key": _DEPLOYMENT_KEY}, f"Bearer {_DEPLOYMENT_KEY}"),
|
||||
(
|
||||
{"api_key": _DEPLOYMENT_KEY, "extra_headers": {"authorization": _CALLER_TOKEN, "X-Embed-Tenant": "t-1"}},
|
||||
_CALLER_TOKEN,
|
||||
),
|
||||
),
|
||||
ids=("api-key", "lowercase-header-overrides-key"),
|
||||
)
|
||||
def test_regression_proxy_ollama_embedding_sends_deployment_credentials(
|
||||
gateway: Gateway, route: str, batched: bool, credentials: Mapping[str, JsonValue], authorization: str
|
||||
) -> None:
|
||||
texts: Final = _texts(2 if batched else 1)
|
||||
with wire_server(_embeddings) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url, **credentials)
|
||||
response: Final = gateway.request("POST", route, {"model": model, "input": texts if batched else texts[0]})
|
||||
request: Final = _only(wire)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _vectors(response) == [_vector(i) for i in range(len(texts))]
|
||||
assert _values_of(request, "authorization") == [authorization], request.header_lines
|
||||
assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body
|
||||
expected_tenant: Final = ["t-1"] if "extra_headers" in credentials else []
|
||||
assert _values_of(request, "x-embed-tenant") == expected_tenant, request.header_lines
|
||||
|
||||
|
||||
@pytest.mark.parametrize("caller_timeout", ({}, {"timeout": 0}), ids=("omitted", "zero"))
|
||||
def test_regression_proxy_ollama_deployment_timeout_cuts_a_held_read(
|
||||
gateway: Gateway, caller_timeout: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
release: Final = threading.Event()
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
release.wait(timeout=_DELAY_SECONDS)
|
||||
return _embeddings(request)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url, timeout=0.5)
|
||||
started: Final = time.monotonic()
|
||||
try:
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/embeddings", {"model": model, "input": _texts(1)[0], **caller_timeout}
|
||||
)
|
||||
finally:
|
||||
elapsed: Final = time.monotonic() - started
|
||||
release.set()
|
||||
_only(wire)
|
||||
assert response.status_code == 408, response.text
|
||||
assert_cut_short(elapsed, injected_delay=_DELAY_SECONDS)
|
||||
|
||||
|
||||
def test_guard_proxy_keyless_ollama_embedding_sends_no_authorization(gateway: Gateway) -> None:
|
||||
texts: Final = _texts(2)
|
||||
with wire_server(_embeddings) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=f"{wire.url}/")
|
||||
response: Final = gateway.request("POST", "/v1/embeddings", {"model": model, "input": texts})
|
||||
request: Final = _only(wire)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _vectors(response) == [_vector(0), _vector(1)]
|
||||
assert _values_of(request, "authorization") == [], request.header_lines
|
||||
assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body
|
||||
|
||||
|
||||
def test_guard_proxy_ollama_upstream_error_reaches_the_caller(gateway: Gateway) -> None:
|
||||
marker: Final = f"scripted-bad-request-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(_: Request) -> Reply:
|
||||
return Reply(status=400, body=json.dumps({"error": marker}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"ollama/{_MODEL}", api_base=wire.url)
|
||||
response: Final = gateway.request("POST", "/v1/embeddings", {"model": model, "input": _texts(1)[0]})
|
||||
_only(wire)
|
||||
assert response.status_code == 400, response.text
|
||||
assert marker in response.text, response.text
|
||||
218
tests/integration/sdk/test_ollama_embedding_wire.py
Normal file
218
tests/integration/sdk/test_ollama_embedding_wire.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT
|
||||
from integration._support.timing import assert_cut_short
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
_MODEL: Final = "nomic-embed-text"
|
||||
_EXPLICIT_KEY: Final = "synthetic-ollama-explicit-key"
|
||||
_ENV_KEY: Final = "synthetic-ollama-env-key"
|
||||
_GLOBAL_KEY: Final = "synthetic-global-key-never-sent"
|
||||
_CALLER_TOKEN: Final = "Bearer synthetic-caller-token"
|
||||
_DELAY_SECONDS: Final = 5.0
|
||||
_SHORT_DELAY_SECONDS: Final = 1.0
|
||||
|
||||
Mode = Literal["sync", "async"]
|
||||
_MODES: Final[tuple[Mode, ...]] = ("sync", "async")
|
||||
|
||||
|
||||
def _vector(index: int) -> list[float]:
|
||||
return [0.25, float(index)]
|
||||
|
||||
|
||||
def _embeddings(request: Request) -> Reply:
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
count: Final = len(body["input"]) if isinstance(body["input"], list) else 1
|
||||
return Reply(
|
||||
body=json.dumps({"embeddings": [_vector(index) for index in range(count)], "prompt_eval_count": 3}).encode()
|
||||
)
|
||||
|
||||
|
||||
def _held(release: threading.Event, seconds: float) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
release.wait(timeout=seconds)
|
||||
return _embeddings(request)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_ambient_ollama_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OLLAMA_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OLLAMA_API_BASE", raising=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _peer(respond: Callable[[Request], Reply] = _embeddings) -> Iterator[Wire]:
|
||||
with wire_server(respond) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
async def _call(mode: Mode, **kwargs: object) -> EmbeddingResponse:
|
||||
if mode == "sync":
|
||||
return litellm.embedding(model=f"ollama/{_MODEL}", **kwargs)
|
||||
return await litellm.aembedding(model=f"ollama/{_MODEL}", **kwargs)
|
||||
|
||||
|
||||
async def _seconds_until_timeout(mode: Mode, **kwargs: object) -> float:
|
||||
started: Final = time.monotonic()
|
||||
with pytest.raises(litellm.Timeout):
|
||||
await _call(mode, **kwargs)
|
||||
return time.monotonic() - started
|
||||
|
||||
|
||||
def _only(wire: Wire) -> Request:
|
||||
received: Final = wire.drain()
|
||||
assert [(request.method, request.target) for request in received] == [("POST", "/api/embed")], [
|
||||
(request.method, request.target) for request in received
|
||||
]
|
||||
return received[0]
|
||||
|
||||
|
||||
def _values_of(request: Request, name: str) -> list[str]:
|
||||
return [value for header, value in request.header_lines if header.lower() == name.lower()]
|
||||
|
||||
|
||||
def _text() -> str:
|
||||
return f"ollama embedding {uuid.uuid4().hex}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
@pytest.mark.parametrize(
|
||||
("explicit", "env", "expected"),
|
||||
(
|
||||
(_EXPLICIT_KEY, None, _EXPLICIT_KEY),
|
||||
(None, _ENV_KEY, _ENV_KEY),
|
||||
(_EXPLICIT_KEY, _ENV_KEY, _EXPLICIT_KEY),
|
||||
),
|
||||
ids=("explicit-key", "env-key", "explicit-beats-env"),
|
||||
)
|
||||
async def test_regression_embedding_sends_the_resolved_ollama_key_as_one_bearer_header(
|
||||
monkeypatch: pytest.MonkeyPatch, mode: Mode, explicit: str | None, env: str | None, expected: str
|
||||
) -> None:
|
||||
if env is not None:
|
||||
monkeypatch.setenv("OLLAMA_API_KEY", env)
|
||||
texts: Final = [_text(), _text()]
|
||||
with _peer() as wire:
|
||||
response: Final = await _call(mode, input=texts, api_base=wire.url, api_key=explicit)
|
||||
request: Final = _only(wire)
|
||||
assert [row["embedding"] for row in response.data] == [_vector(0), _vector(1)], response
|
||||
assert _values_of(request, "authorization") == [f"Bearer {expected}"], request.header_lines
|
||||
assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
@pytest.mark.parametrize("auth_name", ("Authorization", "authorization"), ids=("canonical", "lowercase"))
|
||||
@pytest.mark.parametrize("header_kwarg", ("extra_headers", "headers"))
|
||||
async def test_regression_caller_headers_reach_ollama_and_override_the_key_once(
|
||||
mode: Mode, auth_name: str, header_kwarg: str
|
||||
) -> None:
|
||||
caller: Final[dict[str, str]] = {auth_name: _CALLER_TOKEN, "X-Embed-Tenant": "tenant-a"}
|
||||
snapshot: Final = dict(caller)
|
||||
text: Final = _text()
|
||||
with _peer() as wire:
|
||||
response: Final = await _call(
|
||||
mode, input=text, api_base=wire.url, api_key=_EXPLICIT_KEY, **{header_kwarg: caller}
|
||||
)
|
||||
request: Final = _only(wire)
|
||||
assert [row["embedding"] for row in response.data] == [_vector(0)], response
|
||||
assert _values_of(request, "authorization") == [_CALLER_TOKEN], request.header_lines
|
||||
assert _values_of(request, "x-embed-tenant") == ["tenant-a"], request.header_lines
|
||||
assert caller == snapshot, "the caller's header map was changed by the call"
|
||||
assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": [text]}, request.body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
async def test_regression_extra_headers_win_over_headers_and_both_reach_ollama(mode: Mode) -> None:
|
||||
extra: Final[dict[str, str]] = {"X-Embed-Tenant": "from-extra-headers"}
|
||||
extra_snapshot: Final = dict(extra)
|
||||
with _peer() as wire:
|
||||
await _call(
|
||||
mode,
|
||||
input=_text(),
|
||||
api_base=wire.url,
|
||||
headers={"X-Embed-Tenant": "from-headers", "X-Embed-Trace": "trace-1"},
|
||||
extra_headers=extra,
|
||||
)
|
||||
request: Final = _only(wire)
|
||||
assert _values_of(request, "x-embed-tenant") == ["from-extra-headers"], request.header_lines
|
||||
assert _values_of(request, "x-embed-trace") == ["trace-1"], request.header_lines
|
||||
assert extra == extra_snapshot, "the caller's extra_headers map was changed by the call"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
@pytest.mark.parametrize("timeout", (0.5, httpx.Timeout(0.5)), ids=("float", "httpx-timeout"))
|
||||
async def test_regression_short_timeout_cuts_a_held_ollama_read(mode: Mode, timeout: float | httpx.Timeout) -> None:
|
||||
release: Final = threading.Event()
|
||||
with _peer(_held(release, _DELAY_SECONDS)) as wire:
|
||||
try:
|
||||
elapsed: Final = await _seconds_until_timeout(mode, input=_text(), api_base=wire.url, timeout=timeout)
|
||||
finally:
|
||||
release.set()
|
||||
_only(wire)
|
||||
assert_cut_short(elapsed, injected_delay=_DELAY_SECONDS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
@pytest.mark.parametrize("timeout_kwargs", ({}, {"timeout": None}), ids=("omitted", "none"))
|
||||
async def test_guard_default_timeout_does_not_cut_a_slow_ollama_reply(
|
||||
mode: Mode, timeout_kwargs: Mapping[str, None]
|
||||
) -> None:
|
||||
release: Final = threading.Event()
|
||||
with _peer(_held(release, _SHORT_DELAY_SECONDS)) as wire:
|
||||
response: Final = await _call(mode, input=_text(), api_base=wire.url, **timeout_kwargs)
|
||||
_only(wire)
|
||||
assert [row["embedding"] for row in response.data] == [_vector(0)], response
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
async def test_guard_no_ollama_key_sends_no_authorization_even_with_global_keys(
|
||||
monkeypatch: pytest.MonkeyPatch, mode: Mode
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "api_key", _GLOBAL_KEY)
|
||||
monkeypatch.setattr(litellm, "ollama_key", _GLOBAL_KEY, raising=False)
|
||||
monkeypatch.setattr(litellm, "openai_key", _GLOBAL_KEY)
|
||||
texts: Final = [_text(), _text()]
|
||||
with _peer() as wire:
|
||||
response: Final = await _call(mode, input=texts, api_base=f"{wire.url}/")
|
||||
request: Final = _only(wire)
|
||||
assert [row["embedding"] for row in response.data] == [_vector(0), _vector(1)], response
|
||||
assert _values_of(request, "authorization") == [], request.header_lines
|
||||
assert not any(_GLOBAL_KEY in value for _, value in request.header_lines), request.header_lines
|
||||
assert JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": texts}, request.body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
async def test_guard_non_text_input_is_rejected_before_ollama_is_called(mode: Mode) -> None:
|
||||
valid: Final = _text()
|
||||
with _peer() as wire:
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await _call(mode, input=[[1, 2, 3]], api_base=wire.url, api_key=_EXPLICIT_KEY)
|
||||
await _call(mode, input=valid, api_base=wire.url)
|
||||
received: Final = wire.drain()
|
||||
assert [JSON_OBJECT.validate_json(request.body)["input"] for request in received] == [[valid]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", _MODES)
|
||||
async def test_guard_ollama_error_reaches_the_caller_as_its_status(mode: Mode) -> None:
|
||||
marker: Final = f"scripted-unauthorized-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(_: Request) -> Reply:
|
||||
return Reply(status=401, body=json.dumps({"error": marker}).encode())
|
||||
|
||||
with _peer(respond) as wire:
|
||||
with pytest.raises(litellm.AuthenticationError) as caught:
|
||||
await _call(mode, input=_text(), api_base=wire.url)
|
||||
_only(wire)
|
||||
assert marker in str(caught.value), caught.value
|
||||
Loading…
Add table
Reference in a new issue