mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-18 04:44:37 +00:00
fix(proxy)!: delete callbacks configured under callbacks or failure_callback (#42740)
* fix(proxy): delete callbacks configured under callbacks or failure_callback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): assert on configured callback rows and use a callback absent from CI configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): follow logging suite conventions for callback delete tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): flatten remaining callbacks with a helper generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): type callback delete helpers to satisfy basedpyright budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): save callback config without mutating the loaded dict Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): delete callbacks from every litellm_settings list GET shows POST /config/callback/delete only searched success_callback while GET /get/config/callbacks also lists failure_callback and callbacks, so the Admin UI Delete returned 404 for those rows Remove the name from every list that holds it, matching entries case-insensitively, and skip lists the config file owns when the database owns another copy so the save is not refused Replace the e2e test with an integration test: the shared e2e stack config owns litellm_settings.callbacks, so it cannot exercise that list * test(proxy): cover config-owned success_callback on callback delete and wait on runtime state Add the mirror mixed-ownership case where the config file sets success_callback and the database owns failure_callback, which main refused with a 400, plus a string-valued list case Poll GET listings and /active/callbacks instead of reading them once, so a request served by a worker that has not synced yet cannot fail the test, and reuse the existing callback list key map in the route * test(proxy): annotate callback delete test helper locals with Final * fix(proxy): save callback delete through the config it read so concurrent settings edits are kept --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
977a581213
commit
e45f2737d3
3 changed files with 628 additions and 12 deletions
|
|
@ -19698,6 +19698,26 @@ async def delete_config_general_settings(
|
|||
return response
|
||||
|
||||
|
||||
def _configured_callback_names(value: object) -> tuple[JsonValue, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, list):
|
||||
return tuple(cast(Iterable[JsonValue], value)) # cast-ok: config.yaml values are untyped
|
||||
return ()
|
||||
|
||||
|
||||
def _is_callback_name(entry: object, callback_name: str) -> bool:
|
||||
return isinstance(entry, str) and entry.lower() == callback_name
|
||||
|
||||
|
||||
def _callback_lists_to_rewrite(
|
||||
matching_keys: tuple[str, ...], owned_by_config: Callable[[str], bool]
|
||||
) -> tuple[str, ...]:
|
||||
"""The matching lists the database owns, or all of them when the config file owns every one."""
|
||||
database_owned: Final = tuple(key for key in matching_keys if not owned_by_config(key))
|
||||
return database_owned or matching_keys
|
||||
|
||||
|
||||
@router.post(
|
||||
"/config/callback/delete",
|
||||
tags=["config.yaml"],
|
||||
|
|
@ -19733,24 +19753,42 @@ async def delete_callback(
|
|||
|
||||
try:
|
||||
# Get current configuration
|
||||
config: Final = await proxy_config.get_config()
|
||||
config: Final = cast( # cast-ok: get_config returns an untyped config dict
|
||||
dict[str, object],
|
||||
await proxy_config.get_config(),
|
||||
)
|
||||
callback_name: Final = data.callback_name.lower()
|
||||
|
||||
# Check if callback exists in current configuration
|
||||
litellm_settings: Final = config.get("litellm_settings", {})
|
||||
success_callbacks: Final = litellm_settings.get("success_callback", [])
|
||||
litellm_settings: Final[Mapping[str, object]] = cast( # cast-ok: the config dict's values are untyped
|
||||
Mapping[str, object],
|
||||
config.get("litellm_settings", {}),
|
||||
)
|
||||
configured_lists: Final = {
|
||||
key: _configured_callback_names(litellm_settings.get(key)) for key in _DB_CONFIG_CALLBACK_EVENT_TYPES
|
||||
}
|
||||
matching_keys: Final = tuple(
|
||||
key
|
||||
for key, names in configured_lists.items()
|
||||
if any(_is_callback_name(entry, callback_name) for entry in names)
|
||||
)
|
||||
|
||||
if callback_name not in success_callbacks:
|
||||
if not matching_keys:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": f"Callback '{callback_name}' not found in active configuration"},
|
||||
)
|
||||
|
||||
before_success_callbacks: Final = list(success_callbacks)
|
||||
|
||||
# Remove callback from success_callback list
|
||||
success_callbacks.remove(callback_name)
|
||||
config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks
|
||||
rewritten_keys: Final = _callback_lists_to_rewrite(matching_keys, proxy_config.litellm_settings.owned_by_config)
|
||||
remaining_lists: Final = MappingProxyType(
|
||||
{
|
||||
key: tuple(entry for entry in configured_lists[key] if not _is_callback_name(entry, callback_name))
|
||||
for key in rewritten_keys
|
||||
}
|
||||
)
|
||||
before_callbacks: Final[JsonValue] = {key: list(configured_lists[key]) for key in rewritten_keys}
|
||||
after_callbacks: Final[JsonValue] = {key: list(names) for key, names in remaining_lists.items()}
|
||||
config["litellm_settings"] = {**litellm_settings, **after_callbacks}
|
||||
|
||||
# Save the updated configuration
|
||||
await proxy_config.save_config(new_config=config)
|
||||
|
|
@ -19759,8 +19797,8 @@ async def delete_callback(
|
|||
create_config_audit_log(
|
||||
"litellm_settings",
|
||||
"deleted",
|
||||
{"success_callback": before_success_callbacks},
|
||||
{"success_callback": success_callbacks},
|
||||
before_callbacks,
|
||||
after_callbacks,
|
||||
user_api_key_dict,
|
||||
)
|
||||
)
|
||||
|
|
@ -19771,7 +19809,7 @@ async def delete_callback(
|
|||
return {
|
||||
"message": f"Successfully deleted callback: {callback_name}",
|
||||
"removed_callback": callback_name,
|
||||
"remaining_callbacks": success_callbacks,
|
||||
"remaining_callbacks": tuple(chain.from_iterable(remaining_lists.values())),
|
||||
"deleted_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
|
|
|
|||
351
tests/integration/configuration/test_callback_delete.py
Normal file
351
tests/integration/configuration/test_callback_delete.py
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import JSON_OBJECT, Gateway, JsonValue, eventually, list_value, object_value
|
||||
from integration._support.database import read_rows, scratch_database
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
CALLBACK: Final = "generic_api"
|
||||
SIBLING: Final = "langsmith"
|
||||
LIST_TYPES: Final = MappingProxyType(
|
||||
{
|
||||
"success_callback": "success",
|
||||
"failure_callback": "failure",
|
||||
"callbacks": "success_and_failure",
|
||||
}
|
||||
)
|
||||
SINK_ENVIRONMENT: Final = MappingProxyType(
|
||||
{
|
||||
"GENERIC_LOGGER_ENDPOINT": "http://127.0.0.1:9/unused",
|
||||
"LANGSMITH_API_KEY": "synthetic-langsmith-key",
|
||||
"LANGSMITH_BASE_URL": "http://127.0.0.1:9",
|
||||
}
|
||||
)
|
||||
NOT_FOUND: Final = f"Callback '{CALLBACK}' not found in active configuration"
|
||||
WORKER_SYNC_SECONDS: Final = 40
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: Gateway
|
||||
database_url: str
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _rig(gateway: Gateway, tmp_path: Path, file_settings: Mapping[str, JsonValue]) -> Generator[Rig]:
|
||||
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
path: Final = tmp_path / f"callback_delete_{uuid.uuid4().hex}.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump({**config, "litellm_settings": {**object_value(config["litellm_settings"]), **file_settings}})
|
||||
)
|
||||
with (
|
||||
scratch_database() as database_url,
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"DATABASE_URL": database_url, **SINK_ENVIRONMENT},
|
||||
config=path,
|
||||
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield Rig(proxy, database_url)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rig(gateway: Gateway, tmp_path: Path) -> Iterator[Rig]:
|
||||
with _rig(gateway, tmp_path, {}) as created:
|
||||
yield created
|
||||
|
||||
|
||||
def _configured(proxy: Gateway) -> tuple[tuple[str, str], ...]:
|
||||
rows: Final = proxy.get("/get/config/callbacks")["callbacks"]
|
||||
assert isinstance(rows, list)
|
||||
return tuple(
|
||||
sorted(
|
||||
(str(row["name"]), str(row["type"]))
|
||||
for row in (object_value(entry) for entry in rows)
|
||||
if not row.get("read_only")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _listed(proxy: Gateway, expected: tuple[tuple[str, str], ...]) -> None:
|
||||
eventually(lambda: _configured(proxy), lambda rows: rows == expected, seconds=WORKER_SYNC_SECONDS)
|
||||
|
||||
|
||||
def _stored_settings(database_url: str) -> Mapping[str, JsonValue]:
|
||||
rows: Final = read_rows(
|
||||
'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s',
|
||||
("litellm_settings",),
|
||||
database_url=database_url,
|
||||
)
|
||||
assert len(rows) == 1, rows
|
||||
value: Final = rows[0]["param_value"]
|
||||
return object_value(json.loads(value) if isinstance(value, str) else value)
|
||||
|
||||
|
||||
def _configure(
|
||||
proxy: Gateway, settings: Mapping[str, JsonValue], environment: Mapping[str, JsonValue] | None = None
|
||||
) -> None:
|
||||
body: Final[Mapping[str, JsonValue]] = {
|
||||
"litellm_settings": dict(settings),
|
||||
"environment_variables": None if environment is None else dict(environment),
|
||||
}
|
||||
assert proxy.post("/config/update", body) == {"message": "Config updated successfully"}
|
||||
|
||||
|
||||
def _delete(proxy: Gateway, name: str) -> tuple[int, Mapping[str, JsonValue]]:
|
||||
response: Final = proxy.request("POST", "/config/callback/delete", {"callback_name": name})
|
||||
return response.status_code, JSON_OBJECT.validate_json(response.content)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("list_key", tuple(LIST_TYPES))
|
||||
def test_callback_listed_under_any_settings_list_is_deleted_from_the_listing_and_the_store(
|
||||
rig: Rig, list_key: str
|
||||
) -> None:
|
||||
_configure(rig.proxy, {list_key: [CALLBACK]})
|
||||
_listed(rig.proxy, ((CALLBACK, LIST_TYPES[list_key]),))
|
||||
|
||||
status, body = _delete(rig.proxy, CALLBACK)
|
||||
|
||||
assert status == 200, body
|
||||
assert (body["removed_callback"], body["remaining_callbacks"]) == (CALLBACK, [])
|
||||
assert _stored_settings(rig.database_url)[list_key] == []
|
||||
_listed(rig.proxy, ())
|
||||
|
||||
|
||||
def test_callback_in_both_success_and_failure_lists_is_removed_from_both_in_one_delete(rig: Rig) -> None:
|
||||
_configure(rig.proxy, {"success_callback": [CALLBACK], "failure_callback": [CALLBACK]})
|
||||
_listed(rig.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
|
||||
|
||||
status, body = _delete(rig.proxy, CALLBACK)
|
||||
|
||||
assert status == 200, body
|
||||
stored: Final = _stored_settings(rig.database_url)
|
||||
assert (stored["success_callback"], stored["failure_callback"]) == ([], [])
|
||||
_listed(rig.proxy, ())
|
||||
|
||||
|
||||
def test_delete_keeps_other_callbacks_and_reports_what_is_left_in_the_list_it_edited(rig: Rig) -> None:
|
||||
_configure(rig.proxy, {"callbacks": [SIBLING, CALLBACK], "failure_callback": [SIBLING]})
|
||||
_listed(rig.proxy, ((CALLBACK, "success_and_failure"), (SIBLING, "failure"), (SIBLING, "success_and_failure")))
|
||||
|
||||
status, body = _delete(rig.proxy, CALLBACK)
|
||||
|
||||
assert status == 200, body
|
||||
assert body["remaining_callbacks"] == [SIBLING]
|
||||
stored: Final = _stored_settings(rig.database_url)
|
||||
assert (stored["callbacks"], stored["failure_callback"]) == ([SIBLING], [SIBLING])
|
||||
_listed(rig.proxy, ((SIBLING, "failure"), (SIBLING, "success_and_failure")))
|
||||
|
||||
|
||||
def test_delete_matches_the_callback_name_case_insensitively(rig: Rig) -> None:
|
||||
_configure(rig.proxy, {"failure_callback": [CALLBACK.upper()], "callbacks": [CALLBACK]})
|
||||
_listed(rig.proxy, ((CALLBACK.upper(), "failure"), (CALLBACK, "success_and_failure")))
|
||||
|
||||
status, body = _delete(rig.proxy, CALLBACK.upper())
|
||||
|
||||
assert status == 200, body
|
||||
assert body["removed_callback"] == CALLBACK
|
||||
_listed(rig.proxy, ())
|
||||
|
||||
|
||||
def test_delete_of_an_unconfigured_callback_is_a_404_and_changes_nothing(rig: Rig) -> None:
|
||||
_configure(rig.proxy, {"callbacks": [SIBLING]})
|
||||
_listed(rig.proxy, ((SIBLING, "success_and_failure"),))
|
||||
before: Final = _stored_settings(rig.database_url)
|
||||
|
||||
status, body = _delete(rig.proxy, CALLBACK)
|
||||
|
||||
assert (status, body) == (404, {"detail": {"error": NOT_FOUND}})
|
||||
assert _stored_settings(rig.database_url) == before
|
||||
_listed(rig.proxy, ((SIBLING, "success_and_failure"),))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("file_key", "database_key"), (("failure_callback", "success_callback"), ("success_callback", "failure_callback"))
|
||||
)
|
||||
def test_delete_removes_the_database_copy_and_keeps_the_copy_the_config_file_sets(
|
||||
gateway: Gateway, tmp_path: Path, file_key: str, database_key: str
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, {file_key: [CALLBACK]}) as file_owned:
|
||||
_configure(file_owned.proxy, {database_key: [CALLBACK]})
|
||||
_listed(file_owned.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
|
||||
|
||||
status, body = _delete(file_owned.proxy, CALLBACK)
|
||||
|
||||
assert status == 200, body
|
||||
assert body["remaining_callbacks"] == []
|
||||
assert _stored_settings(file_owned.database_url)[database_key] == []
|
||||
_listed(file_owned.proxy, ((CALLBACK, LIST_TYPES[file_key]),))
|
||||
|
||||
|
||||
def test_delete_of_a_callback_only_the_config_file_lists_is_refused_and_names_the_file(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path, {"callbacks": [CALLBACK]}) as file_owned:
|
||||
_listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
|
||||
|
||||
status, body = _delete(file_owned.proxy, CALLBACK)
|
||||
|
||||
assert status == 400, body
|
||||
detail: Final = object_value(body["detail"])
|
||||
assert (detail["keys"], detail["section"]) == (["callbacks"], "litellm_settings")
|
||||
assert "set in the config file" in str(detail["error"])
|
||||
_listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
|
||||
|
||||
|
||||
def _completion(request: Request) -> Reply:
|
||||
message: Final = object_value(list_value(JSON_OBJECT.validate_json(request.body)["messages"])[0])
|
||||
if str(message["content"]).endswith("-failure"):
|
||||
return Reply(
|
||||
status=400,
|
||||
body=json.dumps(
|
||||
{"error": {"type": "invalid_request_error", "code": "synthetic", "message": "synthetic failure"}}
|
||||
).encode(),
|
||||
)
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-" + uuid.uuid4().hex,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Sink:
|
||||
wire: Wire
|
||||
bodies: list[bytes] # mutable-ok: drain() consumes the queue, later reads keep earlier batches
|
||||
|
||||
def received(self, markers: tuple[str, ...]) -> bool:
|
||||
self.bodies.extend(request.body for request in self.wire.drain())
|
||||
return all(any(marker.encode() in body for body in self.bodies) for marker in markers)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pair:
|
||||
first: Gateway
|
||||
peer: Gateway
|
||||
database_url: str
|
||||
provider: Wire
|
||||
generic: Sink
|
||||
langsmith: Sink
|
||||
|
||||
def running(self, name: str) -> tuple[bool, bool]:
|
||||
return (_runs(self.first, name), _runs(self.peer, name))
|
||||
|
||||
|
||||
def _runs(proxy: Gateway, name: str) -> bool:
|
||||
active: Final = list_value(proxy.get("/active/callbacks")["all_litellm_callbacks"])
|
||||
return any(name in str(entry).lower() for entry in active)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _pair(gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[Pair]:
|
||||
with (
|
||||
scratch_database() as database_url,
|
||||
wire_server(_completion) as provider,
|
||||
wire_server(lambda _: Reply()) as generic,
|
||||
wire_server(lambda _: Reply()) as langsmith,
|
||||
):
|
||||
monkeypatch.setenv("DATABASE_URL", database_url)
|
||||
environment: Final = {
|
||||
"DATABASE_URL": database_url,
|
||||
"DEFAULT_FLUSH_INTERVAL_SECONDS": "1",
|
||||
"LITELLM_STORE_AUDIT_LOGS": "True",
|
||||
}
|
||||
removed: Final = (
|
||||
"DATABASE_URL_READ_REPLICA",
|
||||
"GENERIC_LOGGER_ENDPOINT",
|
||||
"GENERIC_LOGGER_HEADERS",
|
||||
"LANGSMITH_API_KEY",
|
||||
"LANGSMITH_BASE_URL",
|
||||
)
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as first,
|
||||
owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as peer,
|
||||
):
|
||||
yield Pair(first, peer, database_url, provider, Sink(generic, []), Sink(langsmith, []))
|
||||
|
||||
|
||||
def _send_on_both(pair: Pair, model: str, key: str, marker: str, outcome: str) -> tuple[str, str]:
|
||||
markers: Final = (f"{marker}-first-{outcome}", f"{marker}-peer-{outcome}")
|
||||
for proxy, text in zip((pair.first, pair.peer), markers, strict=True):
|
||||
response = proxy.request(
|
||||
"POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": text}]}, key=key
|
||||
)
|
||||
assert response.status_code == (400 if outcome == "failure" else 200), response.text
|
||||
return markers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("list_key", "outcome"), (("failure_callback", "failure"), ("callbacks", "success")))
|
||||
def test_deleted_callback_stops_receiving_events_on_both_processes(
|
||||
gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, list_key: str, outcome: str
|
||||
) -> None:
|
||||
assert os.environ.get("LITELLM_LICENSE"), "LITELLM_LICENSE must be set: audit logs are an enterprise feature"
|
||||
marker: Final = "callback-delete-" + uuid.uuid4().hex
|
||||
with _pair(gateway, tmp_path, monkeypatch) as pair, pair.first.scenario() as scenario:
|
||||
_configure(
|
||||
pair.first,
|
||||
{list_key: [SIBLING, CALLBACK]},
|
||||
{
|
||||
"GENERIC_LOGGER_ENDPOINT": pair.generic.wire.url,
|
||||
"LANGSMITH_API_KEY": "langsmith-key-" + marker,
|
||||
"LANGSMITH_BASE_URL": pair.langsmith.wire.url,
|
||||
},
|
||||
)
|
||||
eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
|
||||
model: Final = scenario.model(api_base=pair.provider.url + "/v1")
|
||||
key: Final = scenario.key(models=[model])
|
||||
before: Final = _send_on_both(pair, model, key, marker + "-before", outcome)
|
||||
eventually(lambda: pair.generic.received(before), bool, seconds=25)
|
||||
eventually(lambda: pair.langsmith.received(before), bool, seconds=25)
|
||||
|
||||
status, body = _delete(pair.first, SIBLING)
|
||||
|
||||
assert status == 200, body
|
||||
assert (body["removed_callback"], body["remaining_callbacks"]) == (SIBLING, [CALLBACK])
|
||||
assert _stored_settings(pair.database_url)[list_key] == [CALLBACK]
|
||||
audit: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT before_value, updated_values FROM "LiteLLM_AuditLog" WHERE object_id = %s AND action = %s',
|
||||
("litellm_settings", "deleted"),
|
||||
database_url=pair.database_url,
|
||||
),
|
||||
bool,
|
||||
seconds=10,
|
||||
)
|
||||
assert len(audit) == 1, audit
|
||||
before_value: Final = list_value(object_value(audit[0]["before_value"])[list_key])
|
||||
assert sorted(str(name) for name in before_value) == sorted([SIBLING, CALLBACK]), audit
|
||||
assert audit[0]["updated_values"] == {list_key: [CALLBACK]}, audit
|
||||
eventually(
|
||||
lambda: pair.running(SIBLING), lambda running: running == (False, False), seconds=WORKER_SYNC_SECONDS
|
||||
)
|
||||
|
||||
after: Final = _send_on_both(pair, model, key, marker + "-after", outcome)
|
||||
eventually(lambda: pair.generic.received(after), bool, seconds=25)
|
||||
_configure(pair.first, {list_key: [CALLBACK, SIBLING]})
|
||||
eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
|
||||
barrier: Final = _send_on_both(pair, model, key, marker + "-barrier", outcome)
|
||||
eventually(lambda: pair.langsmith.received(barrier), bool, seconds=25)
|
||||
|
||||
leaked: Final = tuple(text for text in after if any(text.encode() in body for body in pair.langsmith.bodies))
|
||||
assert leaked == (), leaked
|
||||
|
|
@ -14,10 +14,15 @@ Routes covered:
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from .conftest import VOLATILE_KEYS, normalize
|
||||
|
||||
|
|
@ -944,6 +949,228 @@ def test_config_callback_delete_not_found(client, auth_as, mock_prisma, monkeypa
|
|||
assert "langfuse" in str(response.json()).lower() or "not found" in str(response.json()).lower()
|
||||
|
||||
|
||||
def _delete_callback_roundtrip(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
litellm_settings: dict[str, object],
|
||||
callback_name: str,
|
||||
owned_by_config: frozenset[str] = frozenset(),
|
||||
) -> tuple[httpx.Response, dict[str, object]]:
|
||||
"""POST /config/callback/delete as admin against a stubbed config, and
|
||||
return (response, saved litellm_settings) so each case pins the full body
|
||||
plus the exact config persisted."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_install_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(ps, "store_model_in_db", True)
|
||||
|
||||
fake_proxy_config: Final = MagicMock()
|
||||
fake_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": litellm_settings})
|
||||
fake_proxy_config.save_config = AsyncMock()
|
||||
fake_proxy_config.add_deployment = AsyncMock()
|
||||
fake_proxy_config.litellm_settings.owned_by_config = owned_by_config.__contains__
|
||||
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response: Final = client.post("/config/callback/delete", json={"callback_name": callback_name})
|
||||
saved: Final = fake_proxy_config.save_config.await_args.kwargs["new_config"]["litellm_settings"]
|
||||
return response, saved
|
||||
|
||||
|
||||
def test_config_callback_delete_from_failure_callback(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A callback configured only under litellm_settings.failure_callback is
|
||||
listed by GET /get/config/callbacks, so the delete route must remove it."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"success_callback": ["slack"], "failure_callback": ["datadog", "sentry"]},
|
||||
"datadog",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["removed_callback"] == "datadog"
|
||||
assert saved == {"success_callback": ["slack"], "failure_callback": ["sentry"]}
|
||||
assert response.json()["remaining_callbacks"] == ["sentry"]
|
||||
|
||||
|
||||
def test_config_callback_delete_from_callbacks_list(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A callback configured only under litellm_settings.callbacks is deleted
|
||||
while unrelated lists pass through untouched."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"success_callback": ["langfuse"], "callbacks": ["datadog"], "drop_params": True},
|
||||
"datadog",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["removed_callback"] == "datadog"
|
||||
assert saved == {"success_callback": ["langfuse"], "callbacks": [], "drop_params": True}
|
||||
assert response.json()["remaining_callbacks"] == []
|
||||
|
||||
|
||||
def test_config_callback_delete_from_multiple_lists(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A name present in both success_callback and callbacks is removed from
|
||||
both, and remaining_callbacks reports the leftovers of the lists it left."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{
|
||||
"success_callback": ["datadog", "slack"],
|
||||
"failure_callback": ["sentry"],
|
||||
"callbacks": ["langfuse", "datadog"],
|
||||
},
|
||||
"datadog",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert saved == {
|
||||
"success_callback": ["slack"],
|
||||
"failure_callback": ["sentry"],
|
||||
"callbacks": ["langfuse"],
|
||||
}
|
||||
assert response.json()["remaining_callbacks"] == ["slack", "langfuse"]
|
||||
|
||||
|
||||
def test_config_callback_delete_skips_lists_the_config_file_owns(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A name in a database-owned list and a config-file-owned list is removed
|
||||
from the database-owned one only, so the save is not refused."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"success_callback": ["datadog", "slack"], "failure_callback": ["datadog"]},
|
||||
"datadog",
|
||||
owned_by_config=frozenset({"failure_callback"}),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert saved == {"success_callback": ["slack"], "failure_callback": ["datadog"]}
|
||||
assert response.json()["remaining_callbacks"] == ["slack"]
|
||||
|
||||
|
||||
def test_config_callback_delete_removes_database_copy_when_config_file_owns_success_callback(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A name the config file sets in success_callback is left there while its
|
||||
database-owned failure_callback copy is removed."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"success_callback": ["datadog"], "failure_callback": ["datadog", "sentry"]},
|
||||
"datadog",
|
||||
owned_by_config=frozenset({"success_callback"}),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert saved == {"success_callback": ["datadog"], "failure_callback": ["sentry"]}
|
||||
assert response.json()["remaining_callbacks"] == ["sentry"]
|
||||
|
||||
|
||||
def test_config_callback_delete_from_string_valued_list(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A list written as a single string is matched and saved back as an empty list."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"failure_callback": "datadog"},
|
||||
"datadog",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert saved == {"failure_callback": []}
|
||||
|
||||
|
||||
def test_config_callback_delete_case_insensitive(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""A mixed-case entry is deleted by the name GET /get/config/callbacks shows for it."""
|
||||
response, saved = _delete_callback_roundtrip(
|
||||
client,
|
||||
auth_as,
|
||||
mock_prisma,
|
||||
monkeypatch,
|
||||
{"callbacks": ["DataDog", "otel"]},
|
||||
"DataDog",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["removed_callback"] == "datadog"
|
||||
assert saved == {"callbacks": ["otel"]}
|
||||
|
||||
|
||||
def test_config_callback_delete_saves_against_the_baseline_it_read(
|
||||
client: TestClient,
|
||||
auth_as: Callable[..., contextlib.AbstractContextManager[None]],
|
||||
mock_prisma: MagicMock,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""save_config gets the object get_config returned, so it diffs against the
|
||||
settings as they were read and writes only the edited callback list."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_install_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(ps, "store_model_in_db", True)
|
||||
|
||||
read_time: Final = {"litellm_settings": {"failure_callback": ["datadog", "sentry"], "max_budget": 10}}
|
||||
loaded: Final = ps._ConfigWithBaseline(read_time)
|
||||
fake_proxy_config: Final = MagicMock()
|
||||
fake_proxy_config.get_config = AsyncMock(return_value=loaded)
|
||||
fake_proxy_config.save_config = AsyncMock()
|
||||
fake_proxy_config.add_deployment = AsyncMock()
|
||||
fake_proxy_config.litellm_settings.owned_by_config = frozenset[str]().__contains__
|
||||
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response: Final = client.post("/config/callback/delete", json={"callback_name": "datadog"})
|
||||
|
||||
assert response.status_code == 200
|
||||
saved: Final = fake_proxy_config.save_config.await_args.kwargs["new_config"]
|
||||
assert saved is loaded
|
||||
assert saved.baseline == read_time
|
||||
assert saved["litellm_settings"] == {"failure_callback": ["sentry"], "max_budget": 10}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /get/config/callbacks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue