diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 31c3f7433f0..fb4139f62a7 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -25,7 +25,6 @@ import json import random import re from collections.abc import Callable -from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Final, Optional, cast import litellm @@ -35,7 +34,7 @@ from litellm.integrations.custom_logger import CustomLogger if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient from litellm.router import Router - from litellm.types.utils import ModelResponse, StandardLoggingPayload + from litellm.types.utils import StandardLoggingPayload _JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 6ac1e15e7b5..b88ecd4029d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -485,6 +485,8 @@ def test_load_from_azure_key_vault_missing_uri_failure_is_swallowed(monkeypatch) def test_cost_tracking_adds_two_callbacks_when_prisma_set(monkeypatch): + """_ProxyDBLogger and ShadowEvalLogger both register on litellm.callbacks; + only _ProxyDBLogger also registers on _async_success_callback.""" import litellm fake_prisma = MagicMock() @@ -503,7 +505,7 @@ def test_cost_tracking_adds_two_callbacks_when_prisma_set(monkeypatch): "prisma_was_set": True, } assert normalize(observed) == { - "added_to_callbacks": 1, + "added_to_callbacks": 2, "added_to_async_success": 1, "prisma_was_set": True, } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index bee31e04d43..5efe23fa0cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -55,20 +55,16 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ ...overrides, }); -const mockHooks = ({ - jobs = [], - detail = undefined, -}: { - jobs?: ShadowEvalJob[]; - detail?: ShadowEvalJob; -}) => { +const mockHooks = ({ jobs = [], detail = undefined }: { jobs?: ShadowEvalJob[]; detail?: ShadowEvalJob }) => { vi.mocked(useShadowEvalJobs).mockReturnValue({ data: jobs, error: null } as unknown as ReturnType< typeof useShadowEvalJobs >); vi.mocked(useShadowEvalJob).mockReturnValue({ data: detail } as unknown as ReturnType); - vi.mocked(useStartShadowEval).mockReturnValue({ mutate: vi.fn(), isPending: false, error: null } as unknown as ReturnType< - typeof useStartShadowEval - >); + vi.mocked(useStartShadowEval).mockReturnValue({ + mutate: vi.fn(), + isPending: false, + error: null, + } as unknown as ReturnType); vi.mocked(useStopShadowEval).mockReturnValue({ mutate: vi.fn(), isPending: false } as unknown as ReturnType< typeof useStopShadowEval >); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index a8c00fee5f1..d9ddc519ebd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -140,15 +140,16 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) => const start = useStartShadowEval(); const parsedPct = Number.parseFloat(percentage); - const valid = Boolean(accessToken) && apiKeyId.trim() !== "" && routerName.trim() !== "" && parsedPct > 0 && parsedPct <= 100; + const valid = + Boolean(accessToken) && apiKeyId.trim() !== "" && routerName.trim() !== "" && parsedPct > 0 && parsedPct <= 100; return ( Start a shadow eval

- Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare - both answers blind. The router's answers are never served to users. Judge calls bill to the proxy — an + Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare both + answers blind. The router's answers are never served to users. Judge calls bill to the proxy — an estimate is shown before anything runs.