From 3f39b67495dab31d9f82e14e3a588ea2c2ac3ebd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:10:53 +0000 Subject: [PATCH] fix(wizard): surface real credential-check errors instead of blanket invalid key The setup wizard validated keys via check_valid_key, which collapses every exception into a boolean, so any non-auth failure (rate limit, model unavailable to the key, network) was reported as an invalid API key. It also returned before --detailed_debug enabled debug logging, so users could not diagnose the failure. Classify the probe outcome as a tagged union (valid / invalid / unverified) and print the underlying error for non-auth failures. Honor --detailed_debug in --setup, and refresh the stale Gemini test/default models. --- litellm/proxy/proxy_cli.py | 2 +- litellm/setup_wizard.py | 89 +++++++++++++++++++++---- tests/test_litellm/test_setup_wizard.py | 79 +++++++++++++++++++++- 3 files changed, 155 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 74ec0cc8700..5c3e138fe1d 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -887,7 +887,7 @@ def run_server( if setup: from litellm.setup_wizard import run_setup_wizard - run_setup_wizard() + run_setup_wizard(debug=detailed_debug) return args = locals() diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index 10b4fb30f22..d6856a1c873 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -13,8 +13,9 @@ import re import secrets import sys import sysconfig +from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import Callable, Dict, List, Optional, Set # termios / tty are Unix-only; fall back gracefully on Windows try: @@ -27,14 +28,15 @@ except ImportError: tty = None # type: ignore[assignment] _HAS_RAW_TERMINAL = False -from litellm.utils import check_valid_key +import litellm +from litellm.exceptions import AuthenticationError # --------------------------------------------------------------------------- # Provider definitions # --------------------------------------------------------------------------- # Each entry describes one provider card shown in the wizard. # `env_key` — primary env var name (None = no key needed, e.g. Ollama) -# `test_model` — model passed to check_valid_key for credential validation +# `test_model` — model used to validate credentials with a live completion # (None = skip validation, e.g. Azure needs a deployment name) # `models` — default models written into the generated config # --------------------------------------------------------------------------- @@ -69,11 +71,11 @@ PROVIDERS: List[Dict] = [ { "id": "gemini", "name": "Google Gemini", - "description": "Gemini 2.0 Flash, Gemini 2.5 Pro", + "description": "Gemini 3.5 Flash, Gemini 2.5 Pro", "env_key": "GEMINI_API_KEY", "key_hint": "AIza...", - "test_model": "gemini/gemini-2.0-flash", - "models": ["gemini/gemini-2.0-flash", "gemini/gemini-2.5-pro"], + "test_model": "gemini/gemini-3.5-flash", + "models": ["gemini/gemini-3.5-flash", "gemini/gemini-2.5-pro"], }, { "id": "azure", @@ -111,6 +113,31 @@ PROVIDERS: List[Dict] = [ ] +# --------------------------------------------------------------------------- +# Credential-check result (tagged union) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class _KeyValid: + """The provider accepted the key.""" + + +@dataclass(frozen=True, slots=True) +class _KeyInvalid: + """The provider rejected the key with an authentication error.""" + + +@dataclass(frozen=True, slots=True) +class _KeyUnverified: + """Validity is unknown: the probe failed for a non-auth reason.""" + + reason: str + + +_KeyCheck = _KeyValid | _KeyInvalid | _KeyUnverified + + # --------------------------------------------------------------------------- # ANSI colour helpers # --------------------------------------------------------------------------- @@ -432,10 +459,37 @@ class SetupWizard: if _styled_input(grey(" Skip? (y/N): ")).lower() == "y": return "" + @staticmethod + def _classify_key( + test_model: str, + api_key: str, + completion: Callable[..., object], + ) -> "_KeyCheck": + """ + Probe credentials with a live, minimal completion and classify the outcome. + + An authentication error means the key is definitively invalid. Any other + error (rate limit, model unavailable for this key, network, etc.) leaves + the key's validity unknown, so its reason is surfaced verbatim rather than + being mislabelled as an invalid key. + """ + try: + completion( + model=test_model, + messages=[{"role": "user", "content": "Hey, how's it going?"}], + api_key=api_key, + max_tokens=10, + ) + return _KeyValid() + except AuthenticationError: + return _KeyInvalid() + except Exception as exc: # noqa: BLE001 # any provider/network error is surfaced as unverified, never crashes the wizard + return _KeyUnverified(reason=f"{type(exc).__name__}: {exc}") + @staticmethod def _validate_and_report(provider: Dict, api_key: str) -> str: """ - Validate credentials using litellm.utils.check_valid_key and print result. + Validate credentials with a live completion and print the result. Offers a re-entry loop on failure. Returns the final (possibly re-entered) key. """ test_model: Optional[str] = provider.get("test_model") @@ -447,12 +501,19 @@ class SetupWizard: f" {grey('Testing connection to ' + provider['name'] + '...')}", flush=True, ) - valid = check_valid_key(model=test_model, api_key=api_key) - if valid: - print(f" {green(_CHECK)} {bold(provider['name'])} connected successfully") - return api_key + result = SetupWizard._classify_key(test_model, api_key, litellm.completion) + match result: + case _KeyValid(): + print(f" {green(_CHECK)} {bold(provider['name'])} connected successfully") + return api_key + case _KeyInvalid(): + print(f" {_CROSS} {bold(provider['name'])} {grey('— invalid API key')}") + case _KeyUnverified(reason=reason): + print(f" {_CROSS} {bold(provider['name'])} {grey('— could not verify key')}") + print(f" {grey(reason)}") + print(grey(" The key may still be valid; the test request failed for another reason.")) + print(grey(" Re-run with --detailed_debug to see the full request and response.")) - print(f" {_CROSS} {bold(provider['name'])} {grey('— invalid API key')}") if _styled_input(f" {blue('❯')} Re-enter key? {grey('(y/N)')}: ").lower() != "y": return api_key @@ -620,6 +681,8 @@ class SetupWizard: # --------------------------------------------------------------------------- -def run_setup_wizard() -> None: +def run_setup_wizard(debug: bool = False) -> None: """Run the interactive setup wizard. Called by `litellm --setup`.""" + if debug: + litellm._turn_on_debug() SetupWizard.run() diff --git a/tests/test_litellm/test_setup_wizard.py b/tests/test_litellm/test_setup_wizard.py index c96d6d7ed6c..b54fb29acc6 100644 --- a/tests/test_litellm/test_setup_wizard.py +++ b/tests/test_litellm/test_setup_wizard.py @@ -1,6 +1,13 @@ """Unit tests for litellm.setup_wizard — pure functions only, no network calls.""" -from litellm.setup_wizard import SetupWizard, _yaml_escape +from litellm.exceptions import AuthenticationError, RateLimitError +from litellm.setup_wizard import ( + SetupWizard, + _KeyInvalid, + _KeyUnverified, + _KeyValid, + _yaml_escape, +) # --------------------------------------------------------------------------- # _yaml_escape @@ -186,3 +193,73 @@ def test_build_config_internal_sentinel_keys_excluded(): } config = SetupWizard._build_config([_OPENAI], env_vars, "sk-master") assert "_LITELLM_" not in config + + +# --------------------------------------------------------------------------- +# SetupWizard._classify_key +# --------------------------------------------------------------------------- + + +def test_classify_key_valid_on_success(): + def fake_completion(**kwargs): + assert kwargs["model"] == "gemini/gemini-3.5-flash" + assert kwargs["api_key"] == "good-key" + return object() + + result = SetupWizard._classify_key("gemini/gemini-3.5-flash", "good-key", fake_completion) + assert isinstance(result, _KeyValid) + + +def test_classify_key_invalid_only_on_auth_error(): + def fake_completion(**kwargs): + raise AuthenticationError( + message="API key not valid", + llm_provider="gemini", + model="gemini/gemini-3.5-flash", + ) + + result = SetupWizard._classify_key("gemini/gemini-3.5-flash", "bad-key", fake_completion) + assert isinstance(result, _KeyInvalid) + + +def test_classify_key_unverified_on_non_auth_error(): + """A valid key hitting a rate limit must NOT be reported as invalid; the + real reason is surfaced so users can debug it.""" + + def fake_completion(**kwargs): + raise RateLimitError( + message="Resource has been exhausted (e.g. check quota)", + llm_provider="gemini", + model="gemini/gemini-3.5-flash", + ) + + result = SetupWizard._classify_key("gemini/gemini-3.5-flash", "valid-but-throttled", fake_completion) + assert isinstance(result, _KeyUnverified) + assert "RateLimitError" in result.reason + assert "check quota" in result.reason + + +def test_classify_key_unverified_on_generic_error(): + def fake_completion(**kwargs): + raise ValueError("model gemini/gemini-3.5-flash not found for this key") + + result = SetupWizard._classify_key("gemini/gemini-3.5-flash", "valid-key", fake_completion) + assert isinstance(result, _KeyUnverified) + assert "ValueError" in result.reason + assert "not found" in result.reason + + +def test_run_setup_wizard_enables_debug_when_requested(monkeypatch): + import litellm + import litellm.setup_wizard as wiz + + calls = {"debug": 0, "run": 0} + monkeypatch.setattr(litellm, "_turn_on_debug", lambda: calls.__setitem__("debug", calls["debug"] + 1)) + monkeypatch.setattr(wiz.SetupWizard, "run", staticmethod(lambda: calls.__setitem__("run", calls["run"] + 1))) + + wiz.run_setup_wizard(debug=True) + assert calls == {"debug": 1, "run": 1} + + calls["debug"] = 0 + wiz.run_setup_wizard(debug=False) + assert calls["debug"] == 0