From 033f2e5e2a651fed59faf6478fb18314e487738b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 10 Sep 2026 14:19:08 -0700 Subject: [PATCH] feat(wandb): default unmapped W&B models to reasoning-capable W&B's serverless catalog grows faster than the registry names it, so a model they ship today resolves as non-reasoning here until someone edits the cost map, and the caller's reasoning_effort is dropped or rejected. Add a wandb-reasoning-baseline capability rule to fallback_generalizations so any wandb/ id the map has not described defaults to supports_reasoning. Rules lose to exact entries, so mapped non-reasoning models such as wandb/meta-llama/Llama-3.1-8B-Instruct are unaffected. The rule carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver. Claude-Session: https://claude.ai/code/session_01A6SkwJdfZUmkzfUkrEkqX8 --- ...odel_prices_and_context_window_backup.json | 8 +++ model_prices_and_context_window.json | 8 +++ .../test_fallback_generalizations.py | 60 +++++++++++++++++++ .../wandb/test_wandb_chat_transformation.py | 26 +++++++- 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1c0bd32d782..b7726290f0e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1c0bd32d782..b7726290f0e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -57258,6 +57258,14 @@ "model_info": { "supports_mid_conversation_system": true } + }, + { + "name": "wandb-reasoning-baseline", + "pattern": "^wandb/", + "description": "Any Weights & Biases Inference model id, anchored to the wandb/ namespace so only that provider's ids match. W&B's serverless catalog is reasoning-first and grows faster than this registry names it, so an id the map has not described yet is treated as reasoning-capable and keeps the caller's reasoning_effort instead of dropping it or raising UnsupportedParamsError. Rules lose to exact entries, so a mapped non-reasoning model such as wandb/meta-llama/Llama-3.1-8B-Instruct is unaffected. Carries no mode and no pricing, so cost stays on the standard unpriced behavior and the deployment does not read as catalog-mapped to the router's reasoning-effort resolver.", + "model_info": { + "supports_reasoning": true + } } ] }, diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index b1e8163b91d..56552ec20ab 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -571,3 +571,63 @@ def test_shipped_mid_conversation_gate_on_bedrock_ids(shipped_cost_map): ): matched = match_capability_generalizations(unflagged) assert matched is None or not matched.get("supports_mid_conversation_system"), unflagged + + +def test_shipped_rules_flag_unmapped_wandb_ids_as_reasoning(shipped_cost_map): + """W&B ships reasoning models faster than the registry names them, so an unmapped + wandb id resolves as reasoning-capable and its reasoning_effort survives instead of + being dropped. The rule carries no mode and no pricing, so cost stays on the standard + unpriced behavior and the deployment does not read as catalog-mapped.""" + model = "wandb/zai-org/GLM-6-Turbo" + assert model not in litellm.model_cost + + info = litellm.get_model_info(model, custom_llm_provider="wandb") + assert info["litellm_provider"] == "wandb" + assert info["supports_reasoning"] is True + assert info.get("mode") is None + assert not info.get("input_cost_per_token") + assert not info.get("output_cost_per_token") + + assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True + + +def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_map): + """The whole point of a fallback is that it only fills gaps. A wandb model the map + describes as non-reasoning must stay non-reasoning, otherwise the rule silently + re-introduces the blanket supports_reasoning it exists to avoid.""" + for model in ( + "meta-llama/Llama-3.1-8B-Instruct", + "microsoft/Phi-4-mini-instruct", + "moonshotai/Kimi-K2-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + ): + assert f"wandb/{model}" in litellm.model_cost, model + assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model + + +def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): + """``^wandb/`` is anchored, so it cannot leak onto another provider's ids.""" + assert match_capability_generalizations("wandb/some-new-model") == {"supports_reasoning": True} + for foreign in ("openai/some-new-model", "notwandb/some-new-model", "together_ai/wandb/some-new-model"): + matched = match_capability_generalizations(foreign) + assert matched is None or not matched.get("supports_reasoning"), foreign + + +def test_shipped_wandb_rule_keeps_reasoning_effort_on_an_unmapped_model(shipped_cost_map): + """End to end through the provider config: the gate WandbConfig applies reads the + rule, so reasoning_effort is advertised and survives get_optional_params rather than + raising UnsupportedParamsError.""" + model = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + supported = litellm.get_supported_openai_params(model=f"wandb/{model}") + assert supported is not None + assert "reasoning_effort" in supported + + optional_params = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="wandb", + reasoning_effort="medium", + drop_params=False, + ) + assert optional_params["reasoning_effort"] == "medium" diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 481a52bebad..dd0d1bdbb9d 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -249,7 +249,6 @@ class TestWandbConfig: "model,explicit_false", [ ("meta-llama/Llama-3.1-8B-Instruct", False), - ("unknown-model", False), ("openai/gpt-oss-20b", True), ], ) @@ -290,3 +289,28 @@ class TestWandbConfig: supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") assert supported_params is not None assert "reasoning_effort" not in supported_params + + @pytest.mark.respx() + def test_wandb_completion_keeps_reasoning_effort_for_an_unregistered_model( + self, wandb_test_config, wandb_request_mock: respx.Route + ): + """A wandb id the registry has not named yet resolves through the + wandb-reasoning-baseline fallback generalization, so its reasoning_effort reaches + the provider instead of raising. W&B adds reasoning models faster than this + registry names them, and an exact entry still wins wherever one exists.""" + model: Final = "zai-org/GLM-6-Turbo" + assert f"wandb/{model}" not in litellm.model_cost + + completion( + model=f"wandb/{model}", + messages=[{"role": "user", "content": "Hello"}], + api_key="fake-wandb-key", + api_base="https://api.inference.wandb.ai/v1", + reasoning_effort="medium", + drop_params=False, + ) + + assert wandb_request_mock.call_count == 1 + request_body = json.loads(wandb_request_mock.calls[0].request.content) + assert request_body["model"] == model + assert request_body["reasoning_effort"] == "medium"