From 09da8a7bf9e61b5e90f73bbe698b0d442ec98a69 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 17 May 2026 19:01:35 +0000 Subject: [PATCH] feat(triage): default Agent Shin to gpt-5.4-mini with reasoning_effort=none - Bump DEFAULT_MODEL from gpt-4o-mini to gpt-5.4-mini (more modern; 4M total context window per OpenAI catalog, JSON-schema response format, function calling all supported). - For gpt-5.x family models, pass reasoning_effort="none" via extra_body. gpt-5.x rejects temperature != 1 unless reasoning_effort is explicitly "none"; setting it lets us keep temperature=0 for deterministic JSON rubric judgments. extra_body works across openai SDK versions regardless of whether they natively type the kwarg. - For non-gpt5 overrides (TRIAGE_MODEL=gpt-4o-mini etc.), reasoning_effort is not sent. - 4 new unit tests cover: gpt-5.4-mini -> reasoning_effort=none, capitalized/dated gpt-5 variants -> reasoning_effort=none, gpt-4o-mini -> no extra_body, base_url passthrough. Co-authored-by: Mateo Wang --- .github/scripts/triage_with_llm.py | 30 +++++-- .../test_github_triage_with_llm.py | 88 +++++++++++++++++++ 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/.github/scripts/triage_with_llm.py b/.github/scripts/triage_with_llm.py index 1c6c87aeb0b..24c33f3e78b 100644 --- a/.github/scripts/triage_with_llm.py +++ b/.github/scripts/triage_with_llm.py @@ -24,7 +24,7 @@ Environment: GH_TOKEN / GITHUB_TOKEN - for `gh` CLI auth (auto-set in Actions) OPENAI_API_KEY - required when --close is passed OPENAI_BASE_URL - optional (route to any OpenAI-compatible API) - TRIAGE_MODEL - optional model override (default: gpt-4o-mini) + TRIAGE_MODEL - optional model override (default: gpt-5.4-mini) """ from __future__ import annotations @@ -38,10 +38,17 @@ import sys import textwrap from typing import Any -DEFAULT_MODEL = "gpt-4o-mini" +DEFAULT_MODEL = "gpt-5.4-mini" INTERNAL_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"}) +# Model families that require `reasoning_effort` to be set, and that reject +# `temperature != 1` unless `reasoning_effort` is "none". For these models we +# pass `reasoning_effort="none"` so a `temperature=0` deterministic judgment +# is still accepted. See litellm/llms/openai/chat/gpt_5_transformation.py for +# the full set of constraints LiteLLM applies to these models. +GPT5_FAMILY_PREFIX = "gpt-5" + # Regexes for picking off "obvious passes" without burning LLM tokens. LINKED_ISSUE_PATTERN = re.compile( r"\b(?:fixes|fix|closes|close|resolves|resolve|refs|ref|see|addresses)\s+" @@ -272,12 +279,19 @@ def call_llm_judge( if base_url else OpenAI(api_key=api_key) ) - response = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=0, - response_format={"type": "json_object"}, - ) + kwargs: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0, + "response_format": {"type": "json_object"}, + } + # gpt-5.x reasoning models reject `temperature != 1` unless + # `reasoning_effort` is explicitly "none". Set it via `extra_body` so this + # works across openai SDK versions regardless of whether the SDK natively + # types `reasoning_effort` as a top-level chat-completions param yet. + if model.lower().startswith(GPT5_FAMILY_PREFIX): + kwargs["extra_body"] = {"reasoning_effort": "none"} + response = client.chat.completions.create(**kwargs) return response.choices[0].message.content or "" diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index de72cc8d7c9..57518251ea8 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -147,6 +147,94 @@ class TestBuildPrompts: assert "repro here" in prompt +class TestCallLlmJudge: + """call_llm_judge sets gpt-5 specific kwargs correctly.""" + + def _stub_openai(self, monkeypatch, captured: dict): + """Install a fake `openai.OpenAI` client into sys.modules. + + The fake client records the kwargs passed to chat.completions.create + and returns a minimal response object whose .choices[0].message.content + is "ok". + """ + import types + + class FakeMessage: + content = '{"verdict": "pass"}' + + class FakeChoice: + message = FakeMessage() + + class FakeResponse: + choices = [FakeChoice()] + + class FakeCompletions: + def create(self, **kwargs): + captured.update(kwargs) + return FakeResponse() + + class FakeChat: + completions = FakeCompletions() + + class FakeClient: + def __init__(self, api_key, base_url=None): + captured["__client_kwargs__"] = { + "api_key": api_key, + "base_url": base_url, + } + self.chat = FakeChat() + + fake_module = types.ModuleType("openai") + fake_module.OpenAI = FakeClient + monkeypatch.setitem(sys.modules, "openai", fake_module) + + def test_should_set_reasoning_effort_none_for_gpt5_family( + self, triage_module, monkeypatch + ): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model="gpt-5.4-mini", api_key="sk-test", base_url=None + ) + assert captured["model"] == "gpt-5.4-mini" + assert captured["temperature"] == 0 + assert captured["extra_body"] == {"reasoning_effort": "none"} + + def test_should_set_reasoning_effort_for_capitalized_or_dated_gpt5( + self, triage_module, monkeypatch + ): + for model in ("GPT-5.4-mini", "gpt-5.4-mini-2026-03-17", "gpt-5"): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model=model, api_key="sk-test", base_url=None + ) + assert captured["extra_body"] == {"reasoning_effort": "none"}, model + + def test_should_omit_reasoning_effort_for_non_gpt5( + self, triage_module, monkeypatch + ): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "prompt", model="gpt-4o-mini", api_key="sk-test", base_url=None + ) + assert "extra_body" not in captured + + def test_should_pass_base_url_when_provided(self, triage_module, monkeypatch): + captured: dict = {} + self._stub_openai(monkeypatch, captured) + triage_module.call_llm_judge( + "p", + model="gpt-5.4-mini", + api_key="sk-test", + base_url="https://proxy.example.com/v1", + ) + assert ( + captured["__client_kwargs__"]["base_url"] == "https://proxy.example.com/v1" + ) + + class TestTriageOrchestration: """End-to-end-ish tests that mock both gh fetchers and the LLM."""