From 2ebf82b36d1bd45b96a7d2cc62d457a7bc4c66e0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:57:25 +0000 Subject: [PATCH] fix(decisions): log the caller request at the SDK entry so requests rejected before prepare_call are recorded too --- litellm/decisions/call.py | 15 ++-- litellm/decisions/main.py | 11 +++ litellm/systemone/main.py | 9 +- tests/unit/decisions/test_call.py | 135 ++++++++++++++++-------------- 4 files changed, 98 insertions(+), 72 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 1ea0b29ed86..b31cb1f69db 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -425,9 +425,9 @@ def _jsonable_fallback(value: object) -> object: return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value) -def _logged_request_message(fields: RequestFields) -> ChatCompletionUserMessage: +def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage: caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python( - to_jsonable_python(fields, fallback=_jsonable_fallback) + to_jsonable_python(caller_request, fallback=_jsonable_fallback) ) sent_fields: Final = json.dumps( dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {})) @@ -435,6 +435,13 @@ def _logged_request_message(fields: RequestFields) -> ChatCompletionUserMessage: return ChatCompletionUserMessage(role="user", content=sent_fields) +def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None: + """Record every request field the caller sent as the logged `messages`, before any validation.""" + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_messages([_logged_request_message(caller_request)]) + + def prepare_call( *, model: str, @@ -447,9 +454,6 @@ def prepare_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: - logging_obj: Final = kwargs.get("litellm_logging_obj") - if isinstance(logging_obj, LiteLLMLoggingObj): - logging_obj.update_messages([_logged_request_message(fields)]) upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -539,6 +543,7 @@ def prepare_call( model=model, llm_provider=provider, ) + logging_obj: Final = kwargs.get("litellm_logging_obj") if isinstance(logging_obj, LiteLLMLoggingObj): logging_obj.update_from_kwargs( kwargs=dict(kwargs), diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 7d2ee03da0e..4e609fd2ab8 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -17,6 +17,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_images_next_to_input, send, @@ -78,6 +79,16 @@ def _prepare_decisions_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + log_caller_request( + kwargs, + { + "input": decision_input, + "questions": questions, + "audio": audio, + "videos": videos, + **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs}, + }, + ) return prepare_call( model=model, fields=_request_fields(model, decision_input, questions, audio, videos, kwargs), diff --git a/litellm/systemone/main.py b/litellm/systemone/main.py index 01f151ef4d0..27fa84e3f9b 100644 --- a/litellm/systemone/main.py +++ b/litellm/systemone/main.py @@ -14,6 +14,7 @@ from litellm.decisions.call import ( SystemOneQuestions, SystemOneRequestFields, asend, + log_caller_request, prepare_call, reject_other_format, send, @@ -39,12 +40,14 @@ def _prepare_systemone_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + fields: Final = SystemOneRequestFields( + state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos + ) + log_caller_request(kwargs, fields) reject_other_format(model=model, kwargs=kwargs, field_name="input", other_function="decisions") return prepare_call( model=model, - fields=SystemOneRequestFields( - state=state, questions=questions, images=images, assets=assets, audio=audio, videos=videos - ), + fields=fields, safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, diff --git a/tests/unit/decisions/test_call.py b/tests/unit/decisions/test_call.py index a9d946ec4b6..aad37d1cbfd 100644 --- a/tests/unit/decisions/test_call.py +++ b/tests/unit/decisions/test_call.py @@ -5,10 +5,21 @@ from typing import Final import pytest -from litellm.decisions.call import OpenAIDecisionRequestFields, SystemOneRequestFields, prepare_call +import litellm +from litellm.decisions.main import _prepare_decisions_call from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.systemone.main import _prepare_systemone_call QUESTION_TEXT: Final = "Is this about a billing problem?" +CONNECTION: Final = { + "model": "hosted_vllm/jev-fake", + "safety_identifier": None, + "api_key": "fake-provider-key", + "api_base": "http://127.0.0.1:4010", + "timeout": None, + "custom_llm_provider": None, + "extra_headers": None, +} def _logging_obj(call_type: str) -> Logging: @@ -29,79 +40,75 @@ def _logged_request(logging_obj: Logging) -> Mapping[str, object]: return json.loads(messages[0]["content"]) -@pytest.mark.parametrize( - ("call_type", "fields", "input_key", "expected_input"), - [ - ( - "asystemone", - SystemOneRequestFields( - state={"message": "My card was charged twice."}, - questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, - images=None, - assets=None, - audio=None, - videos=None, - ), - "state", - {"message": "My card was charged twice."}, - ), - ( - "adecisions", - OpenAIDecisionRequestFields( - input="My card was charged twice.", - questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], - audio=None, - videos=None, - ), - "input", - "My card was charged twice.", - ), - ], -) -def test_prepare_call_logs_every_request_field_the_caller_sent(call_type, fields, input_key, expected_input): - logging_obj: Final = _logging_obj(call_type) +def test_systemone_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("asystemone") - prepare_call( - model="hosted_vllm/jev-fake", - fields=fields, - safety_identifier=None, - api_key="fake-provider-key", - api_base="http://127.0.0.1:4010", - timeout=None, - custom_llm_provider=None, - extra_headers=None, + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, kwargs={"litellm_logging_obj": logging_obj}, ) logged: Final = _logged_request(logging_obj) - assert logged[input_key] == expected_input - assert QUESTION_TEXT in json.dumps(logged["questions"]) - assert set(logged) == {input_key, "questions"} + assert logged["state"] == {"message": "My card was charged twice."} + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + assert set(logged) == {"state", "questions"} -def test_prepare_call_logs_the_request_even_when_validation_rejects_it(): +def test_decisions_logs_every_request_field_the_caller_sent(): + logging_obj: Final = _logging_obj("adecisions") + + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "predicate", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["input"] == "My card was charged twice." + assert logged["questions"][0]["instructions"] == QUESTION_TEXT + assert set(logged) == {"input", "questions"} + + +def test_request_is_logged_even_when_it_is_rejected_before_the_provider_call(): logging_obj: Final = _logging_obj("asystemone") - with pytest.raises(Exception, match="Invalid Decisions request"): - prepare_call( - model="hosted_vllm/jev-fake", - fields=SystemOneRequestFields( - state={"message": "My card was charged twice."}, - questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, - images="not-a-list-of-images", - assets=None, - audio=None, - videos=None, - ), - safety_identifier=None, - api_key="fake-provider-key", - api_base="http://127.0.0.1:4010", - timeout=None, - custom_llm_provider=None, - extra_headers=None, + with pytest.raises(litellm.BadRequestError): + _prepare_systemone_call( + **CONNECTION, + state={"message": "My card was charged twice."}, + questions={"billing": {"type": "noul", "instructions": QUESTION_TEXT}}, + images=None, + assets=None, + audio=None, + videos=None, + kwargs={"litellm_logging_obj": logging_obj, "input": "sent to the wrong function"}, + ) + + logged: Final = _logged_request(logging_obj) + assert logged["questions"]["billing"]["instructions"] == QUESTION_TEXT + + +def test_decisions_request_is_logged_even_when_its_questions_fail_validation(): + logging_obj: Final = _logging_obj("adecisions") + + with pytest.raises(Exception): + _prepare_decisions_call( + **CONNECTION, + decision_input="My card was charged twice.", + questions=[{"type": "not-a-question-type", "instructions": QUESTION_TEXT}], + audio=None, + videos=None, kwargs={"litellm_logging_obj": logging_obj}, ) logged: Final = _logged_request(logging_obj) - assert set(logged) == {"state", "questions", "images"} - assert QUESTION_TEXT in json.dumps(logged["questions"]) + assert logged["questions"][0]["instructions"] == QUESTION_TEXT