diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index a776c4867ab..b446c79f49b 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5570 }, "reportMissingTypeArgument": { - "limit": 15280 + "limit": 15279 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,10 +105,10 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38282 + "limit": 38281 }, "reportUnknownParameterType": { - "limit": 19583 + "limit": 19582 }, "reportUnknownVariableType": { "limit": 29829 diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index 8d33081e72f..de1cc2b103b 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -77,4 +77,16 @@ spec: volumes: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.migrationJob.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.migrationJob.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.migrationJob.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index c3f3083ece5..2ebb1b44926 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -1,4 +1,4 @@ -suite: test migrations Job ServiceAccount resolution and pod hardening +suite: test migrations Job ServiceAccount resolution, pod hardening, and scheduling templates: - migrations-job.yaml values: @@ -188,3 +188,69 @@ tests: asserts: - notExists: path: spec.activeDeadlineSeconds + + - it: renders no scheduling fields by default + asserts: + - isNull: + path: spec.template.spec.nodeSelector + - isNull: + path: spec.template.spec.tolerations + - isNull: + path: spec.template.spec.affinity + + - it: renders nodeSelector, tolerations, and affinity from the migrationJob values + set: + migrationJob.nodeSelector: + intent: no-csi-nodes + migrationJob.tolerations: + - key: intent + operator: Equal + value: no-csi-nodes + effect: NoSchedule + migrationJob.affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: intent + operator: In + values: + - no-csi-nodes + asserts: + - equal: + path: spec.template.spec.nodeSelector + value: + intent: no-csi-nodes + - equal: + path: spec.template.spec.tolerations + value: + - key: intent + operator: Equal + value: no-csi-nodes + effect: NoSchedule + - equal: + path: spec.template.spec.affinity + value: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: intent + operator: In + values: + - no-csi-nodes + + - it: does not inherit the gateway's scheduling values + set: + gateway.nodeSelector: + intent: no-csi-nodes + gateway.tolerations: + - key: intent + operator: Equal + value: no-csi-nodes + effect: NoSchedule + asserts: + - isNull: + path: spec.template.spec.nodeSelector + - isNull: + path: spec.template.spec.tolerations diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 461330ba491..6c9fb9440c7 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -152,6 +152,13 @@ migrationJob: # the writable scratch space a read-only root filesystem needs. volumes: [] volumeMounts: [] + # Scheduling for the Job pod, same shape as gateway.nodeSelector / + # gateway.tolerations / gateway.affinity. The Job does not inherit the other + # components' scheduling values: a migration usually needs a larger node + # than the gateway, so pin it here explicitly. + nodeSelector: {} + tolerations: [] + affinity: {} image: repository: ghcr.io/berriai/litellm-migrations tag: "" # defaults to .Chart.AppVersion diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b97521b90c2..d7c2fcace09 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( _safe_convert_created_field, ) +from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name @@ -205,29 +206,76 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): `remove_cache_control_flag_from_messages_and_tools`; mirror that here. """ - input = self._validate_input_param(input) - tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) - sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai( - model=model, tools=tools, litellm_params=litellm_params + replay_safe_input, sanitized_tools = self._prepared_input_and_tools( + model=model, + input=input, + tools=response_api_optional_request_params.get("tools"), + litellm_params=litellm_params, ) if sanitized_tools is not None: response_api_optional_request_params["tools"] = sanitized_tools - replay_safe_input: Final = self._drop_foreign_tool_call_item_ids(input) final_request_params: Final = dict( ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params) ) return final_request_params + def _prepared_input_and_tools( + self, + model: str, + input: str | ResponseInputParam, + tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, + litellm_params: GenericLiteLLMParams, + ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: + validated_input: Final = self._validate_input_param(input) + stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( + model=model, input=validated_input, tools=tools + ) + object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) + sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai( + model=model, tools=object_schema_tools, litellm_params=litellm_params + ) + return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + + def _tools_with_object_parameters( + self, model: str, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None + ) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: + """Decode tool schemas handed over already JSON-encoded, which the Responses validator + rejects with a 400 naming the routed model rather than the tool. A null or absent schema + is left alone because the API accepts both.""" + if tools is None: + return None + decoded: Final = [ # mutable-ok: request tools are a JSON list + self._tool_with_object_parameters(model=model, index=index, tool=tool) for index, tool in enumerate(tools) + ] + return cast("Sequence[ALL_RESPONSES_API_TOOL_PARAMS]", decoded) # cast-ok: dict spread keeps each tool's shape + + def _tool_with_object_parameters(self, model: str, index: int, tool: object) -> object: + if not isinstance(tool, dict) or tool.get("parameters") is None: + return tool + parameters: Final = tool["parameters"] + if isinstance(parameters, dict): + return tool + decoded: Final = safe_json_loads(parameters) if isinstance(parameters, str) else None + if isinstance(decoded, dict): + return {**tool, "parameters": decoded} # mutable-ok: request tools are JSON dicts + raise litellm.BadRequestError( + message=( + f"Invalid type for 'tools[{index}].parameters': expected an object, " + f"but got {type(parameters).__name__} instead." + ), + model=model, + llm_provider=self.custom_llm_provider, + ) + def remove_cache_control_flag_from_input_and_tools( self, model: str, # allows overrides to selectively run this input: str | ResponseInputParam, - tools: list[ALL_RESPONSES_API_TOOL_PARAMS] | None = None, + tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None = None, ) -> tuple[ str | ResponseInputParam, - list[ALL_RESPONSES_API_TOOL_PARAMS] | None, + Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, ]: """Sibling of `remove_cache_control_flag_from_messages_and_tools` on the chat path. Strips Anthropic-only `cache_control` markers from @@ -272,9 +320,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def _flatten_tool_schema_combinators_for_openai( self, model: str, - tools: list[ALL_RESPONSES_API_TOOL_PARAMS] | None, # mutable-ok: request tools are a JSON list + tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, litellm_params: GenericLiteLLMParams, - ) -> list[ALL_RESPONSES_API_TOOL_PARAMS] | None: # mutable-ok: request tools are a JSON list + ) -> Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None: """Flatten top-level schema combinators only where OpenAI's validator rejects them. OpenAI-compatible backends reusing this config (and the ChatGPT backend @@ -293,7 +341,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): flattened: Final = [ # mutable-ok: request tools are a JSON list self._flattened_tool_or_passthrough(tool) for tool in tools ] - return cast("list[ALL_RESPONSES_API_TOOL_PARAMS]", flattened) # cast-ok: dict spread keeps each tool's shape + return cast("Sequence[ALL_RESPONSES_API_TOOL_PARAMS]", flattened) # cast-ok: spread keeps each tool's shape @staticmethod def _flattened_tool_or_passthrough(tool: object) -> object: @@ -786,15 +834,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): compact_path: Final = parsed_url.path.rstrip("/") + "/compact" url: Final = str(parsed_url.copy_with(path=compact_path)) - input = self._validate_input_param(input) - tools = response_api_optional_request_params.get("tools") - input, tools = self.remove_cache_control_flag_from_input_and_tools(model=model, input=input, tools=tools) - sanitized_tools: Final = self._flatten_tool_schema_combinators_for_openai( - model=model, tools=tools, litellm_params=litellm_params + replay_safe_input, sanitized_tools = self._prepared_input_and_tools( + model=model, + input=input, + tools=response_api_optional_request_params.get("tools"), + litellm_params=litellm_params, ) if sanitized_tools is not None: response_api_optional_request_params["tools"] = sanitized_tools - replay_safe_input: Final = self._drop_foreign_tool_call_item_ids(input) data: Final = dict( ResponsesAPIRequestParams(model=model, input=replay_safe_input, **response_api_optional_request_params) ) diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index f2b3a68490a..694ce6e86e0 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -201,7 +201,7 @@ "limit": 310 }, "SIM103": { - "limit": 117 + "limit": 116 }, "SIM113": { "limit": 3 diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 4ac072d0ca6..a5b748e391c 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -300,6 +300,89 @@ class TestOpenAIResponsesAPIConfig: assert result["input"][0]["id"] == "toolu_01Foreign" + @pytest.mark.parametrize( + "raw_parameters", + [ + '{"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}', + '{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}', + ], + ) + def test_transform_decodes_json_string_tool_parameters(self, raw_parameters: str): + """A JSON-encoded schema must reach the provider as an object.""" + result = self.config.transform_responses_api_request( + model=self.model, + input="weather in Paris", + response_api_optional_request_params={ + "tools": [{"type": "function", "name": "get_weather", "parameters": raw_parameters}] + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["tools"][0]["parameters"] == { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + } + + def test_transform_decodes_json_string_tool_parameters_on_compact_request(self): + """The compact request path builds the same wire body, so it must decode too.""" + _url, data = self.config.transform_compact_response_api_request( + model=self.model, + input="weather in Paris", + response_api_optional_request_params={ + "tools": [{"type": "function", "name": "get_weather", "parameters": '{"type": "object"}'}] + }, + api_base="https://api.openai.com/v1/responses", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert data["tools"][0]["parameters"] == {"type": "object"} + + @pytest.mark.parametrize("raw_parameters", ['"just a string"', "not json at all", "[1, 2, 3]", 42]) + def test_transform_rejects_tool_parameters_that_are_not_an_object(self, raw_parameters: object): + """Neither an object nor a string encoding one is a client error naming the tool index.""" + with pytest.raises(litellm.BadRequestError) as exc_info: + self.config.transform_responses_api_request( + model=self.model, + input="weather in Paris", + response_api_optional_request_params={ + "tools": [ + {"type": "web_search_preview"}, + {"type": "function", "name": "get_weather", "parameters": raw_parameters}, + ] + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "tools[1].parameters" in str(exc_info.value) + + def test_transform_leaves_object_null_and_absent_tool_parameters_untouched(self): + """The API accepts an object schema, an explicit null, an omitted schema and a built-in + tool, so decoding must forward all four unchanged rather than raising.""" + schema = {"type": "object", "properties": {"city": {"type": "string"}}} + tools = [ + {"type": "function", "name": "get_weather", "parameters": schema}, + {"type": "function", "name": "null_args", "parameters": None}, + {"type": "function", "name": "no_args"}, + {"type": "web_search_preview"}, + ] + + result = self.config.transform_responses_api_request( + model=self.model, + input="weather in Paris", + response_api_optional_request_params={"tools": tools}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["tools"][0]["parameters"] == schema + assert result["tools"][1]["parameters"] is None + assert "parameters" not in result["tools"][2] + assert result["tools"][3] == {"type": "web_search_preview"} + def test_transform_compact_drops_foreign_tool_call_item_ids(self): """The compact request path replays input the same way, so it must apply the same id drop.""" @@ -864,6 +947,20 @@ class TestAzureResponsesAPIConfig: self.model = "gpt-4o" self.logging_obj = MagicMock() + def test_azure_decodes_json_string_tool_parameters(self): + """Azure reaches the same wire through `super()`, after un-nesting a chat-shaped tool.""" + result = self.config.transform_responses_api_request( + model=self.model, + input="weather in Paris", + response_api_optional_request_params={ + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": '{"type":"object"}'}}] + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["tools"][0]["parameters"] == {"type": "object"} + def test_azure_get_complete_url_with_version_types(self): """Test Azure get_complete_url with different API version types""" base_url = "https://litellm8397336933.openai.azure.com" diff --git a/type-discipline-budget.json b/type-discipline-budget.json index d2130e1bcdb..bfa055b43e5 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22182 + "limit": 22180 }, "LIT002": { "limit": 26745 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16468 + "limit": 16464 }, "LIT011": { - "limit": 5510 + "limit": 5506 }, "LIT012": { "limit": 4486