From 6de00e24b49ae309092027ea740c0b593056aa2d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 11 May 2026 21:08:14 -0400 Subject: [PATCH] fix(ci): unbreak realtime + bedrock batch tests (#27690) * fix(tests): drop deprecated OpenAI-Beta realtime header OpenAI deprecated the 'OpenAI-Beta: realtime=v1' header; the live service now returns code 4000 invalid_beta with "Unknown beta requested: 'realtime'.". Two integration tests in tests/llm_translation/realtime/test_realtime_guardrails_openai.py hardcoded the header and started failing across all PRs. Library code is unaffected: the OpenAI realtime handler only forwards 'OpenAI-Beta: realtime=v1' upstream when the proxy *client* sends it (litellm/llms/openai/realtime/handler.py). Default proxy behavior uses the GA protocol. Connect to OpenAI without the deprecated header, and accept the GA event name 'response.output_audio_transcript.delta' alongside the beta-protocol name 'response.audio_transcript.delta' for the transcript-delta assertion. Co-authored-by: Mateo Wang * fix(logging): post_call tolerates non-JSON-serializable values post_call() did json.dumps(original_response) without default=str, so any provider passing a dict containing datetime/Decimal/etc. would raise TypeError. Bedrock batch retrieval hits this with get_model_invocation_job() responses that include datetime fields (submitTime, lastModifiedTime, endTime), failing tests/batches_tests/test_bedrock_files_and_batches.py::test_async_file_and_batch across all PRs. Pass default=str so non-serializable values fall back to str(). Co-authored-by: Mateo Wang * fix(tests): mock boto3 in bedrock retrieve batch test The test patched AsyncHTTPHandler.get, but the bedrock retrieve handler uses boto3.client('bedrock').get_model_invocation_job directly, so the real AWS call was being made on every run, failing with AccessDeniedException because the hardcoded test ARN belongs to a different AWS account. - Mock boto3.client and BedrockBatchesConfig.get_credentials so the test never touches AWS. - Use status=Completed in the mock response so output_file_id is populated (the handler intentionally leaves it None for non-completed jobs). - Assert the predicted per-job output object URI (matches what the handler actually returns) instead of the bare output prefix. Co-authored-by: Mateo Wang * docs(tests): include GA event name in guardrail-block test docstring Co-authored-by: Mateo Wang --------- Co-authored-by: Cursor Agent Co-authored-by: Mateo Wang --- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../test_bedrock_files_and_batches.py | 45 +++++++++---------- .../test_realtime_guardrails_openai.py | 11 +++-- .../test_litellm_logging.py | 13 ++++++ 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a815442c2f9..c73d914e6cc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1212,7 +1212,7 @@ class Logging(LiteLLMLoggingBaseClass): # Log the exact result from the LLM API, for streaming - log the type of response received litellm.error_logs["POST_CALL"] = locals() if isinstance(original_response, dict): - original_response = json.dumps(original_response) + original_response = json.dumps(original_response, default=str) try: self.model_call_details["input"] = input self.model_call_details["api_key"] = api_key diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index da08f9673d4..5148ea4db91 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -157,16 +157,16 @@ async def test_bedrock_retrieve_batch(): """ print("Testing bedrock batch retrieval") - # Mock bedrock batch response mock_bedrock_response = { "jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", "jobName": "test-job-123", "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "roleArn": "arn:aws:iam::123456789012:role/service-role/AmazonBedrockExecutionRoleForAgents_TEST", - "status": "InProgress", - "message": "Job is in progress", + "status": "Completed", + "message": "", "submitTime": "2024-01-01T12:00:00Z", "lastModifiedTime": "2024-01-01T12:30:00Z", + "endTime": "2024-01-01T13:00:00Z", "inputDataConfig": { "s3InputDataConfig": {"s3Uri": "s3://test-bucket/input/test-input.jsonl"} }, @@ -175,43 +175,38 @@ async def test_bedrock_retrieve_batch(): }, } - # Mock the HTTP response - mock_response = MagicMock() - mock_response.json.return_value = mock_bedrock_response - mock_response.status_code = 200 + mock_bedrock_client = MagicMock() + mock_bedrock_client.get_model_invocation_job.return_value = mock_bedrock_response + mock_creds = MagicMock(access_key="ak", secret_key="sk", token="tok") - # Print the mock response to debug - print("MOCK RESPONSE DATA:", mock_bedrock_response) - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" - ) as mock_get: - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response - - # Test retrieve batch + with ( + patch("boto3.client", return_value=mock_bedrock_client), + patch( + "litellm.llms.bedrock.batches.transformation.BedrockBatchesConfig.get_credentials", + return_value=mock_creds, + ), + ): batch_response = await litellm.aretrieve_batch( batch_id="arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", custom_llm_provider="bedrock", model="us.anthropic.claude-haiku-4-5-20251001-v1:0", ) - print("MOCKED BATCH RESPONSE=", batch_response) - - # Validate the response assert ( batch_response.id == "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123" ) assert batch_response.object == "batch" - assert ( - batch_response.status == "in_progress" - ) # Bedrock "InProgress" maps to "in_progress" + assert batch_response.status == "completed" assert batch_response.endpoint == "/v1/chat/completions" - # Validate input and output file IDs in the final transformed response assert batch_response.input_file_id == "s3://test-bucket/input/test-input.jsonl" - assert batch_response.output_file_id == "s3://test-bucket/output/" + # Bedrock returns only the output *prefix*; the handler predicts the + # actual output object as //.out. + assert ( + batch_response.output_file_id + == "s3://test-bucket/output/test-job-123/test-input.jsonl.out" + ) def test_bedrock_batch_with_encryption_key_in_post_request(): diff --git a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py index 884563c6e5c..170440f6b9f 100644 --- a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py +++ b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py @@ -104,7 +104,8 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): Send a text message containing the blocked phrase. Guardrail must: - Send error event (guardrail_violation) to client. - - Send response.audio_transcript.delta with the block message to client. + - Send response.output_audio_transcript.delta (or beta-protocol + response.audio_transcript.delta) with the block message to client. - NOT forward response.create to OpenAI (no AI response). """ import websockets @@ -119,7 +120,6 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) @@ -182,7 +182,11 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): transcript_deltas = [ e for e in client_events - if e.get("type") == "response.audio_transcript.delta" + if e.get("type") + in ( + "response.output_audio_transcript.delta", + "response.audio_transcript.delta", + ) ] assert ( len(transcript_deltas) >= 1 @@ -298,7 +302,6 @@ async def test_clean_text_message_passes_through_to_openai(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1764d9c609f..e84baf5e137 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -36,6 +36,19 @@ def test_get_masked_api_base(logging_obj): assert type(masked_api_base) == str +def test_post_call_serializes_dict_with_datetime(logging_obj): + import datetime + + response = { + "status": "InProgress", + "submitTime": datetime.datetime(2026, 5, 11, 23, 49, 13, 132000), + } + logging_obj.post_call(original_response=response) + serialized = logging_obj.model_call_details["original_response"] + assert isinstance(serialized, str) + assert "2026-05-11" in serialized + + def test_sentry_sample_rate(): existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE") try: