From 3740d48844d35b356a1dab8239c3d5c49ded54c2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:11:13 +0000 Subject: [PATCH] test(cost): add tier 1 claude messages stream twins and recapture drifted usage shapes (#46690) * test(cost): add tier 1 claude messages stream twins and recapture drifted usage shapes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): freeze the login throttle clock so the retry-after assertion is deterministic Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/anthropic.py | 136 ++++++++++++++++++ .../cases/bedrock_converse.py | 67 ++++++--- .../cost_calculation/cases/openai.py | 30 ---- .../proxy_server/test_routes_login_sso.py | 6 + 4 files changed, 192 insertions(+), 47 deletions(-) diff --git a/tests/integration/cost_calculation/cases/anthropic.py b/tests/integration/cost_calculation/cases/anthropic.py index 4d0987ec1f9..c214352fdb0 100644 --- a/tests/integration/cost_calculation/cases/anthropic.py +++ b/tests/integration/cost_calculation/cases/anthropic.py @@ -4443,6 +4443,39 @@ CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( ), ) + +CLAUDE_SONNET_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase( + name="claude-sonnet-5-5-messages_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-sonnet-5-5", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + "stream": True, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-sonnet-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}', + 'event: ping\ndata: {"type":"ping"}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_$REQUEST_ID"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"scripted "}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"response"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":1}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ), + expected=ExactExpected( + spend=0.01287, input_cost=0.006072, output_cost=0.006798, prompt_tokens=1840, completion_tokens=412 + ), +) CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = CostTrackingTestCase( name="claude-haiku-5-5-input_text", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -4806,6 +4839,39 @@ CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( ), ) + +CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase( + name="claude-haiku-5-5-messages_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-haiku-5-5", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + "stream": True, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-haiku-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}', + 'event: ping\ndata: {"type":"ping"}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_$REQUEST_ID"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"scripted "}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"response"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":1}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ), + expected=ExactExpected( + spend=0.00429, input_cost=0.002024, output_cost=0.002266, prompt_tokens=1840, completion_tokens=412 + ), +) CLAUDE_FABLE_5_INPUT_TEXT: Final = CostTrackingTestCase( name="claude-fable-5-input_text", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -5532,6 +5598,35 @@ CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( ), ) + +CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase( + name="claude-fable-5-1-messages_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-fable-5-1", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + "stream": True, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-1","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: ping\ndata: {"type":"ping"}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted "}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"response"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ), + expected=ExactExpected( + spend=0.0624, input_cost=0.02944, output_cost=0.03296, prompt_tokens=1840, completion_tokens=412 + ), +) CLAUDE_OPUS_5_5_STREAM: Final = CostTrackingTestCase( name="claude-opus-5-5-stream", covers="quota_management.spend_tracking.scripted_wire.logs_cost", @@ -5836,6 +5931,39 @@ CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( ), ) + +CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CostTrackingTestCase( + name="claude-opus-5-5-messages_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-opus-5-5", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + "stream": True, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-opus-5-5","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global"},"diagnostics":null}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}', + 'event: ping\ndata: {"type":"ping"}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"sig_$REQUEST_ID"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"scripted "}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"response"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":1}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ), + expected=ExactExpected( + spend=0.0156, input_cost=0.00736, output_cost=0.00824, prompt_tokens=1840, completion_tokens=412 + ), +) CLAUDE_SONNET_5_5_TOOL_CALL: Final = CostTrackingTestCase( name="claude-sonnet-5-5-tool_call", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -6877,12 +7005,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_SONNET_5_5_CACHE_WRITE_5M, CLAUDE_SONNET_5_5_CACHE_WRITE_1H, CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_SONNET_5_5_MESSAGES_STREAM, CLAUDE_HAIKU_5_5_INPUT_TEXT, CLAUDE_HAIKU_5_5_STREAM, CLAUDE_HAIKU_5_5_CACHE_READ, CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_HAIKU_5_5_MESSAGES_STREAM, CLAUDE_FABLE_5_INPUT_TEXT, CLAUDE_FABLE_5_STREAM, CLAUDE_FABLE_5_CACHE_READ, @@ -6895,11 +7025,13 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_FABLE_5_1_CACHE_WRITE_5M, CLAUDE_FABLE_5_1_CACHE_WRITE_1H, CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_STREAM, CLAUDE_OPUS_5_5_STREAM, CLAUDE_OPUS_5_5_CACHE_READ, CLAUDE_OPUS_5_5_CACHE_WRITE_5M, CLAUDE_OPUS_5_5_CACHE_WRITE_1H, CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_OPUS_5_5_MESSAGES_STREAM, CLAUDE_SONNET_5_5_TOOL_CALL, CLAUDE_SONNET_5_5_THINKING, CLAUDE_HAIKU_5_5_TOOL_CALL, @@ -6938,6 +7070,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=CLAUDE_FABLE_5_INPUT_TEXT, streamed=CLAUDE_FABLE_5_STREAM), StreamParityTestCase(plain=CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_STREAM), StreamParityTestCase(plain=CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_STREAM), + StreamParityTestCase(plain=CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_MESSAGES_STREAM), + StreamParityTestCase(plain=CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_SONNET_5_5_MESSAGES_STREAM), + StreamParityTestCase(plain=CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_HAIKU_5_5_MESSAGES_STREAM), + StreamParityTestCase(plain=CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_MESSAGES_STREAM), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( diff --git a/tests/integration/cost_calculation/cases/bedrock_converse.py b/tests/integration/cost_calculation/cases/bedrock_converse.py index 45ac7398d73..3c353a107f4 100644 --- a/tests/integration/cost_calculation/cases/bedrock_converse.py +++ b/tests/integration/cost_calculation/cases/bedrock_converse.py @@ -3097,7 +3097,16 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = CostTrackingTestCase( body={ "output": {"message": {"role": "assistant", "content": [{"text": "Hello."}]}}, "stopReason": "end_turn", - "usage": {"inputTokens": 1840, "outputTokens": 412, "totalTokens": 2252}, + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 1840, + "outputTokens": 412, + "serverToolUsage": {}, + "totalTokens": 2252, + }, "metrics": {"latencyMs": 42}, }, ), @@ -3508,11 +3517,14 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer 5eff7f685f14"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 12288, + "cacheReadInputTokens": 12288, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, + "serverToolUsage": {}, "totalTokens": 13308, - "cacheReadInputTokens": 12288, - "cacheWriteInputTokens": 0, }, }, ), @@ -3972,11 +3984,14 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer bbd4da2e326b"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 12288, + "cacheReadInputTokens": 12288, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, + "serverToolUsage": {}, "totalTokens": 13308, - "cacheReadInputTokens": 12288, - "cacheWriteInputTokens": 0, }, }, ), @@ -4230,11 +4245,14 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer cf757e3e24df"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 12288, + "cacheReadInputTokens": 12288, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, + "serverToolUsage": {}, "totalTokens": 13308, - "cacheReadInputTokens": 12288, - "cacheWriteInputTokens": 0, }, }, ), @@ -4746,11 +4764,14 @@ US_ANTHROPIC_CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase( "output": {"message": {"content": [{"text": "scripted answer 1a5a397f1134"}], "role": "assistant"}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 12288, + "cacheReadInputTokens": 12288, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 640, "outputTokens": 380, + "serverToolUsage": {}, "totalTokens": 13308, - "cacheReadInputTokens": 12288, - "cacheWriteInputTokens": 0, }, }, ), @@ -5134,11 +5155,14 @@ US_ANTHROPIC_CLAUDE_SONNET_5_5_THINKING: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "scripted answer fe9c9714d04d"}]}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 412, + "serverToolUsage": {}, "totalTokens": 1652, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 0, }, "metrics": {"latencyMs": 2071}, }, @@ -5428,11 +5452,14 @@ US_ANTHROPIC_CLAUDE_OPUS_5_5_THINKING: Final = CostTrackingTestCase( }, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 4040, + "serverToolUsage": {}, "totalTokens": 5280, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 0, }, "metrics": {"latencyMs": 2071}, }, @@ -5560,11 +5587,14 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_THINKING: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "scripted answer 90a6bb1996a5"}]}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 412, + "serverToolUsage": {}, "totalTokens": 1652, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 0, }, "metrics": {"latencyMs": 2071}, }, @@ -5839,11 +5869,14 @@ US_ANTHROPIC_CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase( "output": {"message": {"role": "assistant", "content": [{"text": "scripted answer bca700fd52a2"}]}}, "stopReason": "end_turn", "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 1240, "outputTokens": 412, + "serverToolUsage": {}, "totalTokens": 1652, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 0, }, "metrics": {"latencyMs": 2071}, }, diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index c4f56d4be5b..6ad0ff95615 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -8116,16 +8116,6 @@ GPT_6_1_SOL_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( "reasoning_tokens": 0, "rejected_prediction_tokens": 0, }, - "latency_checkpoint": { - "engine_tbt_ms": 23, - "engine_ttft_ms": 484, - "engine_ttlt_ms": 999, - "pre_inference_ms": 151, - "service_tbt_ms": 25, - "service_ttft_ms": 1085, - "service_ttlt_ms": 1610, - "user_visible_ttft_ms": 934, - }, "prompt_tokens": 12928, "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 12288}, "total_tokens": 13308, @@ -8200,16 +8190,6 @@ GPT_6_LUNA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( "reasoning_tokens": 0, "rejected_prediction_tokens": 0, }, - "latency_checkpoint": { - "engine_tbt_ms": 7, - "engine_ttft_ms": 110, - "engine_ttlt_ms": 706, - "pre_inference_ms": 254, - "service_tbt_ms": 7, - "service_ttft_ms": 647, - "service_ttlt_ms": 1238, - "user_visible_ttft_ms": 393, - }, "prompt_tokens": 12928, "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 12288}, "total_tokens": 13308, @@ -8284,16 +8264,6 @@ GPT_6_ASTRA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( "reasoning_tokens": 0, "rejected_prediction_tokens": 0, }, - "latency_checkpoint": { - "engine_tbt_ms": 25, - "engine_ttft_ms": 697, - "engine_ttlt_ms": 1373, - "pre_inference_ms": 142, - "service_tbt_ms": 26, - "service_ttft_ms": 1289, - "service_ttlt_ms": 1956, - "user_visible_ttft_ms": 1147, - }, "prompt_tokens": 12928, "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 12288}, "total_tokens": 13308, diff --git a/tests/unit/proxy/proxy_server/test_routes_login_sso.py b/tests/unit/proxy/proxy_server/test_routes_login_sso.py index 8daea8d0ad2..75e14a89148 100644 --- a/tests/unit/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/unit/proxy/proxy_server/test_routes_login_sso.py @@ -670,6 +670,12 @@ def test_the_master_key_as_a_bearer_token_still_works_while_the_ui_password_is_b def test_a_database_users_correct_password_is_refused_while_blocked(client, monkeypatch, reset_login_throttle): """The block is hard: while it lasts, nothing from that source signs in as that user, right password or not, and the block is not extended by the refused attempts.""" + from types import SimpleNamespace + + from litellm.proxy.auth import login_throttle + + frozen_now = login_throttle.time.time() + monkeypatch.setattr(login_throttle, "time", SimpleNamespace(time=lambda: frozen_now)) _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2, failed_login_block_seconds=64) _db_user(monkeypatch, "user@corp.com")