test(bedrock): retain source regressions on the release harness

This commit is contained in:
Joshua Valluru 2026-10-20 18:30:31 -07:00
parent 97c0f49cac
commit 2923238322
13 changed files with 145 additions and 1673 deletions

View file

@ -3862,9 +3862,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
{"supports_bedrock_native_structured_output"}
)
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_bedrock_native_structured_output"})
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})

View file

@ -267,3 +267,25 @@ def team_admin_permissions(gateway: Gateway, fields: Sequence[str]) -> Iterator[
yield
finally:
_set_team_admin_permissions(gateway, [str(field) for field in original] if isinstance(original, list) else ())
def held(read: Callable[[], T], satisfied: Callable[[T], bool], *, holding: float, seconds: float = 10) -> T:
"""`eventually`, and then `satisfied` must hold on every read for `holding` more seconds; a miss restarts the hold."""
deadline: Final = time.monotonic() + seconds
while True:
converged: Final = eventually(read, satisfied, seconds=max(deadline - time.monotonic(), 0.0))
misses: Final = _misses_within(read, satisfied, holding)
if not misses:
return converged
assert time.monotonic() < deadline, f"State did not hold: {misses[0]!r}"
time.sleep(0.1)
def _misses_within(read: Callable[[], T], satisfied: Callable[[T], bool], holding: float) -> tuple[T, ...]:
until: Final = time.monotonic() + holding
while time.monotonic() < until:
time.sleep(0.1)
observed: Final = read()
if not satisfied(observed):
return (observed,)
return ()

View file

@ -423,3 +423,21 @@ def main() -> None:
if __name__ == "__main__":
main()
def _aws_int_header(name: str, value: int) -> bytes:
name_bytes: Final = name.encode()
return struct.pack("!B", len(name_bytes)) + name_bytes + struct.pack("!B", 4) + struct.pack("!i", value)
def aws_event_stream_frame(headers: Mapping[str, str | int], payload: bytes) -> bytes:
"""One AWS event-stream frame: a string header is wire type 7, an int header wire type 4 (int32)."""
headers_bytes: Final = b"".join(
_aws_str_header(name, value) if isinstance(value, str) else _aws_int_header(name, value)
for name, value in headers.items()
)
total_length: Final = 12 + len(headers_bytes) + len(payload) + 4
prelude: Final = struct.pack("!II", total_length, len(headers_bytes))
prelude_crc: Final = struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF)
message: Final = prelude + prelude_crc + headers_bytes + payload
return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF)

View file

@ -1,12 +1,11 @@
from integration._support.process import owned_proxy
import yaml
from pathlib import Path
import json
import uuid
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway
from integration._support.mcp import (
ENTRY_POINTS,
@ -25,6 +24,7 @@ from integration._support.mcp import (
text_result,
tool_calls,
)
from integration._support.process import owned_proxy
ADD: Final = {"http": "add", "sse": "add", "stdio": "add", "openapi": "getpet"}
ARGUMENTS: Final = {"add": {"a": 3, "b": 4}, "getpet": {"petId": "7"}}

View file

@ -1,12 +1,10 @@
from typing import Final
# What is this?
## Unit testing for the 'get_model_info()' function
import os
import re
from collections.abc import Collection, Mapping
from typing import List, Dict, Any, Final, Literal
from typing import Any, Dict, Final, List, Literal
from unittest.mock import MagicMock, patch
import pytest
@ -15,7 +13,6 @@ from litellm import get_model_info
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.types.utils import ModelInfoBase
from litellm.utils import _invalidate_model_cost_lowercase_map
from unittest.mock import MagicMock, patch
def test_get_model_info_simple_model_name():
@ -251,8 +248,7 @@ def test_get_model_info_custom_provider():
def test_get_model_info_custom_model_router():
from litellm import Router
from litellm import get_model_info
from litellm import Router, get_model_info
litellm._turn_on_debug()

View file

@ -44,17 +44,25 @@ import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at im
import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency
import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency
from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency
from litellm.anthropic_beta_headers_manager import (
reload_beta_headers_config, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency
from litellm.litellm_core_utils.prompt_templates import (
factory as prompt_factory_module, # noqa: E402 # same import-time dependency
)
from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency
image_handling as image_handling_module,
)
from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency
from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency
close_litellm_async_clients,
)
from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module # noqa: E402 # same import-time dependency
from litellm.llms.gemini.chat import (
transformation as gemini_chat_transformation_module, # noqa: E402 # same import-time dependency
)
from litellm.proxy.db import (
tool_registry_writer as tool_registry_writer_module, # noqa: E402 # same import-time dependency
)
LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1", "localhost"]
AMBIENT_AZURE_CREDENTIAL_ENV_VARS: Final = (

View file

@ -1653,7 +1653,9 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co
@pytest.mark.asyncio
async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None:
from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ClientCredentialsBearerAuth
from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import (
ClientCredentialsBearerAuth,
)
seen = []
@ -1908,6 +1910,7 @@ async def test_sse_read_failure_is_preserved() -> None:
@pytest.mark.parametrize("mode", ["ok", "closed", "silent"])
async def test_transport_completion_and_normal_messages(transport: MCPTransport, mode: str, protocol_version: str) -> None:
from mcp import ClientSession
from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message
logging_callback: Final = AsyncMock()

View file

@ -1,4 +1,5 @@
import litellm
"""Tests for `BedrockConverseLLM.completion`.
AWS credential resolution is stubbed so nothing reaches STS.

View file

@ -2255,7 +2255,6 @@ async def test_assistant_message_cache_control():
"""Test that assistant messages with cache_control generate cachePoint blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
# Test assistant message with string content and cache_control
@ -2308,7 +2307,6 @@ async def test_assistant_message_list_content_cache_control():
"""Test assistant messages with list content and cache_control."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2352,7 +2350,6 @@ async def test_tool_message_cache_control():
"""Test that tool messages with cache_control generate cachePoint blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2416,7 +2413,6 @@ async def test_tool_message_string_content_cache_control():
"""Test tool messages with string content and message-level cache_control."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2472,7 +2468,6 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block()
"""OpenAI tool message search_results should map to Bedrock searchResult blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2533,7 +2528,6 @@ async def test_tool_message_empty_search_results_falls_back_to_content():
"""Empty search_results must not skip normal tool content processing."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2702,7 +2696,6 @@ async def test_assistant_tool_calls_cache_control():
"""Test that assistant tool_calls with cache_control generate cachePoint blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2754,7 +2747,6 @@ async def test_multiple_tool_calls_with_mixed_cache_control():
"""Test multiple tool calls where only some have cache_control."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2815,7 +2807,6 @@ async def test_no_cache_control_no_cache_point():
"""Test that messages without cache_control don't generate cachePoint blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -2860,9 +2851,6 @@ async def test_no_cache_control_no_cache_point():
def test_guarded_text_wraps_in_guardrail_converse_content():
"""Test that guarded_text content type gets wrapped in guardContent blocks."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{
@ -2963,9 +2951,6 @@ def test_guarded_text_with_system_messages():
def test_guarded_text_with_mixed_content_types():
"""Test guarded_text with mixed content types including images."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{
@ -3052,9 +3037,6 @@ async def test_async_guarded_text():
def test_guarded_text_with_tool_calls():
"""Test guarded_text with tool calls in the conversation."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{
@ -3704,9 +3686,6 @@ def test_empty_assistant_message_handling():
"""
# Import the litellm module that factory.py uses to ensure we patch the correct reference
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
# Test case 1: Empty string content - test with modify_params=True to prevent merging
messages = [
@ -3796,9 +3775,6 @@ def test_empty_assistant_message_handling():
def test_bedrock_converse_trailing_prefix_assistant_skips_user_continue():
"""Assistant prefill (prefix: true) must not inject a dummy user 'Please continue.' turn."""
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "user", "content": "Hello, how are you?"},
@ -3825,9 +3801,6 @@ def test_bedrock_converse_trailing_prefix_assistant_skips_user_continue():
def test_bedrock_converse_leading_prefix_assistant_skips_user_continue():
"""Leading assistant with prefix: true should not prepend dummy user."""
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "assistant", "content": "Partial", "prefix": True},
@ -6338,9 +6311,6 @@ def test_bedrock_tool_message_openai_file_pdf_becomes_document():
inside a tool message content list should translate to a Bedrock
toolResult.content[].document block.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK" # tiny "%PDF-1.4\n" header
messages = [
@ -6381,9 +6351,6 @@ def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document():
BedrockImageProcessor correctly returns a {"document": ...} block, but the
tool-result wrapper only appended the "image" case, silently dropping documents.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
messages = [
@ -6426,7 +6393,6 @@ def test_bedrock_tool_message_file_id_http_url_becomes_document():
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockImageProcessor,
_bedrock_converse_messages_pt,
)
pdf_url = "https://example.com/whitepaper.pdf"
@ -6480,9 +6446,6 @@ def test_bedrock_tool_message_file_without_data_or_id_raises():
see an empty tool result and obscures the caller bug.
"""
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "user", "content": "Summarize."},
@ -6509,9 +6472,6 @@ def test_bedrock_tool_message_image_url_png_still_becomes_image():
to a Bedrock image block (not document). Locks in existing behavior after
the document-passthrough fix.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
messages = [
@ -6708,7 +6668,6 @@ async def test_grounding_source_and_query_rendered_as_text():
otherwise vanish from the prompt."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = [
@ -7207,7 +7166,6 @@ async def test_message_level_cache_control_honors_ttl_for_supported_model(
hardcoded `{"type": "default"}`), silently downgrading 1h to 5m."""
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockConverseMessagesProcessor,
_bedrock_converse_messages_pt,
)
messages = _agentic_messages_with_ttl(ttl_target)
@ -7233,9 +7191,6 @@ async def test_message_level_cache_control_honors_ttl_for_supported_model(
def test_message_level_cache_control_drops_ttl_for_unsupported_model(ttl_target):
"""Models outside the extended-caching allow-list must keep emitting the
plain `{"type": "default"}` cachePoint (Bedrock rejects `ttl` for them)."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
result = _bedrock_converse_messages_pt(
messages=_agentic_messages_with_ttl(ttl_target),

View file

@ -46,7 +46,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once():
"""
get_bedrock_response_stream_shape() loads from botocore at most once per process.
"""
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import litellm.llms.bedrock.common_utils as mod
@ -74,7 +74,6 @@ def test_bedrock_response_stream_shape_load_failure_returns_none():
should return None rather than propagating the exception, so the module
still imports cleanly.
"""
from unittest.mock import patch
import litellm.llms.bedrock.common_utils as mod
@ -136,7 +135,7 @@ def test_bedrock_parse_message_from_event_raises_on_none_shape():
_parse_message_from_event must raise BedrockError before touching the
botocore parser — not an opaque AttributeError from inside botocore.
"""
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import litellm.llms.bedrock.common_utils as mod
from litellm.llms.bedrock.common_utils import (
@ -260,7 +259,6 @@ def test_context_window_suffix_stripped_for_cost_lookup():
Models configured like `bedrock/us.anthropic.claude-opus-4-6-v1[1m]`
should resolve to the base model name so pricing can be found.
"""
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1"
assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6"
@ -280,7 +278,6 @@ def test_context_window_suffix_stripped_for_cost_lookup():
def test_legacy_mantle_route_prefix_stripped_for_cost_lookup():
"""The mantle/ route token is a routing prefix like openai/, so a bedrock/mantle/<model>
deployment must resolve the bare Bedrock model for cost lookup while still routing to Mantle."""
from litellm.llms.bedrock.common_utils import get_bedrock_base_model, strip_bedrock_routing_prefix
assert strip_bedrock_routing_prefix("mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5"
assert get_bedrock_base_model("bedrock/mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5"
@ -417,7 +414,6 @@ def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_fi
must not shadow a base entry that has it (`get(model) or get(base)` used to
short-circuit on the truthy regional dict and drop the capability).
"""
import litellm
from litellm.llms.bedrock.common_utils import (
bedrock_converse_supports_parallel_tool_use_config,
is_claude_4_5_on_bedrock,
@ -543,7 +539,6 @@ def test_apply_structured_output_prefers_legacy_output_format(local_model_cost_m
def test_sign_aws_request_assumes_role_with_external_id(monkeypatch):
"""A trust policy requiring sts:ExternalId must be satisfied when signing batch API requests."""
import datetime
from unittest.mock import patch
import boto3
from botocore.exceptions import ClientError
@ -596,7 +591,6 @@ def test_sign_aws_request_assumes_role_with_external_id(monkeypatch):
def test_sign_aws_request_assumes_role_with_session_tags(monkeypatch):
"""Batch and file signing must carry the deployment's session tags into the AssumeRole call too."""
import datetime
from unittest.mock import patch
import boto3
from botocore.exceptions import ClientError
@ -855,7 +849,6 @@ def test_bedrock_mantle_get_error_class_keeps_provider_headers(config):
def _bedrock_configs_with_get_error_class():
import importlib
import inspect
import pathlib
@ -940,7 +933,6 @@ def test_unmapped_openai_family_model_routes_to_converse():
"""
from typing import Final
import litellm
unmapped: Final = "bedrock/global.openai.gpt-99-unmapped"
assert unmapped.removeprefix("bedrock/") not in litellm.bedrock_converse_models

View file

@ -1,7 +1,8 @@
import sys
import ast
import importlib
import inspect
import json
import sys
from unittest import mock
import httpx
@ -9,9 +10,6 @@ import pytest
import respx
from fastapi.testclient import TestClient
import importlib
import litellm
from litellm import constants

View file

@ -2,14 +2,11 @@ import asyncio
import base64
import contextlib
import contextvars
import copy
import importlib
import io
import json
import logging
import os
import queue
import re
import threading
from collections.abc import Callable, Iterator, Mapping
from concurrent.futures import Future, ThreadPoolExecutor
@ -40,7 +37,6 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor
from litellm.llms.base_llm.audio_transcription.transformation import BaseAudioTranscriptionConfig
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.proxy.utils import is_valid_api_key
from litellm.types.caching import CachingSupportedCallTypes
@ -76,19 +72,12 @@ from litellm.utils import (
_is_streaming_request,
_run_success_deployment_hook_on_converted_chat_stream,
_snapshot_exception_for_hook,
add_custom_logger_callback_to_specific_event,
async_post_call_failure_deployment_hook,
async_post_call_success_deployment_hook,
calculate_max_parallel_requests,
check_valid_key,
client,
filter_out_litellm_params,
get_applied_guardrails,
get_llm_provider,
get_non_default_completion_params,
get_optional_params_embeddings,
get_optional_params_image_gen,
get_potential_model_names,
get_prompt_cache_min_tokens,
is_cached_message,
is_prompt_caching_valid_prompt,
@ -3166,7 +3155,9 @@ class TestExtraBodyCannotOverrideModel:
}
]
untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools())
untouched = litellm.get_optional_params(
model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()
)
assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched
result = litellm.get_optional_params(
@ -4135,9 +4126,6 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model
)
def test_custom_logger_guards_ignore_subclass_instances(monkeypatch: pytest.MonkeyPatch) -> None:
"""Regression LIT-4392: the success/failure existence guards used isinstance, so a user
subclass of a built-in logger already promoted into the callback lists made the guard
@ -4583,13 +4571,7 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = (
pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"),
pytest.param(
ResponsesAPIResponse(
id="resp_abc",
created_at=1,
output=[],
parallel_tool_calls=False,
tool_choice="auto",
tools=[],
model="gpt-5.6",
id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6"
),
CallTypes.aresponses,
id="responses",
@ -4617,9 +4599,7 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo
assert result is response
assert second_hook.seen_responses == (response,)
failure_logs: Final = tuple(
r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message
)
failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message)
assert len(failure_logs) == 1
assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message
assert str(call_type) in failure_logs[0].message
@ -4743,9 +4723,6 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi
assert success_kwargs["stream"] is True
@pytest.mark.asyncio
async def test_wrapper_async_replays_cached_converted_chat_stream_as_stream(
monkeypatch: pytest.MonkeyPatch,
@ -6546,9 +6523,6 @@ def test_function_setup_never_logs_the_ocr_data_uri_payload() -> None:
assert payload not in str(logged)
@pytest.mark.asyncio
async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None:
"""Chat over the Responses bridge exits two @client wrappers with one logging object. Issue
@ -6611,6 +6585,3 @@ async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch:
assert len(counting_logger.logged_results) == 1, counting_logger.logged_results
assert counting_logger.logged_results[0] is inner_result