From 85585b63b0cedb95a6084e75a70be511a2025480 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:48:10 +0000 Subject: [PATCH] test: add translation characterization corpus pinning v1 transform seams Snapshot corpus for the translation v2 differential gate: 19 OpenAI-format request cases x {anthropic, bedrock converse, bedrock invoke} through the real v1 transform_request path (get_optional_params, validate_environment included), recorded provider responses through transform_response, SSE and event-stream replays through CustomStreamWrapper plus the real decoders, and both /v1/messages surfaces (anthropic<->openai adapter round-trip and the native AnthropicMessagesConfig quirks: max_tokens pop, reasoning_effort rewrite, advisor-block stripping). 89 canonical-JSON snapshots, deterministic via frozen uuid/fastuuid/time, regenerated with --snapshot-update or SNAPSHOT_UPDATE=1. Wired into CI as a job in test-unit-llm-providers.yml and a make test-characterization target --- .github/workflows/test-unit-llm-providers.yml | 13 + Makefile | 4 + tests/translation_characterization/README.md | 73 ++++ .../translation_characterization/__init__.py | 0 tests/translation_characterization/_corpus.py | 311 ++++++++++++++++++ .../translation_characterization/_helpers.py | 55 ++++ tests/translation_characterization/_seams.py | 164 +++++++++ .../cases/cache_control_messages.json | 33 ++ .../cases/cache_control_tools.json | 54 +++ .../cases/full_combo.json | 73 ++++ .../cases/image_base64.json | 24 ++ .../cases/image_url.json | 27 ++ .../cases/multi_turn.json | 21 ++ .../cases/params_sampling.json | 20 ++ .../cases/pdf_base64.json | 25 ++ .../cases/plain_text.json | 13 + .../cases/reasoning_effort_low.json | 14 + .../cases/response_format_json_object.json | 16 + .../cases/response_format_json_schema.json | 32 ++ .../cases/system_prompt.json | 17 + .../cases/thinking_enabled.json | 17 + .../cases/thinking_history_blocks.json | 32 ++ .../cases/tools_basic.json | 34 ++ .../cases/tools_forced_choice.json | 39 +++ .../cases/tools_parallel.json | 52 +++ .../cases/tools_streamed_args_roundtrip.json | 83 +++++ .../translation_characterization/conftest.py | 68 ++++ .../adapter/native/advisor_blocks_kept.json | 19 ++ .../native/advisor_blocks_stripped.json | 16 + .../adapter/native/basic_passthrough.json | 20 ++ .../native/reasoning_effort_rewrite.json | 5 + .../requests/cache_control_system.json | 9 + .../adapter/requests/image_base64.json | 20 ++ .../adapter/requests/simple_text.json | 5 + .../adapter/requests/system_and_tools.json | 20 ++ .../adapter/requests/thinking_enabled.json | 6 + .../requests/tool_result_roundtrip.json | 31 ++ .../responses/cached_usage_reasoning.json | 24 ++ .../adapter/responses/text_basic.json | 14 + .../adapter/responses/tool_calls.json | 29 ++ .../responses/anthropic/cache_usage.json | 16 + .../responses/anthropic/stop_sequence.json | 10 + .../responses/anthropic/text_basic.json | 10 + .../responses/anthropic/thinking.json | 13 + .../anthropic/tool_use_parallel.json | 14 + .../bedrock_converse/cache_usage.json | 17 + .../responses/bedrock_converse/reasoning.json | 14 + .../bedrock_converse/text_basic.json | 11 + .../responses/bedrock_converse/tool_use.json | 14 + .../responses/bedrock_invoke/text_basic.json | 10 + .../responses/bedrock_invoke/thinking.json | 13 + .../responses/bedrock_invoke/tool_use.json | 13 + .../streams/anthropic/text_stream.txt | 23 ++ .../streams/anthropic/thinking_stream.txt | 32 ++ .../streams/anthropic/tool_stream.txt | 32 ++ .../bedrock_converse/cache_usage_stream.json | 7 + .../bedrock_converse/reasoning_stream.json | 11 + .../streams/bedrock_converse/text_stream.json | 8 + .../streams/bedrock_converse/tool_stream.json | 12 + .../streams/bedrock_invoke/text_stream.json | 9 + .../streams/bedrock_invoke/tool_stream.json | 10 + .../adapter/native/advisor_blocks_kept.json | 47 +++ .../native/advisor_blocks_stripped.json | 23 ++ .../adapter/native/basic_passthrough.json | 29 ++ .../native/reasoning_effort_rewrite.json | 14 + .../requests/cache_control_system.json | 23 ++ .../adapter/requests/image_base64.json | 24 ++ .../adapter/requests/simple_text.json | 13 + .../adapter/requests/system_and_tools.json | 42 +++ .../adapter/requests/thinking_enabled.json | 14 + .../requests/tool_result_roundtrip.json | 53 +++ .../responses/cached_usage_reasoning.json | 24 ++ .../adapter/responses/text_basic.json | 18 + .../adapter/responses/tool_calls.json | 32 ++ .../anthropic/cache_control_messages.json | 27 ++ .../anthropic/cache_control_tools.json | 52 +++ .../requests/anthropic/full_combo.json | 74 +++++ .../requests/anthropic/image_base64.json | 23 ++ .../requests/anthropic/image_url.json | 22 ++ .../requests/anthropic/multi_turn.json | 33 ++ .../requests/anthropic/params_sampling.json | 24 ++ .../requests/anthropic/pdf_base64.json | 23 ++ .../requests/anthropic/plain_text.json | 15 + .../anthropic/reasoning_effort_low.json | 19 ++ .../response_format_json_object.json | 15 + .../response_format_json_schema.json | 37 +++ .../requests/anthropic/system_prompt.json | 21 ++ .../requests/anthropic/thinking_enabled.json | 19 ++ .../anthropic/thinking_history_blocks.json | 42 +++ .../requests/anthropic/tools_basic.json | 36 ++ .../anthropic/tools_forced_choice.json | 37 +++ .../requests/anthropic/tools_parallel.json | 53 +++ .../tools_streamed_args_roundtrip.json | 85 +++++ .../cache_control_messages.json | 33 ++ .../bedrock_converse/cache_control_tools.json | 65 ++++ .../requests/bedrock_converse/full_combo.json | 88 +++++ .../bedrock_converse/image_base64.json | 26 ++ .../requests/bedrock_converse/multi_turn.json | 34 ++ .../bedrock_converse/params_sampling.json | 24 ++ .../requests/bedrock_converse/pdf_base64.json | 27 ++ .../requests/bedrock_converse/plain_text.json | 18 + .../reasoning_effort_low.json | 22 ++ .../response_format_json_object.json | 18 + .../response_format_json_schema.json | 47 +++ .../bedrock_converse/system_prompt.json | 23 ++ .../bedrock_converse/thinking_enabled.json | 22 ++ .../thinking_history_blocks.json | 46 +++ .../bedrock_converse/tools_basic.json | 44 +++ .../bedrock_converse/tools_forced_choice.json | 46 +++ .../bedrock_converse/tools_parallel.json | 60 ++++ .../tools_streamed_args_roundtrip.json | 108 ++++++ .../cache_control_messages.json | 27 ++ .../bedrock_invoke/cache_control_tools.json | 52 +++ .../requests/bedrock_invoke/full_combo.json | 74 +++++ .../requests/bedrock_invoke/image_base64.json | 23 ++ .../requests/bedrock_invoke/multi_turn.json | 33 ++ .../bedrock_invoke/params_sampling.json | 24 ++ .../requests/bedrock_invoke/pdf_base64.json | 23 ++ .../requests/bedrock_invoke/plain_text.json | 15 + .../bedrock_invoke/reasoning_effort_low.json | 19 ++ .../response_format_json_object.json | 15 + .../response_format_json_schema.json | 37 +++ .../bedrock_invoke/system_prompt.json | 21 ++ .../bedrock_invoke/thinking_enabled.json | 19 ++ .../thinking_history_blocks.json | 42 +++ .../requests/bedrock_invoke/tools_basic.json | 36 ++ .../bedrock_invoke/tools_forced_choice.json | 37 +++ .../bedrock_invoke/tools_parallel.json | 53 +++ .../tools_streamed_args_roundtrip.json | 85 +++++ .../responses/anthropic/cache_usage.json | 48 +++ .../responses/anthropic/stop_sequence.json | 44 +++ .../responses/anthropic/text_basic.json | 44 +++ .../responses/anthropic/thinking.json | 58 ++++ .../anthropic/tool_use_parallel.json | 63 ++++ .../bedrock_converse/cache_usage.json | 38 +++ .../responses/bedrock_converse/reasoning.json | 56 ++++ .../bedrock_converse/text_basic.json | 38 +++ .../responses/bedrock_converse/tool_use.json | 48 +++ .../responses/bedrock_invoke/text_basic.json | 44 +++ .../responses/bedrock_invoke/thinking.json | 58 ++++ .../responses/bedrock_invoke/tool_use.json | 54 +++ .../streams/anthropic/text_stream.json | 72 ++++ .../streams/anthropic/thinking_stream.json | 168 ++++++++++ .../streams/anthropic/tool_stream.json | 184 +++++++++++ .../bedrock_converse/cache_usage_stream.json | 72 ++++ .../bedrock_converse/reasoning_stream.json | 178 ++++++++++ .../streams/bedrock_converse/text_stream.json | 96 ++++++ .../streams/bedrock_converse/tool_stream.json | 208 ++++++++++++ .../streams/bedrock_invoke/text_stream.json | 72 ++++ .../streams/bedrock_invoke/tool_stream.json | 160 +++++++++ .../test_messages_adapter.py | 101 ++++++ .../test_request_transforms.py | 32 ++ .../test_response_transforms.py | 30 ++ .../test_stream_replay.py | 50 +++ 154 files changed, 6246 insertions(+) create mode 100644 tests/translation_characterization/README.md create mode 100644 tests/translation_characterization/__init__.py create mode 100644 tests/translation_characterization/_corpus.py create mode 100644 tests/translation_characterization/_helpers.py create mode 100644 tests/translation_characterization/_seams.py create mode 100644 tests/translation_characterization/cases/cache_control_messages.json create mode 100644 tests/translation_characterization/cases/cache_control_tools.json create mode 100644 tests/translation_characterization/cases/full_combo.json create mode 100644 tests/translation_characterization/cases/image_base64.json create mode 100644 tests/translation_characterization/cases/image_url.json create mode 100644 tests/translation_characterization/cases/multi_turn.json create mode 100644 tests/translation_characterization/cases/params_sampling.json create mode 100644 tests/translation_characterization/cases/pdf_base64.json create mode 100644 tests/translation_characterization/cases/plain_text.json create mode 100644 tests/translation_characterization/cases/reasoning_effort_low.json create mode 100644 tests/translation_characterization/cases/response_format_json_object.json create mode 100644 tests/translation_characterization/cases/response_format_json_schema.json create mode 100644 tests/translation_characterization/cases/system_prompt.json create mode 100644 tests/translation_characterization/cases/thinking_enabled.json create mode 100644 tests/translation_characterization/cases/thinking_history_blocks.json create mode 100644 tests/translation_characterization/cases/tools_basic.json create mode 100644 tests/translation_characterization/cases/tools_forced_choice.json create mode 100644 tests/translation_characterization/cases/tools_parallel.json create mode 100644 tests/translation_characterization/cases/tools_streamed_args_roundtrip.json create mode 100644 tests/translation_characterization/conftest.py create mode 100644 tests/translation_characterization/fixtures/adapter/native/advisor_blocks_kept.json create mode 100644 tests/translation_characterization/fixtures/adapter/native/advisor_blocks_stripped.json create mode 100644 tests/translation_characterization/fixtures/adapter/native/basic_passthrough.json create mode 100644 tests/translation_characterization/fixtures/adapter/native/reasoning_effort_rewrite.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/cache_control_system.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/image_base64.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/simple_text.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/system_and_tools.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/thinking_enabled.json create mode 100644 tests/translation_characterization/fixtures/adapter/requests/tool_result_roundtrip.json create mode 100644 tests/translation_characterization/fixtures/adapter/responses/cached_usage_reasoning.json create mode 100644 tests/translation_characterization/fixtures/adapter/responses/text_basic.json create mode 100644 tests/translation_characterization/fixtures/adapter/responses/tool_calls.json create mode 100644 tests/translation_characterization/fixtures/responses/anthropic/cache_usage.json create mode 100644 tests/translation_characterization/fixtures/responses/anthropic/stop_sequence.json create mode 100644 tests/translation_characterization/fixtures/responses/anthropic/text_basic.json create mode 100644 tests/translation_characterization/fixtures/responses/anthropic/thinking.json create mode 100644 tests/translation_characterization/fixtures/responses/anthropic/tool_use_parallel.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_converse/cache_usage.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_converse/reasoning.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_converse/text_basic.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_converse/tool_use.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_invoke/text_basic.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_invoke/thinking.json create mode 100644 tests/translation_characterization/fixtures/responses/bedrock_invoke/tool_use.json create mode 100644 tests/translation_characterization/fixtures/streams/anthropic/text_stream.txt create mode 100644 tests/translation_characterization/fixtures/streams/anthropic/thinking_stream.txt create mode 100644 tests/translation_characterization/fixtures/streams/anthropic/tool_stream.txt create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_converse/cache_usage_stream.json create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_converse/reasoning_stream.json create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_converse/text_stream.json create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_converse/tool_stream.json create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_invoke/text_stream.json create mode 100644 tests/translation_characterization/fixtures/streams/bedrock_invoke/tool_stream.json create mode 100644 tests/translation_characterization/snapshots/adapter/native/advisor_blocks_kept.json create mode 100644 tests/translation_characterization/snapshots/adapter/native/advisor_blocks_stripped.json create mode 100644 tests/translation_characterization/snapshots/adapter/native/basic_passthrough.json create mode 100644 tests/translation_characterization/snapshots/adapter/native/reasoning_effort_rewrite.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/cache_control_system.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/image_base64.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/simple_text.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/system_and_tools.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/thinking_enabled.json create mode 100644 tests/translation_characterization/snapshots/adapter/requests/tool_result_roundtrip.json create mode 100644 tests/translation_characterization/snapshots/adapter/responses/cached_usage_reasoning.json create mode 100644 tests/translation_characterization/snapshots/adapter/responses/text_basic.json create mode 100644 tests/translation_characterization/snapshots/adapter/responses/tool_calls.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/cache_control_messages.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/cache_control_tools.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/full_combo.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/image_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/image_url.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/multi_turn.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/params_sampling.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/pdf_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/plain_text.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/reasoning_effort_low.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/response_format_json_object.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/response_format_json_schema.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/system_prompt.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/thinking_enabled.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/thinking_history_blocks.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/tools_basic.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/tools_forced_choice.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/tools_parallel.json create mode 100644 tests/translation_characterization/snapshots/requests/anthropic/tools_streamed_args_roundtrip.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_messages.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_tools.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/full_combo.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/image_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/multi_turn.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/params_sampling.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/pdf_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/plain_text.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/reasoning_effort_low.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_object.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_schema.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/system_prompt.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_enabled.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_history_blocks.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/tools_basic.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/tools_forced_choice.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/tools_parallel.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_converse/tools_streamed_args_roundtrip.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_messages.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_tools.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/full_combo.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/image_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/multi_turn.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/params_sampling.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/pdf_base64.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/plain_text.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/reasoning_effort_low.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_object.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_schema.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/system_prompt.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_enabled.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_history_blocks.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_basic.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_forced_choice.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_parallel.json create mode 100644 tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_streamed_args_roundtrip.json create mode 100644 tests/translation_characterization/snapshots/responses/anthropic/cache_usage.json create mode 100644 tests/translation_characterization/snapshots/responses/anthropic/stop_sequence.json create mode 100644 tests/translation_characterization/snapshots/responses/anthropic/text_basic.json create mode 100644 tests/translation_characterization/snapshots/responses/anthropic/thinking.json create mode 100644 tests/translation_characterization/snapshots/responses/anthropic/tool_use_parallel.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_converse/cache_usage.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_converse/reasoning.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_converse/text_basic.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_converse/tool_use.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_invoke/text_basic.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_invoke/thinking.json create mode 100644 tests/translation_characterization/snapshots/responses/bedrock_invoke/tool_use.json create mode 100644 tests/translation_characterization/snapshots/streams/anthropic/text_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/anthropic/thinking_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/anthropic/tool_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_converse/cache_usage_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_converse/reasoning_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_converse/text_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_converse/tool_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_invoke/text_stream.json create mode 100644 tests/translation_characterization/snapshots/streams/bedrock_invoke/tool_stream.json create mode 100644 tests/translation_characterization/test_messages_adapter.py create mode 100644 tests/translation_characterization/test_request_transforms.py create mode 100644 tests/translation_characterization/test_response_transforms.py create mode 100644 tests/translation_characterization/test_stream_replay.py diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index 2a1912ce92d..41fd7f07010 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -29,6 +29,19 @@ jobs: reruns: 2 artifact-name: llm-vertex-ai + translation-characterization: + name: Translation Characterization (v1 snapshot corpus) + permissions: + contents: read + id-token: write + pull-requests: write + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/translation_characterization" + workers: 1 + reruns: 0 + artifact-name: translation-characterization + other-providers: name: All Other Providers permissions: diff --git a/Makefile b/Makefile index 3d7b51bc745..26183c87c2f 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ help: @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-characterization - Run translation v1 snapshot corpus" @echo " make test-unit-llms - Run LLM provider tests (~225 files)" @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" @@ -141,6 +142,9 @@ test: install-test-deps test-unit: install-test-deps $(UV_RUN) pytest tests/test_litellm -x -vv -n 4 +test-characterization: install-test-deps + $(UV_RUN) pytest tests/translation_characterization --tb=short -vv + # Matrix test targets (matching CI workflow groups) test-unit-llms: install-test-deps $(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 diff --git a/tests/translation_characterization/README.md b/tests/translation_characterization/README.md new file mode 100644 index 00000000000..297dada292d --- /dev/null +++ b/tests/translation_characterization/README.md @@ -0,0 +1,73 @@ +# Translation characterization corpus + +Pins v1 core-translation behavior at the transform seams so the v2 rewrite +(02-mateo-scope.md, "How you prove it's safe") has a differential gate. Plain +pytest, no network, no API keys, fully deterministic. + +## Layout + +- `cases/` - request corpus: OpenAI-format chat cases, one JSON file per stable + case id, generated by `_corpus.py` (the cross-product of message shapes x + features: tools incl. parallel and streamed args, thinking, cache_control, + response_format json_schema/json_object, images, pdf, sampling params) +- `fixtures/` - handwritten recorded provider payloads: `responses/` (provider + response JSON per API spec), `streams/` (anthropic raw SSE lines, bedrock + parsed event payloads), `adapter/` (anthropic-format requests, OpenAI + responses, native /v1/messages cases) +- `snapshots/` - the pinned v1 output, canonical JSON (pretty-printed, sorted + keys, trailing newline): `requests//.json`, + `responses/...`, `streams/...`, `adapter/...` +- `_seams.py` - invokes the REAL v1 entry points the way `completion()` does: + `get_llm_provider` -> `get_optional_params` -> `get_litellm_params` -> + `ProviderConfigManager.get_provider_chat_config` -> `validate_environment` -> + `transform_request`/`transform_response`, and `CustomStreamWrapper` over the + real provider stream decoders + +Providers covered: `anthropic` chat, `bedrock_converse`, `bedrock_invoke` +(anthropic messages route), plus both /v1/messages surfaces (the +anthropic<->openai adapter and the native `AnthropicMessagesConfig` request +transform with its max_tokens/reasoning_effort/advisor-block quirks). + +## Running + +```bash +pytest tests/translation_characterization +``` + +Regenerate snapshots (and the generated `cases/` files): + +```bash +pytest tests/translation_characterization --snapshot-update +# or, when the conftest is not the initial one (e.g. `pytest tests/`): +SNAPSHOT_UPDATE=1 pytest tests/translation_characterization +``` + +Determinism: `uuid.uuid4`, `fastuuid.uuid4` (via `litellm._uuid`) and +`time.time` are frozen per test in `conftest.py` because v1 mints +`chatcmpl-` ids and `created` timestamps inside the seams. + +## How v2 consumes this + +The differential gate runs the SAME corpus (`cases/` + `fixtures/`) through the +v2 implementation and renders its output with the same canonical JSON helper +(`_helpers.canonical_json`); parity means byte-identical files against +`snapshots/`. The diff is the review artifact. + +Rule: behavior changes never ride along with a port. An intentional behavior +change ships as its own PR that edits only the snapshots it means to change, +regenerated with `--snapshot-update`. + +## Seams not pinned bit-exactly, and why + +- Bedrock binary event-stream framing: production bytes pass through + botocore's `EventStreamBuffer` before reaching the decoders. The corpus + replays the parsed event payloads into the real + `AWSEventStreamDecoder.converse_chunk_parser` / + `AmazonAnthropicClaudeStreamDecoder._chunk_parser`; the binary framing is + AWS plumbing, not translation logic. +- URL-sourced media on bedrock (`image_url` case): v1 downloads the URL during + transform, so those provider cells are skipped with a recorded reason in the + case's `skip` map (anthropic passes URLs through and is pinned). +- Anthropic SSE `error` events: the v1 iterator raises mid-stream instead of + yielding a chunk, so there is no JSON output to snapshot; the raise path is + exception-shaped, not corpus-shaped. diff --git a/tests/translation_characterization/__init__.py b/tests/translation_characterization/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/translation_characterization/_corpus.py b/tests/translation_characterization/_corpus.py new file mode 100644 index 00000000000..8367265c050 --- /dev/null +++ b/tests/translation_characterization/_corpus.py @@ -0,0 +1,311 @@ +"""Request-corpus builder: OpenAI-format chat cases covering the feature grid. + +Each case is written to ``cases/.json`` (stable id, committed) and is +pushed through the REAL v1 request transform for every provider in +``_seams.PROVIDERS``. ``skip`` maps a provider key to the reason that provider +cannot run the case hermetically (e.g. v1 downloads URL media for bedrock). + +Regenerate the case files with ``--snapshot-update`` (test_corpus_files_in_sync) +or ``python -m tests.translation_characterization._corpus``. +""" + +import base64 +from typing import Any, Dict + +PNG_1PX = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAC" + "hwGA60e6kgAAAABJRU5ErkJggg==" +) +PDF_MIN = base64.b64encode( + b"%PDF-1.4\n1 0 obj<>endobj\n" + b"2 0 obj<>endobj\n" + b"trailer<>\n%%EOF\n" +).decode() + +_USER = {"role": "user", "content": "What is the capital of France?"} +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} +_TIME_TOOL = { + "type": "function", + "function": { + "name": "get_time", + "description": "Get current time for a timezone.", + "parameters": { + "type": "object", + "properties": {"tz": {"type": "string"}}, + "required": ["tz"], + }, + }, +} +_PARALLEL_TOOL_CALLS = [ + { + "id": "call_char_001", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + { + "id": "call_char_002", + "type": "function", + "function": {"name": "get_time", "arguments": '{"tz": "Europe/Paris"}'}, + }, +] +_URL_MEDIA_SKIP = "v1 downloads URL media for bedrock transforms (network)" + + +def build_request_cases() -> Dict[str, Dict[str, Any]]: + cases: Dict[str, Dict[str, Any]] = {} + + def add(case_id: str, messages: list, params: dict, skip: dict = {}) -> None: + cases[case_id] = { + "id": case_id, + "messages": messages, + "params": params, + "skip": skip, + } + + add("plain_text", [_USER], {"max_tokens": 256}) + add( + "multi_turn", + [ + _USER, + {"role": "assistant", "content": "Paris."}, + {"role": "user", "content": "And of Italy?"}, + ], + {"max_tokens": 256}, + ) + add( + "system_prompt", + [{"role": "system", "content": "You are a terse geographer."}, _USER], + {"max_tokens": 256}, + ) + add( + "params_sampling", + [_USER], + { + "max_tokens": 128, + "temperature": 0.2, + "top_p": 0.9, + "stop": ["\n\n", "END"], + "user": "char-user-1234", + }, + ) + add( + "tools_basic", + [_USER], + {"max_tokens": 256, "tools": [_WEATHER_TOOL], "tool_choice": "auto"}, + ) + add( + "tools_parallel", + [_USER], + { + "max_tokens": 256, + "tools": [_WEATHER_TOOL, _TIME_TOOL], + "parallel_tool_calls": True, + }, + ) + add( + "tools_forced_choice", + [_USER], + { + "max_tokens": 256, + "tools": [_WEATHER_TOOL], + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + }, + ) + add( + "tools_streamed_args_roundtrip", + [ + {"role": "user", "content": "Weather and time in Paris?"}, + {"role": "assistant", "content": None, "tool_calls": _PARALLEL_TOOL_CALLS}, + {"role": "tool", "tool_call_id": "call_char_001", "content": "18C, clear"}, + {"role": "tool", "tool_call_id": "call_char_002", "content": "14:05"}, + ], + {"max_tokens": 256, "tools": [_WEATHER_TOOL, _TIME_TOOL]}, + ) + add( + "thinking_enabled", + [_USER], + {"max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}}, + ) + add( + "thinking_history_blocks", + [ + _USER, + { + "role": "assistant", + "content": "Paris.", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user asks about France; the capital is Paris.", + "signature": "sig-char-deadbeef", + } + ], + }, + {"role": "user", "content": "And of Italy?"}, + ], + {"max_tokens": 2048, "thinking": {"type": "enabled", "budget_tokens": 1024}}, + ) + add("reasoning_effort_low", [_USER], {"max_tokens": 2048, "reasoning_effort": "low"}) + add( + "cache_control_messages", + [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a terse geographer.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is the capital of France?", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ], + {"max_tokens": 256}, + ) + add( + "cache_control_tools", + [_USER], + { + "max_tokens": 256, + "tools": [ + _WEATHER_TOOL, + {**_TIME_TOOL, "cache_control": {"type": "ephemeral"}}, + ], + }, + ) + add( + "response_format_json_schema", + [_USER], + { + "max_tokens": 256, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "capital", + "strict": True, + "schema": { + "type": "object", + "properties": {"capital": {"type": "string"}}, + "required": ["capital"], + "additionalProperties": False, + }, + }, + }, + }, + ) + add( + "response_format_json_object", + [{"role": "user", "content": "Reply in JSON: capital of France?"}], + {"max_tokens": 256, "response_format": {"type": "json_object"}}, + ) + add( + "image_base64", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{PNG_1PX}"}, + }, + ], + } + ], + {"max_tokens": 256}, + ) + add( + "image_url", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.png"}, + }, + ], + } + ], + {"max_tokens": 256}, + skip={"bedrock_converse": _URL_MEDIA_SKIP, "bedrock_invoke": _URL_MEDIA_SKIP}, + ) + add( + "pdf_base64", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document."}, + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{PDF_MIN}", + "filename": "char.pdf", + }, + }, + ], + } + ], + {"max_tokens": 256}, + ) + add( + "full_combo", + [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a terse geographer.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": "Weather in Paris?"}, + {"role": "assistant", "content": None, "tool_calls": [_PARALLEL_TOOL_CALLS[0]]}, + {"role": "tool", "tool_call_id": "call_char_001", "content": "18C, clear"}, + {"role": "user", "content": "Summarize, then check the weather again."}, + ], + { + "max_tokens": 2048, + "temperature": 1, + "tools": [_WEATHER_TOOL], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + }, + ) + return cases + + +def write_case_files() -> None: + from ._helpers import CASES_DIR, canonical_json + + CASES_DIR.mkdir(parents=True, exist_ok=True) + for case_id, case in build_request_cases().items(): + (CASES_DIR / f"{case_id}.json").write_text(canonical_json(case)) + + +if __name__ == "__main__": + write_case_files() diff --git a/tests/translation_characterization/_helpers.py b/tests/translation_characterization/_helpers.py new file mode 100644 index 00000000000..91dc2d97ac4 --- /dev/null +++ b/tests/translation_characterization/_helpers.py @@ -0,0 +1,55 @@ +"""Snapshot machinery shared by all characterization tests. + +A snapshot is canonical JSON: pretty-printed, sorted keys, trailing newline. +v2's differential gate re-runs the same corpus through v2 and asserts the +emitted JSON is byte-identical to these files. +""" + +import json +from pathlib import Path +from typing import Any + +PKG_ROOT = Path(__file__).parent +CASES_DIR = PKG_ROOT / "cases" +FIXTURES_DIR = PKG_ROOT / "fixtures" +SNAPSHOTS_DIR = PKG_ROOT / "snapshots" + + +def jsonable(obj: Any) -> Any: + """Best-effort conversion of v1 output objects into plain JSON data.""" + if hasattr(obj, "model_dump"): + return jsonable(obj.model_dump()) + if isinstance(obj, dict): + return {str(k): jsonable(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [jsonable(v) for v in obj] + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + return repr(obj) + + +def canonical_json(obj: Any) -> str: + return json.dumps(jsonable(obj), indent=2, sort_keys=True) + "\n" + + +def load_json(path: Path) -> Any: + with open(path) as f: + return json.load(f) + + +def assert_snapshot(rel_path: str, obj: Any, update: bool) -> None: + """Compare ``obj`` against ``snapshots/``; rewrite when updating.""" + path = SNAPSHOTS_DIR / rel_path + rendered = canonical_json(obj) + if update: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered) + return + assert path.exists(), ( + f"Missing snapshot {path}. Run with --snapshot-update (or SNAPSHOT_UPDATE=1) " + "to record it, then commit the file." + ) + assert rendered == path.read_text(), ( + f"Characterization drift for {rel_path}. If this change is intentional, " + "regenerate with --snapshot-update and ship the snapshot diff as its own PR." + ) diff --git a/tests/translation_characterization/_seams.py b/tests/translation_characterization/_seams.py new file mode 100644 index 00000000000..d8d34fbf78e --- /dev/null +++ b/tests/translation_characterization/_seams.py @@ -0,0 +1,164 @@ +"""Invokers for the REAL v1 transform seams, called the way completion() calls them. + +Request path mirrors ``main.py`` + ``base_llm_http_handler``: +``get_llm_provider`` -> ``get_optional_params`` (provider ``map_openai_params``) +-> ``get_litellm_params`` -> ``ProviderConfigManager.get_provider_chat_config`` +-> ``validate_environment`` -> ``transform_request``. + +Nothing in the transforms is mocked; the only stubbed ambient context is the +LiteLLM ``Logging`` object (the real class, fixed call ids) and an +``httpx.Response`` carrying the recorded provider payload. +""" + +import copy +import time +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import httpx + +import litellm +from litellm.litellm_core_utils.get_litellm_params import get_litellm_params +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, get_optional_params + +PROVIDERS: Dict[str, str] = { + "anthropic": "anthropic/claude-sonnet-4-20250514", + "bedrock_converse": "bedrock/anthropic.claude-sonnet-4-20250514-v1:0", + "bedrock_invoke": "bedrock/invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", +} + + +def resolve(provider_key: str) -> Tuple[str, str, Any]: + """(model, custom_llm_provider, provider chat config) — v1's own resolution.""" + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=PROVIDERS[provider_key] + ) + config = ProviderConfigManager.get_provider_chat_config( + model=model, provider=LlmProviders(custom_llm_provider) + ) + assert config is not None + return model, custom_llm_provider, config + + +def make_logging(model: str, messages: List[dict], stream: bool = False) -> Logging: + return Logging( + model=model, + messages=messages, + stream=stream, + call_type="completion", + start_time=time.time(), + litellm_call_id="char-litellm-call-id", + function_id="char-function-id", + ) + + +def run_request_transform(provider_key: str, case: Dict[str, Any]) -> Dict[str, Any]: + model, custom_llm_provider, config = resolve(provider_key) + messages = copy.deepcopy(case["messages"]) + optional_params = get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + messages=messages, + **copy.deepcopy(case["params"]), + ) + litellm_params = get_litellm_params(custom_llm_provider=custom_llm_provider) + headers = config.validate_environment( + headers={}, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None if provider_key.startswith("bedrock") else "sk-ant-char-test-key", + api_base=None, + ) + return config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + +def run_response_transform( + provider_key: str, + provider_response: Dict[str, Any], + messages: List[dict], + optional_params: Optional[Dict[str, Any]] = None, +) -> litellm.ModelResponse: + model, _, config = resolve(provider_key) + raw_response = httpx.Response( + status_code=200, + json=provider_response, + request=httpx.Request("POST", "https://characterization.invalid/v1/messages"), + ) + return config.transform_response( + model=model, + raw_response=raw_response, + model_response=litellm.ModelResponse(), + logging_obj=make_logging(model, messages), + request_data={}, + messages=messages, + optional_params=optional_params or {}, + litellm_params={}, + encoding=litellm.encoding, + api_key=None, + json_mode=None, + ) + + +def _wrap_stream( + provider_key: str, model: str, completion_stream: Iterable[Any] +) -> List[dict]: + custom_llm_provider = ( + "anthropic" if provider_key == "anthropic" else "bedrock" + ) + wrapper = CustomStreamWrapper( + completion_stream=iter(completion_stream), + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=make_logging( + model, [{"role": "user", "content": "stream"}], stream=True + ), + ) + return [chunk.model_dump() for chunk in wrapper] + + +def replay_anthropic_sse(sse_lines: List[str]) -> List[dict]: + """Recorded anthropic SSE lines -> provider iterator -> CustomStreamWrapper.""" + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + model, _, _ = resolve("anthropic") + iterator = ModelResponseIterator( + streaming_response=iter(sse_lines), sync_stream=True + ) + return _wrap_stream("anthropic", model, iterator) + + +def replay_bedrock_converse_events(events: List[dict]) -> List[dict]: + """Parsed converse stream event payloads -> AWSEventStreamDecoder parser. + + Pinned at the parsed-event seam: the binary AWS event-stream framing that + precedes ``converse_chunk_parser`` in production is botocore plumbing, not + translation logic (documented in README.md). + """ + from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + + model, _, _ = resolve("bedrock_converse") + decoder = AWSEventStreamDecoder(model=model) + chunks = (decoder.converse_chunk_parser(event) for event in events) + return _wrap_stream("bedrock_converse", model, chunks) + + +def replay_bedrock_invoke_events(events: List[dict]) -> List[dict]: + """Parsed invoke (anthropic messages) event payloads -> claude stream decoder.""" + from litellm.llms.bedrock.chat.invoke_handler import ( + AmazonAnthropicClaudeStreamDecoder, + ) + + model, _, _ = resolve("bedrock_invoke") + decoder = AmazonAnthropicClaudeStreamDecoder(model=model, sync_stream=True) + chunks = (decoder._chunk_parser(event) for event in events) + return _wrap_stream("bedrock_invoke", model, chunks) diff --git a/tests/translation_characterization/cases/cache_control_messages.json b/tests/translation_characterization/cases/cache_control_messages.json new file mode 100644 index 00000000000..2038a02658f --- /dev/null +++ b/tests/translation_characterization/cases/cache_control_messages.json @@ -0,0 +1,33 @@ +{ + "id": "cache_control_messages", + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ], + "role": "system" + }, + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/cache_control_tools.json b/tests/translation_characterization/cases/cache_control_tools.json new file mode 100644 index 00000000000..9b858e741b4 --- /dev/null +++ b/tests/translation_characterization/cases/cache_control_tools.json @@ -0,0 +1,54 @@ +{ + "id": "cache_control_tools", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + }, + { + "cache_control": { + "type": "ephemeral" + }, + "function": { + "description": "Get current time for a timezone.", + "name": "get_time", + "parameters": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/full_combo.json b/tests/translation_characterization/cases/full_combo.json new file mode 100644 index 00000000000..587cc135e13 --- /dev/null +++ b/tests/translation_characterization/cases/full_combo.json @@ -0,0 +1,73 @@ +{ + "id": "full_combo", + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ], + "role": "system" + }, + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "call_char_001", + "type": "function" + } + ] + }, + { + "content": "18C, clear", + "role": "tool", + "tool_call_id": "call_char_001" + }, + { + "content": "Summarize, then check the weather again.", + "role": "user" + } + ], + "params": { + "max_tokens": 2048, + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + }, + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/image_base64.json b/tests/translation_characterization/cases/image_base64.json new file mode 100644 index 00000000000..a22a64db968 --- /dev/null +++ b/tests/translation_characterization/cases/image_base64.json @@ -0,0 +1,24 @@ +{ + "id": "image_base64", + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/image_url.json b/tests/translation_characterization/cases/image_url.json new file mode 100644 index 00000000000..001a8162237 --- /dev/null +++ b/tests/translation_characterization/cases/image_url.json @@ -0,0 +1,27 @@ +{ + "id": "image_url", + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "image_url": { + "url": "https://example.com/cat.png" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": { + "bedrock_converse": "v1 downloads URL media for bedrock transforms (network)", + "bedrock_invoke": "v1 downloads URL media for bedrock transforms (network)" + } +} diff --git a/tests/translation_characterization/cases/multi_turn.json b/tests/translation_characterization/cases/multi_turn.json new file mode 100644 index 00000000000..1c69bffa872 --- /dev/null +++ b/tests/translation_characterization/cases/multi_turn.json @@ -0,0 +1,21 @@ +{ + "id": "multi_turn", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + }, + { + "content": "Paris.", + "role": "assistant" + }, + { + "content": "And of Italy?", + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/params_sampling.json b/tests/translation_characterization/cases/params_sampling.json new file mode 100644 index 00000000000..f065a9170cc --- /dev/null +++ b/tests/translation_characterization/cases/params_sampling.json @@ -0,0 +1,20 @@ +{ + "id": "params_sampling", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 128, + "stop": [ + "\n\n", + "END" + ], + "temperature": 0.2, + "top_p": 0.9, + "user": "char-user-1234" + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/pdf_base64.json b/tests/translation_characterization/cases/pdf_base64.json new file mode 100644 index 00000000000..3db2c791e6b --- /dev/null +++ b/tests/translation_characterization/cases/pdf_base64.json @@ -0,0 +1,25 @@ +{ + "id": "pdf_base64", + "messages": [ + { + "content": [ + { + "text": "Summarize this document.", + "type": "text" + }, + { + "file": { + "file_data": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbXS9Db3VudCAwPj5lbmRvYmoKdHJhaWxlcjw8L1Jvb3QgMSAwIFI+PgolJUVPRgo=", + "filename": "char.pdf" + }, + "type": "file" + } + ], + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/plain_text.json b/tests/translation_characterization/cases/plain_text.json new file mode 100644 index 00000000000..92bd6787eda --- /dev/null +++ b/tests/translation_characterization/cases/plain_text.json @@ -0,0 +1,13 @@ +{ + "id": "plain_text", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/reasoning_effort_low.json b/tests/translation_characterization/cases/reasoning_effort_low.json new file mode 100644 index 00000000000..ce95feed792 --- /dev/null +++ b/tests/translation_characterization/cases/reasoning_effort_low.json @@ -0,0 +1,14 @@ +{ + "id": "reasoning_effort_low", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 2048, + "reasoning_effort": "low" + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/response_format_json_object.json b/tests/translation_characterization/cases/response_format_json_object.json new file mode 100644 index 00000000000..318c8e54615 --- /dev/null +++ b/tests/translation_characterization/cases/response_format_json_object.json @@ -0,0 +1,16 @@ +{ + "id": "response_format_json_object", + "messages": [ + { + "content": "Reply in JSON: capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "response_format": { + "type": "json_object" + } + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/response_format_json_schema.json b/tests/translation_characterization/cases/response_format_json_schema.json new file mode 100644 index 00000000000..5846118e5f6 --- /dev/null +++ b/tests/translation_characterization/cases/response_format_json_schema.json @@ -0,0 +1,32 @@ +{ + "id": "response_format_json_schema", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "response_format": { + "json_schema": { + "name": "capital", + "schema": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + } + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/system_prompt.json b/tests/translation_characterization/cases/system_prompt.json new file mode 100644 index 00000000000..048bbf1a584 --- /dev/null +++ b/tests/translation_characterization/cases/system_prompt.json @@ -0,0 +1,17 @@ +{ + "id": "system_prompt", + "messages": [ + { + "content": "You are a terse geographer.", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256 + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/thinking_enabled.json b/tests/translation_characterization/cases/thinking_enabled.json new file mode 100644 index 00000000000..bb5384c26d0 --- /dev/null +++ b/tests/translation_characterization/cases/thinking_enabled.json @@ -0,0 +1,17 @@ +{ + "id": "thinking_enabled", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 2048, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/thinking_history_blocks.json b/tests/translation_characterization/cases/thinking_history_blocks.json new file mode 100644 index 00000000000..2f72e6c7ea3 --- /dev/null +++ b/tests/translation_characterization/cases/thinking_history_blocks.json @@ -0,0 +1,32 @@ +{ + "id": "thinking_history_blocks", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + }, + { + "content": "Paris.", + "role": "assistant", + "thinking_blocks": [ + { + "signature": "sig-char-deadbeef", + "thinking": "The user asks about France; the capital is Paris.", + "type": "thinking" + } + ] + }, + { + "content": "And of Italy?", + "role": "user" + } + ], + "params": { + "max_tokens": 2048, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/tools_basic.json b/tests/translation_characterization/cases/tools_basic.json new file mode 100644 index 00000000000..cc0f3a7b54b --- /dev/null +++ b/tests/translation_characterization/cases/tools_basic.json @@ -0,0 +1,34 @@ +{ + "id": "tools_basic", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "tool_choice": "auto", + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/tools_forced_choice.json b/tests/translation_characterization/cases/tools_forced_choice.json new file mode 100644 index 00000000000..e40eeaeec33 --- /dev/null +++ b/tests/translation_characterization/cases/tools_forced_choice.json @@ -0,0 +1,39 @@ +{ + "id": "tools_forced_choice", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/tools_parallel.json b/tests/translation_characterization/cases/tools_parallel.json new file mode 100644 index 00000000000..8e90a550fce --- /dev/null +++ b/tests/translation_characterization/cases/tools_parallel.json @@ -0,0 +1,52 @@ +{ + "id": "tools_parallel", + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "params": { + "max_tokens": 256, + "parallel_tool_calls": true, + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + }, + { + "function": { + "description": "Get current time for a timezone.", + "name": "get_time", + "parameters": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/cases/tools_streamed_args_roundtrip.json b/tests/translation_characterization/cases/tools_streamed_args_roundtrip.json new file mode 100644 index 00000000000..887519d1cb3 --- /dev/null +++ b/tests/translation_characterization/cases/tools_streamed_args_roundtrip.json @@ -0,0 +1,83 @@ +{ + "id": "tools_streamed_args_roundtrip", + "messages": [ + { + "content": "Weather and time in Paris?", + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "call_char_001", + "type": "function" + }, + { + "function": { + "arguments": "{\"tz\": \"Europe/Paris\"}", + "name": "get_time" + }, + "id": "call_char_002", + "type": "function" + } + ] + }, + { + "content": "18C, clear", + "role": "tool", + "tool_call_id": "call_char_001" + }, + { + "content": "14:05", + "role": "tool", + "tool_call_id": "call_char_002" + } + ], + "params": { + "max_tokens": 256, + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + }, + { + "function": { + "description": "Get current time for a timezone.", + "name": "get_time", + "parameters": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "skip": {} +} diff --git a/tests/translation_characterization/conftest.py b/tests/translation_characterization/conftest.py new file mode 100644 index 00000000000..fd7423cde3e --- /dev/null +++ b/tests/translation_characterization/conftest.py @@ -0,0 +1,68 @@ +"""Pytest plumbing for the translation characterization corpus. + +Determinism: every test runs with ``uuid.uuid4`` and ``time.time`` frozen +(see ``deterministic_ids``) because v1 injects generated ids +(``chatcmpl-``, streaming response ids) and ``created`` timestamps +into ``ModelResponse``/``ModelResponseStream``. + +Snapshot refresh: ``pytest tests/translation_characterization --snapshot-update`` +or ``SNAPSHOT_UPDATE=1 pytest tests/translation_characterization``. +""" + +import itertools +import os +import time +import uuid + +import pytest + +# Must be set before litellm is imported anywhere: keeps the model cost map +# local (no network) and gives the anthropic config a key for header building. +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +os.environ.setdefault("ANTHROPIC_API_KEY", "sk-ant-char-test-key") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "AKIACHARTESTKEY00000") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "char-test-secret") +os.environ.setdefault("AWS_REGION_NAME", "us-east-1") + +FROZEN_TIME = 1718064000.0 # 2024-06-11T00:00:00Z + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--snapshot-update", + action="store_true", + default=False, + help="Rewrite characterization snapshots (and generated corpus cases).", + ) + + +@pytest.fixture() +def snapshot_update(request: pytest.FixtureRequest) -> bool: + try: + flag = bool(request.config.getoption("--snapshot-update")) + except ValueError: # conftest not loaded as an initial conftest + flag = False + return flag or os.environ.get("SNAPSHOT_UPDATE", "") not in ("", "0", "false") + + +@pytest.fixture(autouse=True) +def deterministic_ids(monkeypatch: pytest.MonkeyPatch): + """Freeze the two ambient generators v1 reaches for when minting ids. + + - ``uuid.uuid4`` -> a per-test deterministic sequence (UUID(int=1), ...) + - ``time.time`` -> FROZEN_TIME (feeds ``ModelResponse.created``) + """ + counter = itertools.count(1) + + def fake_uuid4(): + return uuid.UUID(int=next(counter)) + + monkeypatch.setattr(uuid, "uuid4", fake_uuid4) + import fastuuid # litellm._uuid aliases this module as `uuid` + + import litellm._uuid + + monkeypatch.setattr(fastuuid, "uuid4", fake_uuid4) + monkeypatch.setattr(litellm._uuid, "uuid4", fake_uuid4) + monkeypatch.setattr(time, "time", lambda: FROZEN_TIME) + yield diff --git a/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_kept.json b/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_kept.json new file mode 100644 index 00000000000..b73ebade183 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_kept.json @@ -0,0 +1,19 @@ +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01CharAdvisor1", "name": "advisor", "input": {"query": "check forecast"}}, + {"type": "advisor_tool_result", "tool_use_id": "srvtoolu_01CharAdvisor1", "content": [{"type": "text", "text": "Advise checking the radar."}]}, + {"type": "text", "text": "I'll check the weather."} + ] + }, + {"role": "user", "content": "Go ahead."} + ], + "params": { + "max_tokens": 512, + "tools": [{"type": "advisor_20260301", "name": "advisor"}] + } +} diff --git a/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_stripped.json b/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_stripped.json new file mode 100644 index 00000000000..c0f0093a2c4 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/native/advisor_blocks_stripped.json @@ -0,0 +1,16 @@ +{ + "model": "claude-sonnet-4-20250514", + "messages": [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01CharAdvisor1", "name": "advisor", "input": {"query": "check forecast"}}, + {"type": "advisor_tool_result", "tool_use_id": "srvtoolu_01CharAdvisor1", "content": [{"type": "text", "text": "Advise checking the radar."}]}, + {"type": "text", "text": "I'll check the weather."} + ] + }, + {"role": "user", "content": "Go ahead."} + ], + "params": {"max_tokens": 512} +} diff --git a/tests/translation_characterization/fixtures/adapter/native/basic_passthrough.json b/tests/translation_characterization/fixtures/adapter/native/basic_passthrough.json new file mode 100644 index 00000000000..a77c23eddc9 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/native/basic_passthrough.json @@ -0,0 +1,20 @@ +{ + "model": "claude-sonnet-4-20250514", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "params": { + "max_tokens": 512, + "temperature": 0.2, + "system": "You are a terse geographer.", + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + } + } + ] + } +} diff --git a/tests/translation_characterization/fixtures/adapter/native/reasoning_effort_rewrite.json b/tests/translation_characterization/fixtures/adapter/native/reasoning_effort_rewrite.json new file mode 100644 index 00000000000..2e02f791b11 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/native/reasoning_effort_rewrite.json @@ -0,0 +1,5 @@ +{ + "model": "claude-sonnet-4-20250514", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "params": {"max_tokens": 2048, "reasoning_effort": "low"} +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/cache_control_system.json b/tests/translation_characterization/fixtures/adapter/requests/cache_control_system.json new file mode 100644 index 00000000000..9f6b166ad09 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/cache_control_system.json @@ -0,0 +1,9 @@ +{ + "model": "gpt-4o", + "max_tokens": 512, + "system": [ + {"type": "text", "text": "You are a terse geographer.", "cache_control": {"type": "ephemeral"}} + ], + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "metadata": {"user_id": "char-user-1234"} +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/image_base64.json b/tests/translation_characterization/fixtures/adapter/requests/image_base64.json new file mode 100644 index 00000000000..8cb9560ca09 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/image_base64.json @@ -0,0 +1,20 @@ +{ + "model": "gpt-4o", + "max_tokens": 512, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + } + } + ] + } + ] +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/simple_text.json b/tests/translation_characterization/fixtures/adapter/requests/simple_text.json new file mode 100644 index 00000000000..7a94373e2eb --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/simple_text.json @@ -0,0 +1,5 @@ +{ + "model": "gpt-4o", + "max_tokens": 512, + "messages": [{"role": "user", "content": "What is the capital of France?"}] +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/system_and_tools.json b/tests/translation_characterization/fixtures/adapter/requests/system_and_tools.json new file mode 100644 index 00000000000..2a5e3d86ee3 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/system_and_tools.json @@ -0,0 +1,20 @@ +{ + "model": "gpt-4o", + "max_tokens": 512, + "system": "You are a terse geographer.", + "temperature": 0.2, + "stop_sequences": ["END"], + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + } + } + ], + "tool_choice": {"type": "auto"} +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/thinking_enabled.json b/tests/translation_characterization/fixtures/adapter/requests/thinking_enabled.json new file mode 100644 index 00000000000..e443bb9b4e1 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/thinking_enabled.json @@ -0,0 +1,6 @@ +{ + "model": "gpt-oss-120b", + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "messages": [{"role": "user", "content": "What is the capital of France?"}] +} diff --git a/tests/translation_characterization/fixtures/adapter/requests/tool_result_roundtrip.json b/tests/translation_characterization/fixtures/adapter/requests/tool_result_roundtrip.json new file mode 100644 index 00000000000..0d2bbdf5f89 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/requests/tool_result_roundtrip.json @@ -0,0 +1,31 @@ +{ + "model": "gpt-4o", + "max_tokens": 512, + "messages": [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "I'll check the weather."}, + {"type": "tool_use", "id": "toolu_01CharAdapter01", "name": "get_weather", "input": {"city": "Paris"}} + ] + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01CharAdapter01", "content": "18C, clear"} + ] + } + ], + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + } + } + ] +} diff --git a/tests/translation_characterization/fixtures/adapter/responses/cached_usage_reasoning.json b/tests/translation_characterization/fixtures/adapter/responses/cached_usage_reasoning.json new file mode 100644 index 00000000000..452a5188c2c --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/responses/cached_usage_reasoning.json @@ -0,0 +1,24 @@ +{ + "id": "chatcmpl-char-adapter-cache-0001", + "object": "chat.completion", + "created": 1718064000, + "model": "gpt-oss-120b", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": "Paris.", + "reasoning_content": "The user asks for the capital of France. That is Paris." + } + } + ], + "usage": { + "prompt_tokens": 3078, + "completion_tokens": 35, + "total_tokens": 3113, + "prompt_tokens_details": {"cached_tokens": 2048}, + "completion_tokens_details": {"reasoning_tokens": 24} + } +} diff --git a/tests/translation_characterization/fixtures/adapter/responses/text_basic.json b/tests/translation_characterization/fixtures/adapter/responses/text_basic.json new file mode 100644 index 00000000000..03e27916733 --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/responses/text_basic.json @@ -0,0 +1,14 @@ +{ + "id": "chatcmpl-char-adapter-text-0001", + "object": "chat.completion", + "created": 1718064000, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Paris is the capital of France."} + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 9, "total_tokens": 21} +} diff --git a/tests/translation_characterization/fixtures/adapter/responses/tool_calls.json b/tests/translation_characterization/fixtures/adapter/responses/tool_calls.json new file mode 100644 index 00000000000..fa48b1cc85b --- /dev/null +++ b/tests/translation_characterization/fixtures/adapter/responses/tool_calls.json @@ -0,0 +1,29 @@ +{ + "id": "chatcmpl-char-adapter-tool-0001", + "object": "chat.completion", + "created": 1718064000, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_char_adapter_001", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"} + }, + { + "id": "call_char_adapter_002", + "type": "function", + "function": {"name": "get_time", "arguments": "{\"tz\": \"Europe/Paris\"}"} + } + ] + } + } + ], + "usage": {"prompt_tokens": 410, "completion_tokens": 60, "total_tokens": 470} +} diff --git a/tests/translation_characterization/fixtures/responses/anthropic/cache_usage.json b/tests/translation_characterization/fixtures/responses/anthropic/cache_usage.json new file mode 100644 index 00000000000..5c778099ac8 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/anthropic/cache_usage.json @@ -0,0 +1,16 @@ +{ + "id": "msg_01CharCacheUsage000001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": "Paris."}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 6, + "output_tokens": 4, + "cache_creation_input_tokens": 1024, + "cache_read_input_tokens": 2048, + "cache_creation": {"ephemeral_5m_input_tokens": 1024, "ephemeral_1h_input_tokens": 0} + } +} diff --git a/tests/translation_characterization/fixtures/responses/anthropic/stop_sequence.json b/tests/translation_characterization/fixtures/responses/anthropic/stop_sequence.json new file mode 100644 index 00000000000..a16da752be2 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/anthropic/stop_sequence.json @@ -0,0 +1,10 @@ +{ + "id": "msg_01CharStopSeq00000001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": "Paris"}], + "stop_reason": "stop_sequence", + "stop_sequence": "END", + "usage": {"input_tokens": 12, "output_tokens": 2} +} diff --git a/tests/translation_characterization/fixtures/responses/anthropic/text_basic.json b/tests/translation_characterization/fixtures/responses/anthropic/text_basic.json new file mode 100644 index 00000000000..77946df7c77 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/anthropic/text_basic.json @@ -0,0 +1,10 @@ +{ + "id": "msg_01CharBasic000000000001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [{"type": "text", "text": "Paris is the capital of France."}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 12, "output_tokens": 9} +} diff --git a/tests/translation_characterization/fixtures/responses/anthropic/thinking.json b/tests/translation_characterization/fixtures/responses/anthropic/thinking.json new file mode 100644 index 00000000000..d50621d6ff2 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/anthropic/thinking.json @@ -0,0 +1,13 @@ +{ + "id": "msg_01CharThinking00000001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + {"type": "thinking", "thinking": "The user asks for the capital of France. That is Paris.", "signature": "EuYBCkQYAiJAchar-signature-bytes"}, + {"type": "text", "text": "Paris."} + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 40, "output_tokens": 31} +} diff --git a/tests/translation_characterization/fixtures/responses/anthropic/tool_use_parallel.json b/tests/translation_characterization/fixtures/responses/anthropic/tool_use_parallel.json new file mode 100644 index 00000000000..63121983976 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/anthropic/tool_use_parallel.json @@ -0,0 +1,14 @@ +{ + "id": "msg_01CharToolUse0000000001", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-20250514", + "content": [ + {"type": "text", "text": "I'll check both for you."}, + {"type": "tool_use", "id": "toolu_01CharWeather01", "name": "get_weather", "input": {"city": "Paris"}}, + {"type": "tool_use", "id": "toolu_01CharTime00001", "name": "get_time", "input": {"tz": "Europe/Paris"}} + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": {"input_tokens": 412, "output_tokens": 87} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_converse/cache_usage.json b/tests/translation_characterization/fixtures/responses/bedrock_converse/cache_usage.json new file mode 100644 index 00000000000..97ab92c7092 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_converse/cache_usage.json @@ -0,0 +1,17 @@ +{ + "metrics": {"latencyMs": 430}, + "output": { + "message": { + "role": "assistant", + "content": [{"text": "Paris."}] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 6, + "outputTokens": 4, + "totalTokens": 3082, + "cacheReadInputTokens": 2048, + "cacheWriteInputTokens": 1024 + } +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_converse/reasoning.json b/tests/translation_characterization/fixtures/responses/bedrock_converse/reasoning.json new file mode 100644 index 00000000000..17e987d54cb --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_converse/reasoning.json @@ -0,0 +1,14 @@ +{ + "metrics": {"latencyMs": 1290}, + "output": { + "message": { + "role": "assistant", + "content": [ + {"reasoningContent": {"reasoningText": {"text": "The user asks for the capital of France. That is Paris.", "signature": "char-converse-signature"}}}, + {"text": "Paris."} + ] + } + }, + "stopReason": "end_turn", + "usage": {"inputTokens": 40, "outputTokens": 31, "totalTokens": 71} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_converse/text_basic.json b/tests/translation_characterization/fixtures/responses/bedrock_converse/text_basic.json new file mode 100644 index 00000000000..bd283ab97c6 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_converse/text_basic.json @@ -0,0 +1,11 @@ +{ + "metrics": {"latencyMs": 551}, + "output": { + "message": { + "role": "assistant", + "content": [{"text": "Paris is the capital of France."}] + } + }, + "stopReason": "end_turn", + "usage": {"inputTokens": 12, "outputTokens": 9, "totalTokens": 21} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_converse/tool_use.json b/tests/translation_characterization/fixtures/responses/bedrock_converse/tool_use.json new file mode 100644 index 00000000000..cb7b45c7137 --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_converse/tool_use.json @@ -0,0 +1,14 @@ +{ + "metrics": {"latencyMs": 702}, + "output": { + "message": { + "role": "assistant", + "content": [ + {"text": "I'll check the weather."}, + {"toolUse": {"toolUseId": "tooluse_charweather01", "name": "get_weather", "input": {"city": "Paris"}}} + ] + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 410, "outputTokens": 60, "totalTokens": 470} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_invoke/text_basic.json b/tests/translation_characterization/fixtures/responses/bedrock_invoke/text_basic.json new file mode 100644 index 00000000000..0a9cd03269c --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_invoke/text_basic.json @@ -0,0 +1,10 @@ +{ + "id": "msg_bdrk_01CharInvokeText01", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [{"type": "text", "text": "Paris is the capital of France."}], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 12, "output_tokens": 9} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_invoke/thinking.json b/tests/translation_characterization/fixtures/responses/bedrock_invoke/thinking.json new file mode 100644 index 00000000000..34184d04c5c --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_invoke/thinking.json @@ -0,0 +1,13 @@ +{ + "id": "msg_bdrk_01CharInvokeThink1", + "type": "message", + "role": "assistant", + "model": "claude-3-7-sonnet-20250219", + "content": [ + {"type": "thinking", "thinking": "The user asks for the capital of France. That is Paris.", "signature": "char-invoke-signature"}, + {"type": "text", "text": "Paris."} + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": {"input_tokens": 40, "output_tokens": 31} +} diff --git a/tests/translation_characterization/fixtures/responses/bedrock_invoke/tool_use.json b/tests/translation_characterization/fixtures/responses/bedrock_invoke/tool_use.json new file mode 100644 index 00000000000..a6c3ee7e87c --- /dev/null +++ b/tests/translation_characterization/fixtures/responses/bedrock_invoke/tool_use.json @@ -0,0 +1,13 @@ +{ + "id": "msg_bdrk_01CharInvokeTool01", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [ + {"type": "text", "text": "I'll check the weather."}, + {"type": "tool_use", "id": "toolu_bdrk_01CharWeather", "name": "get_weather", "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": {"input_tokens": 410, "output_tokens": 60} +} diff --git a/tests/translation_characterization/fixtures/streams/anthropic/text_stream.txt b/tests/translation_characterization/fixtures/streams/anthropic/text_stream.txt new file mode 100644 index 00000000000..c17fcb19f2b --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/anthropic/text_stream.txt @@ -0,0 +1,23 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01CharStreamText000001","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":1}}} + +event: ping +data: {"type": "ping"} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Paris is the"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" capital of France."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}} + +event: message_stop +data: {"type":"message_stop"} diff --git a/tests/translation_characterization/fixtures/streams/anthropic/thinking_stream.txt b/tests/translation_characterization/fixtures/streams/anthropic/thinking_stream.txt new file mode 100644 index 00000000000..c9ddde2e049 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/anthropic/thinking_stream.txt @@ -0,0 +1,32 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01CharStreamThink00001","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":40,"output_tokens":1}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user asks for the capital of France."}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" That is Paris."}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EuYBCkQYAiJAchar-stream-signature"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Paris."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":31}} + +event: message_stop +data: {"type":"message_stop"} diff --git a/tests/translation_characterization/fixtures/streams/anthropic/tool_stream.txt b/tests/translation_characterization/fixtures/streams/anthropic/tool_stream.txt new file mode 100644 index 00000000000..89b9c91ca68 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/anthropic/tool_stream.txt @@ -0,0 +1,32 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_01CharStreamTool000001","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":412,"output_tokens":2}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I'll check the weather."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01CharStreamW01","name":"get_weather","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"ci"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ty\": \"Par"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"is\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":87}} + +event: message_stop +data: {"type":"message_stop"} diff --git a/tests/translation_characterization/fixtures/streams/bedrock_converse/cache_usage_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_converse/cache_usage_stream.json new file mode 100644 index 00000000000..fa4921989b0 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_converse/cache_usage_stream.json @@ -0,0 +1,7 @@ +[ + {"role": "assistant"}, + {"contentBlockIndex": 0, "delta": {"text": "Paris."}}, + {"contentBlockIndex": 0}, + {"stopReason": "end_turn"}, + {"usage": {"inputTokens": 6, "outputTokens": 4, "totalTokens": 3082, "cacheReadInputTokens": 2048, "cacheWriteInputTokens": 1024}, "metrics": {"latencyMs": 430}} +] diff --git a/tests/translation_characterization/fixtures/streams/bedrock_converse/reasoning_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_converse/reasoning_stream.json new file mode 100644 index 00000000000..c62c6a67629 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_converse/reasoning_stream.json @@ -0,0 +1,11 @@ +[ + {"role": "assistant"}, + {"contentBlockIndex": 0, "delta": {"reasoningContent": {"text": "The user asks for the capital of France."}}}, + {"contentBlockIndex": 0, "delta": {"reasoningContent": {"text": " That is Paris."}}}, + {"contentBlockIndex": 0, "delta": {"reasoningContent": {"signature": "char-converse-stream-signature"}}}, + {"contentBlockIndex": 0}, + {"contentBlockIndex": 1, "delta": {"text": "Paris."}}, + {"contentBlockIndex": 1}, + {"stopReason": "end_turn"}, + {"usage": {"inputTokens": 40, "outputTokens": 31, "totalTokens": 71}, "metrics": {"latencyMs": 1290}} +] diff --git a/tests/translation_characterization/fixtures/streams/bedrock_converse/text_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_converse/text_stream.json new file mode 100644 index 00000000000..691c4bd3326 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_converse/text_stream.json @@ -0,0 +1,8 @@ +[ + {"role": "assistant"}, + {"contentBlockIndex": 0, "delta": {"text": "Paris is the"}}, + {"contentBlockIndex": 0, "delta": {"text": " capital of France."}}, + {"contentBlockIndex": 0}, + {"stopReason": "end_turn"}, + {"usage": {"inputTokens": 12, "outputTokens": 9, "totalTokens": 21}, "metrics": {"latencyMs": 551}} +] diff --git a/tests/translation_characterization/fixtures/streams/bedrock_converse/tool_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_converse/tool_stream.json new file mode 100644 index 00000000000..fb1e59d5b7a --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_converse/tool_stream.json @@ -0,0 +1,12 @@ +[ + {"role": "assistant"}, + {"contentBlockIndex": 0, "delta": {"text": "I'll check the weather."}}, + {"contentBlockIndex": 0}, + {"contentBlockIndex": 1, "start": {"toolUse": {"toolUseId": "tooluse_charstream01", "name": "get_weather"}}}, + {"contentBlockIndex": 1, "delta": {"toolUse": {"input": "{\"ci"}}}, + {"contentBlockIndex": 1, "delta": {"toolUse": {"input": "ty\": \"Par"}}}, + {"contentBlockIndex": 1, "delta": {"toolUse": {"input": "is\"}"}}}, + {"contentBlockIndex": 1}, + {"stopReason": "tool_use"}, + {"usage": {"inputTokens": 410, "outputTokens": 60, "totalTokens": 470}, "metrics": {"latencyMs": 702}} +] diff --git a/tests/translation_characterization/fixtures/streams/bedrock_invoke/text_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_invoke/text_stream.json new file mode 100644 index 00000000000..7f6e369db0d --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_invoke/text_stream.json @@ -0,0 +1,9 @@ +[ + {"type": "message_start", "message": {"id": "msg_bdrk_01CharInvStream01", "type": "message", "role": "assistant", "model": "claude-3-5-sonnet-20240620", "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 12, "output_tokens": 1}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Paris is the"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " capital of France."}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": null}, "usage": {"output_tokens": 9}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 12, "outputTokenCount": 9, "invocationLatency": 551, "firstByteLatency": 80}}, + {"type": "message_stop"} +] diff --git a/tests/translation_characterization/fixtures/streams/bedrock_invoke/tool_stream.json b/tests/translation_characterization/fixtures/streams/bedrock_invoke/tool_stream.json new file mode 100644 index 00000000000..ab0865e4dd7 --- /dev/null +++ b/tests/translation_characterization/fixtures/streams/bedrock_invoke/tool_stream.json @@ -0,0 +1,10 @@ +[ + {"type": "message_start", "message": {"id": "msg_bdrk_01CharInvStream02", "type": "message", "role": "assistant", "model": "claude-3-5-sonnet-20240620", "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 412, "output_tokens": 2}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_bdrk_01CharStream", "name": "get_weather", "input": {}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"ci"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "ty\": \"Par"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "is\"}"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, "usage": {"output_tokens": 87}, "amazon-bedrock-invocationMetrics": {"inputTokenCount": 412, "outputTokenCount": 87, "invocationLatency": 702, "firstByteLatency": 95}}, + {"type": "message_stop"} +] diff --git a/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_kept.json b/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_kept.json new file mode 100644 index 00000000000..7752b38c3c7 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_kept.json @@ -0,0 +1,47 @@ +{ + "max_tokens": 512, + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": [ + { + "id": "srvtoolu_01CharAdvisor1", + "input": { + "query": "check forecast" + }, + "name": "advisor", + "type": "server_tool_use" + }, + { + "content": [ + { + "text": "Advise checking the radar.", + "type": "text" + } + ], + "tool_use_id": "srvtoolu_01CharAdvisor1", + "type": "advisor_tool_result" + }, + { + "text": "I'll check the weather.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "Go ahead.", + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tools": [ + { + "name": "advisor", + "type": "advisor_20260301" + } + ] +} diff --git a/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_stripped.json b/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_stripped.json new file mode 100644 index 00000000000..b29694de00f --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/native/advisor_blocks_stripped.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 512, + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": [ + { + "text": "I'll check the weather.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "Go ahead.", + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/adapter/native/basic_passthrough.json b/tests/translation_characterization/snapshots/adapter/native/basic_passthrough.json new file mode 100644 index 00000000000..08e75498bcb --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/native/basic_passthrough.json @@ -0,0 +1,29 @@ +{ + "max_tokens": 512, + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "system": "You are a terse geographer.", + "temperature": 0.2, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/tests/translation_characterization/snapshots/adapter/native/reasoning_effort_rewrite.json b/tests/translation_characterization/snapshots/adapter/native/reasoning_effort_rewrite.json new file mode 100644 index 00000000000..2c6b20780fc --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/native/reasoning_effort_rewrite.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 2048, + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/cache_control_system.json b/tests/translation_characterization/snapshots/adapter/requests/cache_control_system.json new file mode 100644 index 00000000000..cb4ef79b799 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/cache_control_system.json @@ -0,0 +1,23 @@ +{ + "openai_request": { + "max_tokens": 512, + "messages": [ + { + "content": [ + { + "text": "You are a terse geographer.", + "type": "text" + } + ], + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-4o", + "user": "char-user-1234" + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/image_base64.json b/tests/translation_characterization/snapshots/adapter/requests/image_base64.json new file mode 100644 index 00000000000..c1e0b1db966 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/image_base64.json @@ -0,0 +1,24 @@ +{ + "openai_request": { + "max_tokens": 512, + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "gpt-4o" + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/simple_text.json b/tests/translation_characterization/snapshots/adapter/requests/simple_text.json new file mode 100644 index 00000000000..6baad9129d7 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/simple_text.json @@ -0,0 +1,13 @@ +{ + "openai_request": { + "max_tokens": 512, + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-4o" + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/system_and_tools.json b/tests/translation_characterization/snapshots/adapter/requests/system_and_tools.json new file mode 100644 index 00000000000..90e363c8bef --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/system_and_tools.json @@ -0,0 +1,42 @@ +{ + "openai_request": { + "max_tokens": 512, + "messages": [ + { + "content": "You are a terse geographer.", + "role": "system" + }, + { + "content": "Weather in Paris?", + "role": "user" + } + ], + "model": "gpt-4o", + "stop_sequences": [ + "END" + ], + "temperature": 0.2, + "tool_choice": "auto", + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/thinking_enabled.json b/tests/translation_characterization/snapshots/adapter/requests/thinking_enabled.json new file mode 100644 index 00000000000..a6ae64f8141 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/thinking_enabled.json @@ -0,0 +1,14 @@ +{ + "openai_request": { + "max_tokens": 2048, + "messages": [ + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-oss-120b", + "reasoning_effort": "minimal" + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/requests/tool_result_roundtrip.json b/tests/translation_characterization/snapshots/adapter/requests/tool_result_roundtrip.json new file mode 100644 index 00000000000..5db6e05baed --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/requests/tool_result_roundtrip.json @@ -0,0 +1,53 @@ +{ + "openai_request": { + "max_tokens": 512, + "messages": [ + { + "content": "Weather in Paris?", + "role": "user" + }, + { + "content": "I'll check the weather.", + "role": "assistant", + "thinking_blocks": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "toolu_01CharAdapter01", + "type": "function" + } + ] + }, + { + "content": "18C, clear", + "role": "tool", + "tool_call_id": "toolu_01CharAdapter01" + } + ], + "model": "gpt-4o", + "tools": [ + { + "function": { + "description": "Get current weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "tool_name_mapping": {} +} diff --git a/tests/translation_characterization/snapshots/adapter/responses/cached_usage_reasoning.json b/tests/translation_characterization/snapshots/adapter/responses/cached_usage_reasoning.json new file mode 100644 index 00000000000..cff5f8cc221 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/responses/cached_usage_reasoning.json @@ -0,0 +1,24 @@ +{ + "content": [ + { + "signature": null, + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + }, + { + "text": "Paris.", + "type": "text" + } + ], + "id": "chatcmpl-char-adapter-cache-0001", + "model": "gpt-oss-120b", + "role": "assistant", + "stop_reason": "end_turn", + "stop_sequence": null, + "type": "message", + "usage": { + "cache_read_input_tokens": 2048, + "input_tokens": 1030, + "output_tokens": 35 + } +} diff --git a/tests/translation_characterization/snapshots/adapter/responses/text_basic.json b/tests/translation_characterization/snapshots/adapter/responses/text_basic.json new file mode 100644 index 00000000000..a4366e35ab6 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/responses/text_basic.json @@ -0,0 +1,18 @@ +{ + "content": [ + { + "text": "Paris is the capital of France.", + "type": "text" + } + ], + "id": "chatcmpl-char-adapter-text-0001", + "model": "gpt-4o-2024-08-06", + "role": "assistant", + "stop_reason": "end_turn", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 12, + "output_tokens": 9 + } +} diff --git a/tests/translation_characterization/snapshots/adapter/responses/tool_calls.json b/tests/translation_characterization/snapshots/adapter/responses/tool_calls.json new file mode 100644 index 00000000000..ae1d741cda4 --- /dev/null +++ b/tests/translation_characterization/snapshots/adapter/responses/tool_calls.json @@ -0,0 +1,32 @@ +{ + "content": [ + { + "id": "call_char_adapter_001", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "provider_specific_fields": null, + "type": "tool_use" + }, + { + "id": "call_char_adapter_002", + "input": { + "tz": "Europe/Paris" + }, + "name": "get_time", + "provider_specific_fields": null, + "type": "tool_use" + } + ], + "id": "chatcmpl-char-adapter-tool-0001", + "model": "gpt-4o-2024-08-06", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 410, + "output_tokens": 60 + } +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/cache_control_messages.json b/tests/translation_characterization/snapshots/requests/anthropic/cache_control_messages.json new file mode 100644 index 00000000000..e614b23d2da --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/cache_control_messages.json @@ -0,0 +1,27 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/cache_control_tools.json b/tests/translation_characterization/snapshots/requests/anthropic/cache_control_tools.json new file mode 100644 index 00000000000..50804254b16 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/cache_control_tools.json @@ -0,0 +1,52 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/full_combo.json b/tests/translation_characterization/snapshots/requests/anthropic/full_combo.json new file mode 100644 index 00000000000..6e5482747b9 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/full_combo.json @@ -0,0 +1,74 @@ +{ + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "Weather in Paris?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "call_char_001", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear", + "tool_use_id": "call_char_001", + "type": "tool_result" + }, + { + "text": "Summarize, then check the weather again.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ], + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/image_base64.json b/tests/translation_characterization/snapshots/requests/anthropic/image_base64.json new file mode 100644 index 00000000000..c11f732ddde --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/image_base64.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "source": { + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/image_url.json b/tests/translation_characterization/snapshots/requests/anthropic/image_url.json new file mode 100644 index 00000000000..4902f90f50e --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/image_url.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "source": { + "type": "url", + "url": "https://example.com/cat.png" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/multi_turn.json b/tests/translation_characterization/snapshots/requests/anthropic/multi_turn.json new file mode 100644 index 00000000000..6c85613d573 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/multi_turn.json @@ -0,0 +1,33 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Paris.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/params_sampling.json b/tests/translation_characterization/snapshots/requests/anthropic/params_sampling.json new file mode 100644 index 00000000000..aa247d789be --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/params_sampling.json @@ -0,0 +1,24 @@ +{ + "max_tokens": 128, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "metadata": { + "user_id": "char-user-1234" + }, + "model": "claude-sonnet-4-20250514", + "stop_sequences": [ + "\n\n", + "END" + ], + "temperature": 0.2, + "top_p": 0.9 +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/pdf_base64.json b/tests/translation_characterization/snapshots/requests/anthropic/pdf_base64.json new file mode 100644 index 00000000000..36f7649e75c --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/pdf_base64.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Summarize this document.", + "type": "text" + }, + { + "source": { + "data": "JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbXS9Db3VudCAwPj5lbmRvYmoKdHJhaWxlcjw8L1Jvb3QgMSAwIFI+PgolJUVPRgo=", + "media_type": "application/pdf", + "type": "base64" + }, + "type": "document" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/plain_text.json b/tests/translation_characterization/snapshots/requests/anthropic/plain_text.json new file mode 100644 index 00000000000..3a859e96942 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/plain_text.json @@ -0,0 +1,15 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/reasoning_effort_low.json b/tests/translation_characterization/snapshots/requests/anthropic/reasoning_effort_low.json new file mode 100644 index 00000000000..d10251eb571 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/reasoning_effort_low.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_object.json b/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_object.json new file mode 100644 index 00000000000..aa2db3b693a --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_object.json @@ -0,0 +1,15 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Reply in JSON: capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514" +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_schema.json b/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_schema.json new file mode 100644 index 00000000000..d986fc0525a --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/response_format_json_schema.json @@ -0,0 +1,37 @@ +{ + "json_mode": true, + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tool_choice": { + "name": "json_tool_call", + "type": "tool" + }, + "tools": [ + { + "input_schema": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + }, + "name": "json_tool_call" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/system_prompt.json b/tests/translation_characterization/snapshots/requests/anthropic/system_prompt.json new file mode 100644 index 00000000000..ce23f625c9f --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/system_prompt.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "system": [ + { + "text": "You are a terse geographer.", + "type": "text" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/thinking_enabled.json b/tests/translation_characterization/snapshots/requests/anthropic/thinking_enabled.json new file mode 100644 index 00000000000..d10251eb571 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/thinking_enabled.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/thinking_history_blocks.json b/tests/translation_characterization/snapshots/requests/anthropic/thinking_history_blocks.json new file mode 100644 index 00000000000..86399064402 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/thinking_history_blocks.json @@ -0,0 +1,42 @@ +{ + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "signature": "sig-char-deadbeef", + "thinking": "The user asks about France; the capital is Paris.", + "type": "thinking" + }, + { + "text": "Paris.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/tools_basic.json b/tests/translation_characterization/snapshots/requests/anthropic/tools_basic.json new file mode 100644 index 00000000000..fbd5e6adcbc --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/tools_basic.json @@ -0,0 +1,36 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tool_choice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/tools_forced_choice.json b/tests/translation_characterization/snapshots/requests/anthropic/tools_forced_choice.json new file mode 100644 index 00000000000..a108b313e4c --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/tools_forced_choice.json @@ -0,0 +1,37 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tool_choice": { + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/tools_parallel.json b/tests/translation_characterization/snapshots/requests/anthropic/tools_parallel.json new file mode 100644 index 00000000000..78d460d17f5 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/tools_parallel.json @@ -0,0 +1,53 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tool_choice": { + "disable_parallel_tool_use": false, + "type": "auto" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/anthropic/tools_streamed_args_roundtrip.json b/tests/translation_characterization/snapshots/requests/anthropic/tools_streamed_args_roundtrip.json new file mode 100644 index 00000000000..6ae49e3a996 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/anthropic/tools_streamed_args_roundtrip.json @@ -0,0 +1,85 @@ +{ + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Weather and time in Paris?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "call_char_001", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + }, + { + "id": "call_char_002", + "input": { + "tz": "Europe/Paris" + }, + "name": "get_time", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear", + "tool_use_id": "call_char_001", + "type": "tool_result" + }, + { + "content": "14:05", + "tool_use_id": "call_char_002", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "claude-sonnet-4-20250514", + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_messages.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_messages.json new file mode 100644 index 00000000000..3056514350a --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_messages.json @@ -0,0 +1,33 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "role": "user" + } + ], + "system": [ + { + "text": "You are a terse geographer." + }, + { + "cachePoint": { + "type": "default" + } + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_tools.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_tools.json new file mode 100644 index 00000000000..1972b96c422 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/cache_control_tools.json @@ -0,0 +1,65 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + }, + { + "toolSpec": { + "description": "Get current time for a timezone.", + "inputSchema": { + "json": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "name": "get_time" + } + }, + { + "cachePoint": { + "type": "default" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/full_combo.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/full_combo.json new file mode 100644 index 00000000000..af4ed2e22e2 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/full_combo.json @@ -0,0 +1,88 @@ +{ + "additionalModelRequestFields": { + "stream": false, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 2048, + "temperature": 1 + }, + "messages": [ + { + "content": [ + { + "text": "Weather in Paris?" + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Paris" + }, + "name": "get_weather", + "toolUseId": "call_char_001" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "text": "18C, clear" + } + ], + "toolUseId": "call_char_001" + } + }, + { + "text": "Summarize, then check the weather again." + } + ], + "role": "user" + } + ], + "system": [ + { + "text": "You are a terse geographer." + }, + { + "cachePoint": { + "type": "default" + } + } + ], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/image_base64.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/image_base64.json new file mode 100644 index 00000000000..2e7e5a5f089 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/image_base64.json @@ -0,0 +1,26 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "Describe this image." + }, + { + "image": { + "format": "png", + "source": { + "bytes": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" + } + } + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/multi_turn.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/multi_turn.json new file mode 100644 index 00000000000..c8d3349f13d --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/multi_turn.json @@ -0,0 +1,34 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Paris." + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/params_sampling.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/params_sampling.json new file mode 100644 index 00000000000..e800b16fdbd --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/params_sampling.json @@ -0,0 +1,24 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 128, + "stopSequences": [ + "\n\n", + "END" + ], + "temperature": 0.2, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/pdf_base64.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/pdf_base64.json new file mode 100644 index 00000000000..98b946bc322 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/pdf_base64.json @@ -0,0 +1,27 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "Summarize this document." + }, + { + "document": { + "format": "pdf", + "name": "DocumentPDFmessages_4b20ba3afdf62e5d_pdf", + "source": { + "bytes": "JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbXS9Db3VudCAwPj5lbmRvYmoKdHJhaWxlcjw8L1Jvb3QgMSAwIFI+PgolJUVPRgo=" + } + } + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/plain_text.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/plain_text.json new file mode 100644 index 00000000000..9cbf02951c9 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/plain_text.json @@ -0,0 +1,18 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/reasoning_effort_low.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/reasoning_effort_low.json new file mode 100644 index 00000000000..edf94f7075d --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/reasoning_effort_low.json @@ -0,0 +1,22 @@ +{ + "additionalModelRequestFields": { + "stream": false, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 2048 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_object.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_object.json new file mode 100644 index 00000000000..cfc416cefec --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_object.json @@ -0,0 +1,18 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "Reply in JSON: capital of France?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_schema.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_schema.json new file mode 100644 index 00000000000..742d331b4c1 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/response_format_json_schema.json @@ -0,0 +1,47 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "toolConfig": { + "toolChoice": { + "tool": { + "name": "json_tool_call" + } + }, + "tools": [ + { + "toolSpec": { + "description": "json_tool_call", + "inputSchema": { + "json": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + } + }, + "name": "json_tool_call" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/system_prompt.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/system_prompt.json new file mode 100644 index 00000000000..0930d7d73fa --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/system_prompt.json @@ -0,0 +1,23 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "system": [ + { + "text": "You are a terse geographer." + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_enabled.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_enabled.json new file mode 100644 index 00000000000..edf94f7075d --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_enabled.json @@ -0,0 +1,22 @@ +{ + "additionalModelRequestFields": { + "stream": false, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 2048 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_history_blocks.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_history_blocks.json new file mode 100644 index 00000000000..ee23bf983b2 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/thinking_history_blocks.json @@ -0,0 +1,46 @@ +{ + "additionalModelRequestFields": { + "stream": false, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "inferenceConfig": { + "maxTokens": 2048 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + }, + { + "content": [ + { + "reasoningContent": { + "reasoningText": { + "signature": "sig-char-deadbeef", + "text": "The user asks about France; the capital is Paris." + } + } + }, + { + "text": "Paris." + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_basic.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_basic.json new file mode 100644 index 00000000000..1fdae03abef --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_basic.json @@ -0,0 +1,44 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "toolConfig": { + "toolChoice": { + "auto": {} + }, + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_forced_choice.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_forced_choice.json new file mode 100644 index 00000000000..9cad53fc554 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_forced_choice.json @@ -0,0 +1,46 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "toolConfig": { + "toolChoice": { + "tool": { + "name": "get_weather" + } + }, + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_parallel.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_parallel.json new file mode 100644 index 00000000000..946aa2993a3 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_parallel.json @@ -0,0 +1,60 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?" + } + ], + "role": "user" + } + ], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + }, + { + "toolSpec": { + "description": "Get current time for a timezone.", + "inputSchema": { + "json": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "name": "get_time" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_streamed_args_roundtrip.json b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_streamed_args_roundtrip.json new file mode 100644 index 00000000000..17d1254097f --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_converse/tools_streamed_args_roundtrip.json @@ -0,0 +1,108 @@ +{ + "additionalModelRequestFields": { + "stream": false + }, + "inferenceConfig": { + "maxTokens": 256 + }, + "messages": [ + { + "content": [ + { + "text": "Weather and time in Paris?" + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Paris" + }, + "name": "get_weather", + "toolUseId": "call_char_001" + } + }, + { + "toolUse": { + "input": { + "tz": "Europe/Paris" + }, + "name": "get_time", + "toolUseId": "call_char_002" + } + } + ], + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "text": "18C, clear" + } + ], + "toolUseId": "call_char_001" + } + }, + { + "toolResult": { + "content": [ + { + "text": "14:05" + } + ], + "toolUseId": "call_char_002" + } + } + ], + "role": "user" + } + ], + "toolConfig": { + "tools": [ + { + "toolSpec": { + "description": "Get current weather for a city.", + "inputSchema": { + "json": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "name": "get_weather" + } + }, + { + "toolSpec": { + "description": "Get current time for a timezone.", + "inputSchema": { + "json": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + } + }, + "name": "get_time" + } + } + ] + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_messages.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_messages.json new file mode 100644 index 00000000000..933731cf9be --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_messages.json @@ -0,0 +1,27 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_tools.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_tools.json new file mode 100644 index 00000000000..99c23d18198 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/cache_control_tools.json @@ -0,0 +1,52 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "cache_control": { + "type": "ephemeral" + }, + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/full_combo.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/full_combo.json new file mode 100644 index 00000000000..94fcc745fdb --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/full_combo.json @@ -0,0 +1,74 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "Weather in Paris?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "call_char_001", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear", + "tool_use_id": "call_char_001", + "type": "tool_result" + }, + { + "text": "Summarize, then check the weather again.", + "type": "text" + } + ], + "role": "user" + } + ], + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "You are a terse geographer.", + "type": "text" + } + ], + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/image_base64.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/image_base64.json new file mode 100644 index 00000000000..265d88a7fc2 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/image_base64.json @@ -0,0 +1,23 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Describe this image.", + "type": "text" + }, + { + "source": { + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/multi_turn.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/multi_turn.json new file mode 100644 index 00000000000..61ee09b4c18 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/multi_turn.json @@ -0,0 +1,33 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Paris.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?", + "type": "text" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/params_sampling.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/params_sampling.json new file mode 100644 index 00000000000..6229cf15da2 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/params_sampling.json @@ -0,0 +1,24 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 128, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "metadata": { + "user_id": "char-user-1234" + }, + "stop_sequences": [ + "\n\n", + "END" + ], + "temperature": 0.2, + "top_p": 0.9 +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/pdf_base64.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/pdf_base64.json new file mode 100644 index 00000000000..41065fcec38 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/pdf_base64.json @@ -0,0 +1,23 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Summarize this document.", + "type": "text" + }, + { + "source": { + "data": "JVBERi0xLjQKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbXS9Db3VudCAwPj5lbmRvYmoKdHJhaWxlcjw8L1Jvb3QgMSAwIFI+PgolJUVPRgo=", + "media_type": "application/pdf", + "type": "base64" + }, + "type": "document" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/plain_text.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/plain_text.json new file mode 100644 index 00000000000..4b40de8cdd2 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/plain_text.json @@ -0,0 +1,15 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/reasoning_effort_low.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/reasoning_effort_low.json new file mode 100644 index 00000000000..b500ccdda33 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/reasoning_effort_low.json @@ -0,0 +1,19 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_object.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_object.json new file mode 100644 index 00000000000..2b2789daa47 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_object.json @@ -0,0 +1,15 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Reply in JSON: capital of France?", + "type": "text" + } + ], + "role": "user" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_schema.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_schema.json new file mode 100644 index 00000000000..99cefbdc0a6 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/response_format_json_schema.json @@ -0,0 +1,37 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "json_mode": true, + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "tool_choice": { + "name": "json_tool_call", + "type": "tool" + }, + "tools": [ + { + "input_schema": { + "additionalProperties": false, + "properties": { + "capital": { + "type": "string" + } + }, + "required": [ + "capital" + ], + "type": "object" + }, + "name": "json_tool_call" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/system_prompt.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/system_prompt.json new file mode 100644 index 00000000000..749c13f46b4 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/system_prompt.json @@ -0,0 +1,21 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "system": [ + { + "text": "You are a terse geographer.", + "type": "text" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_enabled.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_enabled.json new file mode 100644 index 00000000000..b500ccdda33 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_enabled.json @@ -0,0 +1,19 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_history_blocks.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_history_blocks.json new file mode 100644 index 00000000000..a271ebc4616 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/thinking_history_blocks.json @@ -0,0 +1,42 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "signature": "sig-char-deadbeef", + "thinking": "The user asks about France; the capital is Paris.", + "type": "thinking" + }, + { + "text": "Paris.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "And of Italy?", + "type": "text" + } + ], + "role": "user" + } + ], + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_basic.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_basic.json new file mode 100644 index 00000000000..eeb1177a61c --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_basic.json @@ -0,0 +1,36 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "tool_choice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_forced_choice.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_forced_choice.json new file mode 100644 index 00000000000..51087ce2150 --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_forced_choice.json @@ -0,0 +1,37 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "tool_choice": { + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_parallel.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_parallel.json new file mode 100644 index 00000000000..e741d17e3bd --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_parallel.json @@ -0,0 +1,53 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "What is the capital of France?", + "type": "text" + } + ], + "role": "user" + } + ], + "tool_choice": { + "disable_parallel_tool_use": false, + "type": "auto" + }, + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_streamed_args_roundtrip.json b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_streamed_args_roundtrip.json new file mode 100644 index 00000000000..78f5355841f --- /dev/null +++ b/tests/translation_characterization/snapshots/requests/bedrock_invoke/tools_streamed_args_roundtrip.json @@ -0,0 +1,85 @@ +{ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 256, + "messages": [ + { + "content": [ + { + "text": "Weather and time in Paris?", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "id": "call_char_001", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + }, + { + "id": "call_char_002", + "input": { + "tz": "Europe/Paris" + }, + "name": "get_time", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "18C, clear", + "tool_use_id": "call_char_001", + "type": "tool_result" + }, + { + "content": "14:05", + "tool_use_id": "call_char_002", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "tools": [ + { + "description": "Get current weather for a city.", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather", + "type": "custom" + }, + { + "description": "Get current time for a timezone.", + "input_schema": { + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "type": "object" + }, + "name": "get_time", + "type": "custom" + } + ] +} diff --git a/tests/translation_characterization/snapshots/responses/anthropic/cache_usage.json b/tests/translation_characterization/snapshots/responses/anthropic/cache_usage.json new file mode 100644 index 00000000000..828b3d7033f --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/anthropic/cache_usage.json @@ -0,0 +1,48 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 1024, + "cache_read_input_tokens": 2048, + "completion_tokens": 4, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 4 + }, + "inference_geo": null, + "prompt_tokens": 3078, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_token_details": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 1024 + }, + "cache_creation_tokens": 1024, + "cached_tokens": 2048, + "image_tokens": null, + "text_tokens": 6, + "video_tokens": null + }, + "speed": null, + "total_tokens": 3082 + } +} diff --git a/tests/translation_characterization/snapshots/responses/anthropic/stop_sequence.json b/tests/translation_characterization/snapshots/responses/anthropic/stop_sequence.json new file mode 100644 index 00000000000..436af59966c --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/anthropic/stop_sequence.json @@ -0,0 +1,44 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 2, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 2 + }, + "inference_geo": null, + "prompt_tokens": 12, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 12, + "video_tokens": null + }, + "speed": null, + "total_tokens": 14 + } +} diff --git a/tests/translation_characterization/snapshots/responses/anthropic/text_basic.json b/tests/translation_characterization/snapshots/responses/anthropic/text_basic.json new file mode 100644 index 00000000000..34e6ecc41fc --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/anthropic/text_basic.json @@ -0,0 +1,44 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris is the capital of France.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 9, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 9 + }, + "inference_geo": null, + "prompt_tokens": 12, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 12, + "video_tokens": null + }, + "speed": null, + "total_tokens": 21 + } +} diff --git a/tests/translation_characterization/snapshots/responses/anthropic/thinking.json b/tests/translation_characterization/snapshots/responses/anthropic/thinking.json new file mode 100644 index 00000000000..22992cb402a --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/anthropic/thinking.json @@ -0,0 +1,58 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": [ + { + "signature": "EuYBCkQYAiJAchar-signature-bytes", + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + } + ] + }, + "reasoning_content": "The user asks for the capital of France. That is Paris.", + "role": "assistant", + "thinking_blocks": [ + { + "signature": "EuYBCkQYAiJAchar-signature-bytes", + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + } + ], + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 31, + "completion_tokens_details": { + "reasoning_tokens": 13, + "text_tokens": 18 + }, + "inference_geo": null, + "prompt_tokens": 40, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 40, + "video_tokens": null + }, + "speed": null, + "total_tokens": 71 + } +} diff --git a/tests/translation_characterization/snapshots/responses/anthropic/tool_use_parallel.json b/tests/translation_characterization/snapshots/responses/anthropic/tool_use_parallel.json new file mode 100644 index 00000000000..aa64de2e907 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/anthropic/tool_use_parallel.json @@ -0,0 +1,63 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll check both for you.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "toolu_01CharWeather01", + "index": 1, + "type": "function" + }, + { + "function": { + "arguments": "{\"tz\": \"Europe/Paris\"}", + "name": "get_time" + }, + "id": "toolu_01CharTime00001", + "index": 2, + "type": "function" + } + ] + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 87, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 87 + }, + "inference_geo": null, + "prompt_tokens": 412, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 412, + "video_tokens": null + }, + "speed": null, + "total_tokens": 499 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_converse/cache_usage.json b/tests/translation_characterization/snapshots/responses/bedrock_converse/cache_usage.json new file mode 100644 index 00000000000..a0ce9eb2ea4 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_converse/cache_usage.json @@ -0,0 +1,38 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris.", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 1024, + "cache_read_input_tokens": 2048, + "completion_tokens": 4, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 4 + }, + "prompt_tokens": 3078, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 1024, + "cached_tokens": 2048, + "image_tokens": null, + "text_tokens": 6, + "video_tokens": null + }, + "total_tokens": 3082 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_converse/reasoning.json b/tests/translation_characterization/snapshots/responses/bedrock_converse/reasoning.json new file mode 100644 index 00000000000..8ad1e12bea2 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_converse/reasoning.json @@ -0,0 +1,56 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris.", + "function_call": null, + "provider_specific_fields": { + "reasoningContentBlocks": [ + { + "reasoningText": { + "signature": "char-converse-signature", + "text": "The user asks for the capital of France. That is Paris." + } + } + ] + }, + "reasoning_content": "The user asks for the capital of France. That is Paris.", + "role": "assistant", + "thinking_blocks": [ + { + "signature": "char-converse-signature", + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + } + ], + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 31, + "completion_tokens_details": { + "reasoning_tokens": 13, + "text_tokens": 18 + }, + "prompt_tokens": 40, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 40, + "video_tokens": null + }, + "total_tokens": 71 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_converse/text_basic.json b/tests/translation_characterization/snapshots/responses/bedrock_converse/text_basic.json new file mode 100644 index 00000000000..16b5b188547 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_converse/text_basic.json @@ -0,0 +1,38 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris is the capital of France.", + "function_call": null, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 9, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 9 + }, + "prompt_tokens": 12, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 12, + "video_tokens": null + }, + "total_tokens": 21 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_converse/tool_use.json b/tests/translation_characterization/snapshots/responses/bedrock_converse/tool_use.json new file mode 100644 index 00000000000..359f5e1ef83 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_converse/tool_use.json @@ -0,0 +1,48 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll check the weather.", + "function_call": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "tooluse_charweather01", + "index": 1, + "type": "function" + } + ] + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 60, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 60 + }, + "prompt_tokens": 410, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 410, + "video_tokens": null + }, + "total_tokens": 470 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_invoke/text_basic.json b/tests/translation_characterization/snapshots/responses/bedrock_invoke/text_basic.json new file mode 100644 index 00000000000..0b021b14451 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_invoke/text_basic.json @@ -0,0 +1,44 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris is the capital of France.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-3-5-sonnet-20240620", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 9, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 9 + }, + "inference_geo": null, + "prompt_tokens": 12, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 12, + "video_tokens": null + }, + "speed": null, + "total_tokens": 21 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_invoke/thinking.json b/tests/translation_characterization/snapshots/responses/bedrock_invoke/thinking.json new file mode 100644 index 00000000000..7f13668ee89 --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_invoke/thinking.json @@ -0,0 +1,58 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Paris.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": [ + { + "signature": "char-invoke-signature", + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + } + ] + }, + "reasoning_content": "The user asks for the capital of France. That is Paris.", + "role": "assistant", + "thinking_blocks": [ + { + "signature": "char-invoke-signature", + "thinking": "The user asks for the capital of France. That is Paris.", + "type": "thinking" + } + ], + "tool_calls": null + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-3-7-sonnet-20250219", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 31, + "completion_tokens_details": { + "reasoning_tokens": 13, + "text_tokens": 18 + }, + "inference_geo": null, + "prompt_tokens": 40, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 40, + "video_tokens": null + }, + "speed": null, + "total_tokens": 71 + } +} diff --git a/tests/translation_characterization/snapshots/responses/bedrock_invoke/tool_use.json b/tests/translation_characterization/snapshots/responses/bedrock_invoke/tool_use.json new file mode 100644 index 00000000000..c2212985b9b --- /dev/null +++ b/tests/translation_characterization/snapshots/responses/bedrock_invoke/tool_use.json @@ -0,0 +1,54 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "I'll check the weather.", + "function_call": null, + "provider_specific_fields": { + "citations": null, + "thinking_blocks": null + }, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "toolu_bdrk_01CharWeather", + "index": 1, + "type": "function" + } + ] + } + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-3-5-sonnet-20240620", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "completion_tokens": 60, + "completion_tokens_details": { + "reasoning_tokens": 0, + "text_tokens": 60 + }, + "inference_geo": null, + "prompt_tokens": 410, + "prompt_tokens_details": { + "audio_tokens": null, + "cache_creation_tokens": 0, + "cached_tokens": 0, + "image_tokens": null, + "text_tokens": 410, + "video_tokens": null + }, + "speed": null, + "total_tokens": 470 + } +} diff --git a/tests/translation_characterization/snapshots/streams/anthropic/text_stream.json b/tests/translation_characterization/snapshots/streams/anthropic/text_stream.json new file mode 100644 index 00000000000..de9c0893160 --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/anthropic/text_stream.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris is the", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": " capital of France.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/anthropic/thinking_stream.json b/tests/translation_characterization/snapshots/streams/anthropic/thinking_stream.json new file mode 100644 index 00000000000..5e9c03ac1d3 --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/anthropic/thinking_stream.json @@ -0,0 +1,168 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "thinking_blocks": [ + { + "signature": "", + "thinking": "The user asks for the capital of France.", + "type": "thinking" + } + ] + }, + "reasoning_content": "The user asks for the capital of France.", + "role": "assistant", + "thinking_blocks": [ + { + "signature": "", + "thinking": "The user asks for the capital of France.", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "thinking_blocks": [ + { + "signature": "", + "thinking": " That is Paris.", + "type": "thinking" + } + ] + }, + "reasoning_content": " That is Paris.", + "role": null, + "thinking_blocks": [ + { + "signature": "", + "thinking": " That is Paris.", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "thinking_blocks": [ + { + "signature": "EuYBCkQYAiJAchar-stream-signature", + "thinking": "", + "type": "thinking" + } + ] + }, + "reasoning_content": "", + "role": null, + "thinking_blocks": [ + { + "signature": "EuYBCkQYAiJAchar-stream-signature", + "thinking": "", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/anthropic/tool_stream.json b/tests/translation_characterization/snapshots/streams/anthropic/tool_stream.json new file mode 100644 index 00000000000..dc885df37ec --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/anthropic/tool_stream.json @@ -0,0 +1,184 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "I'll check the weather.", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "get_weather" + }, + "id": "toolu_01CharStreamW01", + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"ci", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "ty\": \"Par", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "is\"}", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "claude-sonnet-4-20250514", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_converse/cache_usage_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_converse/cache_usage_stream.json new file mode 100644 index 00000000000..0832977c252 --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_converse/cache_usage_stream.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_converse/reasoning_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_converse/reasoning_stream.json new file mode 100644 index 00000000000..c1c34511dc9 --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_converse/reasoning_stream.json @@ -0,0 +1,178 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "reasoningContent": { + "text": "The user asks for the capital of France." + } + }, + "reasoning_content": "The user asks for the capital of France.", + "role": null, + "thinking_blocks": [ + { + "thinking": "The user asks for the capital of France.", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "reasoningContent": { + "text": " That is Paris." + } + }, + "reasoning_content": " That is Paris.", + "role": null, + "thinking_blocks": [ + { + "thinking": " That is Paris.", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": { + "reasoningContent": { + "signature": "char-converse-stream-signature" + } + }, + "reasoning_content": "", + "role": null, + "thinking_blocks": [ + { + "signature": "char-converse-stream-signature", + "thinking": "", + "type": "thinking" + } + ], + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_converse/text_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_converse/text_stream.json new file mode 100644 index 00000000000..812b4bce89b --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_converse/text_stream.json @@ -0,0 +1,96 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris is the", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": " capital of France.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_converse/tool_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_converse/tool_stream.json new file mode 100644 index 00000000000..c3f32f1315c --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_converse/tool_stream.json @@ -0,0 +1,208 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "I'll check the weather.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "get_weather" + }, + "id": "tooluse_charstream01", + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"ci", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "ty\": \"Par", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "is\"}", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000002", + "model": "anthropic.claude-sonnet-4-20250514-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": {}, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_invoke/text_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_invoke/text_stream.json new file mode 100644 index 00000000000..d1fb65e3711 --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_invoke/text_stream.json @@ -0,0 +1,72 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "Paris is the", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": " capital of France.", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/snapshots/streams/bedrock_invoke/tool_stream.json b/tests/translation_characterization/snapshots/streams/bedrock_invoke/tool_stream.json new file mode 100644 index 00000000000..55bf4c9ba0c --- /dev/null +++ b/tests/translation_characterization/snapshots/streams/bedrock_invoke/tool_stream.json @@ -0,0 +1,160 @@ +[ + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "get_weather" + }, + "id": "toolu_bdrk_01CharStream", + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"ci", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "ty\": \"Par", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": "", + "function_call": null, + "provider_specific_fields": null, + "role": null, + "tool_calls": [ + { + "function": { + "arguments": "is\"}", + "name": null + }, + "id": null, + "index": 0, + "type": "function" + } + ] + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "citations": null, + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + }, + { + "choices": [ + { + "delta": { + "audio": null, + "content": null, + "function_call": null, + "role": null, + "tool_calls": null + }, + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null + } + ], + "created": 1718064000, + "id": "chatcmpl-00000000-0000-0000-0000-000000000001", + "model": "invoke/anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion.chunk", + "provider_specific_fields": null, + "system_fingerprint": null + } +] diff --git a/tests/translation_characterization/test_messages_adapter.py b/tests/translation_characterization/test_messages_adapter.py new file mode 100644 index 00000000000..4c804cf198f --- /dev/null +++ b/tests/translation_characterization/test_messages_adapter.py @@ -0,0 +1,101 @@ +"""Pin the /v1/messages surface, both flavours: + +1. The anthropic<->openai adapter (anthropic-in -> non-anthropic model): + ``LiteLLMAnthropicMessagesAdapter.translate_anthropic_to_openai`` and + ``translate_openai_response_to_anthropic``. +2. The native ``AnthropicMessagesConfig.transform_anthropic_messages_request`` + quirks, despite its "No transformation is needed" docstring: max_tokens is + popped/required, reasoning_effort is rewritten to ``thinking``/adaptive + params, and advisor blocks are stripped when the advisor tool is absent. +""" + +import copy + +import pytest + +from ._helpers import FIXTURES_DIR, assert_snapshot, load_json + +_ADAPTER_DIR = FIXTURES_DIR / "adapter" + + +def _ids(sub: str) -> list: + return sorted(p.stem for p in (_ADAPTER_DIR / sub).glob("*.json")) + + +@pytest.mark.parametrize("fixture_id", _ids("requests")) +def test_adapter_anthropic_request_to_openai( + fixture_id: str, snapshot_update: bool +) -> None: + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + anthropic_request = load_json(_ADAPTER_DIR / "requests" / f"{fixture_id}.json") + openai_request, tool_name_mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request=anthropic_request + ) + assert_snapshot( + f"adapter/requests/{fixture_id}.json", + {"openai_request": openai_request, "tool_name_mapping": tool_name_mapping}, + snapshot_update, + ) + + +@pytest.mark.parametrize("fixture_id", _ids("responses")) +def test_adapter_openai_response_to_anthropic( + fixture_id: str, snapshot_update: bool +) -> None: + import litellm + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + + openai_response = litellm.ModelResponse( + **load_json(_ADAPTER_DIR / "responses" / f"{fixture_id}.json") + ) + anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=openai_response + ) + assert_snapshot( + f"adapter/responses/{fixture_id}.json", anthropic_response, snapshot_update + ) + + +@pytest.mark.parametrize("fixture_id", _ids("native")) +def test_native_messages_request_transform( + fixture_id: str, snapshot_update: bool +) -> None: + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + from litellm.types.router import GenericLiteLLMParams + + case = load_json(_ADAPTER_DIR / "native" / f"{fixture_id}.json") + result = AnthropicMessagesConfig().transform_anthropic_messages_request( + model=case["model"], + messages=copy.deepcopy(case["messages"]), + anthropic_messages_optional_request_params=copy.deepcopy(case["params"]), + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert_snapshot(f"adapter/native/{fixture_id}.json", result, snapshot_update) + + +def test_native_messages_requires_max_tokens() -> None: + """The documented sad path: max_tokens missing -> AnthropicError(400).""" + from litellm.llms.anthropic.common_utils import AnthropicError + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + from litellm.types.router import GenericLiteLLMParams + + with pytest.raises(AnthropicError) as excinfo: + AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-20250514", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert excinfo.value.status_code == 400 + assert "max_tokens is required" in str(excinfo.value) diff --git a/tests/translation_characterization/test_request_transforms.py b/tests/translation_characterization/test_request_transforms.py new file mode 100644 index 00000000000..149196e8185 --- /dev/null +++ b/tests/translation_characterization/test_request_transforms.py @@ -0,0 +1,32 @@ +"""Pin v1 request transforms: corpus cases -> provider wire-format bodies.""" + +import pytest + +from . import _corpus +from ._helpers import CASES_DIR, canonical_json, load_json +from ._helpers import assert_snapshot +from ._seams import PROVIDERS, run_request_transform + +CASES = _corpus.build_request_cases() + + +def test_corpus_files_in_sync(snapshot_update: bool) -> None: + """cases/*.json must be exactly what the builder emits (stable ids).""" + if snapshot_update: + _corpus.write_case_files() + on_disk = sorted(p.stem for p in CASES_DIR.glob("*.json")) + assert on_disk == sorted(CASES), "corpus builder and cases/ dir diverged" + for case_id, case in CASES.items(): + assert canonical_json(case) == (CASES_DIR / f"{case_id}.json").read_text() + + +@pytest.mark.parametrize("case_id", sorted(CASES)) +@pytest.mark.parametrize("provider_key", sorted(PROVIDERS)) +def test_request_transform( + provider_key: str, case_id: str, snapshot_update: bool +) -> None: + case = load_json(CASES_DIR / f"{case_id}.json") + if provider_key in case["skip"]: + pytest.skip(case["skip"][provider_key]) + body = run_request_transform(provider_key, case) + assert_snapshot(f"requests/{provider_key}/{case_id}.json", body, snapshot_update) diff --git a/tests/translation_characterization/test_response_transforms.py b/tests/translation_characterization/test_response_transforms.py new file mode 100644 index 00000000000..7041a386c57 --- /dev/null +++ b/tests/translation_characterization/test_response_transforms.py @@ -0,0 +1,30 @@ +"""Pin v1 response transforms: recorded provider JSON -> OpenAI ModelResponse.""" + +import pytest + +from ._helpers import FIXTURES_DIR, assert_snapshot, load_json +from ._seams import PROVIDERS, run_response_transform + +_MESSAGES = [{"role": "user", "content": "What is the capital of France?"}] + + +def _fixture_ids(provider_key: str) -> list: + return sorted(p.stem for p in (FIXTURES_DIR / "responses" / provider_key).glob("*.json")) + + +@pytest.mark.parametrize( + "provider_key,fixture_id", + [ + (p, f) + for p in sorted(PROVIDERS) + for f in _fixture_ids(p) + ], +) +def test_response_transform( + provider_key: str, fixture_id: str, snapshot_update: bool +) -> None: + payload = load_json(FIXTURES_DIR / "responses" / provider_key / f"{fixture_id}.json") + result = run_response_transform(provider_key, payload, _MESSAGES) + assert_snapshot( + f"responses/{provider_key}/{fixture_id}.json", result, snapshot_update + ) diff --git a/tests/translation_characterization/test_stream_replay.py b/tests/translation_characterization/test_stream_replay.py new file mode 100644 index 00000000000..b863966081f --- /dev/null +++ b/tests/translation_characterization/test_stream_replay.py @@ -0,0 +1,50 @@ +"""Pin v1 streaming: recorded provider events -> sequence of OpenAI chunks. + +Anthropic replays raw SSE lines through the real ``ModelResponseIterator`` +inside ``CustomStreamWrapper``. The two bedrock routes replay parsed event +payloads through the real decoders' chunk parsers — the binary AWS +event-stream framing upstream of that seam is botocore plumbing, documented +as out of corpus scope in README.md. +""" + +import pytest + +from ._helpers import FIXTURES_DIR, assert_snapshot, load_json +from ._seams import ( + replay_anthropic_sse, + replay_bedrock_converse_events, + replay_bedrock_invoke_events, +) + + +def _fixture_paths(provider_key: str, suffix: str) -> list: + return sorted((FIXTURES_DIR / "streams" / provider_key).glob(f"*{suffix}")) + + +@pytest.mark.parametrize( + "path", _fixture_paths("anthropic", ".txt"), ids=lambda p: p.stem +) +def test_anthropic_stream(path, snapshot_update: bool) -> None: + sse_lines = path.read_text().splitlines() + chunks = replay_anthropic_sse(sse_lines) + assert_snapshot(f"streams/anthropic/{path.stem}.json", chunks, snapshot_update) + + +@pytest.mark.parametrize( + "path", _fixture_paths("bedrock_converse", ".json"), ids=lambda p: p.stem +) +def test_bedrock_converse_stream(path, snapshot_update: bool) -> None: + chunks = replay_bedrock_converse_events(load_json(path)) + assert_snapshot( + f"streams/bedrock_converse/{path.stem}.json", chunks, snapshot_update + ) + + +@pytest.mark.parametrize( + "path", _fixture_paths("bedrock_invoke", ".json"), ids=lambda p: p.stem +) +def test_bedrock_invoke_stream(path, snapshot_update: bool) -> None: + chunks = replay_bedrock_invoke_events(load_json(path)) + assert_snapshot( + f"streams/bedrock_invoke/{path.stem}.json", chunks, snapshot_update + )