From 580bb85f5b088f431b635d637b35cf776ce4f8cd Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Wed, 9 Sep 2026 17:26:57 -0600 Subject: [PATCH] Depend on lithos-llm and the published twin-openai crate Add lithos-llm as a pinned git dependency and replace the in-repo `test/twin/openai` crate with the published `twins` revision that lithos-llm verifies its codecs against. Move the Fabro policy overlay (`fabro-policy.toml`) into fabro-llm so Fabro owns its own catalog policy layer. Drop the twin-openai nextest overrides and CI package filter now that the crate is no longer a workspace member. Co-Authored-By: Claude Fable 5.1 --- .config/nextest.toml | 6 - .github/workflows/rust.yml | 2 +- Cargo.lock | 155 +- Cargo.toml | 8 +- .../fabro-llm/catalog/fabro-policy.toml | 883 ++++++++ lib/foundation/fabro-http/src/lib.rs | 1 + lib/foundation/fabro-test/src/lib.rs | 6 +- test/twin/openai/.gitignore | 1 - test/twin/openai/Cargo.toml | 30 - test/twin/openai/README.md | 85 - test/twin/openai/docs/compatibility-matrix.md | 64 - ...-03-27-async-rust-fake-openai-test-plan.md | 204 -- .../2026-03-27-async-rust-fake-openai.md | 579 ------ .../2026-03-28-debug-web-ui-test-plan.md | 154 -- test/twin/openai/src/admin.rs | 50 - test/twin/openai/src/app.rs | 23 - test/twin/openai/src/config.rs | 66 - test/twin/openai/src/debug_ui.rs | 447 ---- test/twin/openai/src/engine/defaults.rs | 113 - test/twin/openai/src/engine/failures.rs | 58 - test/twin/openai/src/engine/mod.rs | 126 -- test/twin/openai/src/engine/plan.rs | 163 -- test/twin/openai/src/engine/scenario.rs | 271 --- test/twin/openai/src/lib.rs | 26 - test/twin/openai/src/logs.rs | 12 - test/twin/openai/src/main.rs | 21 - test/twin/openai/src/openai/auth.rs | 67 - .../openai/src/openai/chat_completions.rs | 82 - test/twin/openai/src/openai/mod.rs | 24 - test/twin/openai/src/openai/models.rs | 1104 ---------- test/twin/openai/src/openai/responses.rs | 77 - test/twin/openai/src/sse.rs | 385 ---- test/twin/openai/src/state.rs | 178 -- .../openai/tests/chat_completions_contract.rs | 535 ----- test/twin/openai/tests/common/mod.rs | 587 ------ test/twin/openai/tests/config_contract.rs | 15 - test/twin/openai/tests/debug_ui.rs | 403 ---- test/twin/openai/tests/failure_modes.rs | 338 --- test/twin/openai/tests/health_and_auth.rs | 102 - .../twin/openai/tests/live_openai_contract.rs | 1849 ----------------- test/twin/openai/tests/responses_contract.rs | 440 ---- .../openai/tests/tool_and_schema_contract.rs | 581 ------ 42 files changed, 969 insertions(+), 9352 deletions(-) create mode 100644 lib/components/fabro-llm/catalog/fabro-policy.toml delete mode 100644 test/twin/openai/.gitignore delete mode 100644 test/twin/openai/Cargo.toml delete mode 100644 test/twin/openai/README.md delete mode 100644 test/twin/openai/docs/compatibility-matrix.md delete mode 100644 test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai-test-plan.md delete mode 100644 test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai.md delete mode 100644 test/twin/openai/docs/plans/2026-03-28-debug-web-ui-test-plan.md delete mode 100644 test/twin/openai/src/admin.rs delete mode 100644 test/twin/openai/src/app.rs delete mode 100644 test/twin/openai/src/config.rs delete mode 100644 test/twin/openai/src/debug_ui.rs delete mode 100644 test/twin/openai/src/engine/defaults.rs delete mode 100644 test/twin/openai/src/engine/failures.rs delete mode 100644 test/twin/openai/src/engine/mod.rs delete mode 100644 test/twin/openai/src/engine/plan.rs delete mode 100644 test/twin/openai/src/engine/scenario.rs delete mode 100644 test/twin/openai/src/lib.rs delete mode 100644 test/twin/openai/src/logs.rs delete mode 100644 test/twin/openai/src/main.rs delete mode 100644 test/twin/openai/src/openai/auth.rs delete mode 100644 test/twin/openai/src/openai/chat_completions.rs delete mode 100644 test/twin/openai/src/openai/mod.rs delete mode 100644 test/twin/openai/src/openai/models.rs delete mode 100644 test/twin/openai/src/openai/responses.rs delete mode 100644 test/twin/openai/src/sse.rs delete mode 100644 test/twin/openai/src/state.rs delete mode 100644 test/twin/openai/tests/chat_completions_contract.rs delete mode 100644 test/twin/openai/tests/common/mod.rs delete mode 100644 test/twin/openai/tests/config_contract.rs delete mode 100644 test/twin/openai/tests/debug_ui.rs delete mode 100644 test/twin/openai/tests/failure_modes.rs delete mode 100644 test/twin/openai/tests/health_and_auth.rs delete mode 100644 test/twin/openai/tests/live_openai_contract.rs delete mode 100644 test/twin/openai/tests/responses_contract.rs delete mode 100644 test/twin/openai/tests/tool_and_schema_contract.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 3af76c17c..0580f3f5a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -15,9 +15,6 @@ leak-timeout = "500ms" filter = "package(fabro-workflow)" slow-timeout = { period = "2s", terminate-after = 3 } - [[profile.default.overrides]] - filter = "package(twin-openai) & test(debug_page_renders_in_headless_chrome)" - slow-timeout = { period = "30s", terminate-after = 1 } [profile.e2e] # E2E (ignored) tests: flag SLOW after 10s, hard-kill after 30s @@ -48,6 +45,3 @@ leak-timeout = "2s" filter = "package(fabro-workflow)" slow-timeout = { period = "30s", terminate-after = 4 } - [[profile.ci.overrides]] - filter = "package(twin-openai) & test(debug_page_renders_in_headless_chrome)" - slow-timeout = { period = "60s", terminate-after = 2 } diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b3686f872..dededfa2b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -131,7 +131,7 @@ jobs: # in twin mode; widen as the remaining suites are fixed up for CI. # Must not use the e2e nextest profile here: NEXTEST_PROFILE=e2e implies # strict mode, which fails (rather than skips) live tests without keys. - - run: cargo nextest run --locked --workspace --status-level slow --profile ci --run-ignored only -E 'package(fabro-agent) + package(fabro-llm) + package(twin-openai)' + - run: cargo nextest run --locked --workspace --status-level slow --profile ci --run-ignored only -E 'package(fabro-agent) + package(fabro-llm)' test-macos: name: Test (macOS) diff --git a/Cargo.lock b/Cargo.lock index c0d32aa79..3d2169ece 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,17 +660,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - [[package]] name = "aws-smithy-http" version = "0.63.6" @@ -1872,7 +1861,7 @@ name = "daytona-api-client" version = "0.1.0" source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ - "reqwest 0.13.2", + "reqwest 0.13.4", "reqwest-middleware", "serde", "serde_json", @@ -1889,7 +1878,7 @@ dependencies = [ "daytona-api-client", "daytona-toolbox-client", "futures-util", - "reqwest 0.13.2", + "reqwest 0.13.4", "reqwest-middleware", "rustls", "serde", @@ -1906,7 +1895,7 @@ name = "daytona-toolbox-client" version = "0.1.0" source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ - "reqwest 0.13.2", + "reqwest 0.13.4", "reqwest-middleware", "serde", "serde_json", @@ -2074,7 +2063,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2201,7 +2190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2289,7 +2278,6 @@ dependencies = [ "fabro-llm", "fabro-macros", "fabro-mcp", - "fabro-model", "fabro-sandbox", "fabro-static", "fabro-template", @@ -2329,14 +2317,13 @@ dependencies = [ "fabro-automation", "fabro-config", "fabro-environment", - "fabro-model", "fabro-types", "openapiv3", "prettyplease", "progenitor", "progenitor-client", "regress", - "reqwest 0.13.2", + "reqwest 0.13.4", "serde", "serde_json", "serde_yaml", @@ -2353,13 +2340,13 @@ dependencies = [ "base64", "chrono", "fabro-http", - "fabro-model", "fabro-oauth", "fabro-redact", "fabro-static", "fabro-types", "fabro-vault", "httpmock", + "lithos-llm", "serde", "serde_json", "tempfile", @@ -2451,7 +2438,6 @@ dependencies = [ "fabro-manifest", "fabro-mcp", "fabro-mcp-server", - "fabro-model", "fabro-oauth", "fabro-proc", "fabro-redact", @@ -2486,7 +2472,7 @@ dependencies = [ "progenitor-client", "rand 0.9.4", "regex", - "reqwest 0.13.2", + "reqwest 0.13.4", "ring", "rustls", "scopeguard", @@ -2521,7 +2507,6 @@ dependencies = [ "chrono", "fabro-api", "fabro-http", - "fabro-model", "fabro-static", "fabro-types", "fabro-util", @@ -2550,7 +2535,6 @@ dependencies = [ "clap", "dirs", "fabro-macros", - "fabro-model", "fabro-options-metadata", "fabro-proc", "fabro-static", @@ -2704,7 +2688,6 @@ dependencies = [ "fabro-auth", "fabro-http", "fabro-llm", - "fabro-model", "fabro-redact", "fabro-types", "fabro-util", @@ -2724,7 +2707,7 @@ version = "0.348.0-nightly.0" dependencies = [ "fabro-static", "http 1.4.0", - "reqwest 0.13.2", + "reqwest 0.13.4", "thiserror 2.0.18", ] @@ -2768,40 +2751,30 @@ version = "0.348.0-nightly.0" dependencies = [ "anyhow", "async-trait", - "aws-config", - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", "base64", "bytes", "fabro-auth", + "fabro-config", "fabro-http", + "fabro-llm", "fabro-macros", - "fabro-model", "fabro-redact", "fabro-static", "fabro-test", "fabro-types", - "fabro-util", "futures", - "http 1.4.0", "httpmock", - "insta", - "rand 0.9.4", + "lithos-llm", + "mime_guess", "serde", "serde_json", - "sha2 0.10.9", "strum 0.28.0", + "tempfile", "thiserror 2.0.18", "tokio", - "tokio-stream", "tokio-util", "toml 0.8.23", "tracing", - "tracing-subscriber", - "uuid", ] [[package]] @@ -2869,7 +2842,6 @@ dependencies = [ "fabro-client", "fabro-config", "fabro-manifest", - "fabro-model", "fabro-server", "fabro-tool", "fabro-types", @@ -2905,23 +2877,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "fabro-model" -version = "0.348.0-nightly.0" -dependencies = [ - "clap", - "fabro-static", - "http 1.4.0", - "insta", - "rust-embed", - "serde", - "serde_json", - "strum 0.28.0", - "thiserror 2.0.18", - "toml 0.8.23", - "tracing", -] - [[package]] name = "fabro-oauth" version = "0.348.0-nightly.0" @@ -3056,7 +3011,6 @@ dependencies = [ "fabro-macros", "fabro-manifest", "fabro-mcp-store", - "fabro-model", "fabro-proc", "fabro-redact", "fabro-sandbox", @@ -3288,10 +3242,10 @@ dependencies = [ "chrono", "clap", "dirs", - "fabro-model", "fabro-types", "fabro-util", "hex", + "lithos-llm", "serde", "serde_json", "sha2 0.10.9", @@ -3333,7 +3287,7 @@ version = "0.348.0-nightly.0" dependencies = [ "fabro-acp", "fabro-graphviz", - "fabro-model", + "fabro-llm", "fabro-types", "serde", "thiserror 2.0.18", @@ -3404,7 +3358,6 @@ dependencies = [ "fabro-llm", "fabro-macros", "fabro-mcp", - "fabro-model", "fabro-redact", "fabro-sandbox", "fabro-static", @@ -4996,6 +4949,31 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "lithos-llm" +version = "0.1.0" +source = "git+https://github.com/lithoscomputer/lithos-llm?rev=4aab27d7d42e7f762a8b6a3871c3db86816b0721#4aab27d7d42e7f762a8b6a3871c3db86816b0721" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", + "aws-smithy-types", + "base64", + "crc32fast", + "futures-core", + "futures-util", + "reqwest 0.13.4", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "toml 0.8.23", + "tracing", + "uuid", +] + [[package]] name = "litrs" version = "1.0.0" @@ -5435,7 +5413,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6298,7 +6276,7 @@ dependencies = [ "bytes", "futures-core", "percent-encoding", - "reqwest 0.13.2", + "reqwest 0.13.4", "serde", "serde_json", "serde_urlencoded", @@ -6432,7 +6410,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6699,9 +6677,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -6757,7 +6735,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.0", - "reqwest 0.13.2", + "reqwest 0.13.4", "serde", "thiserror 2.0.18", "tower-service", @@ -6797,7 +6775,7 @@ dependencies = [ "pastey", "pin-project-lite", "process-wrap", - "reqwest 0.13.2", + "reqwest 0.13.4", "rmcp-macros", "schemars 1.2.1", "serde", @@ -6909,7 +6887,7 @@ dependencies = [ "errno 0.3.14", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6968,7 +6946,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7492,7 +7470,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno 0.2.8", + "errno 0.3.14", "libc", ] @@ -8082,7 +8060,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8128,7 +8106,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8603,6 +8581,21 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twin-core" +version = "0.1.5" +source = "git+https://github.com/lithoscomputer/twins?rev=ca45f0e50a6716d716aa2f638ca3cf767e88f613#ca45f0e50a6716d716aa2f638ca3cf767e88f613" +dependencies = [ + "anyhow", + "async-stream", + "axum", + "futures-util", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "twin-github" version = "0.348.0-nightly.0" @@ -8612,7 +8605,7 @@ dependencies = [ "chrono", "fabro-http", "jsonwebtoken", - "reqwest 0.13.2", + "reqwest 0.13.4", "serde", "serde_json", "tempfile", @@ -8624,21 +8617,21 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.348.0-nightly.0" +version = "0.1.5" +source = "git+https://github.com/lithoscomputer/twins?rev=ca45f0e50a6716d716aa2f638ca3cf767e88f613#ca45f0e50a6716d716aa2f638ca3cf767e88f613" dependencies = [ "anyhow", "async-stream", "axum", - "fabro-http", - "fabro-static", "futures-util", "http 1.4.0", + "reqwest 0.13.4", "serde", "serde_json", "tokio", - "tower", "tracing", "tracing-subscriber", + "twin-core", ] [[package]] @@ -9191,7 +9184,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c642467a5..91acc604e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ members = [ "lib/apps/*", "lib/components/*", "lib/foundation/*", - "test/twin/openai", "test/twin/github", ] default-members = ["lib/apps/fabro-cli"] @@ -93,7 +92,12 @@ sha2 = "0.10" hex = "0.4" insta = "1" fabro-test = { path = "lib/foundation/fabro-test" } -twin-openai = { path = "test/twin/openai" } +# Provider-neutral LLM catalog and client. Pinned to a revision until 0.x is +# published to crates.io. +lithos-llm = { git = "https://github.com/lithoscomputer/lithos-llm", rev = "4aab27d7d42e7f762a8b6a3871c3db86816b0721", default-features = false } +# Deterministic OpenAI twin used by twin-mode E2E tests; the same revision +# lithos-llm verifies its codecs against. +twin-openai = { git = "https://github.com/lithoscomputer/twins", rev = "ca45f0e50a6716d716aa2f638ca3cf767e88f613" } twin-github = { path = "test/twin/github" } tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] } futures-util = "0.3" diff --git a/lib/components/fabro-llm/catalog/fabro-policy.toml b/lib/components/fabro-llm/catalog/fabro-policy.toml new file mode 100644 index 000000000..86cb4ab3c --- /dev/null +++ b/lib/components/fabro-llm/catalog/fabro-policy.toml @@ -0,0 +1,883 @@ +# Fabro's policy layer over the lithos-llm built-in catalog. +# +# Applied after the built-ins and before operator `[llm]` overlays. Everything +# under `metadata.fabro` is Fabro policy that lithos carries verbatim: +# +# - `enabled`: whether Fabro offers the provider or model. +# - `credentials`: ordered credential references (`env:NAME`, `vault:NAME`, +# `aws_sigv4`); the first that resolves wins. +# - `extra_headers`: request headers, literal or `{{ secrets.NAME }}`. +# - `agent_profile`, `family`, `small_default`, `probe`: agent and selection +# policy read by the Fabro model resolver. +# +# Modal's default model comes from a deployment template, not this layer. +# GPT-OSS is excluded at user request. This layer cannot restore that family. +schema_version = 1 + +[providers."anthropic"] +priority = 100 +default_model = "claude-sonnet-5" + +[providers."anthropic".metadata.fabro] +api_key_url = "https://console.anthropic.com/settings/keys" +credentials = ["env:ANTHROPIC_API_KEY", "vault:ANTHROPIC_API_KEY"] +enabled = true + +[providers."anthropic".models."claude-fable-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."anthropic".models."claude-opus-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."anthropic".models."claude-sonnet-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."anthropic".models."claude-opus-4.8".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."anthropic".models."claude-opus-4.7".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."anthropic".models."claude-opus-4.6".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."anthropic".models."claude-sonnet-4.5".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."anthropic".models."claude-sonnet-4.6".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."anthropic".models."claude-haiku-4.5".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = true +probe = true + +[providers."bedrock-openai"] +priority = 19 +default_model = "gpt-5.5" + +[providers."bedrock-openai".metadata.fabro] +api_key_url = "https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html" +credentials = ["env:AWS_BEARER_TOKEN_BEDROCK", "env:BEDROCK_API_KEY", "vault:AWS_BEARER_TOKEN_BEDROCK", "vault:BEDROCK_API_KEY"] +enabled = false + +[providers."bedrock-openai".models."gpt-5.5".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock-openai".models."gpt-5.4".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock"] +priority = 20 +default_model = "claude-sonnet-5" + +[providers."bedrock".metadata.fabro] +api_key_url = "https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html" +credentials = ["env:AWS_BEARER_TOKEN_BEDROCK", "env:BEDROCK_API_KEY", "vault:AWS_BEARER_TOKEN_BEDROCK", "vault:BEDROCK_API_KEY", "aws_sigv4"] +enabled = false + +[providers."bedrock".models."anthropic.claude-sonnet-4-6".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."bedrock".models."claude-opus-4-8".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = false +probe = false + +[providers."bedrock".models."claude-haiku-4-5".metadata.fabro] +family = "claude-4" +agent_profile = "anthropic" +small_default = true +probe = false + +[providers."bedrock".models."nova-2-lite".metadata.fabro] +family = "nova-2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."llama-4-maverick".metadata.fabro] +family = "llama-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."mistral-large-3".metadata.fabro] +family = "mistral-large" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."devstral-2".metadata.fabro] +family = "devstral" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."deepseek-v3.2".metadata.fabro] +family = "deepseek-v3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."kimi-k2.5".metadata.fabro] +family = "kimi-k2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."glm-5".metadata.fabro] +family = "glm" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."minimax-m2.5".metadata.fabro] +family = "minimax-m2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."nemotron-3-super".metadata.fabro] +family = "nemotron-3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."bedrock".models."claude-fable-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."bedrock".models."claude-sonnet-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."deepseek"] +priority = 75 +default_model = "deepseek-v4-flash" + +[providers."deepseek".metadata.fabro] +api_key_url = "https://platform.deepseek.com/api_keys" +credentials = ["env:DEEPSEEK_API_KEY", "vault:DEEPSEEK_API_KEY"] +enabled = true + +[providers."deepseek".models."deepseek-v4-flash".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = true +probe = true + +[providers."deepseek".models."deepseek-v4-pro".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks"] +priority = 30 +default_model = "kimi-k2.7-code" + +[providers."fireworks".metadata.fabro] +api_key_url = "https://app.fireworks.ai/settings/users/api-keys" +credentials = ["env:FIREWORKS_API_KEY", "vault:FIREWORKS_API_KEY"] +enabled = false + +# Moonshot documents K3 as always reasoning with low, high, and max effort; +# lithos leaves the levels unverified, so Fabro records them here. +[providers."fireworks".models."kimi-k3"] +capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } } + +[providers."fireworks".models."kimi-k3".metadata.fabro] +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."fireworks".models."kimi-k3-fast".metadata.fabro] +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."fireworks".models."kimi-k2.7-code".metadata.fabro] +family = "kimi-k2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."kimi-k2.6".metadata.fabro] +family = "kimi-k2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."deepseek-v4-pro".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."deepseek-v4-flash".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."glm-5.2".metadata.fabro] +family = "glm-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."minimax-m2.7".metadata.fabro] +family = "minimax-m2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."fireworks".models."qwen3.7-plus".metadata.fabro] +family = "qwen3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."gemini"] +priority = 80 +default_model = "gemini-3.5-flash" + +[providers."gemini".metadata.fabro] +api_key_url = "https://aistudio.google.com/apikey" +credentials = ["env:GEMINI_API_KEY", "env:GOOGLE_API_KEY", "vault:GEMINI_API_KEY"] +enabled = true + +[providers."gemini".models."gemini-3.1-pro-preview".metadata.fabro] +family = "gemini-3" +agent_profile = "gemini" +small_default = false +probe = false + +[providers."gemini".models."gemini-3.1-pro-preview-customtools".metadata.fabro] +family = "gemini-3" +agent_profile = "gemini" +small_default = false +probe = false + +[providers."gemini".models."gemini-3.5-flash".metadata.fabro] +family = "gemini-3" +agent_profile = "gemini" +small_default = false +probe = false + +[providers."gemini".models."gemini-3-flash-preview".metadata.fabro] +family = "gemini-3" +agent_profile = "gemini" +small_default = false +probe = false + +[providers."gemini".models."gemini-3.1-flash-lite".metadata.fabro] +family = "gemini-3" +agent_profile = "gemini" +small_default = true +probe = false + +[providers."inception"] +priority = 40 +default_model = "mercury-2" + +[providers."inception".metadata.fabro] +api_key_url = "https://console.inceptionlabs.ai/api-keys" +credentials = ["env:INCEPTION_API_KEY", "vault:INCEPTION_API_KEY"] +enabled = true + +[providers."inception".models."mercury-2".metadata.fabro] +family = "mercury" +agent_profile = "openai" +small_default = false +probe = false + +[providers."litellm"] +priority = 50 + +[providers."litellm".metadata.fabro] +credentials = ["env:LITELLM_API_KEY", "vault:LITELLM_API_KEY"] +enabled = false + +[providers."minimax"] +priority = 50 +default_model = "minimax-m2.5" + +[providers."minimax".metadata.fabro] +api_key_url = "https://platform.minimaxi.com/user-center/basic-information/interface-key" +credentials = ["env:MINIMAX_API_KEY", "vault:MINIMAX_API_KEY"] +enabled = true + +[providers."minimax".models."minimax-m2.5".metadata.fabro] +family = "minimax-m2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."modal"] +priority = 75 + +[providers."modal".metadata.fabro] +api_key_url = "https://modal.com/docs/guide/endpoints#proxy-tokens" +enabled = false + +[providers."modal".metadata.fabro.extra_headers] +"Modal-Key" = "{{ secrets.MODAL_TOKEN_ID }}" +"Modal-Secret" = "{{ secrets.MODAL_TOKEN_SECRET }}" + +[providers."moonshot"] +priority = 70 +default_model = "kimi-k3" + +[providers."moonshot".metadata.fabro] +api_key_url = "https://platform.kimi.ai/console/api-keys" +credentials = ["env:MOONSHOT_API_KEY", "env:KIMI_API_KEY", "vault:MOONSHOT_API_KEY", "vault:KIMI_API_KEY"] +enabled = true + +[providers."moonshot".models."kimi-k2.5".metadata.fabro] +family = "kimi-k2" +agent_profile = "kimi" +small_default = false +probe = false + +# Moonshot documents K3 as always reasoning with low, high, and max effort; +# lithos leaves the levels unverified, so Fabro records them here. +[providers."moonshot".models."kimi-k3"] +capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } } + +[providers."moonshot".models."kimi-k3".metadata.fabro] +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."ollama"] +priority = 30 + +[providers."ollama".metadata.fabro] +enabled = false + +[providers."openai"] +priority = 90 +default_model = "gpt-5.6-sol" + +[providers."openai".metadata.fabro] +api_key_url = "https://platform.openai.com/api-keys" +credentials = ["env:OPENAI_API_KEY", "vault:OPENAI_API_KEY"] +enabled = true + +[providers."openai".models."gpt-5.6-sol".metadata.fabro] +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openai".models."gpt-5.6-terra".metadata.fabro] +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openai".models."gpt-5.6-luna".metadata.fabro] +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openai".models."gpt-5.4".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openai".models."gpt-5.5".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openai".models."gpt-5.5-pro".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openai".models."gpt-5.4-pro".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openai".models."gpt-5.4-mini".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = true +probe = true + +[providers."openrouter"] +priority = 25 +default_model = "claude-sonnet-5" + +[providers."openrouter".metadata.fabro] +api_key_url = "https://openrouter.ai/keys" +credentials = ["env:OPENROUTER_API_KEY", "vault:OPENROUTER_API_KEY"] +enabled = false + +[providers."openrouter".models."claude-fable-5".metadata.fabro] +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."openrouter".models."claude-opus-5".metadata.fabro] +training = "2026-05-01" +knowledge_cutoff = "May 2026" +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."openrouter".models."claude-sonnet-5".metadata.fabro] +training = "2026-01-01" +knowledge_cutoff = "Jan 2026" +family = "claude-5" +agent_profile = "claude-5" +small_default = false +probe = false + +[providers."openrouter".models."claude-opus-4.8".metadata.fabro] +training = "2026-01-01" +knowledge_cutoff = "Jan 2026" +family = "claude-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."claude-opus-4.7".metadata.fabro] +family = "claude-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."claude-sonnet-4.6".metadata.fabro] +family = "claude-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."claude-haiku-4.5".metadata.fabro] +family = "claude-4" +agent_profile = "openai" +small_default = true +probe = false + +[providers."openrouter".models."gpt-5.6-sol".metadata.fabro] +training = "2026-02-16" +knowledge_cutoff = "February 16, 2026" +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openrouter".models."gpt-5.6-terra".metadata.fabro] +training = "2026-02-16" +knowledge_cutoff = "February 16, 2026" +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openrouter".models."gpt-5.6-luna".metadata.fabro] +training = "2026-02-16" +knowledge_cutoff = "February 16, 2026" +family = "gpt-5" +agent_profile = "gpt56" +small_default = false +probe = false + +[providers."openrouter".models."gpt-5.4".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."gpt-5.5".metadata.fabro] +family = "gpt-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."gemini-3.1-pro-preview".metadata.fabro] +family = "gemini-3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."gemini-3.5-flash".metadata.fabro] +family = "gemini-3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."mimo-v2.5-pro".metadata.fabro] +family = "mimo-v2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."minimax-m2.7".metadata.fabro] +family = "minimax-m2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."deepseek-v4-pro".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."deepseek-v4-flash".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."kimi-k2.6".metadata.fabro] +family = "kimi-k2" +agent_profile = "kimi" +small_default = false +probe = false + +# Moonshot documents K3 as always reasoning with low, high, and max effort; +# lithos leaves the levels unverified, so Fabro records them here. +[providers."openrouter".models."kimi-k3"] +capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } } + +[providers."openrouter".models."kimi-k3".metadata.fabro] +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."openrouter".models."laguna-s-2.1".metadata.fabro] +family = "laguna-2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."laguna-xs-2.1".metadata.fabro] +family = "laguna-2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."qwen3-coder".metadata.fabro] +family = "qwen3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."qwen3.6-flash".metadata.fabro] +family = "qwen3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."qwen3.8-max".metadata.fabro] +family = "qwen3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."glm-5.2".metadata.fabro] +family = "glm-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."glm-4.6".metadata.fabro] +family = "glm-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."nemotron-3-super-120b-a12b".metadata.fabro] +family = "nemotron-3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."openrouter".models."devstral-2512".metadata.fabro] +family = "devstral" +agent_profile = "openai" +small_default = false +probe = false + +[providers."poolside"] +priority = 65 +default_model = "laguna-s-2.1" + +[providers."poolside".metadata.fabro] +api_key_url = "https://platform.poolside.ai" +credentials = ["env:POOLSIDE_API_KEY", "vault:POOLSIDE_API_KEY"] +enabled = true + +[providers."poolside".models."laguna-s-2.1".metadata.fabro] +family = "laguna-2" +agent_profile = "openai" +small_default = false +probe = false + +[providers."poolside".models."laguna-xs-2.1".metadata.fabro] +family = "laguna-2" +agent_profile = "openai" +small_default = true +probe = true + +[providers."venice"] +priority = 35 +default_model = "deepseek-v4-flash" + +[providers."venice".metadata.fabro] +credentials = ["env:VENICE_API_KEY", "vault:VENICE_API_KEY"] +enabled = true + +[providers."venice".models."kimi-k3".metadata.fabro] +reasoning_by_default = true +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."venice".models."kimi-k3-fast".metadata.fabro] +reasoning_by_default = true +family = "kimi-k3" +agent_profile = "kimi" +small_default = false +probe = false + +[providers."venice".models."grok-4.6".metadata.fabro] +reasoning_by_default = true +family = "grok-4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."venice".models."glm-5.3".metadata.fabro] +reasoning_by_default = true +family = "glm-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."venice".models."deepseek-v4-flash".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."venice".models."deepseek-v4-pro".metadata.fabro] +reasoning_by_default = true +family = "deepseek-v4" +agent_profile = "openai" +small_default = false +probe = false + +[providers."venice".models."qwen3.8-max".metadata.fabro] +reasoning_by_default = true +family = "qwen3" +agent_profile = "openai" +small_default = false +probe = false + +[providers."venice".models."qwen3.8-27b".metadata.fabro] +reasoning_by_default = true +family = "qwen3.8" +agent_profile = "openai" +small_default = false +probe = false + +[providers."zai"] +priority = 60 +default_model = "glm-5.2" + +[providers."zai".metadata.fabro] +api_key_url = "https://open.bigmodel.cn/usercenter/apikeys" +credentials = ["env:ZAI_API_KEY", "vault:ZAI_API_KEY"] +enabled = true + +[providers."zai".models."glm-5.2".metadata.fabro] +family = "glm-5" +agent_profile = "openai" +small_default = false +probe = false + +[providers."zai".models."glm-4.7".metadata.fabro] +family = "glm-4" +agent_profile = "openai" +small_default = false +probe = false + +# ChatGPT OAuth (Codex) access to the OpenAI roster. lithos speaks the Codex +# deployment through the `openai` adapter in codex mode; this provider exists +# so a `fabro provider login --provider openai` device-flow credential routes +# here while an API key keeps routing to `openai`. It is hidden from listings +# and stands in for `openai` when `openai` itself has no credentials. +[providers."openai-codex"] +display_name = "OpenAI (ChatGPT)" +adapter = "openai" +codec = "openai-responses" +base_url = "https://chatgpt.com/backend-api/codex" +priority = 89 +allow_passthrough = true +default_model = "gpt-5.6-sol" +adapter_options = { mode = "codex" } +default_headers = { originator = "fabro" } + +[providers."openai-codex".auth] +type = "bearer" + +[providers."openai-codex".metadata.fabro] +enabled = true +stands_in_for = "openai" +credentials = ["vault:OPENAI_CODEX"] + +[providers."openai-codex".models."gpt-5.6-sol"] +display_name = "GPT-5.6 Sol" +aliases = ["sol", "gpt-sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] +api_model = "gpt-5.6-sol" +limits = { context_tokens = 1050000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 20000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 8000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 800000, cache_write_usd_micros_per_million = 10000000 }, speed = { fast = { input_usd_micros_per_million = 8000000, output_usd_micros_per_million = 40000000, cached_input_usd_micros_per_million = 800000, cache_write_usd_micros_per_million = 10000000 }, economical = { input_usd_micros_per_million = 2000000, output_usd_micros_per_million = 10000000, cached_input_usd_micros_per_million = 200000, cache_write_usd_micros_per_million = 2500000 } } } + +[providers."openai-codex".models."gpt-5.6-sol".metadata.fabro] +agent_profile = "gpt56" +family = "gpt-5" +small_default = false +probe = false + +[providers."openai-codex".models."gpt-5.6-terra"] +display_name = "GPT-5.6 Terra" +aliases = ["terra", "gpt-terra", "gpt56-terra", "gpt-56-terra"] +api_model = "gpt-5.6-terra" +limits = { context_tokens = 1050000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 2000000, output_usd_micros_per_million = 12000000, cached_input_usd_micros_per_million = 200000, cache_write_usd_micros_per_million = 2500000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 18000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000 }, speed = { fast = { input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 24000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000 }, economical = { input_usd_micros_per_million = 1000000, output_usd_micros_per_million = 6000000, cached_input_usd_micros_per_million = 100000, cache_write_usd_micros_per_million = 1250000 } } } + +[providers."openai-codex".models."gpt-5.6-terra".metadata.fabro] +agent_profile = "gpt56" +family = "gpt-5" +small_default = false +probe = false + +[providers."openai-codex".models."gpt-5.6-luna"] +display_name = "GPT-5.6 Luna" +aliases = ["luna", "gpt-luna", "gpt56-luna", "gpt-56-luna"] +api_model = "gpt-5.6-luna" +limits = { context_tokens = 1050000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 200000, output_usd_micros_per_million = 1200000, cached_input_usd_micros_per_million = 20000, cache_write_usd_micros_per_million = 250000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 400000, output_usd_micros_per_million = 1800000, cached_input_usd_micros_per_million = 40000, cache_write_usd_micros_per_million = 500000 }, speed = { fast = { input_usd_micros_per_million = 400000, output_usd_micros_per_million = 2400000, cached_input_usd_micros_per_million = 40000, cache_write_usd_micros_per_million = 500000 }, economical = { input_usd_micros_per_million = 100000, output_usd_micros_per_million = 600000, cached_input_usd_micros_per_million = 10000, cache_write_usd_micros_per_million = 125000 } } } + +[providers."openai-codex".models."gpt-5.6-luna".metadata.fabro] +agent_profile = "gpt56" +family = "gpt-5" +small_default = false +probe = false + +[providers."openai-codex".models."gpt-5.4"] +display_name = "GPT-5.4" +aliases = ["gpt54", "gpt-54", "gpt-5.2", "gpt5", "gpt-5.3-codex", "codex"] +api_model = "gpt-5.4" +limits = { context_tokens = 1050000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, sampling = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 2500000, output_usd_micros_per_million = 15000000, cached_input_usd_micros_per_million = 250000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 22500000, cached_input_usd_micros_per_million = 500000 }, speed = { fast = { input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 500000 }, economical = { input_usd_micros_per_million = 1250000, output_usd_micros_per_million = 7500000, cached_input_usd_micros_per_million = 125000 } } } + +[providers."openai-codex".models."gpt-5.4".metadata.fabro] +agent_profile = "openai" +family = "gpt-5" +small_default = false +probe = false + +[providers."openai-codex".models."gpt-5.5"] +display_name = "GPT-5.5" +aliases = ["gpt55", "gpt-55"] +api_model = "gpt-5.5" +limits = { context_tokens = 1050000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 500000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 10000000, output_usd_micros_per_million = 45000000, cached_input_usd_micros_per_million = 1000000 }, speed = { fast = { input_usd_micros_per_million = 12500000, output_usd_micros_per_million = 75000000, cached_input_usd_micros_per_million = 1250000 }, economical = { input_usd_micros_per_million = 2500000, output_usd_micros_per_million = 15000000, cached_input_usd_micros_per_million = 250000 } } } + +[providers."openai-codex".models."gpt-5.5".metadata.fabro] +agent_profile = "openai" +family = "gpt-5" +small_default = false +probe = false + +[providers."openai-codex".models."gpt-5.4-mini"] +display_name = "GPT-5.4 Mini" +aliases = ["gpt54-mini", "gpt-54-mini", "gpt-5.3-codex-spark", "codex-spark"] +api_model = "gpt-5.4-mini" +limits = { context_tokens = 400000, max_output_tokens = 128000 } +capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, sampling = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } } +protocol_options = { reasoning_effort_levels = true } +pricing = { input_usd_micros_per_million = 750000, output_usd_micros_per_million = 4500000, cached_input_usd_micros_per_million = 75000, speed = { fast = { input_usd_micros_per_million = 1500000, output_usd_micros_per_million = 9000000, cached_input_usd_micros_per_million = 150000 }, economical = { input_usd_micros_per_million = 375000, output_usd_micros_per_million = 2250000, cached_input_usd_micros_per_million = 37500 } } } + +[providers."openai-codex".models."gpt-5.4-mini".metadata.fabro] +agent_profile = "openai" +family = "gpt-5" +small_default = true +probe = true diff --git a/lib/foundation/fabro-http/src/lib.rs b/lib/foundation/fabro-http/src/lib.rs index 27ef1414d..6d89de497 100644 --- a/lib/foundation/fabro-http/src/lib.rs +++ b/lib/foundation/fabro-http/src/lib.rs @@ -15,6 +15,7 @@ pub use reqwest::{ }; pub type HttpClient = reqwest::Client; +pub type HttpError = reqwest::Error; pub type BlockingHttpClient = reqwest::blocking::Client; pub type BlockingRequestBuilder = reqwest::blocking::RequestBuilder; pub type BlockingResponse = reqwest::blocking::Response; diff --git a/lib/foundation/fabro-test/src/lib.rs b/lib/foundation/fabro-test/src/lib.rs index 5d442b411..62dd1a89c 100644 --- a/lib/foundation/fabro-test/src/lib.rs +++ b/lib/foundation/fabro-test/src/lib.rs @@ -2446,11 +2446,13 @@ pub async fn twin_openai() -> &'static TwinOpenAi { let base_url = format!("http://127.0.0.1:{}/v1", addr.port()); let config = TwinConfig { - bind_addr: addr, + bind_addr: addr, require_auth: true, enable_admin: true, + ..TwinConfig::from_lookup(&|_| None).expect("twin-openai defaults should load") }; - let app = twin_openai::build_app_with_config(config); + let app = + twin_openai::build_app_with_config(config).expect("twin-openai app should build"); tokio::spawn(async move { axum::serve(listener, app).await.expect("twin-openai serve"); diff --git a/test/twin/openai/.gitignore b/test/twin/openai/.gitignore deleted file mode 100644 index 434ee8ad1..000000000 --- a/test/twin/openai/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.worktrees diff --git a/test/twin/openai/Cargo.toml b/test/twin/openai/Cargo.toml deleted file mode 100644 index 4641a47ed..000000000 --- a/test/twin/openai/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "twin-openai" -edition.workspace = true -version.workspace = true -publish = false -license.workspace = true -description = "Fake OpenAI-compatible server for local black-box testing" - -[lib] -doctest = false - -[lints] -workspace = true - -[dependencies] -anyhow.workspace = true -async-stream = "0.3" -axum = { workspace = true, features = ["macros"] } -fabro-http.workspace = true -fabro-static.workspace = true -futures-util.workspace = true -http = "1" -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true - -[dev-dependencies] -tower = "0.5" diff --git a/test/twin/openai/README.md b/test/twin/openai/README.md deleted file mode 100644 index 01ec2da38..000000000 --- a/test/twin/openai/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# twin-openai - -Async Rust fake OpenAI-compatible server for local black-box testing. - -## Endpoints - -- `GET /healthz` -- `POST /v1/responses` -- `POST /v1/chat/completions` -- `POST /__admin/scenarios` -- `POST /__admin/reset` -- `GET /__admin/requests` - -`/v1/*` routes require a non-empty bearer token. Scenarios, request logs, and deterministic response IDs are scoped by bearer token so concurrent test clients can share one server safely. - -`/__admin/*` routes are unauthenticated by default, but an optional bearer token selects the same namespace as `/v1/*`. Admin requests with a malformed or empty `Authorization` header are rejected. - -## Run locally - -```bash -cargo run -``` - -The server binds to `127.0.0.1:3000` by default. - -## Admin scripting - -Load deterministic one-shot scenarios: - -```bash -curl -X POST http://127.0.0.1:3000/__admin/scenarios \ - -H 'Authorization: Bearer suite-a' \ - -H 'content-type: application/json' \ - -d '{ - "scenarios": [ - { - "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false }, - "script": { "kind": "success", "response_text": "scripted reply" } - } - ] - }' -``` - -Inspect normalized request logs: - -```bash -curl http://127.0.0.1:3000/__admin/requests \ - -H 'Authorization: Bearer suite-a' -``` - -Reset scenarios, logs, and deterministic counters: - -```bash -curl -X POST http://127.0.0.1:3000/__admin/reset \ - -H 'Authorization: Bearer suite-a' -``` - -## Behavior summary - -- Non-stream and stream success paths are driven from the same canonical response plan. -- `/v1/responses` and `/v1/chat/completions` share the same deterministic fallback behavior. -- Structured output supports `json_object` and a documented `json_schema` subset. -- Scripted failures support OpenAI-shaped application errors, delays, hangs, partial streams, and malformed SSE. - -## Optional Live OpenAI Smoke Suite - -Run the ignored live drift detector only when you explicitly want to compare `twin-openai` against the real OpenAI API: - -```bash -OPENAI_API_KEY=... cargo test --test live_openai_contract -- --ignored --nocapture -``` - -Optional environment variables: - -- `TWIN_OPENAI_LIVE_MODEL` defaults to `gpt-5-nano-2025-08-07` -- `TWIN_OPENAI_LIVE_BASE_URL` defaults to `https://api.openai.com` -- `OPENAI_ORGANIZATION` and `OPENAI_PROJECT` are forwarded when present - -This suite is not part of normal CI. It is intentionally a drift detector for request/response shape and SSE sequencing, so opt-in failures can represent real compatibility gaps rather than a broken local test harness. - -If the supplied OpenAI credentials lack required endpoint scopes or quota, the ignored test will skip the blocked live surface instead of reporting protocol drift. - -Current live coverage includes `responses` and `chat.completions` text, streaming, structured output, function tools, `tool_choice: "none"` behavior, image-input acceptance, and both non-stream and streamed `responses` continuation turns. - -See [docs/compatibility-matrix.md](/Users/bhelmkamp/p/brynary/twin-openai/docs/compatibility-matrix.md) for the supported field matrix and explicit exclusions. diff --git a/test/twin/openai/docs/compatibility-matrix.md b/test/twin/openai/docs/compatibility-matrix.md deleted file mode 100644 index bf6bcebb3..000000000 --- a/test/twin/openai/docs/compatibility-matrix.md +++ /dev/null @@ -1,64 +0,0 @@ -# Compatibility Matrix - -Supported endpoints: - -- `GET /healthz` -- `POST /v1/responses` -- `POST /v1/chat/completions` -- `POST /__admin/scenarios` -- `POST /__admin/reset` -- `GET /__admin/requests` - -State isolation: - -- `/v1/*` request state is scoped by bearer token -- admin routes may include the same bearer token to target that namespace -- admin routes without auth operate on the global namespace - -Supported `/v1/responses` fields: - -- bearer auth -- `stream` -- `metadata` -- `stop` -- `previous_response_id` -- `reasoning` -- `text.format.type = text | json_object | json_schema` -- image inputs in `input[*].content[*].type = input_image` -- scripted tool calls and continuation input items - -Supported `/v1/chat/completions` fields: - -- bearer auth -- `max_tokens` -- `stream` -- `tools` -- `tool_choice` -- `response_format.type = text | json_object | json_schema` -- `stop` -- reasoning-bearing assistant content - -Unknown top-level fields are accepted and ignored. The twin does not simulate the behavior of -fields that are not listed above. - -Structured output subset: - -- object roots -- primitive property types: `string`, `integer`, `number`, `boolean` -- nested object properties - -Unsupported schema constructs fail explicitly: - -- arrays -- `anyOf` -- `oneOf` - -Failure scripting: - -- application errors with explicit status and OpenAI-shaped body -- optional `Retry-After` -- delay before headers -- hang before first byte -- inter-event stream delay -- close stream after N chunks -- malformed/truncated SSE ending diff --git a/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai-test-plan.md b/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai-test-plan.md deleted file mode 100644 index 00c127687..000000000 --- a/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai-test-plan.md +++ /dev/null @@ -1,204 +0,0 @@ -# Async Rust Fake OpenAI Test Plan - -The approved testing strategy still holds after reconciling it with the implementation plan. The plan narrows the generic strategy into a concrete async Rust action surface: `GET /healthz`, `POST /v1/responses`, `POST /v1/chat/completions`, and the unauthenticated admin control routes `POST /__admin/scenarios`, `POST /__admin/reset`, and `GET /__admin/requests`. No strategy change requiring user approval was identified. - -## Harness requirements - -1. **Ephemeral server harness** - - What it does: boots the real `tokio` + `axum` server on an ephemeral localhost port with test config, then drives it through real HTTP. - - What it exposes: base URL, authenticated and unauthenticated HTTP clients, SSE reader utilities, timeout helpers, and optional raw socket/body readers for truncated-stream cases. - - Estimated complexity: medium. - - Tests depending on it: 1 through 14. - -2. **Admin scenario harness** - - What it does: scripts deterministic server behavior through `POST /__admin/scenarios`, clears state with `POST /__admin/reset`, and fetches normalized request logs from `GET /__admin/requests`. - - What it exposes: helpers to enqueue FIFO scenarios, seed failure scripts, reset state between tests, and fetch request-log artifacts for assertions. - - Estimated complexity: medium. - - Tests depending on it: 4 through 13. - -3. **Canonical output comparison helpers** - - What it does: normalizes non-stream JSON and streamed SSE transcripts into comparable observable artifacts so the same canonical plan can be validated across transport modes and endpoints. - - What it exposes: parsed text transcript, tool-call transcript, reasoning transcript, completion marker presence, and required-field assertions. - - Estimated complexity: low to medium. - - Tests depending on it: 3, 6, 8, 9. - -4. **Optional live OpenAI differential harness** - - What it does: when explicit credentials are present outside normal CI, sends normalized requests to both `twin-openai` and the real OpenAI API and compares protocol shape rather than literal generated text. - - What it exposes: paired request runner, field-by-field comparator for required JSON fields and SSE event ordering, and opt-in skip behavior when credentials are absent. - - Estimated complexity: medium. - - Tests depending on it: 12. - -## Test plan - -1. **Name**: health and auth endpoints enforce the public service boundary - - **Type**: integration - - **Disposition**: new - - **Harness**: Ephemeral server harness - - **Preconditions**: server is running with default local-test config and no scenarios loaded. - - **Actions**: `GET /healthz`; `POST /v1/responses` without `Authorization`; `POST /v1/chat/completions` with an empty bearer token; `POST /v1/responses` with a non-empty bearer token and minimal valid JSON. - - **Expected outcome**: `GET /healthz` returns `200`; `/v1/*` rejects missing or empty bearer auth with stable OpenAI-shaped error JSON; a syntactically valid authenticated request reaches endpoint handling rather than failing auth. Source of truth: approved strategy, implementation plan scope and product contract. - - **Interactions**: router wiring, auth middleware, request parsing, error serialization. - -2. **Name**: non-stream responses create returns deterministic OpenAI-shaped JSON - - **Type**: integration - - **Disposition**: new - - **Harness**: Ephemeral server harness - - **Preconditions**: server is running, no scenario matches the request. - - **Actions**: `POST /v1/responses` with bearer auth, `stream=false`, text input, and optional inert `OpenAI-Organization` and `OpenAI-Project` headers. - - **Expected outcome**: `200` with valid `/v1/responses` JSON containing deterministic ID format, timestamp, output items, and usage object; response text is the documented deterministic fallback derived from user text rather than a real model output. Source of truth: implementation plan user-visible behavior, required invariants, OpenAI Responses API shape. - - **Interactions**: request models, deterministic default engine, JSON serialization, inert compatibility header handling. - -3. **Name**: streaming responses emit valid SSE and the same content as non-stream responses - - **Type**: invariant - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Canonical output comparison helpers - - **Preconditions**: server is running, no scenario matches the request, same logical request body is available in both stream and non-stream forms. - - **Actions**: send one `POST /v1/responses` with `stream=false` and one with `stream=true`; collect the full JSON body and full SSE transcript. - - **Expected outcome**: streamed events are valid SSE, ordered, flushed incrementally, and terminate with the supported completion semantics; the user-visible text, reasoning content, and tool-call transcript reconstructed from the stream match the non-stream response derived from the same canonical plan. Source of truth: approved strategy, implementation plan invariants, OpenAI streaming docs. - - **Interactions**: canonical response-plan projection, SSE encoder, chunk flushing, completion signaling. - -4. **Name**: admin-loaded scenarios are consumed once and in FIFO order - - **Type**: scenario - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness - - **Preconditions**: server is running; two matching scenarios are loaded for the same endpoint and matcher. - - **Actions**: `POST /__admin/scenarios` with two matching scripts; call the matching OpenAI endpoint twice; call it a third time after the queue is exhausted; fetch `GET /__admin/requests`; reset via `POST /__admin/reset`. - - **Expected outcome**: first OpenAI request consumes the first script, second consumes the second, third falls back to deterministic default behavior; request log is append-only until reset and empty after reset. Source of truth: implementation plan scenario scripting model, user-visible behavior, required invariants. - - **Interactions**: admin API, scenario matcher, FIFO consumption, fallback engine, request logging, reset behavior. - -5. **Name**: responses accept the declared compatibility fields and reject unsupported combinations clearly - - **Type**: boundary - - **Disposition**: new - - **Harness**: Ephemeral server harness - - **Preconditions**: server is running with no scenarios required. - - **Actions**: `POST /v1/responses` requests covering `metadata`, `stop`, `previous_response_id`, reasoning options, image inputs, `text` response format, `json_object`, supported `json_schema`, and one unsupported field or unsupported schema construct. - - **Expected outcome**: supported fields are accepted without changing the documented deterministic semantics; unsupported combinations fail with stable OpenAI-shaped error JSON rather than silent success. Source of truth: implementation plan endpoint compatibility contract, approved strategy, OpenAI docs for supported fields. - - **Interactions**: request validation, schema subset validation, image-input parsing, explicit unsupported-case handling. - -6. **Name**: scripted tool-call and continuation flows work through responses in stream and non-stream modes - - **Type**: scenario - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness plus Canonical output comparison helpers - - **Preconditions**: server is running; an admin scenario is queued that emits a tool call on the first request and a final answer on a continuation request that includes `function_call_output`. - - **Actions**: load the scripted scenario; call `POST /v1/responses` once non-stream and once stream to observe tool-call emission; call `POST /v1/responses` again with continuation input containing the tool output and optional `previous_response_id`. - - **Expected outcome**: first turn returns or streams a valid tool-call item with deterministic IDs and supported reasoning content; continuation request is accepted and returns the scripted final answer; stream and non-stream transcripts stay equivalent for the same planned turn. Source of truth: implementation plan tool-call and continuation contract, OpenAI function-calling and Responses docs. - - **Interactions**: scenario engine, canonical plan, responses renderer, continuation input parsing, SSE event sequencing. - -7. **Name**: structured output support is explicit, deterministic, and bounded to the documented schema subset - - **Type**: boundary - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness where needed - - **Preconditions**: server is running; supported and unsupported response format payloads are available. - - **Actions**: call `POST /v1/responses` with `json_object`; call with a supported `json_schema` subset; call with an unsupported schema feature such as an out-of-scope construct defined by the compatibility matrix. - - **Expected outcome**: `json_object` and supported `json_schema` requests return deterministic JSON matching the declared format; unsupported schema constructs fail explicitly with stable error JSON. Source of truth: implementation plan deterministic structured-output behavior and compatibility-matrix requirement, OpenAI structured outputs docs. - - **Interactions**: response-format validation, deterministic JSON generation, error-body rendering. - -8. **Name**: chat completions non-stream and stream use the same canonical plan as responses when behavior is equivalent - - **Type**: invariant - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness plus Canonical output comparison helpers - - **Preconditions**: server is running; one default request pair and one scripted scenario pair exist that are semantically representable on both endpoints. - - **Actions**: send equivalent requests to `POST /v1/responses` and `POST /v1/chat/completions` in both stream and non-stream forms. - - **Expected outcome**: both endpoints produce their respective OpenAI-shaped envelopes, but the user-visible text, reasoning transcript, and tool-call intent are equivalent because both render the same canonical plan; chat streaming emits valid delta events and terminal markers. Source of truth: implementation plan required invariants and endpoint compatibility contract, OpenAI Responses and Chat Completions docs. - - **Interactions**: cross-endpoint adapters, canonical plan projection, chat delta SSE rendering, shared validation helpers. - -9. **Name**: chat completions accept supported fields and reject unsupported combinations explicitly - - **Type**: boundary - - **Disposition**: new - - **Harness**: Ephemeral server harness - - **Preconditions**: server is running with no scenario required. - - **Actions**: `POST /v1/chat/completions` with supported `tools`, `tool_choice`, `response_format`, `stop`, and reasoning-bearing assistant content; send one request with an unsupported combination declared outside the compatibility matrix. - - **Expected outcome**: supported requests succeed in stream and non-stream modes; unsupported combinations fail with stable OpenAI-shaped error JSON rather than being ignored. Source of truth: implementation plan endpoint compatibility contract, OpenAI Chat Completions docs. - - **Interactions**: chat request parsing, response-format handling, tool-choice validation, error serialization. - -10. **Name**: scripted application errors preserve status, error shape, and retry metadata - - **Type**: integration - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness - - **Preconditions**: server is running; scenarios are loaded for each scripted application error variant. - - **Actions**: queue and invoke scripted failures for `400`, `401`, `403`, `404`, `408`, `413`, `429`, `500`, `502`, `503`, and `504`; include distinct quota-style and content-filter-style error bodies and a `429` carrying `Retry-After`. - - **Expected outcome**: each request returns the scripted HTTP status, OpenAI-shaped error JSON, and any scripted `Retry-After` header without transport corruption; quota-style and content-filter-style bodies remain observably distinct from generic invalid-request errors. Source of truth: approved strategy failure matrix, implementation plan task 7. - - **Interactions**: admin scripting, error renderer, header propagation, status mapping. - -11. **Name**: delayed first byte and hung requests are observable over real sockets and bounded by client timeouts - - **Type**: scenario - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness - - **Preconditions**: server is running; one scenario delays before headers and one hangs forever. - - **Actions**: invoke the delayed-first-byte scenario with a timeout larger than the scripted delay; invoke the hang scenario with a short explicit client timeout. - - **Expected outcome**: delayed-first-byte requests eventually succeed after the scripted pause; hung requests never produce completion and are terminated by the test client timeout rather than silently succeeding or closing early. Source of truth: approved strategy performance and failure-injection requirements, implementation plan transport-failure mechanics. - - **Interactions**: async timing, response-body start behavior, timeout guards, scenario engine. - -12. **Name**: partial stream close and malformed SSE are distinguishable transport failures - - **Type**: regression - - **Disposition**: new - - **Harness**: Ephemeral server harness plus Admin scenario harness with raw socket/body reader support - - **Preconditions**: server is running; one scenario is configured to close after N chunks and another to emit malformed or truncated SSE. - - **Actions**: invoke both scenarios against streaming `/v1/responses` and streaming `/v1/chat/completions`; capture raw stream bytes and parsed client behavior. - - **Expected outcome**: the partial-close case yields a valid prefix of the transcript followed by observable premature termination; the malformed/truncated case yields an invalid SSE/body artifact rather than a well-formed completion. Both failures are visible at the HTTP/SSE surface, not only through internal state. Source of truth: approved strategy failure matrix, implementation plan task 7 and execution notes. - - **Interactions**: streaming body writer, SSE encoder, abrupt connection teardown, lower-level HTTP client behavior. - -13. **Name**: optional live OpenAI differential checks preserve required protocol shape for the supported matrix - - **Type**: differential - - **Disposition**: new - - **Harness**: Optional live OpenAI differential harness - - **Preconditions**: explicit real OpenAI credentials are available outside required CI; normalized request fixtures exist only for the documented supported matrix. - - **Actions**: send paired non-stream and stream requests for supported `responses` and `chat.completions` cases to `twin-openai` and the real OpenAI API; compare status classes, required fields, SSE event ordering, and header behavior while ignoring literal text. - - **Expected outcome**: for the documented supported matrix, `twin-openai` matches the real API on protocol shape closely enough that generic OpenAI clients can interact with it; any intentional deviations are recorded in the compatibility matrix rather than hidden. Source of truth: approved strategy reference-comparison harness, official OpenAI API docs, real API as reference implementation. - - **Interactions**: external OpenAI service, local compatibility matrix, normalization/comparison utilities. - -14. **Name**: localhost success paths stay fast enough to catch accidental blocking or deadlock - - **Type**: invariant - - **Disposition**: new - - **Harness**: Ephemeral server harness - - **Preconditions**: server is running locally with default deterministic behavior and no injected delays. - - **Actions**: time a representative non-stream `/v1/responses` request, a representative streaming `/v1/responses` request until first event and completion, and a representative `/v1/chat/completions` request. - - **Expected outcome**: success-path requests complete comfortably under the generous local thresholds from the approved strategy, and first stream event arrives quickly enough to catch catastrophic async blocking rather than normal variance. Source of truth: approved strategy performance section. - - **Interactions**: full request path, async scheduling, streaming flush behavior, serialization. - -## Coverage summary - -Covered action space: -- `GET /healthz` -- `POST /v1/responses` with and without auth -- `POST /v1/responses` non-stream success -- `POST /v1/responses` stream success -- `POST /v1/responses` with `metadata` -- `POST /v1/responses` with `stop` -- `POST /v1/responses` with `previous_response_id` -- `POST /v1/responses` with reasoning options -- `POST /v1/responses` with image inputs -- `POST /v1/responses` with `json_object` -- `POST /v1/responses` with supported `json_schema` -- `POST /v1/responses` with unsupported schema or unsupported field combinations -- `POST /v1/responses` scripted tool-call turn -- `POST /v1/responses` continuation turn with `function_call_output` -- `POST /v1/chat/completions` with and without auth -- `POST /v1/chat/completions` non-stream success -- `POST /v1/chat/completions` stream success -- `POST /v1/chat/completions` with `tools` -- `POST /v1/chat/completions` with `tool_choice` -- `POST /v1/chat/completions` with `response_format` -- `POST /v1/chat/completions` with `stop` -- `POST /v1/chat/completions` with reasoning-bearing assistant content -- `POST /v1/chat/completions` with unsupported field combinations -- `POST /__admin/scenarios` -- `POST /__admin/reset` -- `GET /__admin/requests` -- Scripted status-error variants -- Scripted quota/content-filter variants -- Scripted `Retry-After` -- Scripted delayed first byte -- Scripted hang forever -- Scripted partial stream then close -- Scripted malformed or truncated SSE - -Explicit exclusions per the agreed strategy: -- Any downstream-consumer black-box tests, fixtures, scripts, CI jobs, or docs. -- Any OpenAI endpoint outside the documented supported phase-one matrix. -- Undocumented compatibility quirks not captured by the official docs or the optional live differential suite. -- Production-scale performance benchmarking; only generous local guardrail timing assertions are included. - -Risk carried by exclusions: -- Generic client compatibility outside the documented matrix may still drift until exercised by the optional differential suite. -- Consumers relying on undocumented OpenAI edge behavior may discover gaps that this repository intentionally does not claim to support. diff --git a/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai.md b/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai.md deleted file mode 100644 index 3de019316..000000000 --- a/test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai.md +++ /dev/null @@ -1,579 +0,0 @@ -# Async Rust Fake OpenAI Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use trycycle-executing to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build a standalone async Rust HTTP service that is OpenAI-compatible for the supported endpoints, deterministic without using a real LLM, and able to script protocol and transport failures for black-box end-to-end testing. - -**Architecture:** Implement a single `tokio` + `axum` server with one canonical internal turn/stream model that feeds both `/v1/responses` and `/v1/chat/completions`. Drive behavior through a neutral in-memory scenario engine plus admin control endpoints so success and failure paths are deterministic, consumer-agnostic, and identical across streaming and non-streaming transports. - -**Tech Stack:** Rust, Tokio, Axum, Hyper, Serde, Reqwest, futures-util, uuid, tracing, anyhow/thiserror. - ---- - -## Scope and Product Contract - -This repository stays consumer-agnostic. It must not mention, import, script against, or test against any downstream application. The product is the fake OpenAI server itself. - -The supported steady-state surface should be: - -- `POST /v1/responses` -- `POST /v1/chat/completions` -- `GET /healthz` -- `POST /__admin/scenarios` -- `POST /__admin/reset` -- `GET /__admin/requests` - -The service should require a bearer token for `/v1/*` routes and ignore its value beyond presence, so generic clients can use any non-empty API key. The admin routes should be intentionally local-test-focused and unauthenticated by default. - -Within those two OpenAI endpoints, phase-one compatibility must explicitly cover the request features this fake is expected to handle in practice: bearer auth, optional `OpenAI-Organization` and `OpenAI-Project` headers, `stream`, tools, `tool_choice`, `stop`, `metadata`, `previous_response_id`, reasoning requests, text response formats (`text`, `json_object`, `json_schema` subset), and image inputs. Anything outside that matrix must fail clearly and be documented as unsupported. - -### User-visible behavior - -- Non-streaming endpoints return valid OpenAI-shaped JSON with deterministic IDs, timestamps, usage objects, and output items. -- Streaming endpoints emit valid SSE with flush-per-event behavior and proper terminal completion semantics for the supported endpoint. -- If a scenario is preloaded through the admin API, the next matching OpenAI request consumes that scenario exactly once and responds according to its script. -- If no scripted scenario matches, the server falls back to a documented deterministic default behavior so the fake remains usable without setup. -- Tool-call and continuation flows are supported through the canonical internal turn model rather than endpoint-specific hacks. -- Failure injection can simulate application errors and transport failures including hangs, delayed first byte, partial stream then close, and malformed/truncated stream bodies. - -### Required invariants - -- The same internal response plan drives both stream and non-stream rendering for a given request. -- The same internal response plan drives both `/v1/responses` and `/v1/chat/completions` when the underlying behavior is equivalent. -- Scenario matching and consumption are deterministic and thread-safe. -- Request logs are append-only during a test run and resettable through admin control. -- Unsupported request shapes fail explicitly with stable error JSON; they must not silently degrade into a misleading success. -- No code or docs in this repository may reference downstream projects or embed downstream-specific fixtures. - -## Strategy Gate - -The clean path is to treat this as a server product, not a pile of endpoint stubs. The central design decision is to introduce a canonical internal "response plan" that represents output text, tool calls, usage, and failure timing once, then render it into each OpenAI surface. That avoids the most likely long-term bug: streaming and non-streaming behavior drifting apart, or `/v1/responses` and `/v1/chat/completions` diverging because they were implemented separately. - -The other key decision is to build a neutral admin control plane instead of encoding behavior in magic prompts or downstream-specific conventions. A scripted scenario queue is more explicit, easier to test, and robust enough to drive hangs, truncation, and delayed chunks without polluting the OpenAI-compatible surface. - -## File Structure - -Create and own the code with these boundaries: - -- `Cargo.toml`: crate metadata and dependencies. -- `src/lib.rs`: public server bootstrap API for tests and the binary. -- `src/main.rs`: runtime entrypoint and environment-based config loading. -- `src/config.rs`: bind address, auth mode, and admin-route configuration. -- `src/app.rs`: router construction and shared state wiring. -- `src/state.rs`: top-level application state and synchronization primitives. -- `src/openai/mod.rs`: route registration and shared endpoint helpers. -- `src/openai/auth.rs`: bearer-token enforcement for `/v1/*`. -- `src/openai/models.rs`: serde request/response models and shared validation helpers. -- `src/openai/responses.rs`: `/v1/responses` handler and renderer adapter. -- `src/openai/chat_completions.rs`: `/v1/chat/completions` handler and renderer adapter. -- `src/engine/mod.rs`: orchestration entrypoint from HTTP requests into deterministic execution. -- `src/engine/scenario.rs`: scenario definition, matchers, and one-shot consumption rules. -- `src/engine/defaults.rs`: deterministic fallback behavior when no scenario matches. -- `src/engine/plan.rs`: canonical internal response plan and stream event plan. -- `src/engine/failures.rs`: modeled transport/application failure behaviors and timing. -- `src/admin.rs`: admin routes for scenario load/reset/request-log retrieval. -- `src/logs.rs`: request-log structures and admin serialization. -- `src/sse.rs`: SSE encoding, chunk flushing, and stream completion helpers. -- `tests/common/mod.rs`: ephemeral server harness, admin helpers, and HTTP/SSE client helpers. -- `tests/health_and_auth.rs`: healthcheck plus auth and malformed-request coverage. -- `tests/responses_contract.rs`: `/v1/responses` success and validation cases. -- `tests/chat_completions_contract.rs`: `/v1/chat/completions` success and validation cases. -- `tests/tool_and_schema_contract.rs`: tool calls, continuation, and structured output cases. -- `tests/failure_modes.rs`: status-code errors, retry headers, hangs, partial streams, truncation, and malformed SSE. -- `README.md`: generic usage, supported surface, and local run instructions. -- `docs/compatibility-matrix.md`: explicit supported request fields, unsupported fields, and failure-model capabilities. - -## Contracts and Boundaries To Lock Down Before Coding - -1. Canonical plan model - - Represent one request result as a `ResponsePlan`. - - Include output text segments, tool calls, optional structured JSON payload, usage data, delays, and terminal status. - - Make stream rendering a pure projection of `ResponsePlan`, never a separate business path. - -2. Scenario scripting model - - A scenario should contain a matcher and a response script. - - Matchers should support endpoint, model name, stream flag, and optional metadata tags or request-substring checks. - - Scripts should support: success payload, OpenAI-style error response, delay before headers, delay between chunks, hang forever, close after N chunks, and malformed/truncated final body. - - Consumption should be FIFO among matching scenarios so tests can script multi-call flows deterministically. - -3. Deterministic default behavior - - The fallback path should not require admin setup. - - Default text behavior: extract user text inputs/messages, normalize whitespace, and return a stable synthetic summary/echo form. - - Default tool behavior: do not infer tool calls from arbitrary prompts; only emit tool calls when a scripted scenario requests them. - - Default structured-output behavior: if a supported `json_schema` response format is requested, return deterministic JSON matching the schema only for the supported primitive/object subset; reject unsupported schema constructs explicitly. - -4. Endpoint compatibility - - `/v1/responses` must support both stream and non-stream create flows, including continuation input items such as tool outputs when present. - - `/v1/chat/completions` must support both stream and non-stream chat flows using the same canonical plan. - - `/v1/responses` must accept optional `previous_response_id`, `metadata`, `stop`, reasoning requests, and image inputs without requiring downstream-specific behavior. - - `/v1/chat/completions` must accept `tools`, `tool_choice`, `response_format`, `stop`, and assistant reasoning content in both stream and non-stream modes. - - Request validation should reject unsupported combinations with stable OpenAI-shaped error JSON rather than silently ignoring them. - -5. Transport-failure mechanics - - Use real async streaming bodies so timing and truncation are observable over actual sockets. - - Guard hang tests with explicit timeouts in the test harness. - - Keep low-level transport failure code isolated in `engine/failures.rs` and `sse.rs`; handlers should describe failures, not hand-roll socket behavior. - -## Task 1: Bootstrap the async Rust service skeleton - -**Files:** -- Create: `Cargo.toml` -- Create: `src/lib.rs` -- Create: `src/main.rs` -- Create: `src/config.rs` -- Create: `src/app.rs` -- Create: `src/state.rs` -- Create: `src/admin.rs` -- Create: `src/openai/mod.rs` -- Create: `src/openai/auth.rs` -- Create: `tests/common/mod.rs` -- Create: `tests/health_and_auth.rs` -- Create: `README.md` - -- [ ] **Step 1: Identify or write the failing test** - -Write integration tests that prove the server boots, `GET /healthz` returns `200`, `/v1/*` rejects missing bearer auth, and a syntactically valid authenticated request currently fails because the endpoint handlers are not implemented yet. - -```rust -#[tokio::test] -async fn healthz_is_available() { /* ... */ } - -#[tokio::test] -async fn responses_requires_bearer_auth() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test healthz_is_available -- --exact` -Expected: FAIL because the crate and test harness do not exist yet. - -- [ ] **Step 3: Write minimal implementation** - -Create the crate, wire `tokio` + `axum`, add application state, implement `GET /healthz`, add auth middleware for `/v1/*`, expose an app-construction function from `src/lib.rs`, and add a minimal `README.md` that describes the service generically. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test healthz_is_available responses_requires_bearer_auth -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Tighten config loading and router composition, then run the targeted tests and the current full suite. - -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add Cargo.toml src tests README.md -git commit -m "feat: bootstrap async rust fake openai server" -``` - -## Task 2: Define request models and the canonical response-plan engine - -**Files:** -- Create: `src/openai/models.rs` -- Create: `src/engine/mod.rs` -- Create: `src/engine/plan.rs` -- Create: `src/engine/defaults.rs` -- Modify: `src/openai/mod.rs` -- Modify: `src/state.rs` -- Create: `tests/responses_contract.rs` - -- [ ] **Step 1: Identify or write the failing test** - -Write `/v1/responses` contract tests for authenticated non-streaming requests that should return deterministic text, stable `response.id` formatting, output items, usage fields, and acceptance of the request fields the service must support generically: `metadata`, `stop`, `previous_response_id`, optional org/project headers, and image inputs. - -```rust -#[tokio::test] -async fn responses_create_returns_deterministic_non_stream_payload() { /* ... */ } - -#[tokio::test] -async fn responses_accepts_supported_openai_request_fields() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact` -Expected: FAIL with `404` or unimplemented handler. - -- [ ] **Step 3: Write minimal implementation** - -Add serde request/response models for the supported `/v1/responses` subset, including `metadata`, `stop`, `previous_response_id`, reasoning request fields, and image inputs. Implement `ResponsePlan`, deterministic fallback extraction of user text input, stable IDs/timestamps per response, and a valid non-stream JSON payload from a real handler. Accept org/project headers as inert compatibility inputs rather than rejecting them. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Move validation and fallback rendering into focused engine modules so handlers stay thin, then run targeted checks plus the broader suite. - -Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact` -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/openai/models.rs src/engine src/openai/mod.rs src/state.rs tests/responses_contract.rs -git commit -m "feat: add canonical response plan and responses api baseline" -``` - -## Task 3: Add streaming for `/v1/responses` from the canonical plan - -**Files:** -- Create: `src/sse.rs` -- Modify: `src/openai/responses.rs` -- Modify: `src/engine/plan.rs` -- Modify: `src/engine/defaults.rs` -- Modify: `tests/common/mod.rs` -- Modify: `tests/responses_contract.rs` - -- [ ] **Step 1: Identify or write the failing test** - -Extend the `/v1/responses` tests to assert valid SSE framing, ordered events, flush behavior, and terminal completion for `stream=true`, including reasoning deltas and tool-call-related event sequences when the canonical plan contains them. - -```rust -#[tokio::test] -async fn responses_stream_emits_expected_sse_sequence() { /* ... */ } - -#[tokio::test] -async fn responses_stream_emits_reasoning_and_completion_events() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact` -Expected: FAIL because streaming is not implemented or event order is wrong. - -- [ ] **Step 3: Write minimal implementation** - -Add a streaming renderer that projects `ResponsePlan` into SSE events, including terminal completion, reasoning events, and tool-call-related events required by the supported `/v1/responses` subset. Keep the non-stream and stream paths backed by the same `ResponsePlan` instance so content does not drift. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Extract reusable SSE helpers and add assertions that the streamed text and non-stream text are semantically identical for the same request. - -Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact` -Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact` -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/sse.rs src/openai/responses.rs src/engine tests/common/mod.rs tests/responses_contract.rs -git commit -m "feat: stream responses api events from canonical plans" -``` - -## Task 4: Add scenario scripting and request logging through the admin API - -**Files:** -- Create: `src/engine/scenario.rs` -- Create: `src/logs.rs` -- Modify: `src/admin.rs` -- Modify: `src/state.rs` -- Modify: `src/engine/mod.rs` -- Modify: `src/engine/defaults.rs` -- Create: `tests/tool_and_schema_contract.rs` - -- [ ] **Step 1: Identify or write the failing test** - -Write admin tests that preload one-shot scenarios, verify that matching OpenAI requests consume them in FIFO order, and verify that `GET /__admin/requests` returns normalized request logs. - -```rust -#[tokio::test] -async fn admin_loaded_scenarios_are_consumed_fifo() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact` -Expected: FAIL because admin scripting and logs do not exist yet. - -- [ ] **Step 3: Write minimal implementation** - -Implement scenario definitions, matcher evaluation, atomic one-shot consumption, request logging, `/__admin/scenarios`, `/__admin/reset`, and `/__admin/requests`. Make matching generic: endpoint, model, stream flag, metadata tags, and text-substring checks are enough for phase one. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Ensure the admin wire format is stable and documented, then re-run the targeted checks and full suite. - -Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact` -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/engine/scenario.rs src/logs.rs src/admin.rs src/state.rs src/engine tests/tool_and_schema_contract.rs -git commit -m "feat: add scenario scripting and request log admin api" -``` - -## Task 5: Implement tool calls, continuation, and structured outputs - -**Files:** -- Modify: `src/openai/models.rs` -- Modify: `src/openai/responses.rs` -- Modify: `src/engine/plan.rs` -- Modify: `src/engine/scenario.rs` -- Modify: `src/engine/defaults.rs` -- Modify: `tests/tool_and_schema_contract.rs` -- Create: `docs/compatibility-matrix.md` - -- [ ] **Step 1: Identify or write the failing test** - -Add tests for: - -- scripted tool-call output on `/v1/responses` -- continuation input that includes tool output items -- reasoning-bearing assistant turns and `previous_response_id` continuation acceptance -- deterministic `json_schema` structured output for the supported subset -- deterministic `json_object` output for the simpler structured-output mode -- explicit rejection of unsupported schema constructs - -```rust -#[tokio::test] -async fn responses_supports_scripted_tool_call_and_continuation() { /* ... */ } - -#[tokio::test] -async fn responses_structured_output_support_is_explicit() { /* ... */ } - -#[tokio::test] -async fn responses_reasoning_and_continuation_fields_round_trip() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact` -Run: `cargo test responses_structured_output_support_is_explicit -- --exact` -Expected: FAIL because tool and schema flows are incomplete. - -- [ ] **Step 3: Write minimal implementation** - -Extend the canonical plan model to carry tool calls, reasoning segments, and structured JSON outputs. Support scripted tool-call emissions and continuation inputs on `/v1/responses`, including `function_call_output` items and optional `previous_response_id`. Implement explicit `json_object` support plus `json_schema` support for a documented subset only, and reject everything else with stable OpenAI-shaped errors. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact` -Run: `cargo test responses_structured_output_support_is_explicit -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Update the compatibility matrix so supported and unsupported fields are unambiguous. Then re-run all response and tool tests plus the full suite. - -Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact` -Run: `cargo test responses_structured_output_support_is_explicit -- --exact` -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/openai/models.rs src/openai/responses.rs src/engine docs/compatibility-matrix.md tests/tool_and_schema_contract.rs -git commit -m "feat: support scripted tool calls and structured outputs" -``` - -## Task 6: Add `/v1/chat/completions` on the same engine - -**Files:** -- Create: `src/openai/chat_completions.rs` -- Modify: `src/openai/mod.rs` -- Modify: `src/openai/models.rs` -- Modify: `src/engine/plan.rs` -- Create: `tests/chat_completions_contract.rs` - -- [ ] **Step 1: Identify or write the failing test** - -Write non-stream and stream contract tests for `/v1/chat/completions` proving that the same scenario/default behavior can be rendered into chat-completion JSON and delta SSE events while accepting the supported request features for this endpoint: tools, `tool_choice`, `response_format`, `stop`, and reasoning content. - -```rust -#[tokio::test] -async fn chat_completions_non_stream_uses_same_canonical_plan() { /* ... */ } - -#[tokio::test] -async fn chat_completions_stream_uses_same_canonical_plan() { /* ... */ } - -#[tokio::test] -async fn chat_completions_accepts_supported_openai_compatible_fields() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact` -Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact` -Expected: FAIL because the endpoint is not implemented yet. - -- [ ] **Step 3: Write minimal implementation** - -Add request/response models for the supported chat-completions subset and render chat success and stream deltas from the existing canonical plan instead of adding a second behavior engine. Ensure tool calls, `tool_choice`, `response_format`, `stop`, and assistant reasoning content are accepted and rendered consistently between stream and non-stream paths. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact` -Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Eliminate duplication between the endpoint adapters and verify that cross-endpoint equivalence holds where expected. - -Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact` -Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact` -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/openai/chat_completions.rs src/openai/mod.rs src/openai/models.rs src/engine/plan.rs tests/chat_completions_contract.rs -git commit -m "feat: add chat completions compatibility surface" -``` - -## Task 7: Implement application and transport failure injection - -**Files:** -- Create: `src/engine/failures.rs` -- Modify: `src/engine/scenario.rs` -- Modify: `src/admin.rs` -- Modify: `src/sse.rs` -- Modify: `tests/common/mod.rs` -- Create: `tests/failure_modes.rs` -- Modify: `docs/compatibility-matrix.md` -- Modify: `README.md` - -- [ ] **Step 1: Identify or write the failing test** - -Add real-socket tests for: - -- OpenAI-shaped JSON errors with status `400`, `401`, `403`, `404`, `408`, `413`, `429`, `500`, `502`, `503`, `504` -- quota-exceeded and content-filter error bodies that classify differently from generic invalid requests -- `Retry-After` propagation on scripted rate limits -- delayed first byte -- hang forever guarded by timeout -- partial SSE then close -- malformed/truncated SSE body - -```rust -#[tokio::test] -async fn scripted_hang_times_out_client_side() { /* ... */ } - -#[tokio::test] -async fn scripted_partial_stream_then_close_is_observable() { /* ... */ } - -#[tokio::test] -async fn scripted_budget_and_content_filter_errors_are_distinct() { /* ... */ } -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test scripted_hang_times_out_client_side -- --exact` -Run: `cargo test scripted_partial_stream_then_close_is_observable -- --exact` -Expected: FAIL because failure injection is not implemented yet. - -- [ ] **Step 3: Write minimal implementation** - -Implement failure scripts for status errors, quota/content-filter error variants, header delays, inter-event delays, hangs, close-after-N-chunks, and malformed/truncated stream endings. Support explicit `Retry-After` control for rate-limit scenarios. Keep these paths data-driven through scenarios rather than hard-coded test hooks. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test scripted_hang_times_out_client_side -- --exact` -Run: `cargo test scripted_partial_stream_then_close_is_observable -- --exact` -Expected: PASS - -- [ ] **Step 5: Refactor and verify** - -Make failure behavior documentation explicit and re-run the full suite to ensure the low-level transport code has not broken happy paths. - -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add src/engine/failures.rs src/engine/scenario.rs src/admin.rs src/sse.rs tests/common/mod.rs tests/failure_modes.rs docs/compatibility-matrix.md README.md -git commit -m "feat: add scripted application and transport failures" -``` - -## Task 8: Final polish, docs, and whole-project verification - -**Files:** -- Modify: `README.md` -- Modify: `docs/compatibility-matrix.md` -- Modify: any touched source or tests needed for final cleanup - -- [ ] **Step 1: Identify or write the failing test** - -Identify any remaining gaps from the final full-suite pass. If no behavioral gap remains, treat the failing check as documentation incompleteness: make sure the README explains local startup, admin scripting, and the supported compatibility matrix clearly enough that a new engineer can run it without guesswork. - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: any remaining issues are real and concrete; if all pass, proceed directly to Step 3 as a docs-and-cleanup pass. - -- [ ] **Step 3: Write minimal implementation** - -Resolve any final defects, tighten docs, confirm unsupported behaviors are explicitly documented, and remove dead code or duplicated helpers introduced during the build-out. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 5: Refactor and verify** - -Run one final end-to-end verification pass on the complete repository. - -Run: `cargo test` -Run: `cargo fmt --check` -Run: `cargo clippy --all-targets -- -D warnings` -Expected: all PASS - -- [ ] **Step 6: Commit** - -```bash -git add README.md docs/compatibility-matrix.md src tests -git commit -m "docs: finalize fake openai compatibility and usage guidance" -``` - -## Execution Notes - -- Prefer small focused modules over large endpoint files; the canonical plan is the seam that keeps the codebase understandable. -- Do not add persistence, databases, or external queues. In-memory state is the correct steady-state for deterministic local black-box testing. -- Do not implement undocumented endpoint variants "just in case." Add only the explicitly documented supported surface and fail clearly elsewhere. -- Keep the compatibility matrix concrete. The implementation is only done when the docs and tests name the exact supported request fields and failure scripts for both `/v1/responses` and `/v1/chat/completions`. -- For malformed/truncated stream tests, use lower-level response reading when `reqwest` normalizes away the exact transport symptom. -- Keep timestamps and IDs deterministic enough for assertions without freezing the entire clock globally; injecting a clock/ID generator through state is the clean path. -- Never weaken a valid test to get green. If a transport failure test flakes, fix the timing/control mechanism rather than loosening the assertion into uselessness. diff --git a/test/twin/openai/docs/plans/2026-03-28-debug-web-ui-test-plan.md b/test/twin/openai/docs/plans/2026-03-28-debug-web-ui-test-plan.md deleted file mode 100644 index a8f46f236..000000000 --- a/test/twin/openai/docs/plans/2026-03-28-debug-web-ui-test-plan.md +++ /dev/null @@ -1,154 +0,0 @@ -# Debug Web UI Test Plan - -The agreed testing strategy holds after reconciling it with the implementation plan. The plan adds two new routes (`GET /__debug` and `GET /__debug/state.json`) gated behind the existing `enable_admin` config flag, plus new snapshot types in `src/state.rs` and a `script_kind()` accessor on `ScenarioScript`. No strategy change requiring user approval was identified. - -One clarification: the strategy calls for "3-4 HTTP integration tests against the debug endpoint." After reviewing the implementation plan, the highest-value integration tests are: (1) empty-state HTML page serves correctly, (2) JSON state endpoint returns correct snapshot after scenarios are loaded, (3) HTML page reflects loaded scenarios and request logs, and (4) debug routes are hidden when `enable_admin` is false. That is exactly 4 tests. The headless Chrome screenshot and HTML safety check round out the plan at 6 new tests, plus the 50-test regression gate. - -## Harness requirements - -1. **Ephemeral server harness** (existing: `tests/common/mod.rs`) - - What it does: boots the real `tokio` + `axum` server on an ephemeral localhost port with `enable_admin: true` and `require_auth: true`, exposing `base_url`, authenticated client, and unauthenticated client. - - Tests depending on it: 1 through 6. - -2. **Unauthenticated HTTP client** (existing on `TestServer`) - - What it does: the `TestServer.client` field is an unauthenticated `reqwest::Client` used to hit routes that do not require bearer auth (like `/__debug`). No new helper needed. - - Tests depending on it: 1, 2, 3, 4. - -3. **Admin scenario harness** (existing on `TestServer`) - - What it does: scripts server state via `enqueue_scenarios()`, drives OpenAI requests via `post_responses()`, and fetches request logs via `request_logs()`. - - Tests depending on it: 2, 3. - -4. **Headless Chrome harness** (new, lightweight) - - What it does: launches headless Chrome/Chromium via `std::process::Command` to capture a screenshot of the debug page. Falls back to skip if no Chrome binary is found. - - What it exposes: a PNG file written to a temp path; assertion that the file is non-empty and is a valid PNG (starts with the PNG magic bytes). - - Estimated complexity: low. No new crate; uses `chromium --headless --screenshot` CLI. - - Tests depending on it: 5. - -## Test plan - -### 1. Name: debug HTML page serves valid HTML with correct content-type on empty state - -- **Type**: integration -- **Disposition**: new -- **Harness**: Ephemeral server harness -- **Preconditions**: server is running with `enable_admin: true`; no scenarios loaded; no requests made. -- **Actions**: - - `GET /__debug` using the unauthenticated client. -- **Expected outcome**: - - HTTP status is `200`. - - `content-type` header contains `text/html`. - - Response body contains `` (well-formed HTML document). - - Response body contains the page title text `twin-openai` and `debug`. - - Response body contains the empty-state marker text `no active namespaces` (since no scenarios have been loaded and no requests have been made). - - Source of truth: implementation plan (route `GET /__debug`, HTML rendering with empty state indicator). -- **Interactions**: axum routing, `enable_admin` gate, `debug_snapshot()`, `render_html()`. - -### 2. Name: debug JSON endpoint returns correct state snapshot after scenarios are loaded and requests are made - -- **Type**: integration -- **Disposition**: new -- **Harness**: Ephemeral server harness, Admin scenario harness -- **Preconditions**: server is running with `enable_admin: true`. -- **Actions**: - 1. Load two scenarios via `POST /__admin/scenarios` with bearer auth: one `success` script matching `responses` endpoint with model `gpt-test`, and one `error` script matching `responses` endpoint with model `gpt-error`. - 2. Make one `POST /v1/responses` request with bearer auth, model `gpt-test`, input `"hello debug"`, `stream: false` -- this consumes the first scenario and logs a request. - 3. `GET /__debug/state.json` using the unauthenticated client. -- **Expected outcome**: - - HTTP status is `200`. - - `content-type` header contains `application/json`. - - Response body parses as JSON with a top-level `namespaces` array. - - The namespace array contains exactly one entry whose `key` field starts with `"Bearer:"`. - - That namespace's `scenarios` array has exactly 1 remaining scenario (the `error` script; the `success` script was consumed). - - The remaining scenario has `endpoint: "responses"`, `model: "gpt-error"`, `script_kind: "error"`. - - That namespace's `request_logs` array has exactly 1 entry with `endpoint: "responses"`, `model: "gpt-test"`, `input_text` containing `"hello debug"`. - - Source of truth: implementation plan (JSON API shape, `DebugSnapshot` / `NamespaceSnapshot` / `ScenarioSnapshot` structs, FIFO consumption model). -- **Interactions**: admin scenario loading, OpenAI responses endpoint, `debug_snapshot()`, JSON serialization. - -### 3. Name: debug HTML page reflects loaded scenarios and request logs in rendered output - -- **Type**: integration -- **Disposition**: new -- **Harness**: Ephemeral server harness, Admin scenario harness -- **Preconditions**: server is running with `enable_admin: true`. -- **Actions**: - 1. Load one `success` scenario via `POST /__admin/scenarios` with bearer auth, matching `responses` endpoint, model `gpt-html-test`. - 2. Make one `POST /v1/responses` request with bearer auth, model `gpt-other`, input `"check the page"`, `stream: false` (default behavior, does not consume the scenario because model does not match). - 3. `GET /__debug` using the unauthenticated client. -- **Expected outcome**: - - HTTP status is `200`. - - Response body contains the scenario's model name `gpt-html-test` in the rendered HTML (proving scenarios appear). - - Response body contains the text `success` (the `script_kind` of the loaded scenario). - - Response body contains the text `gpt-other` (the model from the request log). - - Response body contains `check the page` (the input text from the request log). - - Response body does NOT contain `no active namespaces` (because there is at least one namespace). - - Source of truth: implementation plan (HTML page structure showing scenarios table and request log table per namespace). -- **Interactions**: admin scenario loading, OpenAI responses endpoint, `debug_snapshot()`, `render_html()`, HTML escaping. - -### 4. Name: debug routes are not accessible when enable_admin is false - -- **Type**: integration -- **Disposition**: new -- **Harness**: Custom server setup (not the default `spawn_server`, which uses `enable_admin: true`) -- **Preconditions**: server is started with `enable_admin: false` (construct `Config` directly and call `build_app_with_config`). -- **Actions**: - - `GET /__debug` using an unauthenticated client. - - `GET /__debug/state.json` using an unauthenticated client. -- **Expected outcome**: - - Both requests return HTTP `404` (the routes are not registered when admin is disabled). - - Source of truth: implementation plan (`debug_ui::router()` is only merged when `enable_admin` is true, same as `admin::router()`). -- **Interactions**: `app::router()` conditional routing, config flag. - -### 5. Name: debug page renders visually in headless Chrome and produces a non-empty screenshot - -- **Type**: scenario -- **Disposition**: new -- **Harness**: Ephemeral server harness, Headless Chrome harness -- **Preconditions**: server is running with `enable_admin: true`; headless Chrome/Chromium is available on PATH. Test is skipped if Chrome is not found. -- **Actions**: - 1. Load one scenario and make one request (to populate state for a non-trivial render). - 2. Run headless Chrome: `chromium --headless --disable-gpu --screenshot=/tmp/.png --window-size=1280,900 /__debug` (or `google-chrome` / `chromium-browser` depending on platform). - 3. Read the output PNG file. -- **Expected outcome**: - - The Chrome process exits with code 0. - - The screenshot file exists and is at least 10 KB (a non-trivial rendered page, not a blank screen). - - The first 8 bytes of the file match the PNG magic number (`\x89PNG\r\n\x1a\n`). - - Source of truth: agreed testing strategy (headless Chrome screenshot capture, 1 test case). -- **Interactions**: full server stack, HTML rendering, CSS rendering in a real browser engine. - -### 6. Name: HTML output escapes user-controlled values to prevent injection - -- **Type**: unit -- **Disposition**: new -- **Harness**: Ephemeral server harness, Admin scenario harness -- **Preconditions**: server is running with `enable_admin: true`. -- **Actions**: - 1. Load a scenario via `POST /__admin/scenarios` where the model field contains an HTML injection attempt: ``. - 2. `GET /__debug` using the unauthenticated client. -- **Expected outcome**: - - HTTP status is `200`. - - Response body contains the escaped form `<script>` (proving the `escape_html` function is applied). - - Response body does NOT contain the literal unescaped string `\n\n"); - - html -} - -fn render_content(html: &mut String, namespaces: &[NamespaceSnapshot]) { - if namespaces.is_empty() { - html.push_str("

(no active namespaces)

\n"); - return; - } - - for ns in namespaces { - let _ = write!( - html, - "
\n

{}

\n", - escape_html(&ns.key) - ); - - // Queued scenarios - html.push_str("

queued scenarios

\n"); - render_scenarios_table(html, &ns.scenarios); - - // Request log - html.push_str("

request log

\n"); - render_requests_table(html, &ns.request_logs); - - html.push_str("
\n"); - } -} - -fn render_scenarios_table(html: &mut String, scenarios: &[ScenarioSnapshot]) { - if scenarios.is_empty() { - html.push_str("

(no queued scenarios)

\n"); - return; - } - - html.push_str( - " \n \n \n \n \n", - ); - - for (i, s) in scenarios.iter().enumerate() { - let model = s - .model - .as_deref() - .map_or_else(|| "--".to_owned(), escape_html); - let stream = match s.stream { - Some(v) => format!("{v}"), - None => "--".to_owned(), - }; - let input_contains = s - .input_contains - .as_deref() - .map_or_else(|| "--".to_owned(), escape_html); - - let _ = writeln!( - html, - " ", - i + 1, - escape_html(&s.endpoint), - model, - stream, - input_contains, - escape_html(&s.script_kind), - ); - } - - html.push_str(" \n
#endpointmodelstreaminput_containsscript
{}{}{}{}{}{}
\n"); -} - -fn render_requests_table(html: &mut String, logs: &[RequestLog]) { - if logs.is_empty() { - html.push_str("

(no requests logged)

\n"); - return; - } - - html.push_str( - " \n \n \n \n \n", - ); - - for (i, r) in logs.iter().enumerate() { - let meta = serde_json::to_string(&r.metadata).unwrap_or_else(|_| "{}".to_owned()); - let meta_display = if meta == "{}" { - "--".to_owned() - } else { - escape_html(&truncate(&meta, 80)) - }; - - let _ = writeln!( - html, - " ", - i + 1, - escape_html(&r.endpoint), - escape_html(&r.model), - r.stream, - escape_html(&truncate(&r.input_text, 120)), - meta_display, - ); - } - - html.push_str(" \n
#endpointmodelstreaminput textmetadata
{}{}{}{}{}{}
\n"); -} - -const CSS: &str = r#" -:root { - --bg: #0a0a0a; - --bg-row: #111; - --bg-row-alt: #0d0d0d; - --bg-hover: #1a1a0a; - --text: #ccc; - --text-bright: #00ff41; - --text-heading: #ffb000; - --text-dim: #666; - --text-error: #ff4444; - --border: #333; - --border-bright: #555; -} - -body { - background: var(--bg); - color: var(--text); - font-family: "IBM Plex Mono", "Fira Code", "Cascadia Code", monospace; - margin: 0; - padding: 24px; -} - -header { - display: flex; - align-items: center; - justify-content: space-between; -} - -h1 { - color: var(--text-heading); - text-transform: uppercase; - letter-spacing: 0.2em; - font-size: 1.1rem; - margin: 0; -} - -h1 .dim { - color: var(--text-dim); -} - -h2.namespace-header { - color: var(--text-heading); - border-top: 2px solid var(--border-bright); - padding-top: 16px; - margin-top: 32px; -} - -h3 { - color: var(--text-dim); - text-transform: uppercase; - font-size: 0.75rem; - letter-spacing: 0.15em; -} - -table { - width: 100%; - border-collapse: collapse; - border: 2px solid var(--border); -} - -th { - color: var(--text-heading); - text-transform: uppercase; - font-size: 0.7rem; - letter-spacing: 0.1em; - padding: 6px 10px; - text-align: left; - border-bottom: 2px solid var(--border); -} - -td { - color: var(--text-bright); - padding: 5px 10px; - font-size: 0.85rem; - border-bottom: 1px solid var(--border); -} - -tr:nth-child(even) { - background: var(--bg-row-alt); -} - -tr:hover td { - background: var(--bg-hover); - transition: background 0.15s; -} - -.empty { - color: var(--text-error); - font-style: italic; - padding: 8px 0; -} - -.summary { - color: var(--text-dim); - margin: 12px 0 24px; - font-size: 0.85rem; -} - -.sep { - margin: 0 8px; - color: var(--border-bright); -} - -footer { - color: var(--text-dim); - font-size: 0.75rem; - margin-top: 40px; - border-top: 1px solid var(--border); - padding-top: 12px; -} - -.status { - display: flex; - align-items: center; -} - -.live-label { - color: var(--text-bright); - text-transform: uppercase; - font-size: 0.75rem; - letter-spacing: 0.1em; -} - -@keyframes pulse { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.3; } -} - -.live-dot { - display: inline-block; - width: 8px; - height: 8px; - background: var(--text-bright); - border-radius: 50%; - animation: pulse 2s ease-in-out infinite; - margin-right: 6px; - vertical-align: middle; -} - -button { - background: transparent; - color: var(--text-dim); - border: 1px solid var(--border); - padding: 2px 10px; - font-family: inherit; - font-size: 0.75rem; - cursor: pointer; - text-transform: uppercase; - letter-spacing: 0.1em; - margin-left: 12px; -} - -button:hover { - color: var(--text-bright); - border-color: var(--text-bright); -} -"#; - -const JS: &str = r#" -async function refresh() { - try { - const res = await fetch('/__debug/state.json'); - const data = await res.json(); - document.getElementById('content').innerHTML = renderState(data); - updateSummary(data); - } catch(e) { /* silent -- next interval will retry */ } -} - -if (new URLSearchParams(window.location.search).get('refresh') !== '0') { - setInterval(refresh, 2000); -} - -function renderState(data) { - if (data.namespaces.length === 0) { - return '

(no active namespaces)

'; - } - return data.namespaces.map(function(ns) { - return '
' - + '

' + esc(ns.key) + '

' - + '

queued scenarios

' - + renderScenariosTable(ns.scenarios) - + '

request log

' - + renderRequestsTable(ns.request_logs) - + '
'; - }).join(''); -} - -function renderScenariosTable(scenarios) { - if (scenarios.length === 0) return '

(no queued scenarios)

'; - var rows = scenarios.map(function(s, i) { - return '' - + '' + (i+1) + '' - + '' + esc(s.endpoint) + '' - + '' + esc(s.model || '--') + '' - + '' + (s.stream === null ? '--' : s.stream) + '' - + '' + esc(s.input_contains || '--') + '' - + '' + esc(s.script_kind) + '' - + ''; - }).join(''); - return '' - + '' - + '' + rows + '
#endpointmodelstreaminput_containsscript
'; -} - -function renderRequestsTable(logs) { - if (logs.length === 0) return '

(no requests logged)

'; - var rows = logs.map(function(r, i) { - var meta = JSON.stringify(r.metadata); - if (meta === '{}') meta = '--'; - return '' - + '' + (i+1) + '' - + '' + esc(r.endpoint) + '' - + '' + esc(r.model) + '' - + '' + r.stream + '' - + '' + esc(trunc(r.input_text, 120)) + '' - + '' + esc(trunc(meta, 80)) + '' - + ''; - }).join(''); - return '' - + '' - + '' + rows + '
#endpointmodelstreaminput textmetadata
'; -} - -function updateSummary(data) { - var sc = 0, rq = 0; - data.namespaces.forEach(function(ns) { - sc += ns.scenarios.length; - rq += ns.request_logs.length; - }); - document.getElementById('ns-count').textContent = data.namespaces.length + ' namespaces'; - document.getElementById('sc-count').textContent = sc + ' queued scenarios'; - document.getElementById('rq-count').textContent = rq + ' logged requests'; -} - -function esc(s) { - return String(s).replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"').replace(/'/g,'''); -} - -function trunc(s, n) { - return s.length > n ? s.slice(0, n) + '...' : s; -} -"#; diff --git a/test/twin/openai/src/engine/defaults.rs b/test/twin/openai/src/engine/defaults.rs deleted file mode 100644 index 169caf4da..000000000 --- a/test/twin/openai/src/engine/defaults.rs +++ /dev/null @@ -1,113 +0,0 @@ -use serde_json::{Value, json}; - -use super::plan::{ResponsePlan, TokenUsage}; -use crate::openai::models::{ResponseFormat, ResponsesRequest, normalize_whitespace}; - -pub fn build_default_response_plan( - response_number: u64, - request: &ResponsesRequest, -) -> ResponsePlan { - let normalized_text = request.extract_user_text(); - let response_text = format!("deterministic: {normalized_text}"); - let input_tokens = normalized_text.split_whitespace().count() as u64; - - let structured_output = request.response_format().and_then(|format| match format { - ResponseFormat::Text => None, - ResponseFormat::JsonObject => Some(json!({ - "message": response_text, - "model": request.model, - })), - ResponseFormat::JsonSchema(schema) => { - Some(generate_json_from_schema(&schema, &response_text)) - } - }); - let reasoning = if request.reasoning.is_some() { - vec![format!("reasoning: {normalized_text}")] - } else { - Vec::new() - }; - - ResponsePlan { - id: format!("resp_{response_number:06}"), - created: response_number, - model: request.model.clone(), - response_text, - structured_output, - reasoning, - tool_calls: Vec::new(), - usage: TokenUsage::new(input_tokens, 5), - } -} - -pub fn build_default_chat_plan( - response_number: u64, - model: String, - input_text: &str, - response_format: Option, - reasoning_requested: bool, -) -> ResponsePlan { - let normalized_text = normalize_whitespace(input_text); - let response_text = format!("deterministic: {normalized_text}"); - let structured_output = response_format.and_then(|format| match format { - ResponseFormat::Text => None, - ResponseFormat::JsonObject => Some(json!({ - "message": response_text, - "model": model, - })), - ResponseFormat::JsonSchema(schema) => { - Some(generate_json_from_schema(&schema, &response_text)) - } - }); - let reasoning = if reasoning_requested { - vec![format!("reasoning: {normalized_text}")] - } else { - Vec::new() - }; - let input_tokens = normalized_text.split_whitespace().count() as u64; - - ResponsePlan { - id: format!("resp_{response_number:06}"), - created: response_number, - model, - response_text, - structured_output, - reasoning, - tool_calls: Vec::new(), - usage: TokenUsage::new(input_tokens, 5), - } -} - -fn generate_json_from_schema(schema: &Value, response_text: &str) -> Value { - let schema = schema.get("schema").unwrap_or(schema); - - match schema.get("type").and_then(Value::as_str) { - Some("object") => { - let properties = schema - .get("properties") - .and_then(Value::as_object) - .cloned() - .unwrap_or_default(); - - let mut object = serde_json::Map::new(); - for (name, property_schema) in properties { - object.insert( - name, - primitive_value_for_schema(&property_schema, response_text), - ); - } - Value::Object(object) - } - _ => json!({ "message": response_text }), - } -} - -fn primitive_value_for_schema(schema: &Value, response_text: &str) -> Value { - match schema.get("type").and_then(Value::as_str) { - Some("string") => Value::String(response_text.to_owned()), - Some("integer") => json!(1), - Some("number") => json!(1.0), - Some("boolean") => json!(true), - Some("object") => generate_json_from_schema(schema, response_text), - _ => Value::Null, - } -} diff --git a/test/twin/openai/src/engine/failures.rs b/test/twin/openai/src/engine/failures.rs deleted file mode 100644 index 8ad78f60f..000000000 --- a/test/twin/openai/src/engine/failures.rs +++ /dev/null @@ -1,58 +0,0 @@ -use axum::http::StatusCode; - -use super::plan::ResponsePlan; -use crate::openai::models::{ErrorBody, ErrorEnvelope}; - -#[derive(Clone, Copy, Debug, Default)] -pub struct TransportOptions { - pub delay_before_headers_ms: u64, - pub inter_event_delay_ms: u64, - pub close_after_chunks: Option, - pub malformed_sse: bool, -} - -#[derive(Clone, Debug)] -pub struct SuccessOutcome { - pub plan: ResponsePlan, - pub transport: TransportOptions, -} - -#[derive(Clone, Debug)] -pub struct ErrorOutcome { - pub status: StatusCode, - pub body: ErrorEnvelope, - pub retry_after: Option, - pub delay_before_headers_ms: u64, -} - -#[derive(Clone, Debug)] -pub enum ExecutionOutcome { - Success(SuccessOutcome), - Error(ErrorOutcome), - Hang { delay_before_headers_ms: u64 }, -} - -impl ErrorOutcome { - pub fn new( - status: StatusCode, - message: String, - error_type: String, - code: String, - retry_after: Option, - delay_before_headers_ms: u64, - ) -> Self { - Self { - status, - body: ErrorEnvelope { - error: ErrorBody { - message, - error_type, - param: serde_json::Value::Null, - code, - }, - }, - retry_after, - delay_before_headers_ms, - } - } -} diff --git a/test/twin/openai/src/engine/mod.rs b/test/twin/openai/src/engine/mod.rs deleted file mode 100644 index 867735baa..000000000 --- a/test/twin/openai/src/engine/mod.rs +++ /dev/null @@ -1,126 +0,0 @@ -pub mod defaults; -pub mod failures; -pub mod plan; -pub mod scenario; - -use self::defaults::{build_default_chat_plan, build_default_response_plan}; -use self::failures::{ExecutionOutcome, SuccessOutcome, TransportOptions}; -use self::plan::ResponsePlan; -use self::scenario::RequestContext; -use crate::openai::models::{ - ChatCompletionsRequest, OpenAiError, ResponsesRequest, ToolChoiceMode, -}; -use crate::state::{AppState, NamespaceKey}; - -pub fn execute_responses_request( - state: &AppState, - namespace: &NamespaceKey, - request: &ResponsesRequest, -) -> Result { - request.validate()?; - let context = RequestContext { - endpoint: "responses".to_owned(), - model: request.model.clone(), - stream: request.stream, - metadata: request.metadata.clone(), - input_text: request.extract_user_text(), - instructions_text: request.extract_instruction_text(), - }; - state.log_request(namespace, context.clone()); - - if let Some(scenario) = state.take_matching_scenario(namespace, &context) { - return match scenario.execute_for_responses(state.next_response_id(namespace), request) { - ExecutionOutcome::Success(success) => Ok(ExecutionOutcome::Success( - enforce_tool_choice(request.tool_choice_mode(), success)?, - )), - outcome => Ok(outcome), - }; - } - - Ok(ExecutionOutcome::Success(enforce_tool_choice( - request.tool_choice_mode(), - SuccessOutcome { - plan: build_default_response_plan(state.next_response_id(namespace), request), - transport: TransportOptions::default(), - }, - )?)) -} - -pub fn execute_chat_request( - state: &AppState, - namespace: &NamespaceKey, - request: &ChatCompletionsRequest, -) -> Result { - request.validate()?; - let context = RequestContext { - endpoint: "chat.completions".to_owned(), - model: request.model.clone(), - stream: request.stream, - metadata: serde_json::Map::new(), - input_text: request.extract_user_text(), - instructions_text: request.extract_instruction_text(), - }; - state.log_request(namespace, context.clone()); - - if let Some(scenario) = state.take_matching_scenario(namespace, &context) { - return match scenario.execute_for_chat(state.next_response_id(namespace), request) { - ExecutionOutcome::Success(success) => Ok(ExecutionOutcome::Success( - enforce_tool_choice(request.tool_choice_mode(), success)?, - )), - outcome => Ok(outcome), - }; - } - - Ok(ExecutionOutcome::Success(enforce_tool_choice( - request.tool_choice_mode(), - SuccessOutcome { - plan: build_default_chat_plan( - state.next_response_id(namespace), - request.model.clone(), - &request.extract_user_text(), - request.response_format(), - request.reasoning_requested(), - ), - transport: TransportOptions::default(), - }, - )?)) -} - -fn enforce_tool_choice( - tool_choice: Option, - success: SuccessOutcome, -) -> Result { - validate_tool_choice_against_plan(tool_choice, &success.plan)?; - Ok(success) -} - -fn validate_tool_choice_against_plan( - tool_choice: Option, - plan: &ResponsePlan, -) -> Result<(), OpenAiError> { - match tool_choice { - None | Some(ToolChoiceMode::Auto) => Ok(()), - Some(ToolChoiceMode::NoTool) if plan.tool_calls.is_empty() => Ok(()), - Some(ToolChoiceMode::NoTool) => Err(OpenAiError::invalid_request( - "tool_choice", - "tool_choice forbids tool calls for this request", - )), - Some(ToolChoiceMode::Required) if !plan.tool_calls.is_empty() => Ok(()), - Some(ToolChoiceMode::Required) => Err(OpenAiError::invalid_request( - "tool_choice", - "tool_choice required a tool call but none was planned", - )), - Some(ToolChoiceMode::Function(name)) - if plan - .tool_calls - .iter() - .any(|tool_call| tool_call.name == name) => - { - Ok(()) - } - Some(ToolChoiceMode::Function(name)) => Err(OpenAiError::invalid_request( - "tool_choice", - &format!("tool_choice requested function `{name}` but it was not planned"), - )), - } -} diff --git a/test/twin/openai/src/engine/plan.rs b/test/twin/openai/src/engine/plan.rs deleted file mode 100644 index f750bb39b..000000000 --- a/test/twin/openai/src/engine/plan.rs +++ /dev/null @@ -1,163 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; - -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct TokenUsage { - pub input_tokens: u64, - pub output_tokens: u64, -} - -impl TokenUsage { - #[must_use] - pub const fn new(input_tokens: u64, output_tokens: u64) -> Self { - Self { - input_tokens, - output_tokens, - } - } - - #[must_use] - pub const fn total_tokens(self) -> u64 { - self.input_tokens + self.output_tokens - } - - #[must_use] - pub fn responses_json(self) -> Value { - json!({ - "input_tokens": self.input_tokens, - "output_tokens": self.output_tokens, - "total_tokens": self.total_tokens(), - }) - } - - #[must_use] - pub fn chat_completions_json(self) -> Value { - json!({ - "prompt_tokens": self.input_tokens, - "completion_tokens": self.output_tokens, - "total_tokens": self.total_tokens(), - }) - } -} - -impl Default for TokenUsage { - fn default() -> Self { - Self::new(1, 5) - } -} - -#[derive(Clone, Debug)] -pub struct ResponsePlan { - pub id: String, - pub created: u64, - pub model: String, - pub response_text: String, - pub structured_output: Option, - pub reasoning: Vec, - pub tool_calls: Vec, - pub usage: TokenUsage, -} - -#[derive(Clone, Debug)] -pub struct ToolCallPlan { - pub id: String, - pub name: String, - pub arguments: Value, - pub raw_arguments: Option, -} - -impl ResponsePlan { - pub fn tool_call_arguments_text(tool_call: &ToolCallPlan) -> String { - tool_call - .raw_arguments - .clone() - .unwrap_or_else(|| tool_call.arguments.to_string()) - } - - fn responses_tool_call_item(tool_call: &ToolCallPlan) -> Value { - json!({ - "id": format!("fc_{}", tool_call.id), - "type": "function_call", - "call_id": tool_call.id, - "name": tool_call.name, - "arguments": Self::tool_call_arguments_text(tool_call), - }) - } - - pub fn chat_content(&self) -> String { - self.structured_output - .as_ref() - .map_or_else(|| self.response_text.clone(), ToString::to_string) - } - - pub fn responses_json(&self) -> Value { - let mut content_items = Vec::new(); - - if !self.response_text.is_empty() { - content_items.push(json!({ - "type": "output_text", - "text": self.response_text, - })); - } - - if let Some(structured_output) = &self.structured_output { - content_items.push(json!({ - "type": "output_json", - "json": structured_output, - })); - } - - let mut output = Vec::new(); - - if !content_items.is_empty() { - output.push(json!({ - "id": format!("msg_{}", self.id), - "type": "message", - "role": "assistant", - "content": content_items, - })); - } - - for tool_call in &self.tool_calls { - output.push(Self::responses_tool_call_item(tool_call)); - } - - json!({ - "id": self.id, - "object": "response", - "created": self.created, - "model": self.model, - "status": "completed", - "reasoning": self.reasoning, - "output": output, - "usage": self.usage.responses_json() - }) - } - - pub fn chat_completions_json(&self) -> Value { - json!({ - "id": format!("chatcmpl_{}", self.id), - "object": "chat.completion", - "created": self.created, - "model": self.model, - "choices": [{ - "index": 0, - "finish_reason": if self.tool_calls.is_empty() { "stop" } else { "tool_calls" }, - "message": { - "role": "assistant", - "content": self.chat_content(), - "reasoning": self.reasoning, - "tool_calls": self.tool_calls.iter().map(|tool_call| json!({ - "id": tool_call.id, - "type": "function", - "function": { - "name": tool_call.name, - "arguments": Self::tool_call_arguments_text(tool_call), - } - })).collect::>(), - } - }], - "usage": self.usage.chat_completions_json() - }) - } -} diff --git a/test/twin/openai/src/engine/scenario.rs b/test/twin/openai/src/engine/scenario.rs deleted file mode 100644 index b30be7eb0..000000000 --- a/test/twin/openai/src/engine/scenario.rs +++ /dev/null @@ -1,271 +0,0 @@ -use axum::http::StatusCode; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use super::failures::{ErrorOutcome, ExecutionOutcome, SuccessOutcome, TransportOptions}; -use super::plan::{ResponsePlan, TokenUsage, ToolCallPlan}; -use crate::openai::models::{ChatCompletionsRequest, ResponsesRequest}; - -#[derive(Clone, Debug, Deserialize)] -pub struct ScenarioEnvelope { - pub scenarios: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct Scenario { - pub matcher: ScenarioMatcher, - pub script: ScenarioScript, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ScenarioMatcher { - pub endpoint: String, - pub model: Option, - pub stream: Option, - #[serde(default)] - pub metadata: Map, - pub input_contains: Option, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum ScenarioScript { - Success { - response_text: Option, - reasoning: Option>, - structured_output: Option, - tool_calls: Option>, - usage: Option, - delay_before_headers_ms: Option, - inter_event_delay_ms: Option, - close_after_chunks: Option, - malformed_sse: Option, - }, - Error { - status: u16, - message: String, - error_type: String, - code: String, - retry_after: Option, - delay_before_headers_ms: Option, - }, - Hang { - delay_before_headers_ms: Option, - }, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct ToolCallTemplate { - pub id: Option, - pub name: String, - pub arguments: Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_arguments: Option, -} - -#[derive(Clone, Debug)] -pub struct RequestContext { - pub endpoint: String, - pub model: String, - pub stream: bool, - pub metadata: Map, - pub input_text: String, - pub instructions_text: String, -} - -impl ScenarioScript { - pub fn script_kind(&self) -> &str { - match self { - Self::Success { .. } => "success", - Self::Error { .. } => "error", - Self::Hang { .. } => "hang", - } - } -} - -impl Scenario { - pub fn matches(&self, request: &RequestContext) -> bool { - if self.matcher.endpoint != request.endpoint { - return false; - } - - if let Some(model) = &self.matcher.model { - if model != &request.model { - return false; - } - } - - if let Some(stream) = self.matcher.stream { - if stream != request.stream { - return false; - } - } - - if let Some(needle) = &self.matcher.input_contains { - if !request.input_text.contains(needle) { - return false; - } - } - - self.matcher.metadata.iter().all(|(key, value)| { - request - .metadata - .get(key) - .is_some_and(|candidate| candidate == value) - }) - } - - pub fn execute_for_responses( - &self, - response_number: u64, - request: &ResponsesRequest, - ) -> ExecutionOutcome { - match &self.script { - ScenarioScript::Success { - response_text, - reasoning, - structured_output, - tool_calls, - usage, - delay_before_headers_ms, - inter_event_delay_ms, - close_after_chunks, - malformed_sse, - } => ExecutionOutcome::Success(SuccessOutcome { - plan: build_plan_from_script( - response_number, - request.model.clone(), - &request.extract_user_text(), - response_text.clone(), - reasoning.clone().unwrap_or_default(), - structured_output.clone(), - tool_calls.clone().unwrap_or_default(), - *usage, - ), - transport: TransportOptions { - delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(), - inter_event_delay_ms: inter_event_delay_ms.unwrap_or_default(), - close_after_chunks: *close_after_chunks, - malformed_sse: malformed_sse.unwrap_or(false), - }, - }), - ScenarioScript::Error { - status, - message, - error_type, - code, - retry_after, - delay_before_headers_ms, - } => ExecutionOutcome::Error(ErrorOutcome::new( - StatusCode::from_u16(*status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), - message.clone(), - error_type.clone(), - code.clone(), - retry_after.clone(), - delay_before_headers_ms.unwrap_or_default(), - )), - ScenarioScript::Hang { - delay_before_headers_ms, - } => ExecutionOutcome::Hang { - delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(), - }, - } - } - - pub fn execute_for_chat( - &self, - response_number: u64, - request: &ChatCompletionsRequest, - ) -> ExecutionOutcome { - match &self.script { - ScenarioScript::Success { - response_text, - reasoning, - structured_output, - tool_calls, - usage, - delay_before_headers_ms, - inter_event_delay_ms, - close_after_chunks, - malformed_sse, - } => ExecutionOutcome::Success(SuccessOutcome { - plan: build_plan_from_script( - response_number, - request.model.clone(), - &request.extract_user_text(), - response_text.clone(), - reasoning.clone().unwrap_or_default(), - structured_output.clone(), - tool_calls.clone().unwrap_or_default(), - *usage, - ), - transport: TransportOptions { - delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(), - inter_event_delay_ms: inter_event_delay_ms.unwrap_or_default(), - close_after_chunks: *close_after_chunks, - malformed_sse: malformed_sse.unwrap_or(false), - }, - }), - ScenarioScript::Error { - status, - message, - error_type, - code, - retry_after, - delay_before_headers_ms, - } => ExecutionOutcome::Error(ErrorOutcome::new( - StatusCode::from_u16(*status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), - message.clone(), - error_type.clone(), - code.clone(), - retry_after.clone(), - delay_before_headers_ms.unwrap_or_default(), - )), - ScenarioScript::Hang { - delay_before_headers_ms, - } => ExecutionOutcome::Hang { - delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(), - }, - } - } -} - -fn build_plan_from_script( - response_number: u64, - model: String, - default_input: &str, - response_text: Option, - reasoning: Vec, - structured_output: Option, - tool_calls: Vec, - usage: Option, -) -> ResponsePlan { - let output_text = match response_text { - Some(response_text) => response_text, - None if tool_calls.is_empty() && structured_output.is_none() => { - format!("deterministic: {default_input}") - } - None => String::new(), - }; - ResponsePlan { - id: format!("resp_{response_number:06}"), - created: response_number, - model, - response_text: output_text, - structured_output, - reasoning, - tool_calls: tool_calls - .into_iter() - .enumerate() - .map(|(index, tool_call)| ToolCallPlan { - id: tool_call - .id - .unwrap_or_else(|| format!("call_{response_number}_{index}")), - name: tool_call.name, - arguments: tool_call.arguments, - raw_arguments: tool_call.raw_arguments, - }) - .collect(), - usage: usage.unwrap_or_default(), - } -} diff --git a/test/twin/openai/src/lib.rs b/test/twin/openai/src/lib.rs deleted file mode 100644 index 1776c54d4..000000000 --- a/test/twin/openai/src/lib.rs +++ /dev/null @@ -1,26 +0,0 @@ -#![allow( - clippy::result_large_err, - reason = "Twin HTTP handlers return full axum::Response errors directly." -)] - -pub mod admin; -pub mod app; -pub mod config; -pub mod debug_ui; -pub mod engine; -pub mod logs; -pub mod openai; -pub mod sse; -pub mod state; - -use axum::Router; -use config::Config; -use state::AppState; - -pub fn build_app() -> Router { - build_app_with_config(Config::from_env().unwrap_or_default()) -} - -pub fn build_app_with_config(config: Config) -> Router { - app::router(AppState::new(config)) -} diff --git a/test/twin/openai/src/logs.rs b/test/twin/openai/src/logs.rs deleted file mode 100644 index 0fce6eafd..000000000 --- a/test/twin/openai/src/logs.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde::Serialize; -use serde_json::{Map, Value}; - -#[derive(Clone, Debug, Serialize)] -pub struct RequestLog { - pub endpoint: String, - pub model: String, - pub stream: bool, - pub input_text: String, - pub instructions_text: String, - pub metadata: Map, -} diff --git a/test/twin/openai/src/main.rs b/test/twin/openai/src/main.rs deleted file mode 100644 index b4941c9e1..000000000 --- a/test/twin/openai/src/main.rs +++ /dev/null @@ -1,21 +0,0 @@ -use anyhow::Result; -use tokio::net::TcpListener; -use tracing_subscriber::fmt; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use twin_openai::app; -use twin_openai::config::Config; -use twin_openai::state::AppState; - -#[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::registry() - .with(tracing_subscriber::EnvFilter::from_default_env()) - .with(fmt::layer()) - .init(); - - let config = Config::from_env()?; - let listener = TcpListener::bind(config.bind_addr).await?; - axum::serve(listener, app::router(AppState::new(config))).await?; - Ok(()) -} diff --git a/test/twin/openai/src/openai/auth.rs b/test/twin/openai/src/openai/auth.rs deleted file mode 100644 index cd3e72d05..000000000 --- a/test/twin/openai/src/openai/auth.rs +++ /dev/null @@ -1,67 +0,0 @@ -use axum::Json; -use axum::extract::Request; -use axum::http::header::AUTHORIZATION; -use axum::http::{HeaderMap, HeaderValue, StatusCode}; -use axum::middleware::Next; -use axum::response::{IntoResponse, Response}; -use serde_json::json; - -use crate::state::NamespaceKey; - -pub async fn require_bearer_auth(request: Request, next: Next) -> Response { - match bearer_token_from_headers(request.headers()) { - Ok(Some(_)) => next.run(request).await, - Ok(None) | Err(()) => missing_bearer_token_response(), - } -} - -pub fn openai_request_namespace( - headers: &HeaderMap, - require_auth: bool, -) -> Result { - match bearer_token_from_headers(headers) { - Ok(Some(token)) => Ok(NamespaceKey::Bearer(token)), - Ok(None) if !require_auth => Ok(NamespaceKey::Global), - Ok(None) | Err(()) => Err(missing_bearer_token_response()), - } -} - -pub fn admin_request_namespace(headers: &HeaderMap) -> Result { - match bearer_token_from_headers(headers) { - Ok(Some(token)) => Ok(NamespaceKey::Bearer(token)), - Ok(None) => Ok(NamespaceKey::Global), - Err(()) => Err(missing_bearer_token_response()), - } -} - -fn bearer_token_from_headers(headers: &HeaderMap) -> Result, ()> { - match headers.get(AUTHORIZATION) { - Some(value) => parse_bearer_token(value).map(Some).ok_or(()), - None => Ok(None), - } -} - -fn parse_bearer_token(value: &HeaderValue) -> Option { - let Ok(value) = value.to_str() else { - return None; - }; - - let token = value.strip_prefix("Bearer ").map(str::trim)?; - - (!token.is_empty()).then(|| token.to_owned()) -} - -fn missing_bearer_token_response() -> Response { - ( - StatusCode::UNAUTHORIZED, - Json(json!({ - "error": { - "message": "missing or empty bearer token", - "type": "invalid_request_error", - "param": "Authorization", - "code": "missing_bearer_token" - } - })), - ) - .into_response() -} diff --git a/test/twin/openai/src/openai/chat_completions.rs b/test/twin/openai/src/openai/chat_completions.rs deleted file mode 100644 index da2d46bf3..000000000 --- a/test/twin/openai/src/openai/chat_completions.rs +++ /dev/null @@ -1,82 +0,0 @@ -use axum::Json; -use axum::extract::State; -use axum::extract::rejection::JsonRejection; -use axum::http::HeaderMap; -use axum::http::header::RETRY_AFTER; -use axum::response::IntoResponse; -use futures_util::future; -use tokio::time::{Duration, sleep}; - -use super::models::ChatCompletionsRequest; -use crate::engine::execute_chat_request; -use crate::engine::failures::ExecutionOutcome; -use crate::openai::auth; -use crate::sse::chat_sse_response; -use crate::state::AppState; - -pub async fn create_chat_completion( - State(state): State, - headers: HeaderMap, - payload: Result, JsonRejection>, -) -> impl IntoResponse { - let namespace = match auth::openai_request_namespace(&headers, state.config.require_auth) { - Ok(namespace) => namespace, - Err(response) => return response, - }; - - let request = match payload { - Ok(Json(request)) => request, - Err(rejection) => { - return super::models::OpenAiError::from_json_rejection(&rejection) - .into_response() - .into_response(); - } - }; - - match execute_chat_request(&state, &namespace, &request) { - Ok(ExecutionOutcome::Success(success)) => { - if success.transport.delay_before_headers_ms > 0 { - sleep(Duration::from_millis( - success.transport.delay_before_headers_ms, - )) - .await; - } - - if request.stream { - chat_sse_response( - &success.plan, - request.include_stream_usage(), - success.transport, - ) - .into_response() - } else { - Json(success.plan.chat_completions_json()).into_response() - } - } - Ok(ExecutionOutcome::Error(error)) => { - if error.delay_before_headers_ms > 0 { - sleep(Duration::from_millis(error.delay_before_headers_ms)).await; - } - - let mut response = Json(error.body).into_response(); - *response.status_mut() = error.status; - if let Some(retry_after) = error.retry_after { - response.headers_mut().insert( - RETRY_AFTER, - retry_after.parse().expect("valid Retry-After header"), - ); - } - response - } - Ok(ExecutionOutcome::Hang { - delay_before_headers_ms, - }) => { - if delay_before_headers_ms > 0 { - sleep(Duration::from_millis(delay_before_headers_ms)).await; - } - future::pending::<()>().await; - unreachable!() - } - Err(error) => error.into_response().into_response(), - } -} diff --git a/test/twin/openai/src/openai/mod.rs b/test/twin/openai/src/openai/mod.rs deleted file mode 100644 index 582da1cff..000000000 --- a/test/twin/openai/src/openai/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -pub mod auth; -pub mod chat_completions; -pub mod models; -pub mod responses; - -use axum::routing::post; -use axum::{Router, middleware}; - -use crate::state::AppState; - -pub fn router(require_auth: bool) -> Router { - let router = Router::new() - .route("/responses", post(responses::create_response)) - .route( - "/chat/completions", - post(chat_completions::create_chat_completion), - ); - - if require_auth { - router.layer(middleware::from_fn(auth::require_bearer_auth)) - } else { - router - } -} diff --git a/test/twin/openai/src/openai/models.rs b/test/twin/openai/src/openai/models.rs deleted file mode 100644 index be41db099..000000000 --- a/test/twin/openai/src/openai/models.rs +++ /dev/null @@ -1,1104 +0,0 @@ -use std::collections::HashSet; - -use axum::Json; -use axum::extract::rejection::JsonRejection; -use axum::http::StatusCode; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -/// Accepts all known OpenAI Responses API fields. Unknown fields are ignored -/// via `#[serde(flatten)]` so the twin stays compatible as the API evolves. -#[derive(Clone, Debug, Deserialize)] -pub struct ResponsesRequest { - pub model: String, - #[serde(default)] - pub input: ResponseInput, - pub instructions: Option, - #[serde(default)] - pub stream: bool, - #[serde(default)] - pub metadata: Map, - pub stop: Option, - pub previous_response_id: Option, - pub reasoning: Option, - pub text: Option, - pub tools: Option>, - pub tool_choice: Option, - /// Catch-all for fields the twin doesn't use (temperature, top_p, etc.) - #[allow( - dead_code, - reason = "Serde captures unknown request fields for forward compatibility." - )] - #[serde(flatten)] - extra: Map, -} - -impl ResponsesRequest { - pub fn extract_user_text(&self) -> String { - let text = self.input.extract_text(); - if text.is_empty() { - "empty input".to_owned() - } else { - text - } - } - - pub fn extract_instruction_text(&self) -> String { - self.instructions - .as_deref() - .map(normalize_whitespace) - .unwrap_or_default() - } - - pub fn response_format(&self) -> Option { - let format = self.text.as_ref()?.format.as_ref()?; - response_format_from_kind( - "text.format.type", - &format.kind, - format.json_schema_payload(), - ) - .ok() - } - - pub fn tool_choice_mode(&self) -> Option { - tool_choice_mode(self.tool_choice.as_ref(), ToolSurface::Responses) - } - - pub fn validate(&self) -> Result<(), OpenAiError> { - if self.model.trim().is_empty() { - return Err(OpenAiError::invalid_request( - "model", - "model must not be empty", - )); - } - - if let Some(text) = &self.text { - let Some(format) = &text.format else { - return Err(OpenAiError::invalid_request( - "text.format", - "text.format must be present when text is provided", - )); - }; - - if let ResponseFormat::JsonSchema(schema) = response_format_from_kind( - "text.format.type", - &format.kind, - format.json_schema_payload(), - )? { - validate_json_schema_subset(&schema)?; - } - } - - if let Some(ResponseFormat::JsonSchema(schema)) = self.response_format() { - validate_json_schema_subset(&schema)?; - } - - validate_tools(self.tools.as_ref(), "tools", ToolSurface::Responses)?; - validate_tool_choice( - self.tool_choice.as_ref(), - "tool_choice", - ToolSurface::Responses, - )?; - validate_tool_choice_requires_tools(self.tool_choice.as_ref(), self.tools.as_ref())?; - validate_stop(self.stop.as_ref(), "stop")?; - validate_response_input(&self.input, self.previous_response_id.as_deref())?; - - Ok(()) - } -} - -#[derive(Clone, Debug)] -pub enum ResponseFormat { - Text, - JsonObject, - JsonSchema(Value), -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ToolSurface { - Responses, - ChatCompletions, -} - -#[derive(Clone, Debug, Deserialize, Default)] -#[serde(untagged)] -pub enum ResponseInput { - Text(String), - Items(Vec), - #[default] - Empty, -} - -impl ResponseInput { - fn extract_text(&self) -> String { - match self { - Self::Text(text) => normalize_whitespace(text), - Self::Items(items) => { - let pieces: Vec = items - .iter() - .flat_map(InputItem::extract_texts_for_fallback) - .collect(); - normalize_whitespace(&pieces.join(" ")) - } - Self::Empty => String::new(), - } - } -} - -#[derive(Clone, Debug, Deserialize)] -pub struct InputItem { - #[serde(default)] - pub role: Option, - #[serde(default)] - pub content: InputContent, - #[serde(default)] - #[serde(rename = "type")] - pub item_type: Option, - #[serde(default)] - pub output: Option, - #[serde(default)] - pub call_id: Option, -} - -impl InputItem { - fn extract_texts_for_fallback(&self) -> Vec { - if self.kind().is_tool_output() { - return self - .output - .as_ref() - .map(|output| vec![normalize_whitespace(output)]) - .unwrap_or_default(); - } - - if self.role.as_deref() == Some("user") { - return self.content.extract_texts(); - } - - Vec::new() - } - - fn kind(&self) -> InputItemKind { - InputItemKind::from_wire(self.item_type.as_deref()) - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum InputItemKind { - Message, - FunctionCall, - CustomToolCall, - FunctionCallOutput, - CustomToolCallOutput, - Other, -} - -impl InputItemKind { - fn from_wire(item_type: Option<&str>) -> Self { - match item_type { - None | Some("message") => Self::Message, - Some("function_call") => Self::FunctionCall, - Some("custom_tool_call") => Self::CustomToolCall, - Some("function_call_output") => Self::FunctionCallOutput, - Some("custom_tool_call_output") => Self::CustomToolCallOutput, - Some(_) => Self::Other, - } - } - - fn is_tool_call(self) -> bool { - matches!(self, Self::FunctionCall | Self::CustomToolCall) - } - - fn is_tool_output(self) -> bool { - matches!(self, Self::FunctionCallOutput | Self::CustomToolCallOutput) - } -} - -#[derive(Clone, Debug, Deserialize, Default)] -#[serde(untagged)] -pub enum InputContent { - Text(String), - Parts(Vec), - #[default] - Empty, -} - -impl InputContent { - fn extract_texts(&self) -> Vec { - match self { - Self::Text(text) => vec![normalize_whitespace(text)], - Self::Parts(parts) => parts.iter().filter_map(ContentPart::text_value).collect(), - Self::Empty => Vec::new(), - } - } - - fn is_empty(&self) -> bool { - match self { - Self::Text(text) => text.is_empty(), - Self::Parts(parts) => parts.is_empty(), - Self::Empty => true, - } - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ContentPart { - #[serde(rename = "type")] - pub kind: String, - #[serde(default)] - pub text: Option, - #[serde(default)] - pub image_url: Option, -} - -impl ContentPart { - fn text_value(&self) -> Option { - match self.kind.as_str() { - "input_text" | "text" => self.text.as_ref().map(|text| normalize_whitespace(text)), - _ => None, - } - } -} - -fn validate_response_input( - input: &ResponseInput, - previous_response_id: Option<&str>, -) -> Result<(), OpenAiError> { - let ResponseInput::Items(items) = input else { - return Ok(()); - }; - - let mut function_call_ids = HashSet::new(); - for item in items { - validate_input_item(item)?; - let kind = item.kind(); - if kind.is_tool_call() { - if let Some(call_id) = item.call_id.as_deref().filter(|id| !id.is_empty()) { - function_call_ids.insert(call_id); - } - } else if kind.is_tool_output() && previous_response_id.is_none() { - let call_id = item.call_id.as_deref().unwrap_or_default(); - if !function_call_ids.contains(call_id) { - return Err(OpenAiError::invalid_request( - "input", - &format!("No tool call found for tool call output with call_id {call_id}."), - )); - } - } - } - - Ok(()) -} - -fn validate_input_item(item: &InputItem) -> Result<(), OpenAiError> { - match item.kind() { - InputItemKind::FunctionCallOutput | InputItemKind::CustomToolCallOutput => { - validate_function_call_output_item(item) - } - InputItemKind::Message => validate_message_input_item(item), - // Accept any other item type — the twin extracts user text for fallback - // responses and ignores items it doesn't understand. - InputItemKind::FunctionCall | InputItemKind::CustomToolCall | InputItemKind::Other => { - Ok(()) - } - } -} - -fn validate_function_call_output_item(item: &InputItem) -> Result<(), OpenAiError> { - if item.role.is_some() || !item.content.is_empty() { - return Err(OpenAiError::invalid_request( - "input", - "function_call_output items may not include role or content", - )); - } - - if item.call_id.as_deref().is_none_or(str::is_empty) { - return Err(OpenAiError::invalid_request( - "input", - "function_call_output items require call_id", - )); - } - - if item.output.as_deref().is_none_or(str::is_empty) { - return Err(OpenAiError::invalid_request( - "input", - "function_call_output items require output", - )); - } - - Ok(()) -} - -fn validate_message_input_item(item: &InputItem) -> Result<(), OpenAiError> { - if item.role.as_deref().is_none_or(str::is_empty) { - return Err(OpenAiError::invalid_request( - "input", - "message input items require role", - )); - } - - if item.output.is_some() || item.call_id.is_some() { - return Err(OpenAiError::invalid_request( - "input", - "message input items may not include function_call_output fields", - )); - } - - let role = item.role.as_deref().unwrap_or_default(); - validate_input_content(role, &item.content) -} - -fn validate_input_content(role: &str, content: &InputContent) -> Result<(), OpenAiError> { - match content { - InputContent::Text(_) => Ok(()), - InputContent::Parts(parts) if !parts.is_empty() => { - for part in parts { - validate_input_content_part(role, part)?; - } - Ok(()) - } - _ => Err(OpenAiError::invalid_request( - "input", - "message input items require supported content", - )), - } -} - -fn validate_input_content_part(role: &str, part: &ContentPart) -> Result<(), OpenAiError> { - match part.kind.as_str() { - "input_text" | "text" if part.text.as_deref().is_some() => Ok(()), - // Assistant history items are replayed with their original output - // parts; the real API accepts output_text on assistant messages. - "output_text" if role == "assistant" && part.text.as_deref().is_some() => Ok(()), - "input_image" - if part - .image_url - .as_deref() - .is_some_and(is_supported_image_reference) => - { - Ok(()) - } - "input_text" | "text" => Err(OpenAiError::invalid_request( - "input", - "text input parts require text", - )), - "output_text" if role == "assistant" => Err(OpenAiError::invalid_request( - "input", - "text input parts require text", - )), - "output_text" => Err(OpenAiError::invalid_request( - "input", - "output_text parts are only valid on assistant messages", - )), - "input_image" => Err(OpenAiError::invalid_request( - "input", - "image input parts require a supported image_url", - )), - _ => Err(OpenAiError::invalid_request( - "input", - "unsupported input content part type", - )), - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TextOptions { - pub format: Option, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TextFormat { - #[serde(rename = "type")] - pub kind: String, - #[serde(default)] - pub json_schema: Option, - #[serde(default)] - pub name: Option, - #[serde(default)] - pub schema: Option, - #[serde(default)] - pub description: Option, - #[serde(default)] - pub strict: Option, -} - -impl TextFormat { - fn json_schema_payload(&self) -> Option { - self.json_schema.clone().or_else(|| { - json_schema_config_value( - self.name.as_ref(), - self.schema.as_ref(), - self.description.as_ref(), - self.strict, - ) - }) - } -} - -#[derive(Clone, Debug, Serialize)] -pub struct ErrorEnvelope { - pub error: ErrorBody, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ErrorBody { - pub message: String, - #[serde(rename = "type")] - pub error_type: String, - pub param: Value, - pub code: String, -} - -#[derive(Clone, Debug)] -pub struct OpenAiError { - pub status: StatusCode, - pub body: ErrorEnvelope, -} - -impl OpenAiError { - pub fn invalid_request(param: &str, message: &str) -> Self { - Self { - status: StatusCode::BAD_REQUEST, - body: ErrorEnvelope { - error: ErrorBody { - message: message.to_owned(), - error_type: "invalid_request_error".to_owned(), - param: Value::String(param.to_owned()), - code: "invalid_request".to_owned(), - }, - }, - } - } - - pub fn into_response(self) -> (StatusCode, Json) { - (self.status, Json(self.body)) - } - - pub fn from_json_rejection(rejection: &JsonRejection) -> Self { - Self::invalid_request("body", &rejection.body_text()) - } -} - -fn validate_json_schema_subset(schema: &Value) -> Result<(), OpenAiError> { - let schema = schema.get("schema").unwrap_or(schema); - match schema.get("type").and_then(Value::as_str) { - Some("object") => { - let Some(properties) = schema.get("properties").and_then(Value::as_object) else { - return Err(OpenAiError::invalid_request( - "text.format.json_schema", - "json_schema object types must define properties", - )); - }; - - for property in properties.values() { - validate_schema_node(property)?; - } - Ok(()) - } - _ => Err(OpenAiError::invalid_request( - "text.format.json_schema", - "unsupported json_schema root type", - )), - } -} - -fn validate_schema_node(node: &Value) -> Result<(), OpenAiError> { - if node.get("items").is_some() || node.get("anyOf").is_some() || node.get("oneOf").is_some() { - return Err(OpenAiError::invalid_request( - "text.format.json_schema", - "unsupported json_schema construct", - )); - } - - match node.get("type").and_then(Value::as_str) { - Some("string" | "number" | "integer" | "boolean") => Ok(()), - Some("object") => validate_json_schema_subset(node), - _ => Err(OpenAiError::invalid_request( - "text.format.json_schema", - "unsupported json_schema property type", - )), - } -} - -pub fn normalize_whitespace(input: &str) -> String { - input.split_whitespace().collect::>().join(" ") -} - -/// Accepts all known OpenAI Chat Completions API fields. Unknown top-level -/// fields are ignored via `#[serde(flatten)]` so the twin stays compatible as -/// clients add request options. -#[derive(Clone, Debug, Deserialize)] -pub struct ChatCompletionsRequest { - pub model: String, - pub messages: Vec, - pub max_tokens: Option, - #[serde(default)] - pub stream: bool, - stream_options: Option, - pub tools: Option>, - pub tool_choice: Option, - pub response_format: Option, - pub stop: Option, - /// Catch-all for fields the twin doesn't use (temperature, top_p, etc.) - #[allow( - dead_code, - reason = "Serde captures unknown request fields for forward compatibility." - )] - #[serde(flatten)] - extra: Map, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct ChatStreamOptions { - #[serde(default)] - include_usage: bool, -} - -impl ChatCompletionsRequest { - pub fn include_stream_usage(&self) -> bool { - self.stream_options - .as_ref() - .is_some_and(|options| options.include_usage) - } - - pub fn extract_user_text(&self) -> String { - let pieces: Vec = self - .messages - .iter() - .filter(|message| message.role == "user") - .flat_map(ChatMessage::extract_texts) - .collect(); - let text = normalize_whitespace(&pieces.join(" ")); - if text.is_empty() { - "empty input".to_owned() - } else { - text - } - } - - pub fn extract_instruction_text(&self) -> String { - let pieces: Vec = self - .messages - .iter() - .filter(|message| message.role == "system" || message.role == "developer") - .flat_map(ChatMessage::extract_texts) - .collect(); - normalize_whitespace(&pieces.join(" ")) - } - - pub fn response_format(&self) -> Option { - let format = self.response_format.as_ref()?; - response_format_from_kind( - "response_format.type", - &format.kind, - format.json_schema_payload(), - ) - .ok() - } - - pub fn reasoning_requested(&self) -> bool { - self.messages - .iter() - .any(ChatMessage::contains_reasoning_content) - } - - pub fn tool_choice_mode(&self) -> Option { - tool_choice_mode(self.tool_choice.as_ref(), ToolSurface::ChatCompletions) - } - - pub fn validate(&self) -> Result<(), OpenAiError> { - if self.model.trim().is_empty() { - return Err(OpenAiError::invalid_request( - "model", - "model must not be empty", - )); - } - - if let Some(format) = &self.response_format { - if let ResponseFormat::JsonSchema(schema) = response_format_from_kind( - "response_format.type", - &format.kind, - format.json_schema_payload(), - )? { - validate_json_schema_subset(&schema)?; - } - } - - if let Some(ResponseFormat::JsonSchema(schema)) = self.response_format() { - validate_json_schema_subset(&schema)?; - } - - validate_tools(self.tools.as_ref(), "tools", ToolSurface::ChatCompletions)?; - validate_tool_choice( - self.tool_choice.as_ref(), - "tool_choice", - ToolSurface::ChatCompletions, - )?; - validate_tool_choice_requires_tools(self.tool_choice.as_ref(), self.tools.as_ref())?; - validate_stop(self.stop.as_ref(), "stop")?; - validate_chat_messages(&self.messages)?; - - Ok(()) - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChatMessage { - pub role: String, - pub content: Option, - pub reasoning_content: Option, - pub tool_call_id: Option, - pub tool_calls: Option>, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChatMessageToolCall { - pub id: String, - #[serde(rename = "type")] - pub kind: String, - pub function: ChatMessageToolCallFunction, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChatMessageToolCallFunction { - pub name: String, - pub arguments: String, -} - -impl ChatMessage { - fn extract_texts(&self) -> Vec { - match self.content.as_ref() { - Some(Value::String(text)) => vec![normalize_whitespace(text)], - Some(Value::Array(parts)) => parts - .iter() - .filter_map(|part| { - part.get("text") - .and_then(Value::as_str) - .map(normalize_whitespace) - }) - .collect(), - _ => Vec::new(), - } - } - - fn contains_reasoning_content(&self) -> bool { - self.role == "assistant" - && (self - .reasoning_content - .as_deref() - .is_some_and(|reasoning| !reasoning.trim().is_empty()) - || self - .content - .as_ref() - .and_then(Value::as_array) - .is_some_and(|parts| { - parts.iter().any(|part| { - part.get("type") - .and_then(Value::as_str) - .is_some_and(|kind| kind == "reasoning") - }) - })) - } -} - -fn validate_chat_messages(messages: &[ChatMessage]) -> Result<(), OpenAiError> { - if messages.is_empty() { - return Err(OpenAiError::invalid_request( - "messages", - "messages must not be empty", - )); - } - - for message in messages { - validate_chat_message(message)?; - } - - Ok(()) -} - -fn validate_chat_message(message: &ChatMessage) -> Result<(), OpenAiError> { - if message.role.trim().is_empty() { - return Err(OpenAiError::invalid_request( - "messages", - "message role must not be empty", - )); - } - - validate_chat_tool_fields(message)?; - - match message.content.as_ref() { - Some(Value::String(_)) => Ok(()), - Some(Value::Array(parts)) if !parts.is_empty() => { - for part in parts { - validate_chat_message_part(part, &message.role)?; - } - Ok(()) - } - None if message.role == "assistant" - && message - .tool_calls - .as_ref() - .is_some_and(|tool_calls| !tool_calls.is_empty()) => - { - Ok(()) - } - _ => Err(OpenAiError::invalid_request( - "messages", - "unsupported message content shape", - )), - } -} - -fn validate_chat_tool_fields(message: &ChatMessage) -> Result<(), OpenAiError> { - if message.role == "tool" { - if message - .tool_call_id - .as_deref() - .is_none_or(|tool_call_id| tool_call_id.trim().is_empty()) - { - return Err(OpenAiError::invalid_request( - "messages", - "tool messages require a tool_call_id", - )); - } - } else if message.tool_call_id.is_some() { - return Err(OpenAiError::invalid_request( - "messages", - "tool_call_id is only supported on tool messages", - )); - } - - let Some(tool_calls) = &message.tool_calls else { - return Ok(()); - }; - if message.role != "assistant" { - return Err(OpenAiError::invalid_request( - "messages", - "tool_calls are only supported on assistant messages", - )); - } - if tool_calls.is_empty() { - return Err(OpenAiError::invalid_request( - "messages", - "tool_calls must not be empty", - )); - } - for tool_call in tool_calls { - if tool_call.id.trim().is_empty() - || tool_call.kind != "function" - || tool_call.function.name.trim().is_empty() - || tool_call.function.arguments.trim().is_empty() - { - return Err(OpenAiError::invalid_request( - "messages", - "invalid assistant tool call", - )); - } - } - - Ok(()) -} - -fn validate_chat_message_part(part: &Value, role: &str) -> Result<(), OpenAiError> { - let Some(object) = part.as_object() else { - return Err(OpenAiError::invalid_request( - "messages", - "message content parts must be objects", - )); - }; - - let Some(kind) = object.get("type").and_then(Value::as_str) else { - return Err(OpenAiError::invalid_request( - "messages", - "message content part type is required", - )); - }; - - match kind { - "text" => { - if object.get("text").and_then(Value::as_str).is_none() { - return Err(OpenAiError::invalid_request( - "messages", - "text-bearing message content parts require text", - )); - } - Ok(()) - } - "reasoning" => { - if role != "assistant" { - return Err(OpenAiError::invalid_request( - "messages", - "reasoning content parts are only supported on assistant messages", - )); - } - if object.get("text").and_then(Value::as_str).is_none() { - return Err(OpenAiError::invalid_request( - "messages", - "text-bearing message content parts require text", - )); - } - Ok(()) - } - "image_url" => { - if !object - .get("image_url") - .is_some_and(is_valid_chat_image_reference) - { - return Err(OpenAiError::invalid_request( - "messages", - "image_url parts require a supported image_url object", - )); - } - Ok(()) - } - _ => Err(OpenAiError::invalid_request( - "messages", - "unsupported message content part type", - )), - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChatResponseFormat { - #[serde(rename = "type")] - pub kind: String, - #[serde(default)] - pub schema: Option, - #[serde(default)] - pub json_schema: Option, -} - -impl ChatResponseFormat { - fn json_schema_payload(&self) -> Option { - self.json_schema.clone().or_else(|| self.schema.clone()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ToolChoiceMode { - Auto, - NoTool, - Required, - Function(String), -} - -fn response_format_from_kind( - param: &str, - kind: &str, - schema: Option, -) -> Result { - match kind { - "text" => Ok(ResponseFormat::Text), - "json_object" => Ok(ResponseFormat::JsonObject), - "json_schema" => schema - .map(ResponseFormat::JsonSchema) - .ok_or_else(|| OpenAiError::invalid_request(param, "json_schema requires schema")), - _ => Err(OpenAiError::invalid_request( - param, - "unsupported response format type", - )), - } -} - -fn validate_tools( - tools: Option<&Vec>, - param: &str, - surface: ToolSurface, -) -> Result<(), OpenAiError> { - let Some(tools) = tools else { - return Ok(()); - }; - - for tool in tools { - let Some(tool_type) = tool.get("type").and_then(Value::as_str) else { - return Err(OpenAiError::invalid_request(param, "tool type is required")); - }; - - match tool_type { - "function" => { - if function_tool_name(tool, surface).is_none() { - return Err(OpenAiError::invalid_request( - param, - "function tool name is required", - )); - } - } - "custom" if surface == ToolSurface::Responses => { - if function_tool_name(tool, surface).is_none() { - return Err(OpenAiError::invalid_request( - param, - "custom tool name is required", - )); - } - } - _ => { - return Err(OpenAiError::invalid_request(param, "unsupported tool type")); - } - } - } - - Ok(()) -} - -fn validate_tool_choice( - tool_choice: Option<&Value>, - param: &str, - surface: ToolSurface, -) -> Result<(), OpenAiError> { - let Some(tool_choice) = tool_choice else { - return Ok(()); - }; - - match tool_choice { - Value::String(value) if matches!(value.as_str(), "auto" | "none" | "required") => Ok(()), - Value::Object(object) - if object - .get("type") - .and_then(Value::as_str) - .is_some_and(|value| value == "function") - && function_tool_choice_name(tool_choice, surface).is_some() => - { - Ok(()) - } - _ => Err(OpenAiError::invalid_request( - param, - "unsupported tool_choice shape", - )), - } -} - -fn validate_tool_choice_requires_tools( - tool_choice: Option<&Value>, - tools: Option<&Vec>, -) -> Result<(), OpenAiError> { - let Some(tool_choice) = tool_choice else { - return Ok(()); - }; - - let requires_tools = match tool_choice { - Value::String(value) => value == "required", - Value::Object(_) => true, - _ => false, - }; - - if requires_tools && tools.is_none_or(Vec::is_empty) { - return Err(OpenAiError::invalid_request( - "tool_choice", - "tool_choice requires tools to be provided", - )); - } - - Ok(()) -} - -fn validate_stop(stop: Option<&Value>, param: &str) -> Result<(), OpenAiError> { - let Some(stop) = stop else { - return Ok(()); - }; - - match stop { - Value::String(_) => Ok(()), - Value::Array(values) if values.iter().all(Value::is_string) => Ok(()), - _ => Err(OpenAiError::invalid_request( - param, - "stop must be a string or array of strings", - )), - } -} - -fn tool_choice_mode(tool_choice: Option<&Value>, surface: ToolSurface) -> Option { - match tool_choice? { - Value::String(value) => match value.as_str() { - "auto" => Some(ToolChoiceMode::Auto), - "none" => Some(ToolChoiceMode::NoTool), - "required" => Some(ToolChoiceMode::Required), - _ => None, - }, - Value::Object(_) => function_tool_choice_name(tool_choice?, surface) - .map(|name| ToolChoiceMode::Function(name.to_owned())), - _ => None, - } -} - -fn function_tool_name(tool: &Value, surface: ToolSurface) -> Option<&str> { - if surface == ToolSurface::Responses { - if let Some(name) = tool.get("name").and_then(Value::as_str) { - return Some(name); - } - } - - tool.get("function") - .and_then(Value::as_object) - .and_then(|function| function.get("name")) - .and_then(Value::as_str) -} - -fn function_tool_choice_name(tool_choice: &Value, surface: ToolSurface) -> Option<&str> { - if surface == ToolSurface::Responses { - if let Some(name) = tool_choice.get("name").and_then(Value::as_str) { - return Some(name); - } - } - - tool_choice - .get("function") - .and_then(Value::as_object) - .and_then(|function| function.get("name")) - .and_then(Value::as_str) -} - -fn json_schema_config_value( - name: Option<&String>, - schema: Option<&Value>, - description: Option<&String>, - strict: Option, -) -> Option { - if name.is_none() && schema.is_none() && description.is_none() && strict.is_none() { - return None; - } - - let mut object = Map::new(); - if let Some(name) = name { - object.insert("name".to_owned(), Value::String(name.clone())); - } - if let Some(schema) = schema { - object.insert("schema".to_owned(), schema.clone()); - } - if let Some(description) = description { - object.insert("description".to_owned(), Value::String(description.clone())); - } - if let Some(strict) = strict { - object.insert("strict".to_owned(), Value::Bool(strict)); - } - Some(Value::Object(object)) -} - -fn is_supported_image_reference(image_url: &str) -> bool { - !image_url.trim().is_empty() - && (image_url.starts_with("http://") - || image_url.starts_with("https://") - || image_url.starts_with("data:")) -} - -fn is_valid_chat_image_reference(image_url: &Value) -> bool { - let Some(object) = image_url.as_object() else { - return false; - }; - - let Some(url) = object.get("url").and_then(Value::as_str) else { - return false; - }; - - if !is_supported_image_reference(url) { - return false; - } - - object.get("detail").is_none_or(Value::is_string) -} diff --git a/test/twin/openai/src/openai/responses.rs b/test/twin/openai/src/openai/responses.rs deleted file mode 100644 index 0031759c1..000000000 --- a/test/twin/openai/src/openai/responses.rs +++ /dev/null @@ -1,77 +0,0 @@ -use axum::Json; -use axum::extract::State; -use axum::extract::rejection::JsonRejection; -use axum::http::HeaderMap; -use axum::http::header::RETRY_AFTER; -use axum::response::IntoResponse; -use futures_util::future; -use tokio::time::{Duration, sleep}; - -use super::models::ResponsesRequest; -use crate::engine::execute_responses_request; -use crate::engine::failures::ExecutionOutcome; -use crate::openai::auth; -use crate::sse::responses_sse_response; -use crate::state::AppState; - -pub async fn create_response( - State(state): State, - headers: HeaderMap, - payload: Result, JsonRejection>, -) -> impl IntoResponse { - let namespace = match auth::openai_request_namespace(&headers, state.config.require_auth) { - Ok(namespace) => namespace, - Err(response) => return response, - }; - - let request = match payload { - Ok(Json(request)) => request, - Err(rejection) => { - return super::models::OpenAiError::from_json_rejection(&rejection) - .into_response() - .into_response(); - } - }; - - match execute_responses_request(&state, &namespace, &request) { - Ok(ExecutionOutcome::Success(success)) => { - if success.transport.delay_before_headers_ms > 0 { - sleep(Duration::from_millis( - success.transport.delay_before_headers_ms, - )) - .await; - } - - if request.stream { - responses_sse_response(&success.plan, success.transport).into_response() - } else { - Json(success.plan.responses_json()).into_response() - } - } - Ok(ExecutionOutcome::Error(error)) => { - if error.delay_before_headers_ms > 0 { - sleep(Duration::from_millis(error.delay_before_headers_ms)).await; - } - - let mut response = Json(error.body).into_response(); - *response.status_mut() = error.status; - if let Some(retry_after) = error.retry_after { - response.headers_mut().insert( - RETRY_AFTER, - retry_after.parse().expect("valid Retry-After header"), - ); - } - response - } - Ok(ExecutionOutcome::Hang { - delay_before_headers_ms, - }) => { - if delay_before_headers_ms > 0 { - sleep(Duration::from_millis(delay_before_headers_ms)).await; - } - future::pending::<()>().await; - unreachable!() - } - Err(error) => error.into_response().into_response(), - } -} diff --git a/test/twin/openai/src/sse.rs b/test/twin/openai/src/sse.rs deleted file mode 100644 index 2d30a0f88..000000000 --- a/test/twin/openai/src/sse.rs +++ /dev/null @@ -1,385 +0,0 @@ -use async_stream::stream; -use axum::body::Body; -use axum::http::{HeaderValue, Response, StatusCode, header}; -use serde_json::{Value, json}; -use tokio::time::{Duration, sleep}; - -use crate::engine::failures::TransportOptions; -use crate::engine::plan::ResponsePlan; - -pub fn responses_sse_response(plan: &ResponsePlan, transport: TransportOptions) -> Response { - let mut events = Vec::new(); - let reasoning_item_id = format!("rs_{}", plan.id); - let message_item_id = format!("msg_{}", plan.id); - let mut next_output_index = 0; - let streamed_text = plan.structured_output.as_ref().map(Value::to_string); - - events.push(sse_event( - "response.created", - &json!({ - "type": "response.created", - "response": { - "id": plan.id, - "object": "response", - "created": plan.created, - "model": plan.model, - "status": "in_progress", - "output": [], - }, - }), - )); - events.push(sse_event( - "response.in_progress", - &json!({ - "type": "response.in_progress", - "response": { - "id": plan.id, - "object": "response", - "created": plan.created, - "model": plan.model, - "status": "in_progress", - "output": [], - }, - }), - )); - - events.push(sse_event( - "response.output_item.added", - &json!({ - "type": "response.output_item.added", - "item": { - "id": reasoning_item_id, - "type": "reasoning", - "summary": [], - }, - "output_index": next_output_index, - }), - )); - for reasoning in &plan.reasoning { - events.push(sse_event( - "response.reasoning.delta", - &json!({ - "type": "response.reasoning.delta", - "delta": reasoning, - "item_id": reasoning_item_id, - "output_index": next_output_index, - }), - )); - } - events.push(sse_event( - "response.output_item.done", - &json!({ - "type": "response.output_item.done", - "item": { - "id": reasoning_item_id, - "type": "reasoning", - "summary": [], - }, - "output_index": next_output_index, - }), - )); - next_output_index += 1; - - if !plan.response_text.is_empty() || streamed_text.is_some() { - events.push(sse_event( - "response.output_item.added", - &json!({ - "type": "response.output_item.added", - "item": { - "id": message_item_id, - "type": "message", - "status": "in_progress", - "content": [], - "role": "assistant", - }, - "output_index": next_output_index, - }), - )); - - let message_text = streamed_text - .as_deref() - .unwrap_or(plan.response_text.as_str()); - - if !message_text.is_empty() { - events.push(sse_event( - "response.content_part.added", - &json!({ - "type": "response.content_part.added", - "content_index": 0, - "item_id": message_item_id, - "output_index": next_output_index, - "part": { - "type": "output_text", - "text": "", - }, - }), - )); - events.push(sse_event( - "response.output_text.delta", - &json!({ - "type": "response.output_text.delta", - "content_index": 0, - "item_id": message_item_id, - "output_index": next_output_index, - "delta": message_text, - }), - )); - events.push(sse_event( - "response.output_text.done", - &json!({ - "type": "response.output_text.done", - "content_index": 0, - "item_id": message_item_id, - "output_index": next_output_index, - "text": message_text, - }), - )); - events.push(sse_event( - "response.content_part.done", - &json!({ - "type": "response.content_part.done", - "content_index": 0, - "item_id": message_item_id, - "output_index": next_output_index, - "part": { - "type": "output_text", - "text": message_text, - }, - }), - )); - } - - // The completed item carries its full content, like the real API. - // Adapters round-trip this item verbatim into the next request's - // input, so omitting content here produces an invalid replay. - events.push(sse_event( - "response.output_item.done", - &json!({ - "type": "response.output_item.done", - "item": { - "id": message_item_id, - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{ - "type": "output_text", - "text": message_text, - }], - }, - "output_index": next_output_index, - }), - )); - next_output_index += 1; - } - - for tool_call in &plan.tool_calls { - let item_id = format!("fc_{}", tool_call.id); - events.push(sse_event( - "response.output_item.added", - &json!({ - "type": "response.output_item.added", - "item": { - "id": item_id, - "type": "function_call", - "call_id": tool_call.id, - "name": tool_call.name, - "arguments": "", - }, - "output_index": next_output_index, - }), - )); - events.push(sse_event( - "response.function_call_arguments.delta", - &json!({ - "type": "response.function_call_arguments.delta", - "item_id": item_id, - "delta": ResponsePlan::tool_call_arguments_text(tool_call), - "output_index": next_output_index, - }), - )); - events.push(sse_event( - "response.function_call_arguments.done", - &json!({ - "type": "response.function_call_arguments.done", - "item_id": item_id, - "arguments": ResponsePlan::tool_call_arguments_text(tool_call), - "output_index": next_output_index, - }), - )); - events.push(sse_event( - "response.output_item.done", - &json!({ - "type": "response.output_item.done", - "item": { - "id": item_id, - "type": "function_call", - "call_id": tool_call.id, - "name": tool_call.name, - "arguments": ResponsePlan::tool_call_arguments_text(tool_call), - }, - "output_index": next_output_index, - }), - )); - next_output_index += 1; - } - - if !transport.malformed_sse { - events.push(sse_event( - "response.completed", - &json!({ - "type": "response.completed", - "response": plan.responses_json(), - }), - )); - } - - stream_response(events, transport) -} - -pub fn chat_sse_response( - plan: &ResponsePlan, - include_usage: bool, - transport: TransportOptions, -) -> Response { - let mut events = Vec::new(); - let content = plan.chat_content(); - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [{ - "index": 0, - "delta": { - "role": "assistant" - }, - "finish_reason": Value::Null, - }] - }))); - - if !content.is_empty() { - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [{ - "index": 0, - "delta": { - "content": content - }, - "finish_reason": Value::Null, - }] - }))); - } - - for reasoning in &plan.reasoning { - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [{ - "index": 0, - "delta": { - "reasoning": reasoning - }, - "finish_reason": Value::Null, - }] - }))); - } - - if !plan.tool_calls.is_empty() { - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [{ - "index": 0, - "delta": { - "tool_calls": plan.tool_calls.iter().enumerate().map(|(index, tool_call)| json!({ - "index": index, - "id": tool_call.id, - "type": "function", - "function": { - "name": tool_call.name, - "arguments": ResponsePlan::tool_call_arguments_text(tool_call), - } - })).collect::>() - }, - "finish_reason": Value::Null, - }] - }))); - } - - if !transport.malformed_sse { - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": if plan.tool_calls.is_empty() { "stop" } else { "tool_calls" }, - }] - }))); - if include_usage { - events.push(chat_chunk(&json!({ - "id": format!("chatcmpl_{}", plan.id), - "object": "chat.completion.chunk", - "created": plan.created, - "model": plan.model, - "choices": [], - "usage": plan.usage.chat_completions_json(), - }))); - } - events.push("data: [DONE]\n\n".to_owned()); - } - - stream_response(events, transport) -} - -fn stream_response(events: Vec, transport: TransportOptions) -> Response { - let limit = transport.close_after_chunks.unwrap_or(events.len()); - let malformed_sse = transport.malformed_sse; - let inter_event_delay_ms = transport.inter_event_delay_ms; - - let body = Body::from_stream(stream! { - for (index, event) in events.into_iter().enumerate() { - if index >= limit { - break; - } - - if inter_event_delay_ms > 0 { - sleep(Duration::from_millis(inter_event_delay_ms)).await; - } - - yield Ok::<_, std::convert::Infallible>(event.into_bytes()); - } - - if malformed_sse { - yield Ok::<_, std::convert::Infallible>(b"event: malformed\ndata: {".to_vec()); - } - }); - - let mut response = Response::new(body); - *response.status_mut() = StatusCode::OK; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/event-stream"), - ); - response - .headers_mut() - .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")); - response -} - -fn sse_event(event: &str, data: &Value) -> String { - format!("event: {event}\ndata: {data}\n\n") -} - -fn chat_chunk(data: &Value) -> String { - format!("data: {data}\n\n") -} diff --git a/test/twin/openai/src/state.rs b/test/twin/openai/src/state.rs deleted file mode 100644 index 29cece646..000000000 --- a/test/twin/openai/src/state.rs +++ /dev/null @@ -1,178 +0,0 @@ -use std::collections::HashMap; -use std::fmt; -use std::sync::{Arc, Mutex}; - -use serde::Serialize; -use serde_json::Value; - -use crate::config::Config; -use crate::engine::scenario::{RequestContext, Scenario}; -use crate::logs::RequestLog; - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub enum NamespaceKey { - Global, - Bearer(String), -} - -impl fmt::Display for NamespaceKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Global => write!(f, "Global"), - Self::Bearer(token) => write!(f, "Bearer: {token}"), - } - } -} - -#[derive(Clone, Debug, Serialize)] -pub struct DebugSnapshot { - pub namespaces: Vec, -} - -#[derive(Clone, Debug, Serialize)] -pub struct NamespaceSnapshot { - pub key: String, - pub scenarios: Vec, - pub request_logs: Vec, -} - -#[derive(Clone, Debug, Serialize)] -pub struct ScenarioSnapshot { - pub endpoint: String, - pub model: Option, - pub stream: Option, - pub input_contains: Option, - pub metadata: serde_json::Map, - pub script_kind: String, -} - -#[derive(Clone, Debug)] -pub struct AppState { - pub config: Config, - inner: Arc, -} - -#[derive(Debug)] -struct AppStateInner { - namespaces: Mutex>, -} - -#[derive(Debug)] -struct NamespaceState { - next_response_number: u64, - scenarios: Vec, - request_logs: Vec, -} - -impl Default for NamespaceState { - fn default() -> Self { - Self { - next_response_number: 1, - scenarios: Vec::new(), - request_logs: Vec::new(), - } - } -} - -impl AppState { - pub fn new(config: Config) -> Self { - Self { - config, - inner: Arc::new(AppStateInner { - namespaces: Mutex::new(HashMap::new()), - }), - } - } - - pub fn next_response_id(&self, namespace: &NamespaceKey) -> u64 { - let mut namespaces = self.inner.namespaces.lock().expect("namespaces lock"); - let namespace_state = namespaces.entry(namespace.clone()).or_default(); - let response_id = namespace_state.next_response_number; - namespace_state.next_response_number += 1; - response_id - } - - pub fn enqueue_scenarios(&self, namespace: &NamespaceKey, mut scenarios: Vec) { - self.inner - .namespaces - .lock() - .expect("namespaces lock") - .entry(namespace.clone()) - .or_default() - .scenarios - .append(&mut scenarios); - } - - pub fn take_matching_scenario( - &self, - namespace: &NamespaceKey, - request: &RequestContext, - ) -> Option { - let mut namespaces = self.inner.namespaces.lock().expect("namespaces lock"); - let scenarios = &mut namespaces.entry(namespace.clone()).or_default().scenarios; - let position = scenarios - .iter() - .position(|scenario| scenario.matches(request))?; - Some(scenarios.remove(position)) - } - - pub fn log_request(&self, namespace: &NamespaceKey, request: RequestContext) { - self.inner - .namespaces - .lock() - .expect("namespaces lock") - .entry(namespace.clone()) - .or_default() - .request_logs - .push(RequestLog { - endpoint: request.endpoint, - model: request.model, - stream: request.stream, - input_text: request.input_text, - instructions_text: request.instructions_text, - metadata: request.metadata, - }); - } - - pub fn request_logs(&self, namespace: &NamespaceKey) -> Vec { - self.inner - .namespaces - .lock() - .expect("namespaces lock") - .get(namespace) - .map(|namespace_state| namespace_state.request_logs.clone()) - .unwrap_or_default() - } - - pub fn reset(&self, namespace: &NamespaceKey) { - self.inner - .namespaces - .lock() - .expect("namespaces lock") - .remove(namespace); - } - - pub fn debug_snapshot(&self) -> DebugSnapshot { - let namespaces = self.inner.namespaces.lock().expect("namespaces lock"); - let mut result = Vec::new(); - for (key, ns) in namespaces.iter() { - result.push(NamespaceSnapshot { - key: key.to_string(), - scenarios: ns - .scenarios - .iter() - .map(|s| ScenarioSnapshot { - endpoint: s.matcher.endpoint.clone(), - model: s.matcher.model.clone(), - stream: s.matcher.stream, - input_contains: s.matcher.input_contains.clone(), - metadata: s.matcher.metadata.clone(), - script_kind: s.script.script_kind().to_owned(), - }) - .collect(), - request_logs: ns.request_logs.clone(), - }); - } - DebugSnapshot { namespaces: result } - } -} diff --git a/test/twin/openai/tests/chat_completions_contract.rs b/test/twin/openai/tests/chat_completions_contract.rs deleted file mode 100644 index 7837dfe5b..000000000 --- a/test/twin/openai/tests/chat_completions_contract.rs +++ /dev/null @@ -1,535 +0,0 @@ -mod common; - -use serde_json::json; - -#[tokio::test] -async fn chat_completions_non_stream_uses_same_canonical_plan() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_responses(json!({ - "model": "gpt-test", - "input": "same plan", - "stream": false - })) - .await - .json::() - .await - .expect("json"); - - let chat = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "same plan" }], - "stream": false - })) - .await - .json::() - .await - .expect("json"); - - assert_eq!( - response["output"][0]["content"][0]["text"], - chat["choices"][0]["message"]["content"] - ); -} - -#[tokio::test] -async fn chat_completions_stream_uses_same_canonical_plan() { - let server = common::spawn_server().await.expect("server should start"); - - let (status, chunks) = server - .post_chat_stream(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "stream same plan" }], - "stream": true - })) - .await; - - let joined = chunks.join(""); - - assert_eq!(status, 200); - assert!(joined.contains("\"content\":\"deterministic: stream same plan\"")); - assert!(!joined.contains("\"usage\"")); - assert!(joined.contains("data: [DONE]")); -} - -#[tokio::test] -async fn chat_completions_stream_includes_usage_when_requested() { - let server = common::spawn_server().await.expect("server should start"); - - let (status, chunks) = server - .post_chat_stream(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "stream with usage" }], - "stream": true, - "stream_options": { "include_usage": true } - })) - .await; - - assert_eq!(status, 200); - let transcript = - common::parse_sse_transcript(chunks.join("").as_bytes()).expect("valid SSE transcript"); - let usage_chunk = transcript - .events - .iter() - .filter(|event| event.data != "[DONE]") - .map(|event| { - serde_json::from_str::(&event.data).expect("valid JSON chunk") - }) - .find(|chunk| chunk.get("usage").is_some()) - .expect("trailing usage chunk"); - - assert_eq!(usage_chunk["choices"], json!([])); - assert_eq!( - usage_chunk["usage"], - json!({ - "prompt_tokens": 3, - "completion_tokens": 5, - "total_tokens": 8 - }) - ); - assert!(transcript.done); -} - -#[tokio::test] -async fn chat_completions_accepts_supported_openai_compatible_fields() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [ - { - "role": "assistant", - "content": [ - { "type": "reasoning", "text": "reasoning trace" } - ] - }, - { "role": "user", "content": "structured chat" } - ], - "max_tokens": 128, - "stream": false, - "tools": [{ "type": "function", "function": { "name": "lookup" } }], - "tool_choice": "auto", - "stop": ["END"], - "response_format": { "type": "json_object" } - })) - .await; - - assert_eq!(response.status(), 200); - let body = response.json::().await.expect("json"); - assert_eq!( - body["choices"][0]["message"]["content"], - "{\"message\":\"deterministic: structured chat\",\"model\":\"gpt-test\"}" - ); - assert_eq!( - body["choices"][0]["message"]["reasoning"][0], - "reasoning: structured chat" - ); -} - -#[tokio::test] -async fn chat_completions_accepts_tool_call_history() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [ - { "role": "user", "content": "replace old with new" }, - { - "role": "assistant", - "tool_calls": [{ - "id": "call_edit", - "type": "function", - "function": { - "name": "edit_file", - "arguments": "{\"old\":\"old\",\"new\":\"new\"}" - } - }] - }, - { - "role": "tool", - "content": "Updated data.txt", - "tool_call_id": "call_edit" - } - ], - "stream": false - })) - .await; - - assert_eq!(response.status(), 200); - let body = response.json::().await.expect("json"); - assert_eq!( - body["choices"][0]["message"]["content"], - "deterministic: replace old with new" - ); -} - -#[tokio::test] -async fn chat_completions_supports_scripted_tool_call_and_json_schema() { - let server = common::spawn_server().await.expect("server should start"); - server - .enqueue_scenarios(json!({ - "scenarios": [ - { - "matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": false, "input_contains": "tool please" }, - "script": { - "kind": "success", - "tool_calls": [ - { - "id": "call_weather", - "name": "lookup_weather", - "arguments": { "city": "Boston" } - } - ] - } - }, - { - "matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": true, "input_contains": "tool please" }, - "script": { - "kind": "success", - "tool_calls": [ - { - "id": "call_weather", - "name": "lookup_weather", - "arguments": { "city": "Boston" } - } - ] - } - } - ] - })) - .await; - - let non_stream = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "tool please" }], - "tools": [{ "type": "function", "function": { "name": "lookup_weather" } }], - "tool_choice": { - "type": "function", - "function": { "name": "lookup_weather" } - }, - "stream": false - })) - .await - .json::() - .await - .expect("json"); - - assert_eq!(non_stream["choices"][0]["finish_reason"], "tool_calls"); - assert_eq!( - non_stream["choices"][0]["message"]["tool_calls"][0]["function"]["name"], - "lookup_weather" - ); - assert_eq!(non_stream["choices"][0]["message"]["content"], ""); - - let (status, chunks) = server - .post_chat_stream(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "tool please" }], - "tools": [{ "type": "function", "function": { "name": "lookup_weather" } }], - "tool_choice": { - "type": "function", - "function": { "name": "lookup_weather" } - }, - "stream": true - })) - .await; - let joined = chunks.join(""); - assert_eq!(status, 200); - assert!(joined.contains("\"tool_calls\"")); - assert!(!joined.contains("\"content\":\"deterministic:")); - - let structured = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "schema chat" }], - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "chat_schema", - "schema": { - "type": "object", - "properties": { - "message": { "type": "string" }, - "ok": { "type": "boolean" } - } - } - , - "strict": true - } - }, - "stream": false - })) - .await - .json::() - .await - .expect("json"); - - assert_eq!( - structured["choices"][0]["message"]["content"], - "{\"message\":\"deterministic: schema chat\",\"ok\":true}" - ); -} - -#[tokio::test] -async fn chat_completions_stream_preserves_reasoning_transcript() { - let server = common::spawn_server().await.expect("server should start"); - - let (status, chunks) = server - .post_chat_stream(json!({ - "model": "gpt-test", - "messages": [ - { - "role": "assistant", - "content": [ - { "type": "reasoning", "text": "reasoning trace" } - ] - }, - { "role": "user", "content": "stream same plan" } - ], - "stream": true - })) - .await; - - let joined = chunks.join(""); - - assert_eq!(status, 200); - assert!(joined.contains("\"reasoning\":\"reasoning: stream same plan\"")); - assert!(joined.contains("\"content\":\"deterministic: stream same plan\"")); - assert!(joined.contains("data: [DONE]")); -} - -#[tokio::test] -async fn chat_completions_do_not_infer_reasoning_from_user_text() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "Please explain your reasoning plainly" }], - "stream": false - })) - .await - .json::() - .await - .expect("json"); - - assert_eq!(response["choices"][0]["message"]["reasoning"], json!([])); - assert_eq!( - response["choices"][0]["message"]["content"], - "deterministic: Please explain your reasoning plainly" - ); -} - -#[tokio::test] -async fn chat_completions_reject_reasoning_parts_on_non_assistant_messages() { - let server = common::spawn_server().await.expect("server should start"); - - let user_reasoning = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ - "role": "user", - "content": [{ "type": "reasoning", "text": "not allowed here" }] - }], - "stream": false - })) - .await; - - assert_eq!(user_reasoning.status(), 400); - let body = user_reasoning - .json::() - .await - .expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "messages"); - - let system_reasoning = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ - "role": "system", - "content": [{ "type": "reasoning", "text": "not allowed here either" }] - }], - "stream": false - })) - .await; - - assert_eq!(system_reasoning.status(), 400); - let body = system_reasoning - .json::() - .await - .expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "messages"); -} - -#[tokio::test] -async fn chat_completions_accept_unknown_top_level_fields() { - let server = common::spawn_server().await.expect("server should start"); - - let (status, chunks) = server - .post_chat_stream(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "hello" }], - "stream": true, - "temperature": 0.7, - "top_p": 0.9, - "prompt_cache_key": "conversation-123" - })) - .await; - - assert_eq!(status, 200); - let transcript = - common::parse_sse_transcript(chunks.join("").as_bytes()).expect("valid SSE transcript"); - assert!(transcript.done); -} - -#[tokio::test] -async fn chat_completions_reject_unsupported_tool_choice_shape() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "hello" }], - "tools": [{ "type": "function", "function": { "name": "lookup_weather" } }], - "tool_choice": { "type": "required" } - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); -} - -#[tokio::test] -async fn chat_completions_reject_required_tool_choice_without_tools() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "hello" }], - "tool_choice": "required" - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); -} - -#[tokio::test] -async fn chat_completions_reject_unfulfilled_tool_choice_requirements() { - let server = common::spawn_server().await.expect("server should start"); - - let required = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "plain text please" }], - "tools": [{ "type": "function", "function": { "name": "lookup_weather" } }], - "tool_choice": "required" - })) - .await; - - assert_eq!(required.status(), 400); - let body = required.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "tool_choice"); - - let named = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "plain text please" }], - "tools": [{ "type": "function", "function": { "name": "lookup_weather" } }], - "tool_choice": { - "type": "function", - "function": { "name": "lookup_weather" } - } - })) - .await; - - assert_eq!(named.status(), 400); - let body = named.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "tool_choice"); -} - -#[tokio::test] -async fn chat_completions_rejects_unsupported_response_format() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": "bad format" }], - "response_format": { "type": "xml" } - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); -} - -#[tokio::test] -async fn chat_completions_reject_empty_messages() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [] - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "messages"); -} - -#[tokio::test] -async fn chat_completions_reject_null_message_content() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ "role": "user", "content": null }] - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "messages"); -} - -#[tokio::test] -async fn chat_completions_reject_malformed_image_input() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .post_chat(json!({ - "model": "gpt-test", - "messages": [{ - "role": "user", - "content": [{ - "type": "image_url", - "image_url": {} - }] - }] - })) - .await; - - assert_eq!(response.status(), 400); - let body = response.json::().await.expect("json"); - assert_eq!(body["error"]["type"], "invalid_request_error"); - assert_eq!(body["error"]["param"], "messages"); -} diff --git a/test/twin/openai/tests/common/mod.rs b/test/twin/openai/tests/common/mod.rs deleted file mode 100644 index 86625972a..000000000 --- a/test/twin/openai/tests/common/mod.rs +++ /dev/null @@ -1,587 +0,0 @@ -#![allow( - dead_code, - unreachable_pub, - reason = "Shared test helpers stay public within the test crate and not every helper is used everywhere." -)] - -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use anyhow::Result; -use fabro_http::header::AUTHORIZATION; -use fabro_http::{HttpClient as Client, HttpClientBuilder}; -use futures_util::StreamExt; -use serde_json::Value; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use twin_openai::config::Config; - -pub struct TestServer { - pub base_url: String, - pub client: Client, - pub auth_client: Client, - pub bearer_token: String, -} - -#[derive(Clone)] -pub struct ApiClient { - pub base_url: String, - client: Client, - bearer_token: Option, - organization: Option, - project: Option, -} - -pub struct RecordedResponse { - pub status: fabro_http::StatusCode, - pub headers: HashMap, - pub body: Vec, -} - -pub struct RawStreamResponse { - pub status: u16, - pub headers: HashMap, - pub body: Vec, -} - -pub struct TimedStreamResponse { - pub status: fabro_http::StatusCode, - pub first_event_elapsed: Duration, - pub chunks: Vec, -} - -#[derive(Debug)] -pub struct ParsedSseTranscript { - pub blocks: Vec, - pub events: Vec, - pub done: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ParsedSseEvent { - pub event: Option, - pub data: String, -} - -static NEXT_BEARER_TOKEN: AtomicU64 = AtomicU64::new(1); - -pub fn test_http_client() -> Result { - fabro_http::test_http_client().map_err(Into::into) -} - -pub async fn spawn_server() -> Result { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr: SocketAddr = listener.local_addr()?; - let app = twin_openai::build_app_with_config(Config { - bind_addr: "127.0.0.1:0".parse().expect("valid addr"), - require_auth: true, - enable_admin: true, - }); - - tokio::spawn(async move { - axum::serve(listener, app).await.expect("server should run"); - }); - - TestServer::new(format!("http://{addr}"), next_bearer_token()) -} - -fn next_bearer_token() -> String { - format!( - "test-key-{}", - NEXT_BEARER_TOKEN.fetch_add(1, Ordering::SeqCst) - ) -} - -fn authorization_header_value(bearer_token: &str) -> String { - format!("Bearer {bearer_token}") -} - -fn build_authenticated_client(bearer_token: &str) -> Result { - HttpClientBuilder::new() - .proxy_policy(fabro_http::ProxyPolicy::Disabled) - .default_headers( - [( - AUTHORIZATION, - authorization_header_value(bearer_token) - .parse() - .expect("valid header"), - )] - .into_iter() - .collect(), - ) - .build() - .map_err(Into::into) -} - -impl ApiClient { - pub fn new( - base_url: impl Into, - bearer_token: Option, - organization: Option, - project: Option, - ) -> Result { - Ok(Self { - base_url: base_url.into(), - client: HttpClientBuilder::new() - .proxy_policy(fabro_http::ProxyPolicy::Disabled) - .timeout(Duration::from_secs(30)) - .build()?, - bearer_token, - organization, - project, - }) - } - - pub fn with_client( - base_url: impl Into, - client: Client, - bearer_token: Option, - organization: Option, - project: Option, - ) -> Self { - Self { - base_url: base_url.into(), - client, - bearer_token, - organization, - project, - } - } - - pub async fn post_json(&self, path: &str, body: &Value) -> fabro_http::Response { - self.post(path) - .json(body) - .send() - .await - .expect("request should complete") - } - - pub async fn post_json_recorded(&self, path: &str, body: &Value) -> RecordedResponse { - record_response(self.post_json(path, body).await).await - } - - pub async fn get_recorded(&self, path: &str) -> RecordedResponse { - record_response( - self.get(path) - .send() - .await - .expect("request should complete"), - ) - .await - } - - pub fn post(&self, path: &str) -> fabro_http::RequestBuilder { - self.request(self.client.post(format!("{}{}", self.base_url, path))) - } - - pub fn get(&self, path: &str) -> fabro_http::RequestBuilder { - self.request(self.client.get(format!("{}{}", self.base_url, path))) - } - - fn request(&self, mut request: fabro_http::RequestBuilder) -> fabro_http::RequestBuilder { - if let Some(token) = &self.bearer_token { - request = request.bearer_auth(token); - } - if let Some(org) = &self.organization { - request = request.header("OpenAI-Organization", org); - } - if let Some(project) = &self.project { - request = request.header("OpenAI-Project", project); - } - - request - } -} - -impl TestServer { - fn new(base_url: String, bearer_token: String) -> Result { - let client = test_http_client()?; - let auth_client = build_authenticated_client(&bearer_token)?; - - Ok(Self { - base_url, - client, - auth_client, - bearer_token, - }) - } - - pub fn authorization_header_value(&self) -> String { - authorization_header_value(&self.bearer_token) - } - - pub fn api_client(&self) -> ApiClient { - ApiClient::with_client( - self.base_url.clone(), - self.client.clone(), - Some(self.bearer_token.clone()), - None, - None, - ) - } - - pub fn fork_namespace(&self) -> Result { - Self::new(self.base_url.clone(), next_bearer_token()) - } -} - -impl TestServer { - pub async fn post_responses(&self, body: Value) -> fabro_http::Response { - self.auth_client - .post(format!("{}/v1/responses", self.base_url)) - .json(&body) - .send() - .await - .expect("request should complete") - } - - pub async fn post_responses_with_headers( - &self, - body: Value, - org: Option<&str>, - project: Option<&str>, - ) -> fabro_http::Response { - let mut request = self - .auth_client - .post(format!("{}/v1/responses", self.base_url)); - - if let Some(org) = org { - request = request.header("OpenAI-Organization", org); - } - - if let Some(project) = project { - request = request.header("OpenAI-Project", project); - } - - request - .json(&body) - .send() - .await - .expect("request should complete") - } - - pub async fn post_responses_stream( - &self, - body: Value, - ) -> (fabro_http::StatusCode, Vec) { - let response = self - .auth_client - .post(format!("{}/v1/responses", self.base_url)) - .json(&body) - .send() - .await - .expect("request should complete"); - - let status = response.status(); - let mut stream = response.bytes_stream(); - let mut chunks = Vec::new(); - - while let Some(chunk) = stream.next().await { - chunks.push( - String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"), - ); - } - - (status, chunks) - } - - pub async fn post_chat(&self, body: Value) -> fabro_http::Response { - self.auth_client - .post(format!("{}/v1/chat/completions", self.base_url)) - .json(&body) - .send() - .await - .expect("request should complete") - } - - pub async fn post_chat_stream(&self, body: Value) -> (fabro_http::StatusCode, Vec) { - let response = self.post_chat(body).await; - let status = response.status(); - let mut stream = response.bytes_stream(); - let mut chunks = Vec::new(); - - while let Some(chunk) = stream.next().await { - chunks.push( - String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"), - ); - } - - (status, chunks) - } - - pub async fn post_chat_with_auth_header( - &self, - body: Value, - authorization: Option<&str>, - ) -> fabro_http::Response { - let mut request = self - .client - .post(format!("{}/v1/chat/completions", self.base_url)); - - if let Some(value) = authorization { - request = request.header(AUTHORIZATION, value); - } - - request - .json(&body) - .send() - .await - .expect("request should complete") - } - - pub async fn post_responses_stream_timed(&self, body: Value) -> TimedStreamResponse { - let started = Instant::now(); - let response = self - .auth_client - .post(format!("{}/v1/responses", self.base_url)) - .json(&body) - .send() - .await - .expect("request should complete"); - let status = response.status(); - let mut stream = response.bytes_stream(); - let mut chunks = Vec::new(); - let mut first_event_elapsed = Duration::ZERO; - - if let Some(chunk) = stream.next().await { - chunks.push( - String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"), - ); - first_event_elapsed = started.elapsed(); - } - - while let Some(chunk) = stream.next().await { - chunks.push( - String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"), - ); - } - - TimedStreamResponse { - status, - first_event_elapsed, - chunks, - } - } - - pub async fn post_responses_stream_raw(&self, body: Value) -> RawStreamResponse { - self.raw_stream_request("/v1/responses", &body).await - } - - pub async fn post_chat_stream_raw(&self, body: Value) -> RawStreamResponse { - self.raw_stream_request("/v1/chat/completions", &body).await - } - - pub async fn enqueue_scenarios(&self, scenarios: Value) { - let response = self - .auth_client - .post(format!("{}/__admin/scenarios", self.base_url)) - .json(&scenarios) - .send() - .await - .expect("admin request should complete"); - - assert_eq!(response.status(), 200); - } - - pub async fn reset(&self) { - let response = self - .auth_client - .post(format!("{}/__admin/reset", self.base_url)) - .send() - .await - .expect("admin reset should complete"); - - assert_eq!(response.status(), 200); - } - - pub async fn request_logs(&self) -> Value { - self.auth_client - .get(format!("{}/__admin/requests", self.base_url)) - .send() - .await - .expect("admin logs should complete") - .json() - .await - .expect("logs json should parse") - } - - async fn raw_stream_request(&self, path: &str, body: &Value) -> RawStreamResponse { - let authority = self - .base_url - .strip_prefix("http://") - .expect("http base url"); - let mut stream = TcpStream::connect(authority) - .await - .expect("socket should connect"); - let body = serde_json::to_vec(body).expect("json body"); - let authorization = self.authorization_header_value(); - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {authority}\r\nAuthorization: {authorization}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - - stream - .write_all(request.as_bytes()) - .await - .expect("request headers should write"); - stream - .write_all(&body) - .await - .expect("request body should write"); - stream.flush().await.expect("request should flush"); - - let mut response = Vec::new(); - stream - .read_to_end(&mut response) - .await - .expect("response should read"); - - decode_http_response(&response) - } -} - -pub fn parse_sse_transcript(body: &[u8]) -> Result { - let text = std::str::from_utf8(body).map_err(|_| "sse body was not valid utf-8".to_owned())?; - let mut blocks = Vec::new(); - let mut events = Vec::new(); - let mut done = false; - let mut remainder = text; - - while let Some((block, rest)) = remainder.split_once("\n\n") { - if !block.is_empty() { - let event = parse_sse_block(block)?; - if event.data == "[DONE]" { - done = true; - } - blocks.push(block.to_owned()); - events.push(event); - } - remainder = rest; - } - - if !remainder.is_empty() { - return Err("sse stream ended with an incomplete event".to_owned()); - } - - Ok(ParsedSseTranscript { - blocks, - events, - done, - }) -} - -fn parse_sse_block(block: &str) -> Result { - let mut event = None; - let mut data_lines = Vec::new(); - - for line in block.lines() { - if let Some(value) = line.strip_prefix("event: ") { - event = Some(value.to_owned()); - continue; - } - if let Some(value) = line.strip_prefix("data: ") { - data_lines.push(value.to_owned()); - continue; - } - if line.starts_with("id: ") || line.starts_with(':') { - continue; - } - - return Err(format!("unsupported sse line: {line}")); - } - - Ok(ParsedSseEvent { - event, - data: data_lines.join("\n"), - }) -} - -pub async fn record_response(response: fabro_http::Response) -> RecordedResponse { - let status = response.status(); - let headers = response - .headers() - .iter() - .map(|(name, value)| { - ( - name.as_str().to_ascii_lowercase(), - value.to_str().unwrap_or_default().to_owned(), - ) - }) - .collect(); - let body = response - .bytes() - .await - .expect("response body should read") - .to_vec(); - - RecordedResponse { - status, - headers, - body, - } -} - -fn decode_http_response(response: &[u8]) -> RawStreamResponse { - let header_end = response - .windows(4) - .position(|window| window == b"\r\n\r\n") - .expect("http header terminator") - + 4; - let head = std::str::from_utf8(&response[..header_end]).expect("header utf8"); - let mut lines = head.split("\r\n"); - let status_line = lines.next().expect("status line"); - let status = status_line - .split_whitespace() - .nth(1) - .expect("status code") - .parse::() - .expect("numeric status"); - let mut headers = HashMap::new(); - - for line in lines.filter(|line| !line.is_empty()) { - let (name, value) = line.split_once(':').expect("header format"); - headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_owned()); - } - - let body = if headers - .get("transfer-encoding") - .is_some_and(|value| value.eq_ignore_ascii_case("chunked")) - { - decode_chunked_body(&response[header_end..]) - } else { - response[header_end..].to_vec() - }; - - RawStreamResponse { - status, - headers, - body, - } -} - -fn decode_chunked_body(encoded: &[u8]) -> Vec { - let mut cursor = 0; - let mut decoded = Vec::new(); - - loop { - let size_end = encoded[cursor..] - .windows(2) - .position(|window| window == b"\r\n") - .expect("chunk size terminator") - + cursor; - let size = std::str::from_utf8(&encoded[cursor..size_end]).expect("chunk size utf8"); - let size = usize::from_str_radix(size.trim(), 16).expect("hex chunk size"); - cursor = size_end + 2; - - if size == 0 { - break; - } - - decoded.extend_from_slice(&encoded[cursor..cursor + size]); - cursor += size + 2; - } - - decoded -} diff --git a/test/twin/openai/tests/config_contract.rs b/test/twin/openai/tests/config_contract.rs deleted file mode 100644 index 283c14538..000000000 --- a/test/twin/openai/tests/config_contract.rs +++ /dev/null @@ -1,15 +0,0 @@ -use twin_openai::config::Config; - -#[test] -fn config_loads_from_environment() { - let config = Config::from_lookup(&|name| match name { - "TWIN_OPENAI_BIND_ADDR" => Some("127.0.0.1:4100".to_string()), - "TWIN_OPENAI_REQUIRE_AUTH" | "TWIN_OPENAI_ENABLE_ADMIN" => Some("false".to_string()), - _ => None, - }) - .expect("config should load"); - - assert_eq!(config.bind_addr.to_string(), "127.0.0.1:4100"); - assert!(!config.require_auth); - assert!(!config.enable_admin); -} diff --git a/test/twin/openai/tests/debug_ui.rs b/test/twin/openai/tests/debug_ui.rs deleted file mode 100644 index 73d856ad3..000000000 --- a/test/twin/openai/tests/debug_ui.rs +++ /dev/null @@ -1,403 +0,0 @@ -#![expect( - clippy::disallowed_methods, - reason = "These browser-debug integration tests synchronously probe for Chrome binaries before launching external tooling." -)] - -mod common; - -use std::process::Stdio; - -use serde_json::json; -use tokio::net::TcpListener; -use tokio::process::Command as TokioCommand; -use tokio::time::{Duration, timeout}; -use twin_openai::config::Config; - -#[tokio::test] -async fn debug_html_page_serves_valid_html_on_empty_state() { - let server = common::spawn_server().await.expect("server should start"); - - let response = server - .client - .get(format!("{}/__debug", server.base_url)) - .send() - .await - .expect("debug page request should complete"); - - assert_eq!(response.status(), 200); - let content_type = response - .headers() - .get("content-type") - .expect("content-type header should be present") - .to_str() - .expect("content-type should be valid string"); - assert!( - content_type.contains("text/html"), - "content-type should contain text/html, got: {content_type}" - ); - - let body = response.text().await.expect("body should read"); - assert!( - body.contains(""), - "response should contain DOCTYPE" - ); - assert!( - body.contains("twin-openai"), - "response should contain project name" - ); - assert!(body.contains("debug"), "response should contain 'debug'"); - assert!( - body.contains("no active namespaces"), - "empty state should show 'no active namespaces'" - ); -} - -#[tokio::test] -async fn debug_json_endpoint_returns_correct_state_snapshot() { - let server = common::spawn_server().await.expect("server should start"); - - // Load two scenarios: one success, one error - server - .enqueue_scenarios(json!({ - "scenarios": [ - { - "matcher": { "endpoint": "responses", "model": "gpt-test" }, - "script": { "kind": "success" } - }, - { - "matcher": { "endpoint": "responses", "model": "gpt-error" }, - "script": { - "kind": "error", - "status": 500, - "message": "test error", - "error_type": "server_error", - "code": "server_error" - } - } - ] - })) - .await; - - // Make one request that consumes the first (success) scenario - let response = server - .post_responses(json!({ - "model": "gpt-test", - "input": "hello debug", - "stream": false - })) - .await; - assert_eq!(response.status(), 200); - - // GET the debug JSON endpoint (unauthenticated) - let response = server - .client - .get(format!("{}/__debug/state.json", server.base_url)) - .send() - .await - .expect("debug json request should complete"); - - assert_eq!(response.status(), 200); - let content_type = response - .headers() - .get("content-type") - .expect("content-type header should be present") - .to_str() - .expect("content-type should be valid string"); - assert!( - content_type.contains("application/json"), - "content-type should contain application/json, got: {content_type}" - ); - - let body: serde_json::Value = response.json().await.expect("json should parse"); - - // Should have a top-level namespaces array - let namespaces = body["namespaces"] - .as_array() - .expect("namespaces should be an array"); - assert_eq!(namespaces.len(), 1, "should have exactly one namespace"); - - let ns = &namespaces[0]; - assert!( - ns["key"].as_str().unwrap().starts_with("Bearer:"), - "namespace key should start with 'Bearer:', got: {}", - ns["key"] - ); - - // Should have 1 remaining scenario (the error one; the success was consumed) - let scenarios = ns["scenarios"] - .as_array() - .expect("scenarios should be an array"); - assert_eq!(scenarios.len(), 1, "should have 1 remaining scenario"); - assert_eq!(scenarios[0]["endpoint"], "responses"); - assert_eq!(scenarios[0]["model"], "gpt-error"); - assert_eq!(scenarios[0]["script_kind"], "error"); - - // Should have 1 request log - let request_logs = ns["request_logs"] - .as_array() - .expect("request_logs should be an array"); - assert_eq!(request_logs.len(), 1, "should have 1 request log"); - assert_eq!(request_logs[0]["endpoint"], "responses"); - assert_eq!(request_logs[0]["model"], "gpt-test"); - assert!( - request_logs[0]["input_text"] - .as_str() - .unwrap() - .contains("hello debug"), - "request log should contain input text 'hello debug'" - ); -} - -#[tokio::test] -async fn debug_html_page_reflects_loaded_scenarios_and_request_logs() { - let server = common::spawn_server().await.expect("server should start"); - - // Load one success scenario - server - .enqueue_scenarios(json!({ - "scenarios": [ - { - "matcher": { "endpoint": "responses", "model": "gpt-html-test" }, - "script": { "kind": "success" } - } - ] - })) - .await; - - // Make one request with a different model (won't match, but still gets logged - // via the default behavior) - let response = server - .post_responses(json!({ - "model": "gpt-other", - "input": "check the page", - "stream": false - })) - .await; - // The request gets a deterministic response (no matching scenario consumed - // since model doesn't match). Status should be 200 (default behavior). - assert_eq!(response.status(), 200); - - // GET the debug HTML page - let response = server - .client - .get(format!("{}/__debug", server.base_url)) - .send() - .await - .expect("debug page request should complete"); - - assert_eq!(response.status(), 200); - let body = response.text().await.expect("body should read"); - - // Should contain the scenario's model name - assert!( - body.contains("gpt-html-test"), - "HTML should contain scenario model 'gpt-html-test'" - ); - // Should contain the script kind - assert!( - body.contains("success"), - "HTML should contain script kind 'success'" - ); - // Should contain the request log model - assert!( - body.contains("gpt-other"), - "HTML should contain request log model 'gpt-other'" - ); - // Should contain the request log input text - assert!( - body.contains("check the page"), - "HTML should contain request log input text 'check the page'" - ); - // Verify the server-rendered content section does not show empty state. - // The JS source always includes the "no active namespaces" string as a - // template, so we check that the server-rendered content div contains - // namespace sections rather than the empty-state paragraph. - assert!( - body.contains("namespace-header"), - "HTML should contain a namespace-header element (proving non-empty rendering)" - ); -} - -#[tokio::test] -async fn debug_routes_not_accessible_when_admin_disabled() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind should succeed"); - let addr = listener.local_addr().expect("should have addr"); - let app = twin_openai::build_app_with_config(Config { - bind_addr: "127.0.0.1:0".parse().expect("valid addr"), - require_auth: false, - enable_admin: false, - }); - - tokio::spawn(async move { - axum::serve(listener, app).await.expect("server should run"); - }); - - let base_url = format!("http://{addr}"); - let client = common::test_http_client().expect("test client"); - - let html_response = client - .get(format!("{base_url}/__debug")) - .send() - .await - .expect("request should complete"); - assert_eq!( - html_response.status(), - 404, - "debug HTML should be 404 when admin disabled" - ); - - let json_response = client - .get(format!("{base_url}/__debug/state.json")) - .send() - .await - .expect("request should complete"); - assert_eq!( - json_response.status(), - 404, - "debug JSON should be 404 when admin disabled" - ); -} - -#[tokio::test] -async fn debug_page_renders_in_headless_chrome() { - // Find Chrome binary - let chrome_binary = ["chromium", "google-chrome", "chromium-browser"] - .iter() - .find(|name| { - std::process::Command::new("which") - .arg(name) - .output() - .is_ok_and(|o| o.status.success()) - }); - - let Some(chrome_binary) = chrome_binary.copied() else { - return; - }; - - let server = common::spawn_server().await.expect("server should start"); - - // Load a scenario and make a request so the page has content - server - .enqueue_scenarios(json!({ - "scenarios": [ - { - "matcher": { "endpoint": "responses", "model": "gpt-screenshot" }, - "script": { "kind": "success" } - } - ] - })) - .await; - - let response = server - .post_responses(json!({ - "model": "gpt-screenshot", - "input": "screenshot test", - "stream": false - })) - .await; - assert_eq!(response.status(), 200); - - let screenshot_path = format!( - "/tmp/twin-openai-debug-screenshot-{}.png", - std::process::id() - ); - let mut command = TokioCommand::new(chrome_binary); - command.args([ - "--headless", - "--disable-gpu", - &format!("--screenshot={screenshot_path}"), - "--window-size=1280,900", - ]); - if cfg!(target_os = "linux") { - // Ubuntu 24.04 GitHub runners block Chrome's default sandbox unless it - // is launched with a compatible user namespace or disabled explicitly. - command.arg("--no-sandbox"); - } - let mut child = command - .stdout(Stdio::null()) - .stderr(Stdio::null()) - // Static mode keeps the page visually identical for the screenshot - // while avoiding a live refresh loop that can stall headless Chrome - // on Linux CI. - .arg(format!("{}/__debug?refresh=0", server.base_url)) - .spawn() - .expect("Chrome should start"); - - // Chrome with --screenshot exits once the file is written, so waiting on - // the process is the deterministic completion signal. - let wait_result = timeout(Duration::from_mins(2), child.wait()).await; - let status = match wait_result { - Ok(Ok(status)) => status, - Ok(Err(err)) => panic!("Chrome wait failed: {err}"), - Err(_) => { - let _ = child.start_kill(); - let _ = child.wait().await; - panic!("Chrome did not exit within 120s while taking screenshot"); - } - }; - assert!( - status.success(), - "Chrome exited with non-success status: {status}" - ); - - let screenshot_data = std::fs::read(&screenshot_path) - .expect("Chrome exited successfully but screenshot file is missing"); - - assert!( - screenshot_data.len() >= 10_000, - "screenshot should be at least 10KB, got {} bytes", - screenshot_data.len() - ); - // Check PNG magic bytes - assert_eq!( - &screenshot_data[..8], - b"\x89PNG\r\n\x1a\n", - "screenshot should be a valid PNG" - ); - - // Clean up - let _ = std::fs::remove_file(&screenshot_path); -} - -#[tokio::test] -async fn debug_html_escapes_user_controlled_values() { - let server = common::spawn_server().await.expect("server should start"); - - // Load a scenario with an XSS attempt in the model name - server - .enqueue_scenarios(json!({ - "scenarios": [ - { - "matcher": { - "endpoint": "responses", - "model": "" - }, - "script": { "kind": "success" } - } - ] - })) - .await; - - let response = server - .client - .get(format!("{}/__debug", server.base_url)) - .send() - .await - .expect("debug page request should complete"); - - assert_eq!(response.status(), 200); - let body = response.text().await.expect("body should read"); - - // Should contain the escaped form - assert!( - body.contains("<script>"), - "HTML should contain escaped '