scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-11 22:53:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Depend on lithos-llm and the published twin-openai crate

Browse source 
Add lithos-llm as a pinned git dependency and replace the in-repo
`test/twin/openai` crate with the published `twins` revision that
lithos-llm verifies its codecs against. Move the Fabro policy overlay
(`fabro-policy.toml`) into fabro-llm so Fabro owns its own catalog
policy layer.

Drop the twin-openai nextest overrides and CI package filter now that the
crate is no longer a workspace member.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-09-09 17:26:57 -06:00
parent 2f326a13c4
commit 580bb85f5b
No known key found for this signature in database
42 changed files with 969 additions and 9352 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.config/nextest.toml
View file

@ -15,9 +15,6 @@ leak-timeout = "500ms"
filter = "package(fabro-workflow)"
slow-timeout = { period = "2s", terminate-after = 3 }
[[profile.default.overrides]]
filter = "package(twin-openai) & test(debug_page_renders_in_headless_chrome)"
slow-timeout = { period = "30s", terminate-after = 1 }
[profile.e2e]
# E2E (ignored) tests: flag SLOW after 10s, hard-kill after 30s
@ -48,6 +45,3 @@ leak-timeout = "2s"
filter = "package(fabro-workflow)"
slow-timeout = { period = "30s", terminate-after = 4 }
[[profile.ci.overrides]]
filter = "package(twin-openai) & test(debug_page_renders_in_headless_chrome)"
slow-timeout = { period = "60s", terminate-after = 2 }

2
.github/workflows/rust.yml vendored
View file

@ -131,7 +131,7 @@ jobs:
# in twin mode; widen as the remaining suites are fixed up for CI.
# Must not use the e2e nextest profile here: NEXTEST_PROFILE=e2e implies
# strict mode, which fails (rather than skips) live tests without keys.
- run: cargo nextest run --locked --workspace --status-level slow --profile ci --run-ignored only -E 'package(fabro-agent) + package(fabro-llm) + package(twin-openai)'
- run: cargo nextest run --locked --workspace --status-level slow --profile ci --run-ignored only -E 'package(fabro-agent) + package(fabro-llm)'
test-macos:
name: Test (macOS)

155
Cargo.lock generated
View file

@ -660,17 +660,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "aws-smithy-eventstream"
version = "0.60.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78d8391e65fcea47c586a22e1a41f173b38615b112b2c6b7a44e80cec3e6b706"
dependencies = [
"aws-smithy-types",
"bytes",
"crc32fast",
]
[[package]]
name = "aws-smithy-http"
version = "0.63.6"
@ -1872,7 +1861,7 @@ name = "daytona-api-client"
version = "0.1.0"
source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511"
dependencies = [
"reqwest 0.13.2",
"reqwest 0.13.4",
"reqwest-middleware",
"serde",
"serde_json",
@ -1889,7 +1878,7 @@ dependencies = [
"daytona-api-client",
"daytona-toolbox-client",
"futures-util",
"reqwest 0.13.2",
"reqwest 0.13.4",
"reqwest-middleware",
"rustls",
"serde",
@ -1906,7 +1895,7 @@ name = "daytona-toolbox-client"
version = "0.1.0"
source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511"
dependencies = [
"reqwest 0.13.2",
"reqwest 0.13.4",
"reqwest-middleware",
"serde",
"serde_json",
@ -2074,7 +2063,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -2201,7 +2190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -2289,7 +2278,6 @@ dependencies = [
"fabro-llm",
"fabro-macros",
"fabro-mcp",
"fabro-model",
"fabro-sandbox",
"fabro-static",
"fabro-template",
@ -2329,14 +2317,13 @@ dependencies = [
"fabro-automation",
"fabro-config",
"fabro-environment",
"fabro-model",
"fabro-types",
"openapiv3",
"prettyplease",
"progenitor",
"progenitor-client",
"regress",
"reqwest 0.13.2",
"reqwest 0.13.4",
"serde",
"serde_json",
"serde_yaml",
@ -2353,13 +2340,13 @@ dependencies = [
"base64",
"chrono",
"fabro-http",
"fabro-model",
"fabro-oauth",
"fabro-redact",
"fabro-static",
"fabro-types",
"fabro-vault",
"httpmock",
"lithos-llm",
"serde",
"serde_json",
"tempfile",
@ -2451,7 +2438,6 @@ dependencies = [
"fabro-manifest",
"fabro-mcp",
"fabro-mcp-server",
"fabro-model",
"fabro-oauth",
"fabro-proc",
"fabro-redact",
@ -2486,7 +2472,7 @@ dependencies = [
"progenitor-client",
"rand 0.9.4",
"regex",
"reqwest 0.13.2",
"reqwest 0.13.4",
"ring",
"rustls",
"scopeguard",
@ -2521,7 +2507,6 @@ dependencies = [
"chrono",
"fabro-api",
"fabro-http",
"fabro-model",
"fabro-static",
"fabro-types",
"fabro-util",
@ -2550,7 +2535,6 @@ dependencies = [
"clap",
"dirs",
"fabro-macros",
"fabro-model",
"fabro-options-metadata",
"fabro-proc",
"fabro-static",
@ -2704,7 +2688,6 @@ dependencies = [
"fabro-auth",
"fabro-http",
"fabro-llm",
"fabro-model",
"fabro-redact",
"fabro-types",
"fabro-util",
@ -2724,7 +2707,7 @@ version = "0.348.0-nightly.0"
dependencies = [
"fabro-static",
"http 1.4.0",
"reqwest 0.13.2",
"reqwest 0.13.4",
"thiserror 2.0.18",
]
@ -2768,40 +2751,30 @@ version = "0.348.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
"aws-config",
"aws-credential-types",
"aws-sigv4",
"aws-smithy-eventstream",
"aws-smithy-runtime-api",
"aws-smithy-types",
"base64",
"bytes",
"fabro-auth",
"fabro-config",
"fabro-http",
"fabro-llm",
"fabro-macros",
"fabro-model",
"fabro-redact",
"fabro-static",
"fabro-test",
"fabro-types",
"fabro-util",
"futures",
"http 1.4.0",
"httpmock",
"insta",
"rand 0.9.4",
"lithos-llm",
"mime_guess",
"serde",
"serde_json",
"sha2 0.10.9",
"strum 0.28.0",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-util",
"toml 0.8.23",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
@ -2869,7 +2842,6 @@ dependencies = [
"fabro-client",
"fabro-config",
"fabro-manifest",
"fabro-model",
"fabro-server",
"fabro-tool",
"fabro-types",
@ -2905,23 +2877,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "fabro-model"
version = "0.348.0-nightly.0"
dependencies = [
"clap",
"fabro-static",
"http 1.4.0",
"insta",
"rust-embed",
"serde",
"serde_json",
"strum 0.28.0",
"thiserror 2.0.18",
"toml 0.8.23",
"tracing",
]
[[package]]
name = "fabro-oauth"
version = "0.348.0-nightly.0"
@ -3056,7 +3011,6 @@ dependencies = [
"fabro-macros",
"fabro-manifest",
"fabro-mcp-store",
"fabro-model",
"fabro-proc",
"fabro-redact",
"fabro-sandbox",
@ -3288,10 +3242,10 @@ dependencies = [
"chrono",
"clap",
"dirs",
"fabro-model",
"fabro-types",
"fabro-util",
"hex",
"lithos-llm",
"serde",
"serde_json",
"sha2 0.10.9",
@ -3333,7 +3287,7 @@ version = "0.348.0-nightly.0"
dependencies = [
"fabro-acp",
"fabro-graphviz",
"fabro-model",
"fabro-llm",
"fabro-types",
"serde",
"thiserror 2.0.18",
@ -3404,7 +3358,6 @@ dependencies = [
"fabro-llm",
"fabro-macros",
"fabro-mcp",
"fabro-model",
"fabro-redact",
"fabro-sandbox",
"fabro-static",
@ -4996,6 +4949,31 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
[[package]]
name = "lithos-llm"
version = "0.1.0"
source = "git+https://github.com/lithoscomputer/lithos-llm?rev=4aab27d7d42e7f762a8b6a3871c3db86816b0721#4aab27d7d42e7f762a8b6a3871c3db86816b0721"
dependencies = [
"async-trait",
"aws-config",
"aws-credential-types",
"aws-sigv4",
"aws-smithy-runtime-api",
"aws-smithy-types",
"base64",
"crc32fast",
"futures-core",
"futures-util",
"reqwest 0.13.4",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"toml 0.8.23",
"tracing",
"uuid",
]
[[package]]
name = "litrs"
version = "1.0.0"
@ -5435,7 +5413,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -6298,7 +6276,7 @@ dependencies = [
"bytes",
"futures-core",
"percent-encoding",
"reqwest 0.13.2",
"reqwest 0.13.4",
"serde",
"serde_json",
"serde_urlencoded",
@ -6432,7 +6410,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@ -6699,9 +6677,9 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.13.2"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64",
"bytes",
@ -6757,7 +6735,7 @@ dependencies = [
"anyhow",
"async-trait",
"http 1.4.0",
"reqwest 0.13.2",
"reqwest 0.13.4",
"serde",
"thiserror 2.0.18",
"tower-service",
@ -6797,7 +6775,7 @@ dependencies = [
"pastey",
"pin-project-lite",
"process-wrap",
"reqwest 0.13.2",
"reqwest 0.13.4",
"rmcp-macros",
"schemars 1.2.1",
"serde",
@ -6909,7 +6887,7 @@ dependencies = [
"errno 0.3.14",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -6968,7 +6946,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -7492,7 +7470,7 @@ version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno 0.2.8",
"errno 0.3.14",
"libc",
]
@ -8082,7 +8060,7 @@ dependencies = [
"getrandom 0.4.1",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -8128,7 +8106,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
dependencies = [
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@ -8603,6 +8581,21 @@ dependencies = [
"utf-8",
]
[[package]]
name = "twin-core"
version = "0.1.5"
source = "git+https://github.com/lithoscomputer/twins?rev=ca45f0e50a6716d716aa2f638ca3cf767e88f613#ca45f0e50a6716d716aa2f638ca3cf767e88f613"
dependencies = [
"anyhow",
"async-stream",
"axum",
"futures-util",
"serde",
"serde_json",
"tokio",
"tracing",
]
[[package]]
name = "twin-github"
version = "0.348.0-nightly.0"
@ -8612,7 +8605,7 @@ dependencies = [
"chrono",
"fabro-http",
"jsonwebtoken",
"reqwest 0.13.2",
"reqwest 0.13.4",
"serde",
"serde_json",
"tempfile",
@ -8624,21 +8617,21 @@ dependencies = [
[[package]]
name = "twin-openai"
version = "0.348.0-nightly.0"
version = "0.1.5"
source = "git+https://github.com/lithoscomputer/twins?rev=ca45f0e50a6716d716aa2f638ca3cf767e88f613#ca45f0e50a6716d716aa2f638ca3cf767e88f613"
dependencies = [
"anyhow",
"async-stream",
"axum",
"fabro-http",
"fabro-static",
"futures-util",
"http 1.4.0",
"reqwest 0.13.4",
"serde",
"serde_json",
"tokio",
"tower",
"tracing",
"tracing-subscriber",
"twin-core",
]
[[package]]
@ -9191,7 +9184,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]

8
Cargo.toml
View file

@ -3,7 +3,6 @@ members = [
"lib/apps/*",
"lib/components/*",
"lib/foundation/*",
"test/twin/openai",
"test/twin/github",
]
default-members = ["lib/apps/fabro-cli"]
@ -93,7 +92,12 @@ sha2 = "0.10"
hex = "0.4"
insta = "1"
fabro-test = { path = "lib/foundation/fabro-test" }
twin-openai = { path = "test/twin/openai" }
# Provider-neutral LLM catalog and client. Pinned to a revision until 0.x is
# published to crates.io.
lithos-llm = { git = "https://github.com/lithoscomputer/lithos-llm", rev = "4aab27d7d42e7f762a8b6a3871c3db86816b0721", default-features = false }
# Deterministic OpenAI twin used by twin-mode E2E tests; the same revision
# lithos-llm verifies its codecs against.
twin-openai = { git = "https://github.com/lithoscomputer/twins", rev = "ca45f0e50a6716d716aa2f638ca3cf767e88f613" }
twin-github = { path = "test/twin/github" }
tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] }
futures-util = "0.3"

883
lib/components/fabro-llm/catalog/fabro-policy.toml Normal file
View file

@ -0,0 +1,883 @@
# Fabro's policy layer over the lithos-llm built-in catalog.
#
# Applied after the built-ins and before operator `[llm]` overlays. Everything
# under `metadata.fabro` is Fabro policy that lithos carries verbatim:
#
# - `enabled`: whether Fabro offers the provider or model.
# - `credentials`: ordered credential references (`env:NAME`, `vault:NAME`,
# `aws_sigv4`); the first that resolves wins.
# - `extra_headers`: request headers, literal or `{{ secrets.NAME }}`.
# - `agent_profile`, `family`, `small_default`, `probe`: agent and selection
# policy read by the Fabro model resolver.
#
# Modal's default model comes from a deployment template, not this layer.
# GPT-OSS is excluded at user request. This layer cannot restore that family.
schema_version = 1
[providers."anthropic"]
priority = 100
default_model = "claude-sonnet-5"
[providers."anthropic".metadata.fabro]
api_key_url = "https://console.anthropic.com/settings/keys"
credentials = ["env:ANTHROPIC_API_KEY", "vault:ANTHROPIC_API_KEY"]
enabled = true
[providers."anthropic".models."claude-fable-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."anthropic".models."claude-opus-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."anthropic".models."claude-sonnet-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."anthropic".models."claude-opus-4.8".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."anthropic".models."claude-opus-4.7".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."anthropic".models."claude-opus-4.6".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."anthropic".models."claude-sonnet-4.5".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."anthropic".models."claude-sonnet-4.6".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."anthropic".models."claude-haiku-4.5".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = true
probe = true
[providers."bedrock-openai"]
priority = 19
default_model = "gpt-5.5"
[providers."bedrock-openai".metadata.fabro]
api_key_url = "https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html"
credentials = ["env:AWS_BEARER_TOKEN_BEDROCK", "env:BEDROCK_API_KEY", "vault:AWS_BEARER_TOKEN_BEDROCK", "vault:BEDROCK_API_KEY"]
enabled = false
[providers."bedrock-openai".models."gpt-5.5".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock-openai".models."gpt-5.4".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock"]
priority = 20
default_model = "claude-sonnet-5"
[providers."bedrock".metadata.fabro]
api_key_url = "https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html"
credentials = ["env:AWS_BEARER_TOKEN_BEDROCK", "env:BEDROCK_API_KEY", "vault:AWS_BEARER_TOKEN_BEDROCK", "vault:BEDROCK_API_KEY", "aws_sigv4"]
enabled = false
[providers."bedrock".models."anthropic.claude-sonnet-4-6".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."bedrock".models."claude-opus-4-8".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = false
probe = false
[providers."bedrock".models."claude-haiku-4-5".metadata.fabro]
family = "claude-4"
agent_profile = "anthropic"
small_default = true
probe = false
[providers."bedrock".models."nova-2-lite".metadata.fabro]
family = "nova-2"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."llama-4-maverick".metadata.fabro]
family = "llama-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."mistral-large-3".metadata.fabro]
family = "mistral-large"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."devstral-2".metadata.fabro]
family = "devstral"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."deepseek-v3.2".metadata.fabro]
family = "deepseek-v3"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."kimi-k2.5".metadata.fabro]
family = "kimi-k2"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."glm-5".metadata.fabro]
family = "glm"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."minimax-m2.5".metadata.fabro]
family = "minimax-m2"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."nemotron-3-super".metadata.fabro]
family = "nemotron-3"
agent_profile = "openai"
small_default = false
probe = false
[providers."bedrock".models."claude-fable-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."bedrock".models."claude-sonnet-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."deepseek"]
priority = 75
default_model = "deepseek-v4-flash"
[providers."deepseek".metadata.fabro]
api_key_url = "https://platform.deepseek.com/api_keys"
credentials = ["env:DEEPSEEK_API_KEY", "vault:DEEPSEEK_API_KEY"]
enabled = true
[providers."deepseek".models."deepseek-v4-flash".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = true
probe = true
[providers."deepseek".models."deepseek-v4-pro".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks"]
priority = 30
default_model = "kimi-k2.7-code"
[providers."fireworks".metadata.fabro]
api_key_url = "https://app.fireworks.ai/settings/users/api-keys"
credentials = ["env:FIREWORKS_API_KEY", "vault:FIREWORKS_API_KEY"]
enabled = false
# Moonshot documents K3 as always reasoning with low, high, and max effort;
# lithos leaves the levels unverified, so Fabro records them here.
[providers."fireworks".models."kimi-k3"]
capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } }
[providers."fireworks".models."kimi-k3".metadata.fabro]
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."fireworks".models."kimi-k3-fast".metadata.fabro]
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."fireworks".models."kimi-k2.7-code".metadata.fabro]
family = "kimi-k2"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."kimi-k2.6".metadata.fabro]
family = "kimi-k2"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."deepseek-v4-pro".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."deepseek-v4-flash".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."glm-5.2".metadata.fabro]
family = "glm-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."minimax-m2.7".metadata.fabro]
family = "minimax-m2"
agent_profile = "openai"
small_default = false
probe = false
[providers."fireworks".models."qwen3.7-plus".metadata.fabro]
family = "qwen3"
agent_profile = "openai"
small_default = false
probe = false
[providers."gemini"]
priority = 80
default_model = "gemini-3.5-flash"
[providers."gemini".metadata.fabro]
api_key_url = "https://aistudio.google.com/apikey"
credentials = ["env:GEMINI_API_KEY", "env:GOOGLE_API_KEY", "vault:GEMINI_API_KEY"]
enabled = true
[providers."gemini".models."gemini-3.1-pro-preview".metadata.fabro]
family = "gemini-3"
agent_profile = "gemini"
small_default = false
probe = false
[providers."gemini".models."gemini-3.1-pro-preview-customtools".metadata.fabro]
family = "gemini-3"
agent_profile = "gemini"
small_default = false
probe = false
[providers."gemini".models."gemini-3.5-flash".metadata.fabro]
family = "gemini-3"
agent_profile = "gemini"
small_default = false
probe = false
[providers."gemini".models."gemini-3-flash-preview".metadata.fabro]
family = "gemini-3"
agent_profile = "gemini"
small_default = false
probe = false
[providers."gemini".models."gemini-3.1-flash-lite".metadata.fabro]
family = "gemini-3"
agent_profile = "gemini"
small_default = true
probe = false
[providers."inception"]
priority = 40
default_model = "mercury-2"
[providers."inception".metadata.fabro]
api_key_url = "https://console.inceptionlabs.ai/api-keys"
credentials = ["env:INCEPTION_API_KEY", "vault:INCEPTION_API_KEY"]
enabled = true
[providers."inception".models."mercury-2".metadata.fabro]
family = "mercury"
agent_profile = "openai"
small_default = false
probe = false
[providers."litellm"]
priority = 50
[providers."litellm".metadata.fabro]
credentials = ["env:LITELLM_API_KEY", "vault:LITELLM_API_KEY"]
enabled = false
[providers."minimax"]
priority = 50
default_model = "minimax-m2.5"
[providers."minimax".metadata.fabro]
api_key_url = "https://platform.minimaxi.com/user-center/basic-information/interface-key"
credentials = ["env:MINIMAX_API_KEY", "vault:MINIMAX_API_KEY"]
enabled = true
[providers."minimax".models."minimax-m2.5".metadata.fabro]
family = "minimax-m2"
agent_profile = "openai"
small_default = false
probe = false
[providers."modal"]
priority = 75
[providers."modal".metadata.fabro]
api_key_url = "https://modal.com/docs/guide/endpoints#proxy-tokens"
enabled = false
[providers."modal".metadata.fabro.extra_headers]
"Modal-Key" = "{{ secrets.MODAL_TOKEN_ID }}"
"Modal-Secret" = "{{ secrets.MODAL_TOKEN_SECRET }}"
[providers."moonshot"]
priority = 70
default_model = "kimi-k3"
[providers."moonshot".metadata.fabro]
api_key_url = "https://platform.kimi.ai/console/api-keys"
credentials = ["env:MOONSHOT_API_KEY", "env:KIMI_API_KEY", "vault:MOONSHOT_API_KEY", "vault:KIMI_API_KEY"]
enabled = true
[providers."moonshot".models."kimi-k2.5".metadata.fabro]
family = "kimi-k2"
agent_profile = "kimi"
small_default = false
probe = false
# Moonshot documents K3 as always reasoning with low, high, and max effort;
# lithos leaves the levels unverified, so Fabro records them here.
[providers."moonshot".models."kimi-k3"]
capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } }
[providers."moonshot".models."kimi-k3".metadata.fabro]
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."ollama"]
priority = 30
[providers."ollama".metadata.fabro]
enabled = false
[providers."openai"]
priority = 90
default_model = "gpt-5.6-sol"
[providers."openai".metadata.fabro]
api_key_url = "https://platform.openai.com/api-keys"
credentials = ["env:OPENAI_API_KEY", "vault:OPENAI_API_KEY"]
enabled = true
[providers."openai".models."gpt-5.6-sol".metadata.fabro]
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openai".models."gpt-5.6-terra".metadata.fabro]
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openai".models."gpt-5.6-luna".metadata.fabro]
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openai".models."gpt-5.4".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openai".models."gpt-5.5".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openai".models."gpt-5.5-pro".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openai".models."gpt-5.4-pro".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openai".models."gpt-5.4-mini".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = true
probe = true
[providers."openrouter"]
priority = 25
default_model = "claude-sonnet-5"
[providers."openrouter".metadata.fabro]
api_key_url = "https://openrouter.ai/keys"
credentials = ["env:OPENROUTER_API_KEY", "vault:OPENROUTER_API_KEY"]
enabled = false
[providers."openrouter".models."claude-fable-5".metadata.fabro]
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."openrouter".models."claude-opus-5".metadata.fabro]
training = "2026-05-01"
knowledge_cutoff = "May 2026"
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."openrouter".models."claude-sonnet-5".metadata.fabro]
training = "2026-01-01"
knowledge_cutoff = "Jan 2026"
family = "claude-5"
agent_profile = "claude-5"
small_default = false
probe = false
[providers."openrouter".models."claude-opus-4.8".metadata.fabro]
training = "2026-01-01"
knowledge_cutoff = "Jan 2026"
family = "claude-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."claude-opus-4.7".metadata.fabro]
family = "claude-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."claude-sonnet-4.6".metadata.fabro]
family = "claude-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."claude-haiku-4.5".metadata.fabro]
family = "claude-4"
agent_profile = "openai"
small_default = true
probe = false
[providers."openrouter".models."gpt-5.6-sol".metadata.fabro]
training = "2026-02-16"
knowledge_cutoff = "February 16, 2026"
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openrouter".models."gpt-5.6-terra".metadata.fabro]
training = "2026-02-16"
knowledge_cutoff = "February 16, 2026"
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openrouter".models."gpt-5.6-luna".metadata.fabro]
training = "2026-02-16"
knowledge_cutoff = "February 16, 2026"
family = "gpt-5"
agent_profile = "gpt56"
small_default = false
probe = false
[providers."openrouter".models."gpt-5.4".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."gpt-5.5".metadata.fabro]
family = "gpt-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."gemini-3.1-pro-preview".metadata.fabro]
family = "gemini-3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."gemini-3.5-flash".metadata.fabro]
family = "gemini-3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."mimo-v2.5-pro".metadata.fabro]
family = "mimo-v2"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."minimax-m2.7".metadata.fabro]
family = "minimax-m2"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."deepseek-v4-pro".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."deepseek-v4-flash".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."kimi-k2.6".metadata.fabro]
family = "kimi-k2"
agent_profile = "kimi"
small_default = false
probe = false
# Moonshot documents K3 as always reasoning with low, high, and max effort;
# lithos leaves the levels unverified, so Fabro records them here.
[providers."openrouter".models."kimi-k3"]
capabilities = { reasoning_effort = { minimal = false, low = true, medium = false, high = true, xhigh = false, max = true } }
[providers."openrouter".models."kimi-k3".metadata.fabro]
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."openrouter".models."laguna-s-2.1".metadata.fabro]
family = "laguna-2"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."laguna-xs-2.1".metadata.fabro]
family = "laguna-2"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."qwen3-coder".metadata.fabro]
family = "qwen3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."qwen3.6-flash".metadata.fabro]
family = "qwen3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."qwen3.8-max".metadata.fabro]
family = "qwen3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."glm-5.2".metadata.fabro]
family = "glm-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."glm-4.6".metadata.fabro]
family = "glm-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."nemotron-3-super-120b-a12b".metadata.fabro]
family = "nemotron-3"
agent_profile = "openai"
small_default = false
probe = false
[providers."openrouter".models."devstral-2512".metadata.fabro]
family = "devstral"
agent_profile = "openai"
small_default = false
probe = false
[providers."poolside"]
priority = 65
default_model = "laguna-s-2.1"
[providers."poolside".metadata.fabro]
api_key_url = "https://platform.poolside.ai"
credentials = ["env:POOLSIDE_API_KEY", "vault:POOLSIDE_API_KEY"]
enabled = true
[providers."poolside".models."laguna-s-2.1".metadata.fabro]
family = "laguna-2"
agent_profile = "openai"
small_default = false
probe = false
[providers."poolside".models."laguna-xs-2.1".metadata.fabro]
family = "laguna-2"
agent_profile = "openai"
small_default = true
probe = true
[providers."venice"]
priority = 35
default_model = "deepseek-v4-flash"
[providers."venice".metadata.fabro]
credentials = ["env:VENICE_API_KEY", "vault:VENICE_API_KEY"]
enabled = true
[providers."venice".models."kimi-k3".metadata.fabro]
reasoning_by_default = true
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."venice".models."kimi-k3-fast".metadata.fabro]
reasoning_by_default = true
family = "kimi-k3"
agent_profile = "kimi"
small_default = false
probe = false
[providers."venice".models."grok-4.6".metadata.fabro]
reasoning_by_default = true
family = "grok-4"
agent_profile = "openai"
small_default = false
probe = false
[providers."venice".models."glm-5.3".metadata.fabro]
reasoning_by_default = true
family = "glm-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."venice".models."deepseek-v4-flash".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."venice".models."deepseek-v4-pro".metadata.fabro]
reasoning_by_default = true
family = "deepseek-v4"
agent_profile = "openai"
small_default = false
probe = false
[providers."venice".models."qwen3.8-max".metadata.fabro]
reasoning_by_default = true
family = "qwen3"
agent_profile = "openai"
small_default = false
probe = false
[providers."venice".models."qwen3.8-27b".metadata.fabro]
reasoning_by_default = true
family = "qwen3.8"
agent_profile = "openai"
small_default = false
probe = false
[providers."zai"]
priority = 60
default_model = "glm-5.2"
[providers."zai".metadata.fabro]
api_key_url = "https://open.bigmodel.cn/usercenter/apikeys"
credentials = ["env:ZAI_API_KEY", "vault:ZAI_API_KEY"]
enabled = true
[providers."zai".models."glm-5.2".metadata.fabro]
family = "glm-5"
agent_profile = "openai"
small_default = false
probe = false
[providers."zai".models."glm-4.7".metadata.fabro]
family = "glm-4"
agent_profile = "openai"
small_default = false
probe = false
# ChatGPT OAuth (Codex) access to the OpenAI roster. lithos speaks the Codex
# deployment through the `openai` adapter in codex mode; this provider exists
# so a `fabro provider login --provider openai` device-flow credential routes
# here while an API key keeps routing to `openai`. It is hidden from listings
# and stands in for `openai` when `openai` itself has no credentials.
[providers."openai-codex"]
display_name = "OpenAI (ChatGPT)"
adapter = "openai"
codec = "openai-responses"
base_url = "https://chatgpt.com/backend-api/codex"
priority = 89
allow_passthrough = true
default_model = "gpt-5.6-sol"
adapter_options = { mode = "codex" }
default_headers = { originator = "fabro" }
[providers."openai-codex".auth]
type = "bearer"
[providers."openai-codex".metadata.fabro]
enabled = true
stands_in_for = "openai"
credentials = ["vault:OPENAI_CODEX"]
[providers."openai-codex".models."gpt-5.6-sol"]
display_name = "GPT-5.6 Sol"
aliases = ["sol", "gpt-sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"]
api_model = "gpt-5.6-sol"
limits = { context_tokens = 1050000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 20000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 8000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 800000, cache_write_usd_micros_per_million = 10000000 }, speed = { fast = { input_usd_micros_per_million = 8000000, output_usd_micros_per_million = 40000000, cached_input_usd_micros_per_million = 800000, cache_write_usd_micros_per_million = 10000000 }, economical = { input_usd_micros_per_million = 2000000, output_usd_micros_per_million = 10000000, cached_input_usd_micros_per_million = 200000, cache_write_usd_micros_per_million = 2500000 } } }
[providers."openai-codex".models."gpt-5.6-sol".metadata.fabro]
agent_profile = "gpt56"
family = "gpt-5"
small_default = false
probe = false
[providers."openai-codex".models."gpt-5.6-terra"]
display_name = "GPT-5.6 Terra"
aliases = ["terra", "gpt-terra", "gpt56-terra", "gpt-56-terra"]
api_model = "gpt-5.6-terra"
limits = { context_tokens = 1050000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 2000000, output_usd_micros_per_million = 12000000, cached_input_usd_micros_per_million = 200000, cache_write_usd_micros_per_million = 2500000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 18000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000 }, speed = { fast = { input_usd_micros_per_million = 4000000, output_usd_micros_per_million = 24000000, cached_input_usd_micros_per_million = 400000, cache_write_usd_micros_per_million = 5000000 }, economical = { input_usd_micros_per_million = 1000000, output_usd_micros_per_million = 6000000, cached_input_usd_micros_per_million = 100000, cache_write_usd_micros_per_million = 1250000 } } }
[providers."openai-codex".models."gpt-5.6-terra".metadata.fabro]
agent_profile = "gpt56"
family = "gpt-5"
small_default = false
probe = false
[providers."openai-codex".models."gpt-5.6-luna"]
display_name = "GPT-5.6 Luna"
aliases = ["luna", "gpt-luna", "gpt56-luna", "gpt-56-luna"]
api_model = "gpt-5.6-luna"
limits = { context_tokens = 1050000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 200000, output_usd_micros_per_million = 1200000, cached_input_usd_micros_per_million = 20000, cache_write_usd_micros_per_million = 250000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 400000, output_usd_micros_per_million = 1800000, cached_input_usd_micros_per_million = 40000, cache_write_usd_micros_per_million = 500000 }, speed = { fast = { input_usd_micros_per_million = 400000, output_usd_micros_per_million = 2400000, cached_input_usd_micros_per_million = 40000, cache_write_usd_micros_per_million = 500000 }, economical = { input_usd_micros_per_million = 100000, output_usd_micros_per_million = 600000, cached_input_usd_micros_per_million = 10000, cache_write_usd_micros_per_million = 125000 } } }
[providers."openai-codex".models."gpt-5.6-luna".metadata.fabro]
agent_profile = "gpt56"
family = "gpt-5"
small_default = false
probe = false
[providers."openai-codex".models."gpt-5.4"]
display_name = "GPT-5.4"
aliases = ["gpt54", "gpt-54", "gpt-5.2", "gpt5", "gpt-5.3-codex", "codex"]
api_model = "gpt-5.4"
limits = { context_tokens = 1050000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, sampling = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 2500000, output_usd_micros_per_million = 15000000, cached_input_usd_micros_per_million = 250000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 22500000, cached_input_usd_micros_per_million = 500000 }, speed = { fast = { input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 500000 }, economical = { input_usd_micros_per_million = 1250000, output_usd_micros_per_million = 7500000, cached_input_usd_micros_per_million = 125000 } } }
[providers."openai-codex".models."gpt-5.4".metadata.fabro]
agent_profile = "openai"
family = "gpt-5"
small_default = false
probe = false
[providers."openai-codex".models."gpt-5.5"]
display_name = "GPT-5.5"
aliases = ["gpt55", "gpt-55"]
api_model = "gpt-5.5"
limits = { context_tokens = 1050000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 5000000, output_usd_micros_per_million = 30000000, cached_input_usd_micros_per_million = 500000, long_context = { above_input_tokens = 272000, input_usd_micros_per_million = 10000000, output_usd_micros_per_million = 45000000, cached_input_usd_micros_per_million = 1000000 }, speed = { fast = { input_usd_micros_per_million = 12500000, output_usd_micros_per_million = 75000000, cached_input_usd_micros_per_million = 1250000 }, economical = { input_usd_micros_per_million = 2500000, output_usd_micros_per_million = 15000000, cached_input_usd_micros_per_million = 250000 } } }
[providers."openai-codex".models."gpt-5.5".metadata.fabro]
agent_profile = "openai"
family = "gpt-5"
small_default = false
probe = false
[providers."openai-codex".models."gpt-5.4-mini"]
display_name = "GPT-5.4 Mini"
aliases = ["gpt54-mini", "gpt-54-mini", "gpt-5.3-codex-spark", "codex-spark"]
api_model = "gpt-5.4-mini"
limits = { context_tokens = 400000, max_output_tokens = 128000 }
capabilities = { text = true, images = true, documents = true, tools = true, response_format = { json_object = true, json_schema = true }, reasoning = true, caching = true, cache_routing = true, sampling = true, tool_choice = { required = true, named = true }, speed = { fast = true, economical = true } }
protocol_options = { reasoning_effort_levels = true }
pricing = { input_usd_micros_per_million = 750000, output_usd_micros_per_million = 4500000, cached_input_usd_micros_per_million = 75000, speed = { fast = { input_usd_micros_per_million = 1500000, output_usd_micros_per_million = 9000000, cached_input_usd_micros_per_million = 150000 }, economical = { input_usd_micros_per_million = 375000, output_usd_micros_per_million = 2250000, cached_input_usd_micros_per_million = 37500 } } }
[providers."openai-codex".models."gpt-5.4-mini".metadata.fabro]
agent_profile = "openai"
family = "gpt-5"
small_default = true
probe = true

1
lib/foundation/fabro-http/src/lib.rs
View file

@ -15,6 +15,7 @@ pub use reqwest::{
};
pub type HttpClient = reqwest::Client;
pub type HttpError = reqwest::Error;
pub type BlockingHttpClient = reqwest::blocking::Client;
pub type BlockingRequestBuilder = reqwest::blocking::RequestBuilder;
pub type BlockingResponse = reqwest::blocking::Response;

6
lib/foundation/fabro-test/src/lib.rs
View file

@ -2446,11 +2446,13 @@ pub async fn twin_openai() -> &'static TwinOpenAi {
let base_url = format!("http://127.0.0.1:{}/v1", addr.port());
let config = TwinConfig {
bind_addr: addr,
bind_addr: addr,
require_auth: true,
enable_admin: true,
..TwinConfig::from_lookup(&|_| None).expect("twin-openai defaults should load")
};
let app = twin_openai::build_app_with_config(config);
let app =
twin_openai::build_app_with_config(config).expect("twin-openai app should build");
tokio::spawn(async move {
axum::serve(listener, app).await.expect("twin-openai serve");

1
test/twin/openai/.gitignore vendored
View file

@ -1 +0,0 @@
.worktrees

30
test/twin/openai/Cargo.toml
View file

@ -1,30 +0,0 @@
[package]
name = "twin-openai"
edition.workspace = true
version.workspace = true
publish = false
license.workspace = true
description = "Fake OpenAI-compatible server for local black-box testing"
[lib]
doctest = false
[lints]
workspace = true
[dependencies]
anyhow.workspace = true
async-stream = "0.3"
axum = { workspace = true, features = ["macros"] }
fabro-http.workspace = true
fabro-static.workspace = true
futures-util.workspace = true
http = "1"
serde.workspace = true
serde_json.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
[dev-dependencies]
tower = "0.5"

85
test/twin/openai/README.md
View file

@ -1,85 +0,0 @@
# twin-openai
Async Rust fake OpenAI-compatible server for local black-box testing.
## Endpoints
- `GET /healthz`
- `POST /v1/responses`
- `POST /v1/chat/completions`
- `POST /__admin/scenarios`
- `POST /__admin/reset`
- `GET /__admin/requests`
`/v1/*` routes require a non-empty bearer token. Scenarios, request logs, and deterministic response IDs are scoped by bearer token so concurrent test clients can share one server safely.
`/__admin/*` routes are unauthenticated by default, but an optional bearer token selects the same namespace as `/v1/*`. Admin requests with a malformed or empty `Authorization` header are rejected.
## Run locally
```bash
cargo run
```
The server binds to `127.0.0.1:3000` by default.
## Admin scripting
Load deterministic one-shot scenarios:
```bash
curl -X POST http://127.0.0.1:3000/__admin/scenarios \
-H 'Authorization: Bearer suite-a' \
-H 'content-type: application/json' \
-d '{
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "scripted reply" }
}
]
}'
```
Inspect normalized request logs:
```bash
curl http://127.0.0.1:3000/__admin/requests \
-H 'Authorization: Bearer suite-a'
```
Reset scenarios, logs, and deterministic counters:
```bash
curl -X POST http://127.0.0.1:3000/__admin/reset \
-H 'Authorization: Bearer suite-a'
```
## Behavior summary
- Non-stream and stream success paths are driven from the same canonical response plan.
- `/v1/responses` and `/v1/chat/completions` share the same deterministic fallback behavior.
- Structured output supports `json_object` and a documented `json_schema` subset.
- Scripted failures support OpenAI-shaped application errors, delays, hangs, partial streams, and malformed SSE.
## Optional Live OpenAI Smoke Suite
Run the ignored live drift detector only when you explicitly want to compare `twin-openai` against the real OpenAI API:
```bash
OPENAI_API_KEY=... cargo test --test live_openai_contract -- --ignored --nocapture
```
Optional environment variables:
- `TWIN_OPENAI_LIVE_MODEL` defaults to `gpt-5-nano-2025-08-07`
- `TWIN_OPENAI_LIVE_BASE_URL` defaults to `https://api.openai.com`
- `OPENAI_ORGANIZATION` and `OPENAI_PROJECT` are forwarded when present
This suite is not part of normal CI. It is intentionally a drift detector for request/response shape and SSE sequencing, so opt-in failures can represent real compatibility gaps rather than a broken local test harness.
If the supplied OpenAI credentials lack required endpoint scopes or quota, the ignored test will skip the blocked live surface instead of reporting protocol drift.
Current live coverage includes `responses` and `chat.completions` text, streaming, structured output, function tools, `tool_choice: "none"` behavior, image-input acceptance, and both non-stream and streamed `responses` continuation turns.
See [docs/compatibility-matrix.md](/Users/bhelmkamp/p/brynary/twin-openai/docs/compatibility-matrix.md) for the supported field matrix and explicit exclusions.

64
test/twin/openai/docs/compatibility-matrix.md
View file

@ -1,64 +0,0 @@
# Compatibility Matrix
Supported endpoints:
- `GET /healthz`
- `POST /v1/responses`
- `POST /v1/chat/completions`
- `POST /__admin/scenarios`
- `POST /__admin/reset`
- `GET /__admin/requests`
State isolation:
- `/v1/*` request state is scoped by bearer token
- admin routes may include the same bearer token to target that namespace
- admin routes without auth operate on the global namespace
Supported `/v1/responses` fields:
- bearer auth
- `stream`
- `metadata`
- `stop`
- `previous_response_id`
- `reasoning`
- `text.format.type = text | json_object | json_schema`
- image inputs in `input[*].content[*].type = input_image`
- scripted tool calls and continuation input items
Supported `/v1/chat/completions` fields:
- bearer auth
- `max_tokens`
- `stream`
- `tools`
- `tool_choice`
- `response_format.type = text | json_object | json_schema`
- `stop`
- reasoning-bearing assistant content
Unknown top-level fields are accepted and ignored. The twin does not simulate the behavior of
fields that are not listed above.
Structured output subset:
- object roots
- primitive property types: `string`, `integer`, `number`, `boolean`
- nested object properties
Unsupported schema constructs fail explicitly:
- arrays
- `anyOf`
- `oneOf`
Failure scripting:
- application errors with explicit status and OpenAI-shaped body
- optional `Retry-After`
- delay before headers
- hang before first byte
- inter-event stream delay
- close stream after N chunks
- malformed/truncated SSE ending

204
test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai-test-plan.md
View file

@ -1,204 +0,0 @@
# Async Rust Fake OpenAI Test Plan
The approved testing strategy still holds after reconciling it with the implementation plan. The plan narrows the generic strategy into a concrete async Rust action surface: `GET /healthz`, `POST /v1/responses`, `POST /v1/chat/completions`, and the unauthenticated admin control routes `POST /__admin/scenarios`, `POST /__admin/reset`, and `GET /__admin/requests`. No strategy change requiring user approval was identified.
## Harness requirements
1. **Ephemeral server harness**
- What it does: boots the real `tokio` + `axum` server on an ephemeral localhost port with test config, then drives it through real HTTP.
- What it exposes: base URL, authenticated and unauthenticated HTTP clients, SSE reader utilities, timeout helpers, and optional raw socket/body readers for truncated-stream cases.
- Estimated complexity: medium.
- Tests depending on it: 1 through 14.
2. **Admin scenario harness**
- What it does: scripts deterministic server behavior through `POST /__admin/scenarios`, clears state with `POST /__admin/reset`, and fetches normalized request logs from `GET /__admin/requests`.
- What it exposes: helpers to enqueue FIFO scenarios, seed failure scripts, reset state between tests, and fetch request-log artifacts for assertions.
- Estimated complexity: medium.
- Tests depending on it: 4 through 13.
3. **Canonical output comparison helpers**
- What it does: normalizes non-stream JSON and streamed SSE transcripts into comparable observable artifacts so the same canonical plan can be validated across transport modes and endpoints.
- What it exposes: parsed text transcript, tool-call transcript, reasoning transcript, completion marker presence, and required-field assertions.
- Estimated complexity: low to medium.
- Tests depending on it: 3, 6, 8, 9.
4. **Optional live OpenAI differential harness**
- What it does: when explicit credentials are present outside normal CI, sends normalized requests to both `twin-openai` and the real OpenAI API and compares protocol shape rather than literal generated text.
- What it exposes: paired request runner, field-by-field comparator for required JSON fields and SSE event ordering, and opt-in skip behavior when credentials are absent.
- Estimated complexity: medium.
- Tests depending on it: 12.
## Test plan
1. **Name**: health and auth endpoints enforce the public service boundary
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running with default local-test config and no scenarios loaded.
- **Actions**: `GET /healthz`; `POST /v1/responses` without `Authorization`; `POST /v1/chat/completions` with an empty bearer token; `POST /v1/responses` with a non-empty bearer token and minimal valid JSON.
- **Expected outcome**: `GET /healthz` returns `200`; `/v1/*` rejects missing or empty bearer auth with stable OpenAI-shaped error JSON; a syntactically valid authenticated request reaches endpoint handling rather than failing auth. Source of truth: approved strategy, implementation plan scope and product contract.
- **Interactions**: router wiring, auth middleware, request parsing, error serialization.
2. **Name**: non-stream responses create returns deterministic OpenAI-shaped JSON
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running, no scenario matches the request.
- **Actions**: `POST /v1/responses` with bearer auth, `stream=false`, text input, and optional inert `OpenAI-Organization` and `OpenAI-Project` headers.
- **Expected outcome**: `200` with valid `/v1/responses` JSON containing deterministic ID format, timestamp, output items, and usage object; response text is the documented deterministic fallback derived from user text rather than a real model output. Source of truth: implementation plan user-visible behavior, required invariants, OpenAI Responses API shape.
- **Interactions**: request models, deterministic default engine, JSON serialization, inert compatibility header handling.
3. **Name**: streaming responses emit valid SSE and the same content as non-stream responses
- **Type**: invariant
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Canonical output comparison helpers
- **Preconditions**: server is running, no scenario matches the request, same logical request body is available in both stream and non-stream forms.
- **Actions**: send one `POST /v1/responses` with `stream=false` and one with `stream=true`; collect the full JSON body and full SSE transcript.
- **Expected outcome**: streamed events are valid SSE, ordered, flushed incrementally, and terminate with the supported completion semantics; the user-visible text, reasoning content, and tool-call transcript reconstructed from the stream match the non-stream response derived from the same canonical plan. Source of truth: approved strategy, implementation plan invariants, OpenAI streaming docs.
- **Interactions**: canonical response-plan projection, SSE encoder, chunk flushing, completion signaling.
4. **Name**: admin-loaded scenarios are consumed once and in FIFO order
- **Type**: scenario
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness
- **Preconditions**: server is running; two matching scenarios are loaded for the same endpoint and matcher.
- **Actions**: `POST /__admin/scenarios` with two matching scripts; call the matching OpenAI endpoint twice; call it a third time after the queue is exhausted; fetch `GET /__admin/requests`; reset via `POST /__admin/reset`.
- **Expected outcome**: first OpenAI request consumes the first script, second consumes the second, third falls back to deterministic default behavior; request log is append-only until reset and empty after reset. Source of truth: implementation plan scenario scripting model, user-visible behavior, required invariants.
- **Interactions**: admin API, scenario matcher, FIFO consumption, fallback engine, request logging, reset behavior.
5. **Name**: responses accept the declared compatibility fields and reject unsupported combinations clearly
- **Type**: boundary
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running with no scenarios required.
- **Actions**: `POST /v1/responses` requests covering `metadata`, `stop`, `previous_response_id`, reasoning options, image inputs, `text` response format, `json_object`, supported `json_schema`, and one unsupported field or unsupported schema construct.
- **Expected outcome**: supported fields are accepted without changing the documented deterministic semantics; unsupported combinations fail with stable OpenAI-shaped error JSON rather than silent success. Source of truth: implementation plan endpoint compatibility contract, approved strategy, OpenAI docs for supported fields.
- **Interactions**: request validation, schema subset validation, image-input parsing, explicit unsupported-case handling.
6. **Name**: scripted tool-call and continuation flows work through responses in stream and non-stream modes
- **Type**: scenario
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness plus Canonical output comparison helpers
- **Preconditions**: server is running; an admin scenario is queued that emits a tool call on the first request and a final answer on a continuation request that includes `function_call_output`.
- **Actions**: load the scripted scenario; call `POST /v1/responses` once non-stream and once stream to observe tool-call emission; call `POST /v1/responses` again with continuation input containing the tool output and optional `previous_response_id`.
- **Expected outcome**: first turn returns or streams a valid tool-call item with deterministic IDs and supported reasoning content; continuation request is accepted and returns the scripted final answer; stream and non-stream transcripts stay equivalent for the same planned turn. Source of truth: implementation plan tool-call and continuation contract, OpenAI function-calling and Responses docs.
- **Interactions**: scenario engine, canonical plan, responses renderer, continuation input parsing, SSE event sequencing.
7. **Name**: structured output support is explicit, deterministic, and bounded to the documented schema subset
- **Type**: boundary
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness where needed
- **Preconditions**: server is running; supported and unsupported response format payloads are available.
- **Actions**: call `POST /v1/responses` with `json_object`; call with a supported `json_schema` subset; call with an unsupported schema feature such as an out-of-scope construct defined by the compatibility matrix.
- **Expected outcome**: `json_object` and supported `json_schema` requests return deterministic JSON matching the declared format; unsupported schema constructs fail explicitly with stable error JSON. Source of truth: implementation plan deterministic structured-output behavior and compatibility-matrix requirement, OpenAI structured outputs docs.
- **Interactions**: response-format validation, deterministic JSON generation, error-body rendering.
8. **Name**: chat completions non-stream and stream use the same canonical plan as responses when behavior is equivalent
- **Type**: invariant
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness plus Canonical output comparison helpers
- **Preconditions**: server is running; one default request pair and one scripted scenario pair exist that are semantically representable on both endpoints.
- **Actions**: send equivalent requests to `POST /v1/responses` and `POST /v1/chat/completions` in both stream and non-stream forms.
- **Expected outcome**: both endpoints produce their respective OpenAI-shaped envelopes, but the user-visible text, reasoning transcript, and tool-call intent are equivalent because both render the same canonical plan; chat streaming emits valid delta events and terminal markers. Source of truth: implementation plan required invariants and endpoint compatibility contract, OpenAI Responses and Chat Completions docs.
- **Interactions**: cross-endpoint adapters, canonical plan projection, chat delta SSE rendering, shared validation helpers.
9. **Name**: chat completions accept supported fields and reject unsupported combinations explicitly
- **Type**: boundary
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running with no scenario required.
- **Actions**: `POST /v1/chat/completions` with supported `tools`, `tool_choice`, `response_format`, `stop`, and reasoning-bearing assistant content; send one request with an unsupported combination declared outside the compatibility matrix.
- **Expected outcome**: supported requests succeed in stream and non-stream modes; unsupported combinations fail with stable OpenAI-shaped error JSON rather than being ignored. Source of truth: implementation plan endpoint compatibility contract, OpenAI Chat Completions docs.
- **Interactions**: chat request parsing, response-format handling, tool-choice validation, error serialization.
10. **Name**: scripted application errors preserve status, error shape, and retry metadata
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness
- **Preconditions**: server is running; scenarios are loaded for each scripted application error variant.
- **Actions**: queue and invoke scripted failures for `400`, `401`, `403`, `404`, `408`, `413`, `429`, `500`, `502`, `503`, and `504`; include distinct quota-style and content-filter-style error bodies and a `429` carrying `Retry-After`.
- **Expected outcome**: each request returns the scripted HTTP status, OpenAI-shaped error JSON, and any scripted `Retry-After` header without transport corruption; quota-style and content-filter-style bodies remain observably distinct from generic invalid-request errors. Source of truth: approved strategy failure matrix, implementation plan task 7.
- **Interactions**: admin scripting, error renderer, header propagation, status mapping.
11. **Name**: delayed first byte and hung requests are observable over real sockets and bounded by client timeouts
- **Type**: scenario
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness
- **Preconditions**: server is running; one scenario delays before headers and one hangs forever.
- **Actions**: invoke the delayed-first-byte scenario with a timeout larger than the scripted delay; invoke the hang scenario with a short explicit client timeout.
- **Expected outcome**: delayed-first-byte requests eventually succeed after the scripted pause; hung requests never produce completion and are terminated by the test client timeout rather than silently succeeding or closing early. Source of truth: approved strategy performance and failure-injection requirements, implementation plan transport-failure mechanics.
- **Interactions**: async timing, response-body start behavior, timeout guards, scenario engine.
12. **Name**: partial stream close and malformed SSE are distinguishable transport failures
- **Type**: regression
- **Disposition**: new
- **Harness**: Ephemeral server harness plus Admin scenario harness with raw socket/body reader support
- **Preconditions**: server is running; one scenario is configured to close after N chunks and another to emit malformed or truncated SSE.
- **Actions**: invoke both scenarios against streaming `/v1/responses` and streaming `/v1/chat/completions`; capture raw stream bytes and parsed client behavior.
- **Expected outcome**: the partial-close case yields a valid prefix of the transcript followed by observable premature termination; the malformed/truncated case yields an invalid SSE/body artifact rather than a well-formed completion. Both failures are visible at the HTTP/SSE surface, not only through internal state. Source of truth: approved strategy failure matrix, implementation plan task 7 and execution notes.
- **Interactions**: streaming body writer, SSE encoder, abrupt connection teardown, lower-level HTTP client behavior.
13. **Name**: optional live OpenAI differential checks preserve required protocol shape for the supported matrix
- **Type**: differential
- **Disposition**: new
- **Harness**: Optional live OpenAI differential harness
- **Preconditions**: explicit real OpenAI credentials are available outside required CI; normalized request fixtures exist only for the documented supported matrix.
- **Actions**: send paired non-stream and stream requests for supported `responses` and `chat.completions` cases to `twin-openai` and the real OpenAI API; compare status classes, required fields, SSE event ordering, and header behavior while ignoring literal text.
- **Expected outcome**: for the documented supported matrix, `twin-openai` matches the real API on protocol shape closely enough that generic OpenAI clients can interact with it; any intentional deviations are recorded in the compatibility matrix rather than hidden. Source of truth: approved strategy reference-comparison harness, official OpenAI API docs, real API as reference implementation.
- **Interactions**: external OpenAI service, local compatibility matrix, normalization/comparison utilities.
14. **Name**: localhost success paths stay fast enough to catch accidental blocking or deadlock
- **Type**: invariant
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running locally with default deterministic behavior and no injected delays.
- **Actions**: time a representative non-stream `/v1/responses` request, a representative streaming `/v1/responses` request until first event and completion, and a representative `/v1/chat/completions` request.
- **Expected outcome**: success-path requests complete comfortably under the generous local thresholds from the approved strategy, and first stream event arrives quickly enough to catch catastrophic async blocking rather than normal variance. Source of truth: approved strategy performance section.
- **Interactions**: full request path, async scheduling, streaming flush behavior, serialization.
## Coverage summary
Covered action space:
- `GET /healthz`
- `POST /v1/responses` with and without auth
- `POST /v1/responses` non-stream success
- `POST /v1/responses` stream success
- `POST /v1/responses` with `metadata`
- `POST /v1/responses` with `stop`
- `POST /v1/responses` with `previous_response_id`
- `POST /v1/responses` with reasoning options
- `POST /v1/responses` with image inputs
- `POST /v1/responses` with `json_object`
- `POST /v1/responses` with supported `json_schema`
- `POST /v1/responses` with unsupported schema or unsupported field combinations
- `POST /v1/responses` scripted tool-call turn
- `POST /v1/responses` continuation turn with `function_call_output`
- `POST /v1/chat/completions` with and without auth
- `POST /v1/chat/completions` non-stream success
- `POST /v1/chat/completions` stream success
- `POST /v1/chat/completions` with `tools`
- `POST /v1/chat/completions` with `tool_choice`
- `POST /v1/chat/completions` with `response_format`
- `POST /v1/chat/completions` with `stop`
- `POST /v1/chat/completions` with reasoning-bearing assistant content
- `POST /v1/chat/completions` with unsupported field combinations
- `POST /__admin/scenarios`
- `POST /__admin/reset`
- `GET /__admin/requests`
- Scripted status-error variants
- Scripted quota/content-filter variants
- Scripted `Retry-After`
- Scripted delayed first byte
- Scripted hang forever
- Scripted partial stream then close
- Scripted malformed or truncated SSE
Explicit exclusions per the agreed strategy:
- Any downstream-consumer black-box tests, fixtures, scripts, CI jobs, or docs.
- Any OpenAI endpoint outside the documented supported phase-one matrix.
- Undocumented compatibility quirks not captured by the official docs or the optional live differential suite.
- Production-scale performance benchmarking; only generous local guardrail timing assertions are included.
Risk carried by exclusions:
- Generic client compatibility outside the documented matrix may still drift until exercised by the optional differential suite.
- Consumers relying on undocumented OpenAI edge behavior may discover gaps that this repository intentionally does not claim to support.

579
test/twin/openai/docs/plans/2026-03-27-async-rust-fake-openai.md
View file

@ -1,579 +0,0 @@
# Async Rust Fake OpenAI Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use trycycle-executing to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Build a standalone async Rust HTTP service that is OpenAI-compatible for the supported endpoints, deterministic without using a real LLM, and able to script protocol and transport failures for black-box end-to-end testing.
**Architecture:** Implement a single `tokio` + `axum` server with one canonical internal turn/stream model that feeds both `/v1/responses` and `/v1/chat/completions`. Drive behavior through a neutral in-memory scenario engine plus admin control endpoints so success and failure paths are deterministic, consumer-agnostic, and identical across streaming and non-streaming transports.
**Tech Stack:** Rust, Tokio, Axum, Hyper, Serde, Reqwest, futures-util, uuid, tracing, anyhow/thiserror.
---
## Scope and Product Contract
This repository stays consumer-agnostic. It must not mention, import, script against, or test against any downstream application. The product is the fake OpenAI server itself.
The supported steady-state surface should be:
- `POST /v1/responses`
- `POST /v1/chat/completions`
- `GET /healthz`
- `POST /__admin/scenarios`
- `POST /__admin/reset`
- `GET /__admin/requests`
The service should require a bearer token for `/v1/*` routes and ignore its value beyond presence, so generic clients can use any non-empty API key. The admin routes should be intentionally local-test-focused and unauthenticated by default.
Within those two OpenAI endpoints, phase-one compatibility must explicitly cover the request features this fake is expected to handle in practice: bearer auth, optional `OpenAI-Organization` and `OpenAI-Project` headers, `stream`, tools, `tool_choice`, `stop`, `metadata`, `previous_response_id`, reasoning requests, text response formats (`text`, `json_object`, `json_schema` subset), and image inputs. Anything outside that matrix must fail clearly and be documented as unsupported.
### User-visible behavior
- Non-streaming endpoints return valid OpenAI-shaped JSON with deterministic IDs, timestamps, usage objects, and output items.
- Streaming endpoints emit valid SSE with flush-per-event behavior and proper terminal completion semantics for the supported endpoint.
- If a scenario is preloaded through the admin API, the next matching OpenAI request consumes that scenario exactly once and responds according to its script.
- If no scripted scenario matches, the server falls back to a documented deterministic default behavior so the fake remains usable without setup.
- Tool-call and continuation flows are supported through the canonical internal turn model rather than endpoint-specific hacks.
- Failure injection can simulate application errors and transport failures including hangs, delayed first byte, partial stream then close, and malformed/truncated stream bodies.
### Required invariants
- The same internal response plan drives both stream and non-stream rendering for a given request.
- The same internal response plan drives both `/v1/responses` and `/v1/chat/completions` when the underlying behavior is equivalent.
- Scenario matching and consumption are deterministic and thread-safe.
- Request logs are append-only during a test run and resettable through admin control.
- Unsupported request shapes fail explicitly with stable error JSON; they must not silently degrade into a misleading success.
- No code or docs in this repository may reference downstream projects or embed downstream-specific fixtures.
## Strategy Gate
The clean path is to treat this as a server product, not a pile of endpoint stubs. The central design decision is to introduce a canonical internal "response plan" that represents output text, tool calls, usage, and failure timing once, then render it into each OpenAI surface. That avoids the most likely long-term bug: streaming and non-streaming behavior drifting apart, or `/v1/responses` and `/v1/chat/completions` diverging because they were implemented separately.
The other key decision is to build a neutral admin control plane instead of encoding behavior in magic prompts or downstream-specific conventions. A scripted scenario queue is more explicit, easier to test, and robust enough to drive hangs, truncation, and delayed chunks without polluting the OpenAI-compatible surface.
## File Structure
Create and own the code with these boundaries:
- `Cargo.toml`: crate metadata and dependencies.
- `src/lib.rs`: public server bootstrap API for tests and the binary.
- `src/main.rs`: runtime entrypoint and environment-based config loading.
- `src/config.rs`: bind address, auth mode, and admin-route configuration.
- `src/app.rs`: router construction and shared state wiring.
- `src/state.rs`: top-level application state and synchronization primitives.
- `src/openai/mod.rs`: route registration and shared endpoint helpers.
- `src/openai/auth.rs`: bearer-token enforcement for `/v1/*`.
- `src/openai/models.rs`: serde request/response models and shared validation helpers.
- `src/openai/responses.rs`: `/v1/responses` handler and renderer adapter.
- `src/openai/chat_completions.rs`: `/v1/chat/completions` handler and renderer adapter.
- `src/engine/mod.rs`: orchestration entrypoint from HTTP requests into deterministic execution.
- `src/engine/scenario.rs`: scenario definition, matchers, and one-shot consumption rules.
- `src/engine/defaults.rs`: deterministic fallback behavior when no scenario matches.
- `src/engine/plan.rs`: canonical internal response plan and stream event plan.
- `src/engine/failures.rs`: modeled transport/application failure behaviors and timing.
- `src/admin.rs`: admin routes for scenario load/reset/request-log retrieval.
- `src/logs.rs`: request-log structures and admin serialization.
- `src/sse.rs`: SSE encoding, chunk flushing, and stream completion helpers.
- `tests/common/mod.rs`: ephemeral server harness, admin helpers, and HTTP/SSE client helpers.
- `tests/health_and_auth.rs`: healthcheck plus auth and malformed-request coverage.
- `tests/responses_contract.rs`: `/v1/responses` success and validation cases.
- `tests/chat_completions_contract.rs`: `/v1/chat/completions` success and validation cases.
- `tests/tool_and_schema_contract.rs`: tool calls, continuation, and structured output cases.
- `tests/failure_modes.rs`: status-code errors, retry headers, hangs, partial streams, truncation, and malformed SSE.
- `README.md`: generic usage, supported surface, and local run instructions.
- `docs/compatibility-matrix.md`: explicit supported request fields, unsupported fields, and failure-model capabilities.
## Contracts and Boundaries To Lock Down Before Coding
1. Canonical plan model
- Represent one request result as a `ResponsePlan`.
- Include output text segments, tool calls, optional structured JSON payload, usage data, delays, and terminal status.
- Make stream rendering a pure projection of `ResponsePlan`, never a separate business path.
2. Scenario scripting model
- A scenario should contain a matcher and a response script.
- Matchers should support endpoint, model name, stream flag, and optional metadata tags or request-substring checks.
- Scripts should support: success payload, OpenAI-style error response, delay before headers, delay between chunks, hang forever, close after N chunks, and malformed/truncated final body.
- Consumption should be FIFO among matching scenarios so tests can script multi-call flows deterministically.
3. Deterministic default behavior
- The fallback path should not require admin setup.
- Default text behavior: extract user text inputs/messages, normalize whitespace, and return a stable synthetic summary/echo form.
- Default tool behavior: do not infer tool calls from arbitrary prompts; only emit tool calls when a scripted scenario requests them.
- Default structured-output behavior: if a supported `json_schema` response format is requested, return deterministic JSON matching the schema only for the supported primitive/object subset; reject unsupported schema constructs explicitly.
4. Endpoint compatibility
- `/v1/responses` must support both stream and non-stream create flows, including continuation input items such as tool outputs when present.
- `/v1/chat/completions` must support both stream and non-stream chat flows using the same canonical plan.
- `/v1/responses` must accept optional `previous_response_id`, `metadata`, `stop`, reasoning requests, and image inputs without requiring downstream-specific behavior.
- `/v1/chat/completions` must accept `tools`, `tool_choice`, `response_format`, `stop`, and assistant reasoning content in both stream and non-stream modes.
- Request validation should reject unsupported combinations with stable OpenAI-shaped error JSON rather than silently ignoring them.
5. Transport-failure mechanics
- Use real async streaming bodies so timing and truncation are observable over actual sockets.
- Guard hang tests with explicit timeouts in the test harness.
- Keep low-level transport failure code isolated in `engine/failures.rs` and `sse.rs`; handlers should describe failures, not hand-roll socket behavior.
## Task 1: Bootstrap the async Rust service skeleton
**Files:**
- Create: `Cargo.toml`
- Create: `src/lib.rs`
- Create: `src/main.rs`
- Create: `src/config.rs`
- Create: `src/app.rs`
- Create: `src/state.rs`
- Create: `src/admin.rs`
- Create: `src/openai/mod.rs`
- Create: `src/openai/auth.rs`
- Create: `tests/common/mod.rs`
- Create: `tests/health_and_auth.rs`
- Create: `README.md`
- [ ] **Step 1: Identify or write the failing test**
Write integration tests that prove the server boots, `GET /healthz` returns `200`, `/v1/*` rejects missing bearer auth, and a syntactically valid authenticated request currently fails because the endpoint handlers are not implemented yet.
```rust
#[tokio::test]
async fn healthz_is_available() { /* ... */ }
#[tokio::test]
async fn responses_requires_bearer_auth() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test healthz_is_available -- --exact`
Expected: FAIL because the crate and test harness do not exist yet.
- [ ] **Step 3: Write minimal implementation**
Create the crate, wire `tokio` + `axum`, add application state, implement `GET /healthz`, add auth middleware for `/v1/*`, expose an app-construction function from `src/lib.rs`, and add a minimal `README.md` that describes the service generically.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test healthz_is_available responses_requires_bearer_auth -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Tighten config loading and router composition, then run the targeted tests and the current full suite.
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add Cargo.toml src tests README.md
git commit -m "feat: bootstrap async rust fake openai server"
```
## Task 2: Define request models and the canonical response-plan engine
**Files:**
- Create: `src/openai/models.rs`
- Create: `src/engine/mod.rs`
- Create: `src/engine/plan.rs`
- Create: `src/engine/defaults.rs`
- Modify: `src/openai/mod.rs`
- Modify: `src/state.rs`
- Create: `tests/responses_contract.rs`
- [ ] **Step 1: Identify or write the failing test**
Write `/v1/responses` contract tests for authenticated non-streaming requests that should return deterministic text, stable `response.id` formatting, output items, usage fields, and acceptance of the request fields the service must support generically: `metadata`, `stop`, `previous_response_id`, optional org/project headers, and image inputs.
```rust
#[tokio::test]
async fn responses_create_returns_deterministic_non_stream_payload() { /* ... */ }
#[tokio::test]
async fn responses_accepts_supported_openai_request_fields() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact`
Expected: FAIL with `404` or unimplemented handler.
- [ ] **Step 3: Write minimal implementation**
Add serde request/response models for the supported `/v1/responses` subset, including `metadata`, `stop`, `previous_response_id`, reasoning request fields, and image inputs. Implement `ResponsePlan`, deterministic fallback extraction of user text input, stable IDs/timestamps per response, and a valid non-stream JSON payload from a real handler. Accept org/project headers as inert compatibility inputs rather than rejecting them.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Move validation and fallback rendering into focused engine modules so handlers stay thin, then run targeted checks plus the broader suite.
Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact`
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/openai/models.rs src/engine src/openai/mod.rs src/state.rs tests/responses_contract.rs
git commit -m "feat: add canonical response plan and responses api baseline"
```
## Task 3: Add streaming for `/v1/responses` from the canonical plan
**Files:**
- Create: `src/sse.rs`
- Modify: `src/openai/responses.rs`
- Modify: `src/engine/plan.rs`
- Modify: `src/engine/defaults.rs`
- Modify: `tests/common/mod.rs`
- Modify: `tests/responses_contract.rs`
- [ ] **Step 1: Identify or write the failing test**
Extend the `/v1/responses` tests to assert valid SSE framing, ordered events, flush behavior, and terminal completion for `stream=true`, including reasoning deltas and tool-call-related event sequences when the canonical plan contains them.
```rust
#[tokio::test]
async fn responses_stream_emits_expected_sse_sequence() { /* ... */ }
#[tokio::test]
async fn responses_stream_emits_reasoning_and_completion_events() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact`
Expected: FAIL because streaming is not implemented or event order is wrong.
- [ ] **Step 3: Write minimal implementation**
Add a streaming renderer that projects `ResponsePlan` into SSE events, including terminal completion, reasoning events, and tool-call-related events required by the supported `/v1/responses` subset. Keep the non-stream and stream paths backed by the same `ResponsePlan` instance so content does not drift.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Extract reusable SSE helpers and add assertions that the streamed text and non-stream text are semantically identical for the same request.
Run: `cargo test responses_stream_emits_expected_sse_sequence -- --exact`
Run: `cargo test responses_create_returns_deterministic_non_stream_payload -- --exact`
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/sse.rs src/openai/responses.rs src/engine tests/common/mod.rs tests/responses_contract.rs
git commit -m "feat: stream responses api events from canonical plans"
```
## Task 4: Add scenario scripting and request logging through the admin API
**Files:**
- Create: `src/engine/scenario.rs`
- Create: `src/logs.rs`
- Modify: `src/admin.rs`
- Modify: `src/state.rs`
- Modify: `src/engine/mod.rs`
- Modify: `src/engine/defaults.rs`
- Create: `tests/tool_and_schema_contract.rs`
- [ ] **Step 1: Identify or write the failing test**
Write admin tests that preload one-shot scenarios, verify that matching OpenAI requests consume them in FIFO order, and verify that `GET /__admin/requests` returns normalized request logs.
```rust
#[tokio::test]
async fn admin_loaded_scenarios_are_consumed_fifo() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact`
Expected: FAIL because admin scripting and logs do not exist yet.
- [ ] **Step 3: Write minimal implementation**
Implement scenario definitions, matcher evaluation, atomic one-shot consumption, request logging, `/__admin/scenarios`, `/__admin/reset`, and `/__admin/requests`. Make matching generic: endpoint, model, stream flag, metadata tags, and text-substring checks are enough for phase one.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Ensure the admin wire format is stable and documented, then re-run the targeted checks and full suite.
Run: `cargo test admin_loaded_scenarios_are_consumed_fifo -- --exact`
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/engine/scenario.rs src/logs.rs src/admin.rs src/state.rs src/engine tests/tool_and_schema_contract.rs
git commit -m "feat: add scenario scripting and request log admin api"
```
## Task 5: Implement tool calls, continuation, and structured outputs
**Files:**
- Modify: `src/openai/models.rs`
- Modify: `src/openai/responses.rs`
- Modify: `src/engine/plan.rs`
- Modify: `src/engine/scenario.rs`
- Modify: `src/engine/defaults.rs`
- Modify: `tests/tool_and_schema_contract.rs`
- Create: `docs/compatibility-matrix.md`
- [ ] **Step 1: Identify or write the failing test**
Add tests for:
- scripted tool-call output on `/v1/responses`
- continuation input that includes tool output items
- reasoning-bearing assistant turns and `previous_response_id` continuation acceptance
- deterministic `json_schema` structured output for the supported subset
- deterministic `json_object` output for the simpler structured-output mode
- explicit rejection of unsupported schema constructs
```rust
#[tokio::test]
async fn responses_supports_scripted_tool_call_and_continuation() { /* ... */ }
#[tokio::test]
async fn responses_structured_output_support_is_explicit() { /* ... */ }
#[tokio::test]
async fn responses_reasoning_and_continuation_fields_round_trip() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact`
Run: `cargo test responses_structured_output_support_is_explicit -- --exact`
Expected: FAIL because tool and schema flows are incomplete.
- [ ] **Step 3: Write minimal implementation**
Extend the canonical plan model to carry tool calls, reasoning segments, and structured JSON outputs. Support scripted tool-call emissions and continuation inputs on `/v1/responses`, including `function_call_output` items and optional `previous_response_id`. Implement explicit `json_object` support plus `json_schema` support for a documented subset only, and reject everything else with stable OpenAI-shaped errors.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact`
Run: `cargo test responses_structured_output_support_is_explicit -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Update the compatibility matrix so supported and unsupported fields are unambiguous. Then re-run all response and tool tests plus the full suite.
Run: `cargo test responses_supports_scripted_tool_call_and_continuation -- --exact`
Run: `cargo test responses_structured_output_support_is_explicit -- --exact`
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/openai/models.rs src/openai/responses.rs src/engine docs/compatibility-matrix.md tests/tool_and_schema_contract.rs
git commit -m "feat: support scripted tool calls and structured outputs"
```
## Task 6: Add `/v1/chat/completions` on the same engine
**Files:**
- Create: `src/openai/chat_completions.rs`
- Modify: `src/openai/mod.rs`
- Modify: `src/openai/models.rs`
- Modify: `src/engine/plan.rs`
- Create: `tests/chat_completions_contract.rs`
- [ ] **Step 1: Identify or write the failing test**
Write non-stream and stream contract tests for `/v1/chat/completions` proving that the same scenario/default behavior can be rendered into chat-completion JSON and delta SSE events while accepting the supported request features for this endpoint: tools, `tool_choice`, `response_format`, `stop`, and reasoning content.
```rust
#[tokio::test]
async fn chat_completions_non_stream_uses_same_canonical_plan() { /* ... */ }
#[tokio::test]
async fn chat_completions_stream_uses_same_canonical_plan() { /* ... */ }
#[tokio::test]
async fn chat_completions_accepts_supported_openai_compatible_fields() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact`
Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact`
Expected: FAIL because the endpoint is not implemented yet.
- [ ] **Step 3: Write minimal implementation**
Add request/response models for the supported chat-completions subset and render chat success and stream deltas from the existing canonical plan instead of adding a second behavior engine. Ensure tool calls, `tool_choice`, `response_format`, `stop`, and assistant reasoning content are accepted and rendered consistently between stream and non-stream paths.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact`
Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Eliminate duplication between the endpoint adapters and verify that cross-endpoint equivalence holds where expected.
Run: `cargo test chat_completions_non_stream_uses_same_canonical_plan -- --exact`
Run: `cargo test chat_completions_stream_uses_same_canonical_plan -- --exact`
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/openai/chat_completions.rs src/openai/mod.rs src/openai/models.rs src/engine/plan.rs tests/chat_completions_contract.rs
git commit -m "feat: add chat completions compatibility surface"
```
## Task 7: Implement application and transport failure injection
**Files:**
- Create: `src/engine/failures.rs`
- Modify: `src/engine/scenario.rs`
- Modify: `src/admin.rs`
- Modify: `src/sse.rs`
- Modify: `tests/common/mod.rs`
- Create: `tests/failure_modes.rs`
- Modify: `docs/compatibility-matrix.md`
- Modify: `README.md`
- [ ] **Step 1: Identify or write the failing test**
Add real-socket tests for:
- OpenAI-shaped JSON errors with status `400`, `401`, `403`, `404`, `408`, `413`, `429`, `500`, `502`, `503`, `504`
- quota-exceeded and content-filter error bodies that classify differently from generic invalid requests
- `Retry-After` propagation on scripted rate limits
- delayed first byte
- hang forever guarded by timeout
- partial SSE then close
- malformed/truncated SSE body
```rust
#[tokio::test]
async fn scripted_hang_times_out_client_side() { /* ... */ }
#[tokio::test]
async fn scripted_partial_stream_then_close_is_observable() { /* ... */ }
#[tokio::test]
async fn scripted_budget_and_content_filter_errors_are_distinct() { /* ... */ }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test scripted_hang_times_out_client_side -- --exact`
Run: `cargo test scripted_partial_stream_then_close_is_observable -- --exact`
Expected: FAIL because failure injection is not implemented yet.
- [ ] **Step 3: Write minimal implementation**
Implement failure scripts for status errors, quota/content-filter error variants, header delays, inter-event delays, hangs, close-after-N-chunks, and malformed/truncated stream endings. Support explicit `Retry-After` control for rate-limit scenarios. Keep these paths data-driven through scenarios rather than hard-coded test hooks.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test scripted_hang_times_out_client_side -- --exact`
Run: `cargo test scripted_partial_stream_then_close_is_observable -- --exact`
Expected: PASS
- [ ] **Step 5: Refactor and verify**
Make failure behavior documentation explicit and re-run the full suite to ensure the low-level transport code has not broken happy paths.
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add src/engine/failures.rs src/engine/scenario.rs src/admin.rs src/sse.rs tests/common/mod.rs tests/failure_modes.rs docs/compatibility-matrix.md README.md
git commit -m "feat: add scripted application and transport failures"
```
## Task 8: Final polish, docs, and whole-project verification
**Files:**
- Modify: `README.md`
- Modify: `docs/compatibility-matrix.md`
- Modify: any touched source or tests needed for final cleanup
- [ ] **Step 1: Identify or write the failing test**
Identify any remaining gaps from the final full-suite pass. If no behavioral gap remains, treat the failing check as documentation incompleteness: make sure the README explains local startup, admin scripting, and the supported compatibility matrix clearly enough that a new engineer can run it without guesswork.
- [ ] **Step 2: Run test to verify it fails**
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: any remaining issues are real and concrete; if all pass, proceed directly to Step 3 as a docs-and-cleanup pass.
- [ ] **Step 3: Write minimal implementation**
Resolve any final defects, tighten docs, confirm unsupported behaviors are explicitly documented, and remove dead code or duplicated helpers introduced during the build-out.
- [ ] **Step 4: Run test to verify it passes**
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 5: Refactor and verify**
Run one final end-to-end verification pass on the complete repository.
Run: `cargo test`
Run: `cargo fmt --check`
Run: `cargo clippy --all-targets -- -D warnings`
Expected: all PASS
- [ ] **Step 6: Commit**
```bash
git add README.md docs/compatibility-matrix.md src tests
git commit -m "docs: finalize fake openai compatibility and usage guidance"
```
## Execution Notes
- Prefer small focused modules over large endpoint files; the canonical plan is the seam that keeps the codebase understandable.
- Do not add persistence, databases, or external queues. In-memory state is the correct steady-state for deterministic local black-box testing.
- Do not implement undocumented endpoint variants "just in case." Add only the explicitly documented supported surface and fail clearly elsewhere.
- Keep the compatibility matrix concrete. The implementation is only done when the docs and tests name the exact supported request fields and failure scripts for both `/v1/responses` and `/v1/chat/completions`.
- For malformed/truncated stream tests, use lower-level response reading when `reqwest` normalizes away the exact transport symptom.
- Keep timestamps and IDs deterministic enough for assertions without freezing the entire clock globally; injecting a clock/ID generator through state is the clean path.
- Never weaken a valid test to get green. If a transport failure test flakes, fix the timing/control mechanism rather than loosening the assertion into uselessness.

154
test/twin/openai/docs/plans/2026-03-28-debug-web-ui-test-plan.md
View file

@ -1,154 +0,0 @@
# Debug Web UI Test Plan
The agreed testing strategy holds after reconciling it with the implementation plan. The plan adds two new routes (`GET /__debug` and `GET /__debug/state.json`) gated behind the existing `enable_admin` config flag, plus new snapshot types in `src/state.rs` and a `script_kind()` accessor on `ScenarioScript`. No strategy change requiring user approval was identified.
One clarification: the strategy calls for "3-4 HTTP integration tests against the debug endpoint." After reviewing the implementation plan, the highest-value integration tests are: (1) empty-state HTML page serves correctly, (2) JSON state endpoint returns correct snapshot after scenarios are loaded, (3) HTML page reflects loaded scenarios and request logs, and (4) debug routes are hidden when `enable_admin` is false. That is exactly 4 tests. The headless Chrome screenshot and HTML safety check round out the plan at 6 new tests, plus the 50-test regression gate.
## Harness requirements
1. **Ephemeral server harness** (existing: `tests/common/mod.rs`)
- What it does: boots the real `tokio` + `axum` server on an ephemeral localhost port with `enable_admin: true` and `require_auth: true`, exposing `base_url`, authenticated client, and unauthenticated client.
- Tests depending on it: 1 through 6.
2. **Unauthenticated HTTP client** (existing on `TestServer`)
- What it does: the `TestServer.client` field is an unauthenticated `reqwest::Client` used to hit routes that do not require bearer auth (like `/__debug`). No new helper needed.
- Tests depending on it: 1, 2, 3, 4.
3. **Admin scenario harness** (existing on `TestServer`)
- What it does: scripts server state via `enqueue_scenarios()`, drives OpenAI requests via `post_responses()`, and fetches request logs via `request_logs()`.
- Tests depending on it: 2, 3.
4. **Headless Chrome harness** (new, lightweight)
- What it does: launches headless Chrome/Chromium via `std::process::Command` to capture a screenshot of the debug page. Falls back to skip if no Chrome binary is found.
- What it exposes: a PNG file written to a temp path; assertion that the file is non-empty and is a valid PNG (starts with the PNG magic bytes).
- Estimated complexity: low. No new crate; uses `chromium --headless --screenshot` CLI.
- Tests depending on it: 5.
## Test plan
### 1. Name: debug HTML page serves valid HTML with correct content-type on empty state
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness
- **Preconditions**: server is running with `enable_admin: true`; no scenarios loaded; no requests made.
- **Actions**:
- `GET /__debug` using the unauthenticated client.
- **Expected outcome**:
- HTTP status is `200`.
- `content-type` header contains `text/html`.
- Response body contains `<!DOCTYPE html>` (well-formed HTML document).
- Response body contains the page title text `twin-openai` and `debug`.
- Response body contains the empty-state marker text `no active namespaces` (since no scenarios have been loaded and no requests have been made).
- Source of truth: implementation plan (route `GET /__debug`, HTML rendering with empty state indicator).
- **Interactions**: axum routing, `enable_admin` gate, `debug_snapshot()`, `render_html()`.
### 2. Name: debug JSON endpoint returns correct state snapshot after scenarios are loaded and requests are made
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness, Admin scenario harness
- **Preconditions**: server is running with `enable_admin: true`.
- **Actions**:
1. Load two scenarios via `POST /__admin/scenarios` with bearer auth: one `success` script matching `responses` endpoint with model `gpt-test`, and one `error` script matching `responses` endpoint with model `gpt-error`.
2. Make one `POST /v1/responses` request with bearer auth, model `gpt-test`, input `"hello debug"`, `stream: false` -- this consumes the first scenario and logs a request.
3. `GET /__debug/state.json` using the unauthenticated client.
- **Expected outcome**:
- HTTP status is `200`.
- `content-type` header contains `application/json`.
- Response body parses as JSON with a top-level `namespaces` array.
- The namespace array contains exactly one entry whose `key` field starts with `"Bearer:"`.
- That namespace's `scenarios` array has exactly 1 remaining scenario (the `error` script; the `success` script was consumed).
- The remaining scenario has `endpoint: "responses"`, `model: "gpt-error"`, `script_kind: "error"`.
- That namespace's `request_logs` array has exactly 1 entry with `endpoint: "responses"`, `model: "gpt-test"`, `input_text` containing `"hello debug"`.
- Source of truth: implementation plan (JSON API shape, `DebugSnapshot` / `NamespaceSnapshot` / `ScenarioSnapshot` structs, FIFO consumption model).
- **Interactions**: admin scenario loading, OpenAI responses endpoint, `debug_snapshot()`, JSON serialization.
### 3. Name: debug HTML page reflects loaded scenarios and request logs in rendered output
- **Type**: integration
- **Disposition**: new
- **Harness**: Ephemeral server harness, Admin scenario harness
- **Preconditions**: server is running with `enable_admin: true`.
- **Actions**:
1. Load one `success` scenario via `POST /__admin/scenarios` with bearer auth, matching `responses` endpoint, model `gpt-html-test`.
2. Make one `POST /v1/responses` request with bearer auth, model `gpt-other`, input `"check the page"`, `stream: false` (default behavior, does not consume the scenario because model does not match).
3. `GET /__debug` using the unauthenticated client.
- **Expected outcome**:
- HTTP status is `200`.
- Response body contains the scenario's model name `gpt-html-test` in the rendered HTML (proving scenarios appear).
- Response body contains the text `success` (the `script_kind` of the loaded scenario).
- Response body contains the text `gpt-other` (the model from the request log).
- Response body contains `check the page` (the input text from the request log).
- Response body does NOT contain `no active namespaces` (because there is at least one namespace).
- Source of truth: implementation plan (HTML page structure showing scenarios table and request log table per namespace).
- **Interactions**: admin scenario loading, OpenAI responses endpoint, `debug_snapshot()`, `render_html()`, HTML escaping.
### 4. Name: debug routes are not accessible when enable_admin is false
- **Type**: integration
- **Disposition**: new
- **Harness**: Custom server setup (not the default `spawn_server`, which uses `enable_admin: true`)
- **Preconditions**: server is started with `enable_admin: false` (construct `Config` directly and call `build_app_with_config`).
- **Actions**:
- `GET /__debug` using an unauthenticated client.
- `GET /__debug/state.json` using an unauthenticated client.
- **Expected outcome**:
- Both requests return HTTP `404` (the routes are not registered when admin is disabled).
- Source of truth: implementation plan (`debug_ui::router()` is only merged when `enable_admin` is true, same as `admin::router()`).
- **Interactions**: `app::router()` conditional routing, config flag.
### 5. Name: debug page renders visually in headless Chrome and produces a non-empty screenshot
- **Type**: scenario
- **Disposition**: new
- **Harness**: Ephemeral server harness, Headless Chrome harness
- **Preconditions**: server is running with `enable_admin: true`; headless Chrome/Chromium is available on PATH. Test is skipped if Chrome is not found.
- **Actions**:
1. Load one scenario and make one request (to populate state for a non-trivial render).
2. Run headless Chrome: `chromium --headless --disable-gpu --screenshot=/tmp/<unique>.png --window-size=1280,900 <base_url>/__debug` (or `google-chrome` / `chromium-browser` depending on platform).
3. Read the output PNG file.
- **Expected outcome**:
- The Chrome process exits with code 0.
- The screenshot file exists and is at least 10 KB (a non-trivial rendered page, not a blank screen).
- The first 8 bytes of the file match the PNG magic number (`\x89PNG\r\n\x1a\n`).
- Source of truth: agreed testing strategy (headless Chrome screenshot capture, 1 test case).
- **Interactions**: full server stack, HTML rendering, CSS rendering in a real browser engine.
### 6. Name: HTML output escapes user-controlled values to prevent injection
- **Type**: unit
- **Disposition**: new
- **Harness**: Ephemeral server harness, Admin scenario harness
- **Preconditions**: server is running with `enable_admin: true`.
- **Actions**:
1. Load a scenario via `POST /__admin/scenarios` where the model field contains an HTML injection attempt: `<script>alert('xss')</script>`.
2. `GET /__debug` using the unauthenticated client.
- **Expected outcome**:
- HTTP status is `200`.
- Response body contains the escaped form `&lt;script&gt;` (proving the `escape_html` function is applied).
- Response body does NOT contain the literal unescaped string `<script>alert` (proving no raw injection).
- Source of truth: implementation plan (`escape_html` helper replaces `<`, `>`, `&`, `"`, `'` with entities), agreed testing strategy (HTML safety check).
- **Interactions**: admin scenario loading, `render_html()`, `escape_html()`.
### 7. Name: all 50 existing tests pass as regression gate
- **Type**: regression
- **Disposition**: existing
- **Harness**: `cargo test`
- **Preconditions**: all implementation changes are complete.
- **Actions**:
- Run `cargo test` (the full test suite).
- **Expected outcome**:
- All 50 existing tests pass. Zero failures, zero errors.
- The new debug UI code does not break any existing endpoint behavior, auth handling, scenario consumption, streaming, or admin routes.
- Source of truth: agreed testing strategy (existing test suite as regression gate).
- **Interactions**: entire codebase.
## Test file placement
All new tests should be placed in a single new file: `tests/debug_ui.rs`. This follows the existing convention where each test file in `tests/` covers a distinct feature area (e.g., `tests/health_and_auth.rs`, `tests/responses_contract.rs`, `tests/failure_modes.rs`). The file should `mod common;` to reuse the existing test server harness.
For test 4 (admin disabled), the test should construct a custom `Config` with `enable_admin: false` and use `twin_openai::build_app_with_config()` directly, similar to the pattern in `tests/config_contract.rs`.
For test 5 (headless Chrome), the test should detect available Chrome binaries at the start and skip (return early with a message) if none are found, ensuring CI environments without Chrome do not fail.

50
test/twin/openai/src/admin.rs
View file

@ -1,50 +0,0 @@
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use serde_json::json;
use crate::engine::scenario::ScenarioEnvelope;
use crate::openai::auth;
use crate::state::AppState;
pub fn router() -> Router<AppState> {
Router::new()
.route("/__admin/scenarios", post(load_scenarios))
.route("/__admin/requests", get(request_logs))
.route("/__admin/reset", post(reset))
}
async fn load_scenarios(
State(state): State<AppState>,
headers: HeaderMap,
Json(payload): Json<ScenarioEnvelope>,
) -> Response {
let namespace = match auth::admin_request_namespace(&headers) {
Ok(namespace) => namespace,
Err(response) => return response,
};
state.enqueue_scenarios(&namespace, payload.scenarios);
Json(json!({ "status": "ok" })).into_response()
}
async fn request_logs(State(state): State<AppState>, headers: HeaderMap) -> Response {
let namespace = match auth::admin_request_namespace(&headers) {
Ok(namespace) => namespace,
Err(response) => return response,
};
Json(json!({ "requests": state.request_logs(&namespace) })).into_response()
}
async fn reset(State(state): State<AppState>, headers: HeaderMap) -> Response {
let namespace = match auth::admin_request_namespace(&headers) {
Ok(namespace) => namespace,
Err(response) => return response,
};
state.reset(&namespace);
Json(json!({ "status": "ok" })).into_response()
}

23
test/twin/openai/src/app.rs
View file

@ -1,23 +0,0 @@
use axum::response::IntoResponse;
use axum::routing::get;
use axum::{Json, Router};
use serde_json::json;
use crate::state::AppState;
use crate::{admin, debug_ui, openai};
pub fn router(state: AppState) -> Router {
let mut router = Router::new()
.route("/healthz", get(healthz))
.nest("/v1", openai::router(state.config.require_auth));
if state.config.enable_admin {
router = router.merge(admin::router()).merge(debug_ui::router());
}
router.with_state(state)
}
async fn healthz() -> impl IntoResponse {
Json(json!({ "status": "ok" }))
}

66
test/twin/openai/src/config.rs
View file

@ -1,66 +0,0 @@
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use anyhow::{Context, Result};
use fabro_static::EnvVars;
#[derive(Clone, Debug)]
pub struct Config {
pub bind_addr: SocketAddr,
pub require_auth: bool,
pub enable_admin: bool,
}
impl Config {
pub fn from_env() -> Result<Self> {
Self::from_lookup(&process_env_var)
}
pub fn from_lookup(lookup: &dyn Fn(&str) -> Option<String>) -> Result<Self> {
let bind_addr = lookup(EnvVars::TWIN_OPENAI_BIND_ADDR)
.map(|value| value.parse().context("invalid TWIN_OPENAI_BIND_ADDR"))
.transpose()?
.unwrap_or_else(|| SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 3000));
let require_auth = lookup(EnvVars::TWIN_OPENAI_REQUIRE_AUTH)
.map(|value| parse_bool_env(&value, EnvVars::TWIN_OPENAI_REQUIRE_AUTH))
.transpose()?
.unwrap_or(true);
let enable_admin = lookup(EnvVars::TWIN_OPENAI_ENABLE_ADMIN)
.map(|value| parse_bool_env(&value, EnvVars::TWIN_OPENAI_ENABLE_ADMIN))
.transpose()?
.unwrap_or(true);
Ok(Self {
bind_addr,
require_auth,
enable_admin,
})
}
}
#[expect(
clippy::disallowed_methods,
reason = "twin-openai config owns a process-env lookup facade for its test server settings."
)]
fn process_env_var(name: &str) -> Option<String> {
std::env::var(name).ok()
}
impl Default for Config {
fn default() -> Self {
Self::from_env().unwrap_or(Self {
bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 3000),
require_auth: true,
enable_admin: true,
})
}
}
fn parse_bool_env(value: &str, name: &str) -> Result<bool> {
match value {
"true" | "1" => Ok(true),
"false" | "0" => Ok(false),
_ => anyhow::bail!("{name} must be true/false or 1/0"),
}
}

447
test/twin/openai/src/debug_ui.rs
View file

@ -1,447 +0,0 @@
use std::fmt::Write;
use axum::extract::State;
use axum::response::{Html, IntoResponse};
use axum::routing::get;
use axum::{Json, Router};
use crate::logs::RequestLog;
use crate::state::{AppState, DebugSnapshot, NamespaceSnapshot, ScenarioSnapshot};
pub fn router() -> Router<AppState> {
Router::new()
.route("/__debug", get(debug_page))
.route("/__debug/state.json", get(debug_state_json))
}
async fn debug_page(State(state): State<AppState>) -> impl IntoResponse {
let snapshot = state.debug_snapshot();
Html(render_html(&snapshot))
}
async fn debug_state_json(State(state): State<AppState>) -> impl IntoResponse {
Json(state.debug_snapshot())
}
fn escape_html(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&#39;")
}
fn truncate(s: &str, max_len: usize) -> String {
if s.len() > max_len {
format!("{}...", &s[..max_len])
} else {
s.to_owned()
}
}
fn render_html(snapshot: &DebugSnapshot) -> String {
let mut html = String::with_capacity(8192);
// Count totals for summary
let ns_count = snapshot.namespaces.len();
let sc_count: usize = snapshot
.namespaces
.iter()
.map(|ns| ns.scenarios.len())
.sum();
let rq_count: usize = snapshot
.namespaces
.iter()
.map(|ns| ns.request_logs.len())
.sum();
html.push_str(
r#"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>twin-openai // debug</title>
<style>"#,
);
html.push_str(CSS);
html.push_str(
r#"</style>
</head>
<body>
<header>
<h1>twin-openai <span class="dim">//</span> debug</h1>
<div class="status">
<span class="live-dot"></span>
<span class="live-label">live</span>
<button onclick="refresh()">refresh now</button>
</div>
</header>
<div class="summary" id="summary">
"#,
);
let _ = write!(
html,
r#" <span id="ns-count">{ns_count} namespaces</span>
<span class="sep">|</span>
<span id="sc-count">{sc_count} queued scenarios</span>
<span class="sep">|</span>
<span id="rq-count">{rq_count} logged requests</span>
"#
);
html.push_str(" </div>\n\n <div id=\"content\">\n");
render_content(&mut html, &snapshot.namespaces);
html.push_str(" </div>\n\n <footer>auto-refreshing every 2s</footer>\n\n <script>\n");
html.push_str(JS);
html.push_str("\n </script>\n</body>\n</html>");
html
}
fn render_content(html: &mut String, namespaces: &[NamespaceSnapshot]) {
if namespaces.is_empty() {
html.push_str(" <p class=\"empty\">(no active namespaces)</p>\n");
return;
}
for ns in namespaces {
let _ = write!(
html,
" <section class=\"namespace\">\n <h2 class=\"namespace-header\">{}</h2>\n",
escape_html(&ns.key)
);
// Queued scenarios
html.push_str(" <h3>queued scenarios</h3>\n");
render_scenarios_table(html, &ns.scenarios);
// Request log
html.push_str(" <h3>request log</h3>\n");
render_requests_table(html, &ns.request_logs);
html.push_str(" </section>\n");
}
}
fn render_scenarios_table(html: &mut String, scenarios: &[ScenarioSnapshot]) {
if scenarios.is_empty() {
html.push_str(" <p class=\"empty\">(no queued scenarios)</p>\n");
return;
}
html.push_str(
" <table>\n <thead><tr>\n <th>#</th><th>endpoint</th><th>model</th><th>stream</th><th>input_contains</th><th>script</th>\n </tr></thead>\n <tbody>\n",
);
for (i, s) in scenarios.iter().enumerate() {
let model = s
.model
.as_deref()
.map_or_else(|| "--".to_owned(), escape_html);
let stream = match s.stream {
Some(v) => format!("{v}"),
None => "--".to_owned(),
};
let input_contains = s
.input_contains
.as_deref()
.map_or_else(|| "--".to_owned(), escape_html);
let _ = writeln!(
html,
" <tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
i + 1,
escape_html(&s.endpoint),
model,
stream,
input_contains,
escape_html(&s.script_kind),
);
}
html.push_str(" </tbody>\n </table>\n");
}
fn render_requests_table(html: &mut String, logs: &[RequestLog]) {
if logs.is_empty() {
html.push_str(" <p class=\"empty\">(no requests logged)</p>\n");
return;
}
html.push_str(
" <table>\n <thead><tr>\n <th>#</th><th>endpoint</th><th>model</th><th>stream</th><th>input text</th><th>metadata</th>\n </tr></thead>\n <tbody>\n",
);
for (i, r) in logs.iter().enumerate() {
let meta = serde_json::to_string(&r.metadata).unwrap_or_else(|_| "{}".to_owned());
let meta_display = if meta == "{}" {
"--".to_owned()
} else {
escape_html(&truncate(&meta, 80))
};
let _ = writeln!(
html,
" <tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
i + 1,
escape_html(&r.endpoint),
escape_html(&r.model),
r.stream,
escape_html(&truncate(&r.input_text, 120)),
meta_display,
);
}
html.push_str(" </tbody>\n </table>\n");
}
const CSS: &str = r#"
:root {
--bg: #0a0a0a;
--bg-row: #111;
--bg-row-alt: #0d0d0d;
--bg-hover: #1a1a0a;
--text: #ccc;
--text-bright: #00ff41;
--text-heading: #ffb000;
--text-dim: #666;
--text-error: #ff4444;
--border: #333;
--border-bright: #555;
}
body {
background: var(--bg);
color: var(--text);
font-family: "IBM Plex Mono", "Fira Code", "Cascadia Code", monospace;
margin: 0;
padding: 24px;
}
header {
display: flex;
align-items: center;
justify-content: space-between;
}
h1 {
color: var(--text-heading);
text-transform: uppercase;
letter-spacing: 0.2em;
font-size: 1.1rem;
margin: 0;
}
h1 .dim {
color: var(--text-dim);
}
h2.namespace-header {
color: var(--text-heading);
border-top: 2px solid var(--border-bright);
padding-top: 16px;
margin-top: 32px;
}
h3 {
color: var(--text-dim);
text-transform: uppercase;
font-size: 0.75rem;
letter-spacing: 0.15em;
}
table {
width: 100%;
border-collapse: collapse;
border: 2px solid var(--border);
}
th {
color: var(--text-heading);
text-transform: uppercase;
font-size: 0.7rem;
letter-spacing: 0.1em;
padding: 6px 10px;
text-align: left;
border-bottom: 2px solid var(--border);
}
td {
color: var(--text-bright);
padding: 5px 10px;
font-size: 0.85rem;
border-bottom: 1px solid var(--border);
}
tr:nth-child(even) {
background: var(--bg-row-alt);
}
tr:hover td {
background: var(--bg-hover);
transition: background 0.15s;
}
.empty {
color: var(--text-error);
font-style: italic;
padding: 8px 0;
}
.summary {
color: var(--text-dim);
margin: 12px 0 24px;
font-size: 0.85rem;
}
.sep {
margin: 0 8px;
color: var(--border-bright);
}
footer {
color: var(--text-dim);
font-size: 0.75rem;
margin-top: 40px;
border-top: 1px solid var(--border);
padding-top: 12px;
}
.status {
display: flex;
align-items: center;
}
.live-label {
color: var(--text-bright);
text-transform: uppercase;
font-size: 0.75rem;
letter-spacing: 0.1em;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
.live-dot {
display: inline-block;
width: 8px;
height: 8px;
background: var(--text-bright);
border-radius: 50%;
animation: pulse 2s ease-in-out infinite;
margin-right: 6px;
vertical-align: middle;
}
button {
background: transparent;
color: var(--text-dim);
border: 1px solid var(--border);
padding: 2px 10px;
font-family: inherit;
font-size: 0.75rem;
cursor: pointer;
text-transform: uppercase;
letter-spacing: 0.1em;
margin-left: 12px;
}
button:hover {
color: var(--text-bright);
border-color: var(--text-bright);
}
"#;
const JS: &str = r#"
async function refresh() {
try {
const res = await fetch('/__debug/state.json');
const data = await res.json();
document.getElementById('content').innerHTML = renderState(data);
updateSummary(data);
} catch(e) { /* silent -- next interval will retry */ }
}
if (new URLSearchParams(window.location.search).get('refresh') !== '0') {
setInterval(refresh, 2000);
}
function renderState(data) {
if (data.namespaces.length === 0) {
return '<p class="empty">(no active namespaces)</p>';
}
return data.namespaces.map(function(ns) {
return '<section class="namespace">'
+ '<h2 class="namespace-header">' + esc(ns.key) + '</h2>'
+ '<h3>queued scenarios</h3>'
+ renderScenariosTable(ns.scenarios)
+ '<h3>request log</h3>'
+ renderRequestsTable(ns.request_logs)
+ '</section>';
}).join('');
}
function renderScenariosTable(scenarios) {
if (scenarios.length === 0) return '<p class="empty">(no queued scenarios)</p>';
var rows = scenarios.map(function(s, i) {
return '<tr>'
+ '<td>' + (i+1) + '</td>'
+ '<td>' + esc(s.endpoint) + '</td>'
+ '<td>' + esc(s.model || '--') + '</td>'
+ '<td>' + (s.stream === null ? '--' : s.stream) + '</td>'
+ '<td>' + esc(s.input_contains || '--') + '</td>'
+ '<td>' + esc(s.script_kind) + '</td>'
+ '</tr>';
}).join('');
return '<table><thead><tr>'
+ '<th>#</th><th>endpoint</th><th>model</th><th>stream</th><th>input_contains</th><th>script</th>'
+ '</tr></thead><tbody>' + rows + '</tbody></table>';
}
function renderRequestsTable(logs) {
if (logs.length === 0) return '<p class="empty">(no requests logged)</p>';
var rows = logs.map(function(r, i) {
var meta = JSON.stringify(r.metadata);
if (meta === '{}') meta = '--';
return '<tr>'
+ '<td>' + (i+1) + '</td>'
+ '<td>' + esc(r.endpoint) + '</td>'
+ '<td>' + esc(r.model) + '</td>'
+ '<td>' + r.stream + '</td>'
+ '<td>' + esc(trunc(r.input_text, 120)) + '</td>'
+ '<td>' + esc(trunc(meta, 80)) + '</td>'
+ '</tr>';
}).join('');
return '<table><thead><tr>'
+ '<th>#</th><th>endpoint</th><th>model</th><th>stream</th><th>input text</th><th>metadata</th>'
+ '</tr></thead><tbody>' + rows + '</tbody></table>';
}
function updateSummary(data) {
var sc = 0, rq = 0;
data.namespaces.forEach(function(ns) {
sc += ns.scenarios.length;
rq += ns.request_logs.length;
});
document.getElementById('ns-count').textContent = data.namespaces.length + ' namespaces';
document.getElementById('sc-count').textContent = sc + ' queued scenarios';
document.getElementById('rq-count').textContent = rq + ' logged requests';
}
function esc(s) {
return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;').replace(/'/g,'&#39;');
}
function trunc(s, n) {
return s.length > n ? s.slice(0, n) + '...' : s;
}
"#;

113
test/twin/openai/src/engine/defaults.rs
View file

@ -1,113 +0,0 @@
use serde_json::{Value, json};
use super::plan::{ResponsePlan, TokenUsage};
use crate::openai::models::{ResponseFormat, ResponsesRequest, normalize_whitespace};
pub fn build_default_response_plan(
response_number: u64,
request: &ResponsesRequest,
) -> ResponsePlan {
let normalized_text = request.extract_user_text();
let response_text = format!("deterministic: {normalized_text}");
let input_tokens = normalized_text.split_whitespace().count() as u64;
let structured_output = request.response_format().and_then(|format| match format {
ResponseFormat::Text => None,
ResponseFormat::JsonObject => Some(json!({
"message": response_text,
"model": request.model,
})),
ResponseFormat::JsonSchema(schema) => {
Some(generate_json_from_schema(&schema, &response_text))
}
});
let reasoning = if request.reasoning.is_some() {
vec![format!("reasoning: {normalized_text}")]
} else {
Vec::new()
};
ResponsePlan {
id: format!("resp_{response_number:06}"),
created: response_number,
model: request.model.clone(),
response_text,
structured_output,
reasoning,
tool_calls: Vec::new(),
usage: TokenUsage::new(input_tokens, 5),
}
}
pub fn build_default_chat_plan(
response_number: u64,
model: String,
input_text: &str,
response_format: Option<ResponseFormat>,
reasoning_requested: bool,
) -> ResponsePlan {
let normalized_text = normalize_whitespace(input_text);
let response_text = format!("deterministic: {normalized_text}");
let structured_output = response_format.and_then(|format| match format {
ResponseFormat::Text => None,
ResponseFormat::JsonObject => Some(json!({
"message": response_text,
"model": model,
})),
ResponseFormat::JsonSchema(schema) => {
Some(generate_json_from_schema(&schema, &response_text))
}
});
let reasoning = if reasoning_requested {
vec![format!("reasoning: {normalized_text}")]
} else {
Vec::new()
};
let input_tokens = normalized_text.split_whitespace().count() as u64;
ResponsePlan {
id: format!("resp_{response_number:06}"),
created: response_number,
model,
response_text,
structured_output,
reasoning,
tool_calls: Vec::new(),
usage: TokenUsage::new(input_tokens, 5),
}
}
fn generate_json_from_schema(schema: &Value, response_text: &str) -> Value {
let schema = schema.get("schema").unwrap_or(schema);
match schema.get("type").and_then(Value::as_str) {
Some("object") => {
let properties = schema
.get("properties")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
let mut object = serde_json::Map::new();
for (name, property_schema) in properties {
object.insert(
name,
primitive_value_for_schema(&property_schema, response_text),
);
}
Value::Object(object)
}
_ => json!({ "message": response_text }),
}
}
fn primitive_value_for_schema(schema: &Value, response_text: &str) -> Value {
match schema.get("type").and_then(Value::as_str) {
Some("string") => Value::String(response_text.to_owned()),
Some("integer") => json!(1),
Some("number") => json!(1.0),
Some("boolean") => json!(true),
Some("object") => generate_json_from_schema(schema, response_text),
_ => Value::Null,
}
}

58
test/twin/openai/src/engine/failures.rs
View file

@ -1,58 +0,0 @@
use axum::http::StatusCode;
use super::plan::ResponsePlan;
use crate::openai::models::{ErrorBody, ErrorEnvelope};
#[derive(Clone, Copy, Debug, Default)]
pub struct TransportOptions {
pub delay_before_headers_ms: u64,
pub inter_event_delay_ms: u64,
pub close_after_chunks: Option<usize>,
pub malformed_sse: bool,
}
#[derive(Clone, Debug)]
pub struct SuccessOutcome {
pub plan: ResponsePlan,
pub transport: TransportOptions,
}
#[derive(Clone, Debug)]
pub struct ErrorOutcome {
pub status: StatusCode,
pub body: ErrorEnvelope,
pub retry_after: Option<String>,
pub delay_before_headers_ms: u64,
}
#[derive(Clone, Debug)]
pub enum ExecutionOutcome {
Success(SuccessOutcome),
Error(ErrorOutcome),
Hang { delay_before_headers_ms: u64 },
}
impl ErrorOutcome {
pub fn new(
status: StatusCode,
message: String,
error_type: String,
code: String,
retry_after: Option<String>,
delay_before_headers_ms: u64,
) -> Self {
Self {
status,
body: ErrorEnvelope {
error: ErrorBody {
message,
error_type,
param: serde_json::Value::Null,
code,
},
},
retry_after,
delay_before_headers_ms,
}
}
}

126
test/twin/openai/src/engine/mod.rs
View file

@ -1,126 +0,0 @@
pub mod defaults;
pub mod failures;
pub mod plan;
pub mod scenario;
use self::defaults::{build_default_chat_plan, build_default_response_plan};
use self::failures::{ExecutionOutcome, SuccessOutcome, TransportOptions};
use self::plan::ResponsePlan;
use self::scenario::RequestContext;
use crate::openai::models::{
ChatCompletionsRequest, OpenAiError, ResponsesRequest, ToolChoiceMode,
};
use crate::state::{AppState, NamespaceKey};
pub fn execute_responses_request(
state: &AppState,
namespace: &NamespaceKey,
request: &ResponsesRequest,
) -> Result<ExecutionOutcome, OpenAiError> {
request.validate()?;
let context = RequestContext {
endpoint: "responses".to_owned(),
model: request.model.clone(),
stream: request.stream,
metadata: request.metadata.clone(),
input_text: request.extract_user_text(),
instructions_text: request.extract_instruction_text(),
};
state.log_request(namespace, context.clone());
if let Some(scenario) = state.take_matching_scenario(namespace, &context) {
return match scenario.execute_for_responses(state.next_response_id(namespace), request) {
ExecutionOutcome::Success(success) => Ok(ExecutionOutcome::Success(
enforce_tool_choice(request.tool_choice_mode(), success)?,
)),
outcome => Ok(outcome),
};
}
Ok(ExecutionOutcome::Success(enforce_tool_choice(
request.tool_choice_mode(),
SuccessOutcome {
plan: build_default_response_plan(state.next_response_id(namespace), request),
transport: TransportOptions::default(),
},
)?))
}
pub fn execute_chat_request(
state: &AppState,
namespace: &NamespaceKey,
request: &ChatCompletionsRequest,
) -> Result<ExecutionOutcome, OpenAiError> {
request.validate()?;
let context = RequestContext {
endpoint: "chat.completions".to_owned(),
model: request.model.clone(),
stream: request.stream,
metadata: serde_json::Map::new(),
input_text: request.extract_user_text(),
instructions_text: request.extract_instruction_text(),
};
state.log_request(namespace, context.clone());
if let Some(scenario) = state.take_matching_scenario(namespace, &context) {
return match scenario.execute_for_chat(state.next_response_id(namespace), request) {
ExecutionOutcome::Success(success) => Ok(ExecutionOutcome::Success(
enforce_tool_choice(request.tool_choice_mode(), success)?,
)),
outcome => Ok(outcome),
};
}
Ok(ExecutionOutcome::Success(enforce_tool_choice(
request.tool_choice_mode(),
SuccessOutcome {
plan: build_default_chat_plan(
state.next_response_id(namespace),
request.model.clone(),
&request.extract_user_text(),
request.response_format(),
request.reasoning_requested(),
),
transport: TransportOptions::default(),
},
)?))
}
fn enforce_tool_choice(
tool_choice: Option<ToolChoiceMode>,
success: SuccessOutcome,
) -> Result<SuccessOutcome, OpenAiError> {
validate_tool_choice_against_plan(tool_choice, &success.plan)?;
Ok(success)
}
fn validate_tool_choice_against_plan(
tool_choice: Option<ToolChoiceMode>,
plan: &ResponsePlan,
) -> Result<(), OpenAiError> {
match tool_choice {
None | Some(ToolChoiceMode::Auto) => Ok(()),
Some(ToolChoiceMode::NoTool) if plan.tool_calls.is_empty() => Ok(()),
Some(ToolChoiceMode::NoTool) => Err(OpenAiError::invalid_request(
"tool_choice",
"tool_choice forbids tool calls for this request",
)),
Some(ToolChoiceMode::Required) if !plan.tool_calls.is_empty() => Ok(()),
Some(ToolChoiceMode::Required) => Err(OpenAiError::invalid_request(
"tool_choice",
"tool_choice required a tool call but none was planned",
)),
Some(ToolChoiceMode::Function(name))
if plan
.tool_calls
.iter()
.any(|tool_call| tool_call.name == name) =>
{
Ok(())
}
Some(ToolChoiceMode::Function(name)) => Err(OpenAiError::invalid_request(
"tool_choice",
&format!("tool_choice requested function `{name}` but it was not planned"),
)),
}
}

163
test/twin/openai/src/engine/plan.rs
View file

@ -1,163 +0,0 @@
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct TokenUsage {
pub input_tokens: u64,
pub output_tokens: u64,
}
impl TokenUsage {
#[must_use]
pub const fn new(input_tokens: u64, output_tokens: u64) -> Self {
Self {
input_tokens,
output_tokens,
}
}
#[must_use]
pub const fn total_tokens(self) -> u64 {
self.input_tokens + self.output_tokens
}
#[must_use]
pub fn responses_json(self) -> Value {
json!({
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"total_tokens": self.total_tokens(),
})
}
#[must_use]
pub fn chat_completions_json(self) -> Value {
json!({
"prompt_tokens": self.input_tokens,
"completion_tokens": self.output_tokens,
"total_tokens": self.total_tokens(),
})
}
}
impl Default for TokenUsage {
fn default() -> Self {
Self::new(1, 5)
}
}
#[derive(Clone, Debug)]
pub struct ResponsePlan {
pub id: String,
pub created: u64,
pub model: String,
pub response_text: String,
pub structured_output: Option<Value>,
pub reasoning: Vec<String>,
pub tool_calls: Vec<ToolCallPlan>,
pub usage: TokenUsage,
}
#[derive(Clone, Debug)]
pub struct ToolCallPlan {
pub id: String,
pub name: String,
pub arguments: Value,
pub raw_arguments: Option<String>,
}
impl ResponsePlan {
pub fn tool_call_arguments_text(tool_call: &ToolCallPlan) -> String {
tool_call
.raw_arguments
.clone()
.unwrap_or_else(|| tool_call.arguments.to_string())
}
fn responses_tool_call_item(tool_call: &ToolCallPlan) -> Value {
json!({
"id": format!("fc_{}", tool_call.id),
"type": "function_call",
"call_id": tool_call.id,
"name": tool_call.name,
"arguments": Self::tool_call_arguments_text(tool_call),
})
}
pub fn chat_content(&self) -> String {
self.structured_output
.as_ref()
.map_or_else(|| self.response_text.clone(), ToString::to_string)
}
pub fn responses_json(&self) -> Value {
let mut content_items = Vec::new();
if !self.response_text.is_empty() {
content_items.push(json!({
"type": "output_text",
"text": self.response_text,
}));
}
if let Some(structured_output) = &self.structured_output {
content_items.push(json!({
"type": "output_json",
"json": structured_output,
}));
}
let mut output = Vec::new();
if !content_items.is_empty() {
output.push(json!({
"id": format!("msg_{}", self.id),
"type": "message",
"role": "assistant",
"content": content_items,
}));
}
for tool_call in &self.tool_calls {
output.push(Self::responses_tool_call_item(tool_call));
}
json!({
"id": self.id,
"object": "response",
"created": self.created,
"model": self.model,
"status": "completed",
"reasoning": self.reasoning,
"output": output,
"usage": self.usage.responses_json()
})
}
pub fn chat_completions_json(&self) -> Value {
json!({
"id": format!("chatcmpl_{}", self.id),
"object": "chat.completion",
"created": self.created,
"model": self.model,
"choices": [{
"index": 0,
"finish_reason": if self.tool_calls.is_empty() { "stop" } else { "tool_calls" },
"message": {
"role": "assistant",
"content": self.chat_content(),
"reasoning": self.reasoning,
"tool_calls": self.tool_calls.iter().map(|tool_call| json!({
"id": tool_call.id,
"type": "function",
"function": {
"name": tool_call.name,
"arguments": Self::tool_call_arguments_text(tool_call),
}
})).collect::<Vec<_>>(),
}
}],
"usage": self.usage.chat_completions_json()
})
}
}

271
test/twin/openai/src/engine/scenario.rs
View file

@ -1,271 +0,0 @@
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use super::failures::{ErrorOutcome, ExecutionOutcome, SuccessOutcome, TransportOptions};
use super::plan::{ResponsePlan, TokenUsage, ToolCallPlan};
use crate::openai::models::{ChatCompletionsRequest, ResponsesRequest};
#[derive(Clone, Debug, Deserialize)]
pub struct ScenarioEnvelope {
pub scenarios: Vec<Scenario>,
}
#[derive(Clone, Debug, Deserialize)]
pub struct Scenario {
pub matcher: ScenarioMatcher,
pub script: ScenarioScript,
}
#[derive(Clone, Debug, Deserialize)]
pub struct ScenarioMatcher {
pub endpoint: String,
pub model: Option<String>,
pub stream: Option<bool>,
#[serde(default)]
pub metadata: Map<String, Value>,
pub input_contains: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ScenarioScript {
Success {
response_text: Option<String>,
reasoning: Option<Vec<String>>,
structured_output: Option<Value>,
tool_calls: Option<Vec<ToolCallTemplate>>,
usage: Option<TokenUsage>,
delay_before_headers_ms: Option<u64>,
inter_event_delay_ms: Option<u64>,
close_after_chunks: Option<usize>,
malformed_sse: Option<bool>,
},
Error {
status: u16,
message: String,
error_type: String,
code: String,
retry_after: Option<String>,
delay_before_headers_ms: Option<u64>,
},
Hang {
delay_before_headers_ms: Option<u64>,
},
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ToolCallTemplate {
pub id: Option<String>,
pub name: String,
pub arguments: Value,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw_arguments: Option<String>,
}
#[derive(Clone, Debug)]
pub struct RequestContext {
pub endpoint: String,
pub model: String,
pub stream: bool,
pub metadata: Map<String, Value>,
pub input_text: String,
pub instructions_text: String,
}
impl ScenarioScript {
pub fn script_kind(&self) -> &str {
match self {
Self::Success { .. } => "success",
Self::Error { .. } => "error",
Self::Hang { .. } => "hang",
}
}
}
impl Scenario {
pub fn matches(&self, request: &RequestContext) -> bool {
if self.matcher.endpoint != request.endpoint {
return false;
}
if let Some(model) = &self.matcher.model {
if model != &request.model {
return false;
}
}
if let Some(stream) = self.matcher.stream {
if stream != request.stream {
return false;
}
}
if let Some(needle) = &self.matcher.input_contains {
if !request.input_text.contains(needle) {
return false;
}
}
self.matcher.metadata.iter().all(|(key, value)| {
request
.metadata
.get(key)
.is_some_and(|candidate| candidate == value)
})
}
pub fn execute_for_responses(
&self,
response_number: u64,
request: &ResponsesRequest,
) -> ExecutionOutcome {
match &self.script {
ScenarioScript::Success {
response_text,
reasoning,
structured_output,
tool_calls,
usage,
delay_before_headers_ms,
inter_event_delay_ms,
close_after_chunks,
malformed_sse,
} => ExecutionOutcome::Success(SuccessOutcome {
plan: build_plan_from_script(
response_number,
request.model.clone(),
&request.extract_user_text(),
response_text.clone(),
reasoning.clone().unwrap_or_default(),
structured_output.clone(),
tool_calls.clone().unwrap_or_default(),
*usage,
),
transport: TransportOptions {
delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(),
inter_event_delay_ms: inter_event_delay_ms.unwrap_or_default(),
close_after_chunks: *close_after_chunks,
malformed_sse: malformed_sse.unwrap_or(false),
},
}),
ScenarioScript::Error {
status,
message,
error_type,
code,
retry_after,
delay_before_headers_ms,
} => ExecutionOutcome::Error(ErrorOutcome::new(
StatusCode::from_u16(*status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
message.clone(),
error_type.clone(),
code.clone(),
retry_after.clone(),
delay_before_headers_ms.unwrap_or_default(),
)),
ScenarioScript::Hang {
delay_before_headers_ms,
} => ExecutionOutcome::Hang {
delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(),
},
}
}
pub fn execute_for_chat(
&self,
response_number: u64,
request: &ChatCompletionsRequest,
) -> ExecutionOutcome {
match &self.script {
ScenarioScript::Success {
response_text,
reasoning,
structured_output,
tool_calls,
usage,
delay_before_headers_ms,
inter_event_delay_ms,
close_after_chunks,
malformed_sse,
} => ExecutionOutcome::Success(SuccessOutcome {
plan: build_plan_from_script(
response_number,
request.model.clone(),
&request.extract_user_text(),
response_text.clone(),
reasoning.clone().unwrap_or_default(),
structured_output.clone(),
tool_calls.clone().unwrap_or_default(),
*usage,
),
transport: TransportOptions {
delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(),
inter_event_delay_ms: inter_event_delay_ms.unwrap_or_default(),
close_after_chunks: *close_after_chunks,
malformed_sse: malformed_sse.unwrap_or(false),
},
}),
ScenarioScript::Error {
status,
message,
error_type,
code,
retry_after,
delay_before_headers_ms,
} => ExecutionOutcome::Error(ErrorOutcome::new(
StatusCode::from_u16(*status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
message.clone(),
error_type.clone(),
code.clone(),
retry_after.clone(),
delay_before_headers_ms.unwrap_or_default(),
)),
ScenarioScript::Hang {
delay_before_headers_ms,
} => ExecutionOutcome::Hang {
delay_before_headers_ms: delay_before_headers_ms.unwrap_or_default(),
},
}
}
}
fn build_plan_from_script(
response_number: u64,
model: String,
default_input: &str,
response_text: Option<String>,
reasoning: Vec<String>,
structured_output: Option<Value>,
tool_calls: Vec<ToolCallTemplate>,
usage: Option<TokenUsage>,
) -> ResponsePlan {
let output_text = match response_text {
Some(response_text) => response_text,
None if tool_calls.is_empty() && structured_output.is_none() => {
format!("deterministic: {default_input}")
}
None => String::new(),
};
ResponsePlan {
id: format!("resp_{response_number:06}"),
created: response_number,
model,
response_text: output_text,
structured_output,
reasoning,
tool_calls: tool_calls
.into_iter()
.enumerate()
.map(|(index, tool_call)| ToolCallPlan {
id: tool_call
.id
.unwrap_or_else(|| format!("call_{response_number}_{index}")),
name: tool_call.name,
arguments: tool_call.arguments,
raw_arguments: tool_call.raw_arguments,
})
.collect(),
usage: usage.unwrap_or_default(),
}
}

26
test/twin/openai/src/lib.rs
View file

@ -1,26 +0,0 @@
#![allow(
clippy::result_large_err,
reason = "Twin HTTP handlers return full axum::Response errors directly."
)]
pub mod admin;
pub mod app;
pub mod config;
pub mod debug_ui;
pub mod engine;
pub mod logs;
pub mod openai;
pub mod sse;
pub mod state;
use axum::Router;
use config::Config;
use state::AppState;
pub fn build_app() -> Router {
build_app_with_config(Config::from_env().unwrap_or_default())
}
pub fn build_app_with_config(config: Config) -> Router {
app::router(AppState::new(config))
}

12
test/twin/openai/src/logs.rs
View file

@ -1,12 +0,0 @@
use serde::Serialize;
use serde_json::{Map, Value};
#[derive(Clone, Debug, Serialize)]
pub struct RequestLog {
pub endpoint: String,
pub model: String,
pub stream: bool,
pub input_text: String,
pub instructions_text: String,
pub metadata: Map<String, Value>,
}

21
test/twin/openai/src/main.rs
View file

@ -1,21 +0,0 @@
use anyhow::Result;
use tokio::net::TcpListener;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use twin_openai::app;
use twin_openai::config::Config;
use twin_openai::state::AppState;
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::registry()
.with(tracing_subscriber::EnvFilter::from_default_env())
.with(fmt::layer())
.init();
let config = Config::from_env()?;
let listener = TcpListener::bind(config.bind_addr).await?;
axum::serve(listener, app::router(AppState::new(config))).await?;
Ok(())
}

67
test/twin/openai/src/openai/auth.rs
View file

@ -1,67 +0,0 @@
use axum::Json;
use axum::extract::Request;
use axum::http::header::AUTHORIZATION;
use axum::http::{HeaderMap, HeaderValue, StatusCode};
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use serde_json::json;
use crate::state::NamespaceKey;
pub async fn require_bearer_auth(request: Request, next: Next) -> Response {
match bearer_token_from_headers(request.headers()) {
Ok(Some(_)) => next.run(request).await,
Ok(None) | Err(()) => missing_bearer_token_response(),
}
}
pub fn openai_request_namespace(
headers: &HeaderMap,
require_auth: bool,
) -> Result<NamespaceKey, Response> {
match bearer_token_from_headers(headers) {
Ok(Some(token)) => Ok(NamespaceKey::Bearer(token)),
Ok(None) if !require_auth => Ok(NamespaceKey::Global),
Ok(None) | Err(()) => Err(missing_bearer_token_response()),
}
}
pub fn admin_request_namespace(headers: &HeaderMap) -> Result<NamespaceKey, Response> {
match bearer_token_from_headers(headers) {
Ok(Some(token)) => Ok(NamespaceKey::Bearer(token)),
Ok(None) => Ok(NamespaceKey::Global),
Err(()) => Err(missing_bearer_token_response()),
}
}
fn bearer_token_from_headers(headers: &HeaderMap) -> Result<Option<String>, ()> {
match headers.get(AUTHORIZATION) {
Some(value) => parse_bearer_token(value).map(Some).ok_or(()),
None => Ok(None),
}
}
fn parse_bearer_token(value: &HeaderValue) -> Option<String> {
let Ok(value) = value.to_str() else {
return None;
};
let token = value.strip_prefix("Bearer ").map(str::trim)?;
(!token.is_empty()).then(|| token.to_owned())
}
fn missing_bearer_token_response() -> Response {
(
StatusCode::UNAUTHORIZED,
Json(json!({
"error": {
"message": "missing or empty bearer token",
"type": "invalid_request_error",
"param": "Authorization",
"code": "missing_bearer_token"
}
})),
)
.into_response()
}

82
test/twin/openai/src/openai/chat_completions.rs
View file

@ -1,82 +0,0 @@
use axum::Json;
use axum::extract::State;
use axum::extract::rejection::JsonRejection;
use axum::http::HeaderMap;
use axum::http::header::RETRY_AFTER;
use axum::response::IntoResponse;
use futures_util::future;
use tokio::time::{Duration, sleep};
use super::models::ChatCompletionsRequest;
use crate::engine::execute_chat_request;
use crate::engine::failures::ExecutionOutcome;
use crate::openai::auth;
use crate::sse::chat_sse_response;
use crate::state::AppState;
pub async fn create_chat_completion(
State(state): State<AppState>,
headers: HeaderMap,
payload: Result<Json<ChatCompletionsRequest>, JsonRejection>,
) -> impl IntoResponse {
let namespace = match auth::openai_request_namespace(&headers, state.config.require_auth) {
Ok(namespace) => namespace,
Err(response) => return response,
};
let request = match payload {
Ok(Json(request)) => request,
Err(rejection) => {
return super::models::OpenAiError::from_json_rejection(&rejection)
.into_response()
.into_response();
}
};
match execute_chat_request(&state, &namespace, &request) {
Ok(ExecutionOutcome::Success(success)) => {
if success.transport.delay_before_headers_ms > 0 {
sleep(Duration::from_millis(
success.transport.delay_before_headers_ms,
))
.await;
}
if request.stream {
chat_sse_response(
&success.plan,
request.include_stream_usage(),
success.transport,
)
.into_response()
} else {
Json(success.plan.chat_completions_json()).into_response()
}
}
Ok(ExecutionOutcome::Error(error)) => {
if error.delay_before_headers_ms > 0 {
sleep(Duration::from_millis(error.delay_before_headers_ms)).await;
}
let mut response = Json(error.body).into_response();
*response.status_mut() = error.status;
if let Some(retry_after) = error.retry_after {
response.headers_mut().insert(
RETRY_AFTER,
retry_after.parse().expect("valid Retry-After header"),
);
}
response
}
Ok(ExecutionOutcome::Hang {
delay_before_headers_ms,
}) => {
if delay_before_headers_ms > 0 {
sleep(Duration::from_millis(delay_before_headers_ms)).await;
}
future::pending::<()>().await;
unreachable!()
}
Err(error) => error.into_response().into_response(),
}
}

24
test/twin/openai/src/openai/mod.rs
View file

@ -1,24 +0,0 @@
pub mod auth;
pub mod chat_completions;
pub mod models;
pub mod responses;
use axum::routing::post;
use axum::{Router, middleware};
use crate::state::AppState;
pub fn router(require_auth: bool) -> Router<AppState> {
let router = Router::new()
.route("/responses", post(responses::create_response))
.route(
"/chat/completions",
post(chat_completions::create_chat_completion),
);
if require_auth {
router.layer(middleware::from_fn(auth::require_bearer_auth))
} else {
router
}
}

1104
test/twin/openai/src/openai/models.rs
View file

File diff suppressed because it is too large Load diff

77
test/twin/openai/src/openai/responses.rs
View file

@ -1,77 +0,0 @@
use axum::Json;
use axum::extract::State;
use axum::extract::rejection::JsonRejection;
use axum::http::HeaderMap;
use axum::http::header::RETRY_AFTER;
use axum::response::IntoResponse;
use futures_util::future;
use tokio::time::{Duration, sleep};
use super::models::ResponsesRequest;
use crate::engine::execute_responses_request;
use crate::engine::failures::ExecutionOutcome;
use crate::openai::auth;
use crate::sse::responses_sse_response;
use crate::state::AppState;
pub async fn create_response(
State(state): State<AppState>,
headers: HeaderMap,
payload: Result<Json<ResponsesRequest>, JsonRejection>,
) -> impl IntoResponse {
let namespace = match auth::openai_request_namespace(&headers, state.config.require_auth) {
Ok(namespace) => namespace,
Err(response) => return response,
};
let request = match payload {
Ok(Json(request)) => request,
Err(rejection) => {
return super::models::OpenAiError::from_json_rejection(&rejection)
.into_response()
.into_response();
}
};
match execute_responses_request(&state, &namespace, &request) {
Ok(ExecutionOutcome::Success(success)) => {
if success.transport.delay_before_headers_ms > 0 {
sleep(Duration::from_millis(
success.transport.delay_before_headers_ms,
))
.await;
}
if request.stream {
responses_sse_response(&success.plan, success.transport).into_response()
} else {
Json(success.plan.responses_json()).into_response()
}
}
Ok(ExecutionOutcome::Error(error)) => {
if error.delay_before_headers_ms > 0 {
sleep(Duration::from_millis(error.delay_before_headers_ms)).await;
}
let mut response = Json(error.body).into_response();
*response.status_mut() = error.status;
if let Some(retry_after) = error.retry_after {
response.headers_mut().insert(
RETRY_AFTER,
retry_after.parse().expect("valid Retry-After header"),
);
}
response
}
Ok(ExecutionOutcome::Hang {
delay_before_headers_ms,
}) => {
if delay_before_headers_ms > 0 {
sleep(Duration::from_millis(delay_before_headers_ms)).await;
}
future::pending::<()>().await;
unreachable!()
}
Err(error) => error.into_response().into_response(),
}
}

385
test/twin/openai/src/sse.rs
View file

@ -1,385 +0,0 @@
use async_stream::stream;
use axum::body::Body;
use axum::http::{HeaderValue, Response, StatusCode, header};
use serde_json::{Value, json};
use tokio::time::{Duration, sleep};
use crate::engine::failures::TransportOptions;
use crate::engine::plan::ResponsePlan;
pub fn responses_sse_response(plan: &ResponsePlan, transport: TransportOptions) -> Response<Body> {
let mut events = Vec::new();
let reasoning_item_id = format!("rs_{}", plan.id);
let message_item_id = format!("msg_{}", plan.id);
let mut next_output_index = 0;
let streamed_text = plan.structured_output.as_ref().map(Value::to_string);
events.push(sse_event(
"response.created",
&json!({
"type": "response.created",
"response": {
"id": plan.id,
"object": "response",
"created": plan.created,
"model": plan.model,
"status": "in_progress",
"output": [],
},
}),
));
events.push(sse_event(
"response.in_progress",
&json!({
"type": "response.in_progress",
"response": {
"id": plan.id,
"object": "response",
"created": plan.created,
"model": plan.model,
"status": "in_progress",
"output": [],
},
}),
));
events.push(sse_event(
"response.output_item.added",
&json!({
"type": "response.output_item.added",
"item": {
"id": reasoning_item_id,
"type": "reasoning",
"summary": [],
},
"output_index": next_output_index,
}),
));
for reasoning in &plan.reasoning {
events.push(sse_event(
"response.reasoning.delta",
&json!({
"type": "response.reasoning.delta",
"delta": reasoning,
"item_id": reasoning_item_id,
"output_index": next_output_index,
}),
));
}
events.push(sse_event(
"response.output_item.done",
&json!({
"type": "response.output_item.done",
"item": {
"id": reasoning_item_id,
"type": "reasoning",
"summary": [],
},
"output_index": next_output_index,
}),
));
next_output_index += 1;
if !plan.response_text.is_empty() || streamed_text.is_some() {
events.push(sse_event(
"response.output_item.added",
&json!({
"type": "response.output_item.added",
"item": {
"id": message_item_id,
"type": "message",
"status": "in_progress",
"content": [],
"role": "assistant",
},
"output_index": next_output_index,
}),
));
let message_text = streamed_text
.as_deref()
.unwrap_or(plan.response_text.as_str());
if !message_text.is_empty() {
events.push(sse_event(
"response.content_part.added",
&json!({
"type": "response.content_part.added",
"content_index": 0,
"item_id": message_item_id,
"output_index": next_output_index,
"part": {
"type": "output_text",
"text": "",
},
}),
));
events.push(sse_event(
"response.output_text.delta",
&json!({
"type": "response.output_text.delta",
"content_index": 0,
"item_id": message_item_id,
"output_index": next_output_index,
"delta": message_text,
}),
));
events.push(sse_event(
"response.output_text.done",
&json!({
"type": "response.output_text.done",
"content_index": 0,
"item_id": message_item_id,
"output_index": next_output_index,
"text": message_text,
}),
));
events.push(sse_event(
"response.content_part.done",
&json!({
"type": "response.content_part.done",
"content_index": 0,
"item_id": message_item_id,
"output_index": next_output_index,
"part": {
"type": "output_text",
"text": message_text,
},
}),
));
}
// The completed item carries its full content, like the real API.
// Adapters round-trip this item verbatim into the next request's
// input, so omitting content here produces an invalid replay.
events.push(sse_event(
"response.output_item.done",
&json!({
"type": "response.output_item.done",
"item": {
"id": message_item_id,
"type": "message",
"status": "completed",
"role": "assistant",
"content": [{
"type": "output_text",
"text": message_text,
}],
},
"output_index": next_output_index,
}),
));
next_output_index += 1;
}
for tool_call in &plan.tool_calls {
let item_id = format!("fc_{}", tool_call.id);
events.push(sse_event(
"response.output_item.added",
&json!({
"type": "response.output_item.added",
"item": {
"id": item_id,
"type": "function_call",
"call_id": tool_call.id,
"name": tool_call.name,
"arguments": "",
},
"output_index": next_output_index,
}),
));
events.push(sse_event(
"response.function_call_arguments.delta",
&json!({
"type": "response.function_call_arguments.delta",
"item_id": item_id,
"delta": ResponsePlan::tool_call_arguments_text(tool_call),
"output_index": next_output_index,
}),
));
events.push(sse_event(
"response.function_call_arguments.done",
&json!({
"type": "response.function_call_arguments.done",
"item_id": item_id,
"arguments": ResponsePlan::tool_call_arguments_text(tool_call),
"output_index": next_output_index,
}),
));
events.push(sse_event(
"response.output_item.done",
&json!({
"type": "response.output_item.done",
"item": {
"id": item_id,
"type": "function_call",
"call_id": tool_call.id,
"name": tool_call.name,
"arguments": ResponsePlan::tool_call_arguments_text(tool_call),
},
"output_index": next_output_index,
}),
));
next_output_index += 1;
}
if !transport.malformed_sse {
events.push(sse_event(
"response.completed",
&json!({
"type": "response.completed",
"response": plan.responses_json(),
}),
));
}
stream_response(events, transport)
}
pub fn chat_sse_response(
plan: &ResponsePlan,
include_usage: bool,
transport: TransportOptions,
) -> Response<Body> {
let mut events = Vec::new();
let content = plan.chat_content();
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [{
"index": 0,
"delta": {
"role": "assistant"
},
"finish_reason": Value::Null,
}]
})));
if !content.is_empty() {
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [{
"index": 0,
"delta": {
"content": content
},
"finish_reason": Value::Null,
}]
})));
}
for reasoning in &plan.reasoning {
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [{
"index": 0,
"delta": {
"reasoning": reasoning
},
"finish_reason": Value::Null,
}]
})));
}
if !plan.tool_calls.is_empty() {
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [{
"index": 0,
"delta": {
"tool_calls": plan.tool_calls.iter().enumerate().map(|(index, tool_call)| json!({
"index": index,
"id": tool_call.id,
"type": "function",
"function": {
"name": tool_call.name,
"arguments": ResponsePlan::tool_call_arguments_text(tool_call),
}
})).collect::<Vec<_>>()
},
"finish_reason": Value::Null,
}]
})));
}
if !transport.malformed_sse {
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [{
"index": 0,
"delta": {},
"finish_reason": if plan.tool_calls.is_empty() { "stop" } else { "tool_calls" },
}]
})));
if include_usage {
events.push(chat_chunk(&json!({
"id": format!("chatcmpl_{}", plan.id),
"object": "chat.completion.chunk",
"created": plan.created,
"model": plan.model,
"choices": [],
"usage": plan.usage.chat_completions_json(),
})));
}
events.push("data: [DONE]\n\n".to_owned());
}
stream_response(events, transport)
}
fn stream_response(events: Vec<String>, transport: TransportOptions) -> Response<Body> {
let limit = transport.close_after_chunks.unwrap_or(events.len());
let malformed_sse = transport.malformed_sse;
let inter_event_delay_ms = transport.inter_event_delay_ms;
let body = Body::from_stream(stream! {
for (index, event) in events.into_iter().enumerate() {
if index >= limit {
break;
}
if inter_event_delay_ms > 0 {
sleep(Duration::from_millis(inter_event_delay_ms)).await;
}
yield Ok::<_, std::convert::Infallible>(event.into_bytes());
}
if malformed_sse {
yield Ok::<_, std::convert::Infallible>(b"event: malformed\ndata: {".to_vec());
}
});
let mut response = Response::new(body);
*response.status_mut() = StatusCode::OK;
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("text/event-stream"),
);
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
response
}
fn sse_event(event: &str, data: &Value) -> String {
format!("event: {event}\ndata: {data}\n\n")
}
fn chat_chunk(data: &Value) -> String {
format!("data: {data}\n\n")
}

178
test/twin/openai/src/state.rs
View file

@ -1,178 +0,0 @@
use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, Mutex};
use serde::Serialize;
use serde_json::Value;
use crate::config::Config;
use crate::engine::scenario::{RequestContext, Scenario};
use crate::logs::RequestLog;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub enum NamespaceKey {
Global,
Bearer(String),
}
impl fmt::Display for NamespaceKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Global => write!(f, "Global"),
Self::Bearer(token) => write!(f, "Bearer: {token}"),
}
}
}
#[derive(Clone, Debug, Serialize)]
pub struct DebugSnapshot {
pub namespaces: Vec<NamespaceSnapshot>,
}
#[derive(Clone, Debug, Serialize)]
pub struct NamespaceSnapshot {
pub key: String,
pub scenarios: Vec<ScenarioSnapshot>,
pub request_logs: Vec<RequestLog>,
}
#[derive(Clone, Debug, Serialize)]
pub struct ScenarioSnapshot {
pub endpoint: String,
pub model: Option<String>,
pub stream: Option<bool>,
pub input_contains: Option<String>,
pub metadata: serde_json::Map<String, Value>,
pub script_kind: String,
}
#[derive(Clone, Debug)]
pub struct AppState {
pub config: Config,
inner: Arc<AppStateInner>,
}
#[derive(Debug)]
struct AppStateInner {
namespaces: Mutex<HashMap<NamespaceKey, NamespaceState>>,
}
#[derive(Debug)]
struct NamespaceState {
next_response_number: u64,
scenarios: Vec<Scenario>,
request_logs: Vec<RequestLog>,
}
impl Default for NamespaceState {
fn default() -> Self {
Self {
next_response_number: 1,
scenarios: Vec::new(),
request_logs: Vec::new(),
}
}
}
impl AppState {
pub fn new(config: Config) -> Self {
Self {
config,
inner: Arc::new(AppStateInner {
namespaces: Mutex::new(HashMap::new()),
}),
}
}
pub fn next_response_id(&self, namespace: &NamespaceKey) -> u64 {
let mut namespaces = self.inner.namespaces.lock().expect("namespaces lock");
let namespace_state = namespaces.entry(namespace.clone()).or_default();
let response_id = namespace_state.next_response_number;
namespace_state.next_response_number += 1;
response_id
}
pub fn enqueue_scenarios(&self, namespace: &NamespaceKey, mut scenarios: Vec<Scenario>) {
self.inner
.namespaces
.lock()
.expect("namespaces lock")
.entry(namespace.clone())
.or_default()
.scenarios
.append(&mut scenarios);
}
pub fn take_matching_scenario(
&self,
namespace: &NamespaceKey,
request: &RequestContext,
) -> Option<Scenario> {
let mut namespaces = self.inner.namespaces.lock().expect("namespaces lock");
let scenarios = &mut namespaces.entry(namespace.clone()).or_default().scenarios;
let position = scenarios
.iter()
.position(|scenario| scenario.matches(request))?;
Some(scenarios.remove(position))
}
pub fn log_request(&self, namespace: &NamespaceKey, request: RequestContext) {
self.inner
.namespaces
.lock()
.expect("namespaces lock")
.entry(namespace.clone())
.or_default()
.request_logs
.push(RequestLog {
endpoint: request.endpoint,
model: request.model,
stream: request.stream,
input_text: request.input_text,
instructions_text: request.instructions_text,
metadata: request.metadata,
});
}
pub fn request_logs(&self, namespace: &NamespaceKey) -> Vec<RequestLog> {
self.inner
.namespaces
.lock()
.expect("namespaces lock")
.get(namespace)
.map(|namespace_state| namespace_state.request_logs.clone())
.unwrap_or_default()
}
pub fn reset(&self, namespace: &NamespaceKey) {
self.inner
.namespaces
.lock()
.expect("namespaces lock")
.remove(namespace);
}
pub fn debug_snapshot(&self) -> DebugSnapshot {
let namespaces = self.inner.namespaces.lock().expect("namespaces lock");
let mut result = Vec::new();
for (key, ns) in namespaces.iter() {
result.push(NamespaceSnapshot {
key: key.to_string(),
scenarios: ns
.scenarios
.iter()
.map(|s| ScenarioSnapshot {
endpoint: s.matcher.endpoint.clone(),
model: s.matcher.model.clone(),
stream: s.matcher.stream,
input_contains: s.matcher.input_contains.clone(),
metadata: s.matcher.metadata.clone(),
script_kind: s.script.script_kind().to_owned(),
})
.collect(),
request_logs: ns.request_logs.clone(),
});
}
DebugSnapshot { namespaces: result }
}
}

535
test/twin/openai/tests/chat_completions_contract.rs
View file

@ -1,535 +0,0 @@
mod common;
use serde_json::json;
#[tokio::test]
async fn chat_completions_non_stream_uses_same_canonical_plan() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "same plan",
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let chat = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "same plan" }],
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
response["output"][0]["content"][0]["text"],
chat["choices"][0]["message"]["content"]
);
}
#[tokio::test]
async fn chat_completions_stream_uses_same_canonical_plan() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_chat_stream(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "stream same plan" }],
"stream": true
}))
.await;
let joined = chunks.join("");
assert_eq!(status, 200);
assert!(joined.contains("\"content\":\"deterministic: stream same plan\""));
assert!(!joined.contains("\"usage\""));
assert!(joined.contains("data: [DONE]"));
}
#[tokio::test]
async fn chat_completions_stream_includes_usage_when_requested() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_chat_stream(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "stream with usage" }],
"stream": true,
"stream_options": { "include_usage": true }
}))
.await;
assert_eq!(status, 200);
let transcript =
common::parse_sse_transcript(chunks.join("").as_bytes()).expect("valid SSE transcript");
let usage_chunk = transcript
.events
.iter()
.filter(|event| event.data != "[DONE]")
.map(|event| {
serde_json::from_str::<serde_json::Value>(&event.data).expect("valid JSON chunk")
})
.find(|chunk| chunk.get("usage").is_some())
.expect("trailing usage chunk");
assert_eq!(usage_chunk["choices"], json!([]));
assert_eq!(
usage_chunk["usage"],
json!({
"prompt_tokens": 3,
"completion_tokens": 5,
"total_tokens": 8
})
);
assert!(transcript.done);
}
#[tokio::test]
async fn chat_completions_accepts_supported_openai_compatible_fields() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [
{
"role": "assistant",
"content": [
{ "type": "reasoning", "text": "reasoning trace" }
]
},
{ "role": "user", "content": "structured chat" }
],
"max_tokens": 128,
"stream": false,
"tools": [{ "type": "function", "function": { "name": "lookup" } }],
"tool_choice": "auto",
"stop": ["END"],
"response_format": { "type": "json_object" }
}))
.await;
assert_eq!(response.status(), 200);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(
body["choices"][0]["message"]["content"],
"{\"message\":\"deterministic: structured chat\",\"model\":\"gpt-test\"}"
);
assert_eq!(
body["choices"][0]["message"]["reasoning"][0],
"reasoning: structured chat"
);
}
#[tokio::test]
async fn chat_completions_accepts_tool_call_history() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [
{ "role": "user", "content": "replace old with new" },
{
"role": "assistant",
"tool_calls": [{
"id": "call_edit",
"type": "function",
"function": {
"name": "edit_file",
"arguments": "{\"old\":\"old\",\"new\":\"new\"}"
}
}]
},
{
"role": "tool",
"content": "Updated data.txt",
"tool_call_id": "call_edit"
}
],
"stream": false
}))
.await;
assert_eq!(response.status(), 200);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(
body["choices"][0]["message"]["content"],
"deterministic: replace old with new"
);
}
#[tokio::test]
async fn chat_completions_supports_scripted_tool_call_and_json_schema() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": false, "input_contains": "tool please" },
"script": {
"kind": "success",
"tool_calls": [
{
"id": "call_weather",
"name": "lookup_weather",
"arguments": { "city": "Boston" }
}
]
}
},
{
"matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": true, "input_contains": "tool please" },
"script": {
"kind": "success",
"tool_calls": [
{
"id": "call_weather",
"name": "lookup_weather",
"arguments": { "city": "Boston" }
}
]
}
}
]
}))
.await;
let non_stream = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "tool please" }],
"tools": [{ "type": "function", "function": { "name": "lookup_weather" } }],
"tool_choice": {
"type": "function",
"function": { "name": "lookup_weather" }
},
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(non_stream["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
non_stream["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"lookup_weather"
);
assert_eq!(non_stream["choices"][0]["message"]["content"], "");
let (status, chunks) = server
.post_chat_stream(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "tool please" }],
"tools": [{ "type": "function", "function": { "name": "lookup_weather" } }],
"tool_choice": {
"type": "function",
"function": { "name": "lookup_weather" }
},
"stream": true
}))
.await;
let joined = chunks.join("");
assert_eq!(status, 200);
assert!(joined.contains("\"tool_calls\""));
assert!(!joined.contains("\"content\":\"deterministic:"));
let structured = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "schema chat" }],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "chat_schema",
"schema": {
"type": "object",
"properties": {
"message": { "type": "string" },
"ok": { "type": "boolean" }
}
}
,
"strict": true
}
},
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
structured["choices"][0]["message"]["content"],
"{\"message\":\"deterministic: schema chat\",\"ok\":true}"
);
}
#[tokio::test]
async fn chat_completions_stream_preserves_reasoning_transcript() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_chat_stream(json!({
"model": "gpt-test",
"messages": [
{
"role": "assistant",
"content": [
{ "type": "reasoning", "text": "reasoning trace" }
]
},
{ "role": "user", "content": "stream same plan" }
],
"stream": true
}))
.await;
let joined = chunks.join("");
assert_eq!(status, 200);
assert!(joined.contains("\"reasoning\":\"reasoning: stream same plan\""));
assert!(joined.contains("\"content\":\"deterministic: stream same plan\""));
assert!(joined.contains("data: [DONE]"));
}
#[tokio::test]
async fn chat_completions_do_not_infer_reasoning_from_user_text() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "Please explain your reasoning plainly" }],
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(response["choices"][0]["message"]["reasoning"], json!([]));
assert_eq!(
response["choices"][0]["message"]["content"],
"deterministic: Please explain your reasoning plainly"
);
}
#[tokio::test]
async fn chat_completions_reject_reasoning_parts_on_non_assistant_messages() {
let server = common::spawn_server().await.expect("server should start");
let user_reasoning = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{
"role": "user",
"content": [{ "type": "reasoning", "text": "not allowed here" }]
}],
"stream": false
}))
.await;
assert_eq!(user_reasoning.status(), 400);
let body = user_reasoning
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "messages");
let system_reasoning = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{
"role": "system",
"content": [{ "type": "reasoning", "text": "not allowed here either" }]
}],
"stream": false
}))
.await;
assert_eq!(system_reasoning.status(), 400);
let body = system_reasoning
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "messages");
}
#[tokio::test]
async fn chat_completions_accept_unknown_top_level_fields() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_chat_stream(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "hello" }],
"stream": true,
"temperature": 0.7,
"top_p": 0.9,
"prompt_cache_key": "conversation-123"
}))
.await;
assert_eq!(status, 200);
let transcript =
common::parse_sse_transcript(chunks.join("").as_bytes()).expect("valid SSE transcript");
assert!(transcript.done);
}
#[tokio::test]
async fn chat_completions_reject_unsupported_tool_choice_shape() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "hello" }],
"tools": [{ "type": "function", "function": { "name": "lookup_weather" } }],
"tool_choice": { "type": "required" }
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn chat_completions_reject_required_tool_choice_without_tools() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "hello" }],
"tool_choice": "required"
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn chat_completions_reject_unfulfilled_tool_choice_requirements() {
let server = common::spawn_server().await.expect("server should start");
let required = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "plain text please" }],
"tools": [{ "type": "function", "function": { "name": "lookup_weather" } }],
"tool_choice": "required"
}))
.await;
assert_eq!(required.status(), 400);
let body = required.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
let named = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "plain text please" }],
"tools": [{ "type": "function", "function": { "name": "lookup_weather" } }],
"tool_choice": {
"type": "function",
"function": { "name": "lookup_weather" }
}
}))
.await;
assert_eq!(named.status(), 400);
let body = named.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
}
#[tokio::test]
async fn chat_completions_rejects_unsupported_response_format() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "bad format" }],
"response_format": { "type": "xml" }
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn chat_completions_reject_empty_messages() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": []
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "messages");
}
#[tokio::test]
async fn chat_completions_reject_null_message_content() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": null }]
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "messages");
}
#[tokio::test]
async fn chat_completions_reject_malformed_image_input() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{
"role": "user",
"content": [{
"type": "image_url",
"image_url": {}
}]
}]
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "messages");
}

587
test/twin/openai/tests/common/mod.rs
View file

@ -1,587 +0,0 @@
#![allow(
dead_code,
unreachable_pub,
reason = "Shared test helpers stay public within the test crate and not every helper is used everywhere."
)]
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use anyhow::Result;
use fabro_http::header::AUTHORIZATION;
use fabro_http::{HttpClient as Client, HttpClientBuilder};
use futures_util::StreamExt;
use serde_json::Value;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use twin_openai::config::Config;
pub struct TestServer {
pub base_url: String,
pub client: Client,
pub auth_client: Client,
pub bearer_token: String,
}
#[derive(Clone)]
pub struct ApiClient {
pub base_url: String,
client: Client,
bearer_token: Option<String>,
organization: Option<String>,
project: Option<String>,
}
pub struct RecordedResponse {
pub status: fabro_http::StatusCode,
pub headers: HashMap<String, String>,
pub body: Vec<u8>,
}
pub struct RawStreamResponse {
pub status: u16,
pub headers: HashMap<String, String>,
pub body: Vec<u8>,
}
pub struct TimedStreamResponse {
pub status: fabro_http::StatusCode,
pub first_event_elapsed: Duration,
pub chunks: Vec<String>,
}
#[derive(Debug)]
pub struct ParsedSseTranscript {
pub blocks: Vec<String>,
pub events: Vec<ParsedSseEvent>,
pub done: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParsedSseEvent {
pub event: Option<String>,
pub data: String,
}
static NEXT_BEARER_TOKEN: AtomicU64 = AtomicU64::new(1);
pub fn test_http_client() -> Result<Client> {
fabro_http::test_http_client().map_err(Into::into)
}
pub async fn spawn_server() -> Result<TestServer> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr: SocketAddr = listener.local_addr()?;
let app = twin_openai::build_app_with_config(Config {
bind_addr: "127.0.0.1:0".parse().expect("valid addr"),
require_auth: true,
enable_admin: true,
});
tokio::spawn(async move {
axum::serve(listener, app).await.expect("server should run");
});
TestServer::new(format!("http://{addr}"), next_bearer_token())
}
fn next_bearer_token() -> String {
format!(
"test-key-{}",
NEXT_BEARER_TOKEN.fetch_add(1, Ordering::SeqCst)
)
}
fn authorization_header_value(bearer_token: &str) -> String {
format!("Bearer {bearer_token}")
}
fn build_authenticated_client(bearer_token: &str) -> Result<Client> {
HttpClientBuilder::new()
.proxy_policy(fabro_http::ProxyPolicy::Disabled)
.default_headers(
[(
AUTHORIZATION,
authorization_header_value(bearer_token)
.parse()
.expect("valid header"),
)]
.into_iter()
.collect(),
)
.build()
.map_err(Into::into)
}
impl ApiClient {
pub fn new(
base_url: impl Into<String>,
bearer_token: Option<String>,
organization: Option<String>,
project: Option<String>,
) -> Result<Self> {
Ok(Self {
base_url: base_url.into(),
client: HttpClientBuilder::new()
.proxy_policy(fabro_http::ProxyPolicy::Disabled)
.timeout(Duration::from_secs(30))
.build()?,
bearer_token,
organization,
project,
})
}
pub fn with_client(
base_url: impl Into<String>,
client: Client,
bearer_token: Option<String>,
organization: Option<String>,
project: Option<String>,
) -> Self {
Self {
base_url: base_url.into(),
client,
bearer_token,
organization,
project,
}
}
pub async fn post_json(&self, path: &str, body: &Value) -> fabro_http::Response {
self.post(path)
.json(body)
.send()
.await
.expect("request should complete")
}
pub async fn post_json_recorded(&self, path: &str, body: &Value) -> RecordedResponse {
record_response(self.post_json(path, body).await).await
}
pub async fn get_recorded(&self, path: &str) -> RecordedResponse {
record_response(
self.get(path)
.send()
.await
.expect("request should complete"),
)
.await
}
pub fn post(&self, path: &str) -> fabro_http::RequestBuilder {
self.request(self.client.post(format!("{}{}", self.base_url, path)))
}
pub fn get(&self, path: &str) -> fabro_http::RequestBuilder {
self.request(self.client.get(format!("{}{}", self.base_url, path)))
}
fn request(&self, mut request: fabro_http::RequestBuilder) -> fabro_http::RequestBuilder {
if let Some(token) = &self.bearer_token {
request = request.bearer_auth(token);
}
if let Some(org) = &self.organization {
request = request.header("OpenAI-Organization", org);
}
if let Some(project) = &self.project {
request = request.header("OpenAI-Project", project);
}
request
}
}
impl TestServer {
fn new(base_url: String, bearer_token: String) -> Result<Self> {
let client = test_http_client()?;
let auth_client = build_authenticated_client(&bearer_token)?;
Ok(Self {
base_url,
client,
auth_client,
bearer_token,
})
}
pub fn authorization_header_value(&self) -> String {
authorization_header_value(&self.bearer_token)
}
pub fn api_client(&self) -> ApiClient {
ApiClient::with_client(
self.base_url.clone(),
self.client.clone(),
Some(self.bearer_token.clone()),
None,
None,
)
}
pub fn fork_namespace(&self) -> Result<Self> {
Self::new(self.base_url.clone(), next_bearer_token())
}
}
impl TestServer {
pub async fn post_responses(&self, body: Value) -> fabro_http::Response {
self.auth_client
.post(format!("{}/v1/responses", self.base_url))
.json(&body)
.send()
.await
.expect("request should complete")
}
pub async fn post_responses_with_headers(
&self,
body: Value,
org: Option<&str>,
project: Option<&str>,
) -> fabro_http::Response {
let mut request = self
.auth_client
.post(format!("{}/v1/responses", self.base_url));
if let Some(org) = org {
request = request.header("OpenAI-Organization", org);
}
if let Some(project) = project {
request = request.header("OpenAI-Project", project);
}
request
.json(&body)
.send()
.await
.expect("request should complete")
}
pub async fn post_responses_stream(
&self,
body: Value,
) -> (fabro_http::StatusCode, Vec<String>) {
let response = self
.auth_client
.post(format!("{}/v1/responses", self.base_url))
.json(&body)
.send()
.await
.expect("request should complete");
let status = response.status();
let mut stream = response.bytes_stream();
let mut chunks = Vec::new();
while let Some(chunk) = stream.next().await {
chunks.push(
String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"),
);
}
(status, chunks)
}
pub async fn post_chat(&self, body: Value) -> fabro_http::Response {
self.auth_client
.post(format!("{}/v1/chat/completions", self.base_url))
.json(&body)
.send()
.await
.expect("request should complete")
}
pub async fn post_chat_stream(&self, body: Value) -> (fabro_http::StatusCode, Vec<String>) {
let response = self.post_chat(body).await;
let status = response.status();
let mut stream = response.bytes_stream();
let mut chunks = Vec::new();
while let Some(chunk) = stream.next().await {
chunks.push(
String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"),
);
}
(status, chunks)
}
pub async fn post_chat_with_auth_header(
&self,
body: Value,
authorization: Option<&str>,
) -> fabro_http::Response {
let mut request = self
.client
.post(format!("{}/v1/chat/completions", self.base_url));
if let Some(value) = authorization {
request = request.header(AUTHORIZATION, value);
}
request
.json(&body)
.send()
.await
.expect("request should complete")
}
pub async fn post_responses_stream_timed(&self, body: Value) -> TimedStreamResponse {
let started = Instant::now();
let response = self
.auth_client
.post(format!("{}/v1/responses", self.base_url))
.json(&body)
.send()
.await
.expect("request should complete");
let status = response.status();
let mut stream = response.bytes_stream();
let mut chunks = Vec::new();
let mut first_event_elapsed = Duration::ZERO;
if let Some(chunk) = stream.next().await {
chunks.push(
String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"),
);
first_event_elapsed = started.elapsed();
}
while let Some(chunk) = stream.next().await {
chunks.push(
String::from_utf8(chunk.expect("stream chunk").to_vec()).expect("utf8 stream"),
);
}
TimedStreamResponse {
status,
first_event_elapsed,
chunks,
}
}
pub async fn post_responses_stream_raw(&self, body: Value) -> RawStreamResponse {
self.raw_stream_request("/v1/responses", &body).await
}
pub async fn post_chat_stream_raw(&self, body: Value) -> RawStreamResponse {
self.raw_stream_request("/v1/chat/completions", &body).await
}
pub async fn enqueue_scenarios(&self, scenarios: Value) {
let response = self
.auth_client
.post(format!("{}/__admin/scenarios", self.base_url))
.json(&scenarios)
.send()
.await
.expect("admin request should complete");
assert_eq!(response.status(), 200);
}
pub async fn reset(&self) {
let response = self
.auth_client
.post(format!("{}/__admin/reset", self.base_url))
.send()
.await
.expect("admin reset should complete");
assert_eq!(response.status(), 200);
}
pub async fn request_logs(&self) -> Value {
self.auth_client
.get(format!("{}/__admin/requests", self.base_url))
.send()
.await
.expect("admin logs should complete")
.json()
.await
.expect("logs json should parse")
}
async fn raw_stream_request(&self, path: &str, body: &Value) -> RawStreamResponse {
let authority = self
.base_url
.strip_prefix("http://")
.expect("http base url");
let mut stream = TcpStream::connect(authority)
.await
.expect("socket should connect");
let body = serde_json::to_vec(body).expect("json body");
let authorization = self.authorization_header_value();
let request = format!(
"POST {path} HTTP/1.1\r\nHost: {authority}\r\nAuthorization: {authorization}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream
.write_all(request.as_bytes())
.await
.expect("request headers should write");
stream
.write_all(&body)
.await
.expect("request body should write");
stream.flush().await.expect("request should flush");
let mut response = Vec::new();
stream
.read_to_end(&mut response)
.await
.expect("response should read");
decode_http_response(&response)
}
}
pub fn parse_sse_transcript(body: &[u8]) -> Result<ParsedSseTranscript, String> {
let text = std::str::from_utf8(body).map_err(|_| "sse body was not valid utf-8".to_owned())?;
let mut blocks = Vec::new();
let mut events = Vec::new();
let mut done = false;
let mut remainder = text;
while let Some((block, rest)) = remainder.split_once("\n\n") {
if !block.is_empty() {
let event = parse_sse_block(block)?;
if event.data == "[DONE]" {
done = true;
}
blocks.push(block.to_owned());
events.push(event);
}
remainder = rest;
}
if !remainder.is_empty() {
return Err("sse stream ended with an incomplete event".to_owned());
}
Ok(ParsedSseTranscript {
blocks,
events,
done,
})
}
fn parse_sse_block(block: &str) -> Result<ParsedSseEvent, String> {
let mut event = None;
let mut data_lines = Vec::new();
for line in block.lines() {
if let Some(value) = line.strip_prefix("event: ") {
event = Some(value.to_owned());
continue;
}
if let Some(value) = line.strip_prefix("data: ") {
data_lines.push(value.to_owned());
continue;
}
if line.starts_with("id: ") || line.starts_with(':') {
continue;
}
return Err(format!("unsupported sse line: {line}"));
}
Ok(ParsedSseEvent {
event,
data: data_lines.join("\n"),
})
}
pub async fn record_response(response: fabro_http::Response) -> RecordedResponse {
let status = response.status();
let headers = response
.headers()
.iter()
.map(|(name, value)| {
(
name.as_str().to_ascii_lowercase(),
value.to_str().unwrap_or_default().to_owned(),
)
})
.collect();
let body = response
.bytes()
.await
.expect("response body should read")
.to_vec();
RecordedResponse {
status,
headers,
body,
}
}
fn decode_http_response(response: &[u8]) -> RawStreamResponse {
let header_end = response
.windows(4)
.position(|window| window == b"\r\n\r\n")
.expect("http header terminator")
+ 4;
let head = std::str::from_utf8(&response[..header_end]).expect("header utf8");
let mut lines = head.split("\r\n");
let status_line = lines.next().expect("status line");
let status = status_line
.split_whitespace()
.nth(1)
.expect("status code")
.parse::<u16>()
.expect("numeric status");
let mut headers = HashMap::new();
for line in lines.filter(|line| !line.is_empty()) {
let (name, value) = line.split_once(':').expect("header format");
headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_owned());
}
let body = if headers
.get("transfer-encoding")
.is_some_and(|value| value.eq_ignore_ascii_case("chunked"))
{
decode_chunked_body(&response[header_end..])
} else {
response[header_end..].to_vec()
};
RawStreamResponse {
status,
headers,
body,
}
}
fn decode_chunked_body(encoded: &[u8]) -> Vec<u8> {
let mut cursor = 0;
let mut decoded = Vec::new();
loop {
let size_end = encoded[cursor..]
.windows(2)
.position(|window| window == b"\r\n")
.expect("chunk size terminator")
+ cursor;
let size = std::str::from_utf8(&encoded[cursor..size_end]).expect("chunk size utf8");
let size = usize::from_str_radix(size.trim(), 16).expect("hex chunk size");
cursor = size_end + 2;
if size == 0 {
break;
}
decoded.extend_from_slice(&encoded[cursor..cursor + size]);
cursor += size + 2;
}
decoded
}

15
test/twin/openai/tests/config_contract.rs
View file

@ -1,15 +0,0 @@
use twin_openai::config::Config;
#[test]
fn config_loads_from_environment() {
let config = Config::from_lookup(&|name| match name {
"TWIN_OPENAI_BIND_ADDR" => Some("127.0.0.1:4100".to_string()),
"TWIN_OPENAI_REQUIRE_AUTH" | "TWIN_OPENAI_ENABLE_ADMIN" => Some("false".to_string()),
_ => None,
})
.expect("config should load");
assert_eq!(config.bind_addr.to_string(), "127.0.0.1:4100");
assert!(!config.require_auth);
assert!(!config.enable_admin);
}

403
test/twin/openai/tests/debug_ui.rs
View file

@ -1,403 +0,0 @@
#![expect(
clippy::disallowed_methods,
reason = "These browser-debug integration tests synchronously probe for Chrome binaries before launching external tooling."
)]
mod common;
use std::process::Stdio;
use serde_json::json;
use tokio::net::TcpListener;
use tokio::process::Command as TokioCommand;
use tokio::time::{Duration, timeout};
use twin_openai::config::Config;
#[tokio::test]
async fn debug_html_page_serves_valid_html_on_empty_state() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.client
.get(format!("{}/__debug", server.base_url))
.send()
.await
.expect("debug page request should complete");
assert_eq!(response.status(), 200);
let content_type = response
.headers()
.get("content-type")
.expect("content-type header should be present")
.to_str()
.expect("content-type should be valid string");
assert!(
content_type.contains("text/html"),
"content-type should contain text/html, got: {content_type}"
);
let body = response.text().await.expect("body should read");
assert!(
body.contains("<!DOCTYPE html>"),
"response should contain DOCTYPE"
);
assert!(
body.contains("twin-openai"),
"response should contain project name"
);
assert!(body.contains("debug"), "response should contain 'debug'");
assert!(
body.contains("no active namespaces"),
"empty state should show 'no active namespaces'"
);
}
#[tokio::test]
async fn debug_json_endpoint_returns_correct_state_snapshot() {
let server = common::spawn_server().await.expect("server should start");
// Load two scenarios: one success, one error
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test" },
"script": { "kind": "success" }
},
{
"matcher": { "endpoint": "responses", "model": "gpt-error" },
"script": {
"kind": "error",
"status": 500,
"message": "test error",
"error_type": "server_error",
"code": "server_error"
}
}
]
}))
.await;
// Make one request that consumes the first (success) scenario
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "hello debug",
"stream": false
}))
.await;
assert_eq!(response.status(), 200);
// GET the debug JSON endpoint (unauthenticated)
let response = server
.client
.get(format!("{}/__debug/state.json", server.base_url))
.send()
.await
.expect("debug json request should complete");
assert_eq!(response.status(), 200);
let content_type = response
.headers()
.get("content-type")
.expect("content-type header should be present")
.to_str()
.expect("content-type should be valid string");
assert!(
content_type.contains("application/json"),
"content-type should contain application/json, got: {content_type}"
);
let body: serde_json::Value = response.json().await.expect("json should parse");
// Should have a top-level namespaces array
let namespaces = body["namespaces"]
.as_array()
.expect("namespaces should be an array");
assert_eq!(namespaces.len(), 1, "should have exactly one namespace");
let ns = &namespaces[0];
assert!(
ns["key"].as_str().unwrap().starts_with("Bearer:"),
"namespace key should start with 'Bearer:', got: {}",
ns["key"]
);
// Should have 1 remaining scenario (the error one; the success was consumed)
let scenarios = ns["scenarios"]
.as_array()
.expect("scenarios should be an array");
assert_eq!(scenarios.len(), 1, "should have 1 remaining scenario");
assert_eq!(scenarios[0]["endpoint"], "responses");
assert_eq!(scenarios[0]["model"], "gpt-error");
assert_eq!(scenarios[0]["script_kind"], "error");
// Should have 1 request log
let request_logs = ns["request_logs"]
.as_array()
.expect("request_logs should be an array");
assert_eq!(request_logs.len(), 1, "should have 1 request log");
assert_eq!(request_logs[0]["endpoint"], "responses");
assert_eq!(request_logs[0]["model"], "gpt-test");
assert!(
request_logs[0]["input_text"]
.as_str()
.unwrap()
.contains("hello debug"),
"request log should contain input text 'hello debug'"
);
}
#[tokio::test]
async fn debug_html_page_reflects_loaded_scenarios_and_request_logs() {
let server = common::spawn_server().await.expect("server should start");
// Load one success scenario
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-html-test" },
"script": { "kind": "success" }
}
]
}))
.await;
// Make one request with a different model (won't match, but still gets logged
// via the default behavior)
let response = server
.post_responses(json!({
"model": "gpt-other",
"input": "check the page",
"stream": false
}))
.await;
// The request gets a deterministic response (no matching scenario consumed
// since model doesn't match). Status should be 200 (default behavior).
assert_eq!(response.status(), 200);
// GET the debug HTML page
let response = server
.client
.get(format!("{}/__debug", server.base_url))
.send()
.await
.expect("debug page request should complete");
assert_eq!(response.status(), 200);
let body = response.text().await.expect("body should read");
// Should contain the scenario's model name
assert!(
body.contains("gpt-html-test"),
"HTML should contain scenario model 'gpt-html-test'"
);
// Should contain the script kind
assert!(
body.contains("success"),
"HTML should contain script kind 'success'"
);
// Should contain the request log model
assert!(
body.contains("gpt-other"),
"HTML should contain request log model 'gpt-other'"
);
// Should contain the request log input text
assert!(
body.contains("check the page"),
"HTML should contain request log input text 'check the page'"
);
// Verify the server-rendered content section does not show empty state.
// The JS source always includes the "no active namespaces" string as a
// template, so we check that the server-rendered content div contains
// namespace sections rather than the empty-state paragraph.
assert!(
body.contains("namespace-header"),
"HTML should contain a namespace-header element (proving non-empty rendering)"
);
}
#[tokio::test]
async fn debug_routes_not_accessible_when_admin_disabled() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind should succeed");
let addr = listener.local_addr().expect("should have addr");
let app = twin_openai::build_app_with_config(Config {
bind_addr: "127.0.0.1:0".parse().expect("valid addr"),
require_auth: false,
enable_admin: false,
});
tokio::spawn(async move {
axum::serve(listener, app).await.expect("server should run");
});
let base_url = format!("http://{addr}");
let client = common::test_http_client().expect("test client");
let html_response = client
.get(format!("{base_url}/__debug"))
.send()
.await
.expect("request should complete");
assert_eq!(
html_response.status(),
404,
"debug HTML should be 404 when admin disabled"
);
let json_response = client
.get(format!("{base_url}/__debug/state.json"))
.send()
.await
.expect("request should complete");
assert_eq!(
json_response.status(),
404,
"debug JSON should be 404 when admin disabled"
);
}
#[tokio::test]
async fn debug_page_renders_in_headless_chrome() {
// Find Chrome binary
let chrome_binary = ["chromium", "google-chrome", "chromium-browser"]
.iter()
.find(|name| {
std::process::Command::new("which")
.arg(name)
.output()
.is_ok_and(|o| o.status.success())
});
let Some(chrome_binary) = chrome_binary.copied() else {
return;
};
let server = common::spawn_server().await.expect("server should start");
// Load a scenario and make a request so the page has content
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-screenshot" },
"script": { "kind": "success" }
}
]
}))
.await;
let response = server
.post_responses(json!({
"model": "gpt-screenshot",
"input": "screenshot test",
"stream": false
}))
.await;
assert_eq!(response.status(), 200);
let screenshot_path = format!(
"/tmp/twin-openai-debug-screenshot-{}.png",
std::process::id()
);
let mut command = TokioCommand::new(chrome_binary);
command.args([
"--headless",
"--disable-gpu",
&format!("--screenshot={screenshot_path}"),
"--window-size=1280,900",
]);
if cfg!(target_os = "linux") {
// Ubuntu 24.04 GitHub runners block Chrome's default sandbox unless it
// is launched with a compatible user namespace or disabled explicitly.
command.arg("--no-sandbox");
}
let mut child = command
.stdout(Stdio::null())
.stderr(Stdio::null())
// Static mode keeps the page visually identical for the screenshot
// while avoiding a live refresh loop that can stall headless Chrome
// on Linux CI.
.arg(format!("{}/__debug?refresh=0", server.base_url))
.spawn()
.expect("Chrome should start");
// Chrome with --screenshot exits once the file is written, so waiting on
// the process is the deterministic completion signal.
let wait_result = timeout(Duration::from_mins(2), child.wait()).await;
let status = match wait_result {
Ok(Ok(status)) => status,
Ok(Err(err)) => panic!("Chrome wait failed: {err}"),
Err(_) => {
let _ = child.start_kill();
let _ = child.wait().await;
panic!("Chrome did not exit within 120s while taking screenshot");
}
};
assert!(
status.success(),
"Chrome exited with non-success status: {status}"
);
let screenshot_data = std::fs::read(&screenshot_path)
.expect("Chrome exited successfully but screenshot file is missing");
assert!(
screenshot_data.len() >= 10_000,
"screenshot should be at least 10KB, got {} bytes",
screenshot_data.len()
);
// Check PNG magic bytes
assert_eq!(
&screenshot_data[..8],
b"\x89PNG\r\n\x1a\n",
"screenshot should be a valid PNG"
);
// Clean up
let _ = std::fs::remove_file(&screenshot_path);
}
#[tokio::test]
async fn debug_html_escapes_user_controlled_values() {
let server = common::spawn_server().await.expect("server should start");
// Load a scenario with an XSS attempt in the model name
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": {
"endpoint": "responses",
"model": "<script>alert('xss')</script>"
},
"script": { "kind": "success" }
}
]
}))
.await;
let response = server
.client
.get(format!("{}/__debug", server.base_url))
.send()
.await
.expect("debug page request should complete");
assert_eq!(response.status(), 200);
let body = response.text().await.expect("body should read");
// Should contain the escaped form
assert!(
body.contains("&lt;script&gt;"),
"HTML should contain escaped '<script>' as '&lt;script&gt;'"
);
// Should NOT contain the raw injection
assert!(
!body.contains("<script>alert"),
"HTML should NOT contain raw unescaped '<script>alert'"
);
}

338
test/twin/openai/tests/failure_modes.rs
View file

@ -1,338 +0,0 @@
mod common;
use std::time::Duration;
use fabro_http::header::AUTHORIZATION;
use serde_json::json;
#[tokio::test]
async fn scripted_budget_and_content_filter_errors_are_distinct() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "quota" },
"script": {
"kind": "error",
"status": 429,
"message": "quota exceeded",
"error_type": "insufficient_quota",
"code": "quota_exceeded",
"retry_after": "120"
}
},
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "filter" },
"script": {
"kind": "error",
"status": 400,
"message": "content filtered",
"error_type": "content_filter",
"code": "content_filter"
}
}
]
}))
.await;
let quota = server
.post_responses(json!({ "model": "gpt-test", "input": "quota please", "stream": false }))
.await;
assert_eq!(quota.status(), 429);
assert_eq!(quota.headers()["retry-after"], "120");
let quota_body = quota.json::<serde_json::Value>().await.expect("json");
assert_eq!(quota_body["error"]["type"], "insufficient_quota");
let filter = server
.post_responses(json!({ "model": "gpt-test", "input": "filter please", "stream": false }))
.await;
assert_eq!(filter.status(), 400);
let filter_body = filter.json::<serde_json::Value>().await.expect("json");
assert_eq!(filter_body["error"]["type"], "content_filter");
}
#[tokio::test]
async fn scripted_error_status_matrix_is_preserved() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "400" }, "script": { "kind": "error", "status": 400, "message": "400 error", "error_type": "invalid_request_error", "code": "400_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "401" }, "script": { "kind": "error", "status": 401, "message": "401 error", "error_type": "invalid_api_key", "code": "401_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "403" }, "script": { "kind": "error", "status": 403, "message": "403 error", "error_type": "permission_error", "code": "403_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "404" }, "script": { "kind": "error", "status": 404, "message": "404 error", "error_type": "not_found_error", "code": "404_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "408" }, "script": { "kind": "error", "status": 408, "message": "408 error", "error_type": "timeout_error", "code": "408_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "413" }, "script": { "kind": "error", "status": 413, "message": "413 error", "error_type": "request_too_large", "code": "413_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "429 status" }, "script": { "kind": "error", "status": 429, "message": "429 error", "error_type": "rate_limit_error", "code": "429_code", "retry_after": "30" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "500" }, "script": { "kind": "error", "status": 500, "message": "500 error", "error_type": "server_error", "code": "500_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "502" }, "script": { "kind": "error", "status": 502, "message": "502 error", "error_type": "bad_gateway", "code": "502_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "503" }, "script": { "kind": "error", "status": 503, "message": "503 error", "error_type": "service_unavailable", "code": "503_code" } },
{ "matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "504" }, "script": { "kind": "error", "status": 504, "message": "504 error", "error_type": "gateway_timeout", "code": "504_code" } }
]
}))
.await;
for status in [400_u16, 401, 403, 404, 408, 413, 429, 500, 502, 503, 504] {
let marker = if status == 429 {
"429 status".to_owned()
} else {
status.to_string()
};
let response = server
.post_responses(json!({ "model": "gpt-test", "input": marker, "stream": false }))
.await;
assert_eq!(response.status().as_u16(), status);
let headers = response.headers().clone();
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["message"], format!("{status} error"));
if status == 429 {
assert_eq!(headers["retry-after"], "30");
}
}
}
#[tokio::test]
async fn scripted_hang_times_out_client_side() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "hang" }
}
]
}))
.await;
let client = fabro_http::HttpClientBuilder::new()
.no_proxy()
.timeout(Duration::from_millis(150))
.default_headers(
[(
AUTHORIZATION,
server
.authorization_header_value()
.parse()
.expect("valid header"),
)]
.into_iter()
.collect(),
)
.build()
.expect("client");
let result = client
.post(format!("{}/v1/responses", server.base_url))
.json(&json!({ "model": "gpt-test", "input": "hang", "stream": false }))
.send()
.await;
assert!(result.is_err());
assert!(result.expect_err("timeout").is_timeout());
}
#[tokio::test]
async fn scripted_partial_stream_then_close_is_observable() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"response_text": "partial stream",
"close_after_chunks": 2
}
}
]
}))
.await;
let raw = server
.post_responses_stream_raw(
json!({ "model": "gpt-test", "input": "partial", "stream": true }),
)
.await;
let parsed = common::parse_sse_transcript(&raw.body).expect("valid sse prefix");
let joined = String::from_utf8(raw.body).expect("utf8");
assert_eq!(raw.status, 200);
assert_eq!(
raw.headers
.get("content-type")
.expect("content-type header"),
"text/event-stream"
);
assert!(joined.contains("event: response.created"));
assert!(!joined.contains("event: malformed"));
assert!(!parsed.done);
}
#[tokio::test]
async fn scripted_partial_stream_then_close_is_observable_for_chat() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"response_text": "partial chat stream",
"close_after_chunks": 2
}
}
]
}))
.await;
let raw = server
.post_chat_stream_raw(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "partial chat" }],
"stream": true
}))
.await;
let parsed = common::parse_sse_transcript(&raw.body).expect("valid sse prefix");
let joined = String::from_utf8(raw.body).expect("utf8");
assert_eq!(raw.status, 200);
assert!(joined.contains("chat.completion.chunk"));
assert!(!joined.contains("event: malformed"));
assert!(!parsed.done);
}
#[tokio::test]
async fn scripted_delayed_first_byte_is_observable() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"response_text": "delayed stream",
"delay_before_headers_ms": 120
}
}
]
}))
.await;
let timed = server
.post_responses_stream_timed(
json!({ "model": "gpt-test", "input": "delay", "stream": true }),
)
.await;
assert_eq!(timed.status, 200);
assert!(timed.first_event_elapsed >= Duration::from_millis(100));
assert!(timed.chunks.join("").contains("delayed stream"));
}
#[tokio::test]
async fn scripted_malformed_sse_is_observable() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "chat.completions", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"response_text": "broken stream",
"malformed_sse": true
}
}
]
}))
.await;
let raw = server
.post_chat_stream_raw(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "malformed" }],
"stream": true
}))
.await;
let joined = String::from_utf8(raw.body.clone()).expect("utf8");
let parse_error = common::parse_sse_transcript(&raw.body).expect_err("malformed sse");
assert_eq!(raw.status, 200);
assert!(joined.contains("event: malformed"));
assert!(parse_error.contains("incomplete"));
}
#[tokio::test]
async fn scripted_malformed_sse_is_observable_for_responses() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"response_text": "broken response stream",
"malformed_sse": true
}
}
]
}))
.await;
let raw = server
.post_responses_stream_raw(json!({
"model": "gpt-test",
"input": "malformed responses",
"stream": true
}))
.await;
let joined = String::from_utf8(raw.body.clone()).expect("utf8");
let parse_error = common::parse_sse_transcript(&raw.body).expect_err("malformed sse");
assert_eq!(raw.status, 200);
assert!(joined.contains("event: malformed"));
assert!(parse_error.contains("incomplete"));
}
#[tokio::test]
async fn localhost_success_paths_stay_fast_enough() {
let server = common::spawn_server().await.expect("server should start");
let started = std::time::Instant::now();
let responses = server
.post_responses(json!({ "model": "gpt-test", "input": "speed", "stream": false }))
.await;
let responses_elapsed = started.elapsed();
assert_eq!(responses.status(), 200);
assert!(responses_elapsed < Duration::from_secs(1));
let started = std::time::Instant::now();
let timed = server
.post_responses_stream_timed(
json!({ "model": "gpt-test", "input": "speed stream", "stream": true }),
)
.await;
let stream_elapsed = started.elapsed();
assert_eq!(timed.status, 200);
assert!(timed.first_event_elapsed < Duration::from_secs(1));
assert!(stream_elapsed < Duration::from_secs(1));
assert!(!timed.chunks.is_empty());
let started = std::time::Instant::now();
let chat = server
.post_chat(json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "speed chat" }],
"stream": false
}))
.await;
let chat_elapsed = started.elapsed();
assert_eq!(chat.status(), 200);
assert!(chat_elapsed < Duration::from_secs(1));
}

102
test/twin/openai/tests/health_and_auth.rs
View file

@ -1,102 +0,0 @@
mod common;
use serde_json::json;
#[tokio::test]
async fn healthz_is_available() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.client
.get(format!("{}/healthz", server.base_url))
.send()
.await
.expect("health request should succeed");
assert_eq!(response.status(), 200);
}
#[tokio::test]
async fn responses_requires_bearer_auth() {
let server = common::spawn_server().await.expect("server should start");
let unauthenticated = server
.client
.post(format!("{}/v1/responses", server.base_url))
.json(&json!({
"model": "gpt-test",
"input": "hello"
}))
.send()
.await
.expect("request should complete");
assert_eq!(unauthenticated.status(), 401);
let unauthenticated_body = unauthenticated
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
unauthenticated_body["error"]["type"],
"invalid_request_error"
);
assert_eq!(
unauthenticated_body["error"]["code"],
"missing_bearer_token"
);
let authenticated = server
.auth_client
.post(format!("{}/v1/responses", server.base_url))
.json(&json!({
"model": "gpt-test",
"input": "hello"
}))
.send()
.await
.expect("request should complete");
assert_ne!(authenticated.status(), 401);
}
#[tokio::test]
async fn chat_completions_rejects_empty_bearer_auth() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_chat_with_auth_header(
json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "hello" }]
}),
Some("Bearer "),
)
.await;
assert_eq!(response.status(), 401);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["code"], "missing_bearer_token");
}
#[tokio::test]
async fn chat_completions_requires_bearer_auth() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.client
.post(format!("{}/v1/chat/completions", server.base_url))
.json(&json!({
"model": "gpt-test",
"messages": [{ "role": "user", "content": "hello" }]
}))
.send()
.await
.expect("request should complete");
assert_eq!(response.status(), 401);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["code"], "missing_bearer_token");
}

1849
test/twin/openai/tests/live_openai_contract.rs
View file

File diff suppressed because it is too large Load diff

440
test/twin/openai/tests/responses_contract.rs
View file

@ -1,440 +0,0 @@
mod common;
use serde_json::json;
#[tokio::test]
async fn responses_create_returns_deterministic_non_stream_payload() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses_with_headers(
json!({
"model": "gpt-test",
"input": "Hello from the test suite",
"stream": false
}),
Some("org-test"),
Some("proj-test"),
)
.await;
assert_eq!(response.status(), 200);
let body = response
.json::<serde_json::Value>()
.await
.expect("json body should parse");
assert_eq!(body["object"], "response");
assert_eq!(body["model"], "gpt-test");
assert_eq!(body["status"], "completed");
assert_eq!(body["id"], "resp_000001");
assert_eq!(body["created"], 1);
assert_eq!(body["output"][0]["type"], "message");
assert_eq!(
body["output"][0]["content"][0]["text"],
"deterministic: Hello from the test suite"
);
assert_eq!(body["usage"]["input_tokens"], 5);
assert_eq!(body["usage"]["output_tokens"], 5);
}
#[tokio::test]
async fn responses_accepts_supported_openai_request_fields() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"stream": false,
"input": [
{
"role": "user",
"content": [
{ "type": "input_text", "text": "Summarize this image please" },
{ "type": "input_image", "image_url": "https://example.com/cat.png" }
]
}
],
"metadata": {
"suite": "responses",
"case": "supported-fields"
},
"stop": ["END"],
"previous_response_id": "resp_previous",
"reasoning": {
"effort": "medium"
},
"tools": [
{
"type": "function",
"name": "lookup_weather",
"description": "Look up weather",
"parameters": {
"type": "object",
"properties": {
"city": { "type": "string" }
}
}
}
],
"tool_choice": "auto",
"text": {
"format": {
"type": "text"
}
}
}))
.await;
assert_eq!(response.status(), 200);
let body = response
.json::<serde_json::Value>()
.await
.expect("json body should parse");
assert_eq!(body["object"], "response");
assert_eq!(
body["output"][0]["content"][0]["text"],
"deterministic: Summarize this image please"
);
}
#[tokio::test]
async fn responses_reject_unfulfilled_tool_choice_requirements() {
let server = common::spawn_server().await.expect("server should start");
let required = server
.post_responses(json!({
"model": "gpt-test",
"input": "plain text please",
"stream": false,
"tools": [{ "type": "function", "name": "lookup_weather" }],
"tool_choice": "required"
}))
.await;
assert_eq!(required.status(), 400);
let body = required.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
let named = server
.post_responses(json!({
"model": "gpt-test",
"input": "plain text please",
"stream": false,
"tools": [{ "type": "function", "name": "lookup_weather" }],
"tool_choice": {
"type": "function",
"name": "lookup_weather"
}
}))
.await;
assert_eq!(named.status(), 400);
let body = named.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
}
#[tokio::test]
async fn responses_accept_unknown_top_level_fields() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "hello",
"stream": false,
"unexpected_field": true
}))
.await;
assert_eq!(response.status(), 200);
}
#[tokio::test]
async fn responses_reject_malformed_input_items() {
let server = common::spawn_server().await.expect("server should start");
let missing_call_id = server
.post_responses(json!({
"model": "gpt-test",
"input": [
{
"type": "function_call_output",
"output": "72 and sunny"
}
],
"stream": false
}))
.await;
assert_eq!(missing_call_id.status(), 400);
let body = missing_call_id
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "input");
let missing_content = server
.post_responses(json!({
"model": "gpt-test",
"input": [
{
"role": "user"
}
],
"stream": false
}))
.await;
assert_eq!(missing_content.status(), 400);
let body = missing_content
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "input");
}
#[tokio::test]
async fn responses_reject_malformed_image_input() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": [{
"role": "user",
"content": [{
"type": "input_image",
"image_url": ""
}]
}],
"stream": false
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "input");
}
#[tokio::test]
async fn responses_stream_message_item_done_round_trips_as_input() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_responses_stream(json!({
"model": "gpt-test",
"input": "stream this request",
"stream": true
}))
.await;
assert_eq!(status, 200);
let joined = chunks.join("");
let transcript = common::parse_sse_transcript(joined.as_bytes()).expect("valid sse");
let message_item = transcript
.events
.iter()
.filter(|event| event.event.as_deref() == Some("response.output_item.done"))
.filter_map(|event| serde_json::from_str::<serde_json::Value>(&event.data).ok())
.map(|payload| payload["item"].clone())
.find(|item| item["type"] == "message")
.expect("stream should emit a completed message item");
// The completed item carries its full content, like the real API.
let content = message_item["content"]
.as_array()
.expect("completed message item should include content");
assert_eq!(content.len(), 1);
assert_eq!(content[0]["type"], "output_text");
assert!(
content[0]["text"]
.as_str()
.is_some_and(|text| !text.is_empty())
);
// Adapters replay the completed item verbatim as assistant history on the
// next turn, so the twin must accept its own streamed output as input.
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "stream this request"}]
},
message_item,
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "and again"}]
},
],
"stream": false
}))
.await;
assert_eq!(response.status(), 200);
}
#[tokio::test]
async fn responses_stream_emits_expected_sse_sequence() {
let server = common::spawn_server().await.expect("server should start");
let request = json!({
"model": "gpt-test",
"input": "stream this request",
"stream": true
});
let non_stream = server
.post_responses(json!({
"model": "gpt-test",
"input": "stream this request",
"stream": false
}))
.await
.json::<serde_json::Value>()
.await
.expect("json body should parse");
let (status, chunks) = server.post_responses_stream(request).await;
let joined = chunks.join("");
let transcript = common::parse_sse_transcript(joined.as_bytes()).expect("valid sse");
let events = transcript
.events
.iter()
.filter_map(|event| event.event.as_deref())
.collect::<Vec<_>>();
assert_eq!(status, 200);
assert_eq!(events, vec![
"response.created",
"response.in_progress",
"response.output_item.added",
"response.output_item.done",
"response.output_item.added",
"response.content_part.added",
"response.output_text.delta",
"response.output_text.done",
"response.content_part.done",
"response.output_item.done",
"response.completed",
]);
assert!(!transcript.done);
assert!(joined.contains("deterministic: stream this request"));
assert!(
joined.contains(
non_stream["output"][0]["content"][0]["text"]
.as_str()
.expect("text")
)
);
}
#[tokio::test]
async fn responses_stream_emits_reasoning_and_completion_events() {
let server = common::spawn_server().await.expect("server should start");
let (status, chunks) = server
.post_responses_stream(json!({
"model": "gpt-test",
"input": "show your reasoning",
"stream": true,
"reasoning": {
"effort": "high"
}
}))
.await;
let joined = chunks.join("");
assert_eq!(status, 200);
assert!(joined.contains("event: response.reasoning.delta\n"));
assert!(joined.contains("reasoning: show your reasoning"));
assert!(joined.contains("event: response.completed\n"));
}
#[tokio::test]
async fn responses_stream_emits_structured_output_events() {
let server = common::spawn_server().await.expect("server should start");
let non_stream = server
.post_responses(json!({
"model": "gpt-test",
"input": "structured stream",
"stream": false,
"text": {
"format": { "type": "json_object" }
}
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let (status, chunks) = server
.post_responses_stream(json!({
"model": "gpt-test",
"input": "structured stream",
"stream": true,
"text": {
"format": { "type": "json_object" }
}
}))
.await;
let joined = chunks.join("");
let transcript = common::parse_sse_transcript(joined.as_bytes()).expect("valid sse");
let events = transcript
.events
.iter()
.filter_map(|event| event.event.as_deref())
.collect::<Vec<_>>();
let streamed_json = transcript
.events
.iter()
.find(|event| event.event.as_deref() == Some("response.output_text.done"))
.and_then(|event| serde_json::from_str::<serde_json::Value>(&event.data).ok())
.and_then(|payload| {
payload
.get("text")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned)
})
.and_then(|text| serde_json::from_str::<serde_json::Value>(&text).ok())
.expect("structured stream output text");
assert_eq!(status, 200);
assert_eq!(events, vec![
"response.created",
"response.in_progress",
"response.output_item.added",
"response.output_item.done",
"response.output_item.added",
"response.content_part.added",
"response.output_text.delta",
"response.output_text.done",
"response.content_part.done",
"response.output_item.done",
"response.completed",
]);
assert!(!transcript.done);
assert_eq!(
streamed_json["message"],
non_stream["output"][0]["content"][1]["json"]["message"]
);
assert!(
joined.contains(
non_stream["output"][0]["content"][1]["json"]["message"]
.as_str()
.expect("json message")
)
);
}

581
test/twin/openai/tests/tool_and_schema_contract.rs
View file

@ -1,581 +0,0 @@
mod common;
use fabro_http::header::AUTHORIZATION;
use serde_json::json;
#[tokio::test]
async fn admin_loaded_scenarios_are_consumed_fifo() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "first scripted response" }
},
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "second scripted response" }
}
]
}))
.await;
let first = server
.post_responses(json!({ "model": "gpt-test", "input": "hello", "stream": false }))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let second = server
.post_responses(json!({ "model": "gpt-test", "input": "hello", "stream": false }))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let fallback = server
.post_responses(json!({ "model": "gpt-test", "input": "hello", "stream": false }))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
first["output"][0]["content"][0]["text"],
"first scripted response"
);
assert_eq!(
second["output"][0]["content"][0]["text"],
"second scripted response"
);
assert_eq!(
fallback["output"][0]["content"][0]["text"],
"deterministic: hello"
);
let logs = server.request_logs().await;
assert_eq!(logs["requests"].as_array().expect("request logs").len(), 3);
server.reset().await;
let logs_after_reset = server.request_logs().await;
assert_eq!(
logs_after_reset["requests"]
.as_array()
.expect("request logs")
.len(),
0
);
}
#[tokio::test]
async fn bearer_namespaces_isolate_scenarios_and_request_logs_on_shared_server() {
let primary = common::spawn_server().await.expect("server should start");
let secondary = primary.fork_namespace().expect("secondary namespace");
primary
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "primary scripted response" }
}
]
}))
.await;
secondary
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "secondary scripted response" }
}
]
}))
.await;
let primary_response = primary
.post_responses(
json!({ "model": "gpt-test", "input": "hello from primary", "stream": false }),
)
.await
.json::<serde_json::Value>()
.await
.expect("json");
let secondary_response = secondary
.post_responses(
json!({ "model": "gpt-test", "input": "hello from secondary", "stream": false }),
)
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
primary_response["output"][0]["content"][0]["text"],
"primary scripted response"
);
assert_eq!(
secondary_response["output"][0]["content"][0]["text"],
"secondary scripted response"
);
let primary_logs = primary.request_logs().await;
let secondary_logs = secondary.request_logs().await;
assert_eq!(
primary_logs["requests"]
.as_array()
.expect("primary logs")
.len(),
1
);
assert_eq!(
secondary_logs["requests"]
.as_array()
.expect("secondary logs")
.len(),
1
);
assert_eq!(
primary_logs["requests"][0]["input_text"],
"hello from primary"
);
assert_eq!(
secondary_logs["requests"][0]["input_text"],
"hello from secondary"
);
}
#[tokio::test]
async fn admin_reset_only_clears_the_target_bearer_namespace() {
let primary = common::spawn_server().await.expect("server should start");
let secondary = primary.fork_namespace().expect("secondary namespace");
let primary_before_reset = primary
.post_responses(json!({ "model": "gpt-test", "input": "before reset", "stream": false }))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let secondary_before_reset = secondary
.post_responses(
json!({ "model": "gpt-test", "input": "before secondary", "stream": false }),
)
.await
.json::<serde_json::Value>()
.await
.expect("json");
secondary
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": { "kind": "success", "response_text": "secondary still scripted" }
}
]
}))
.await;
primary.reset().await;
let primary_after_reset = primary
.post_responses(json!({ "model": "gpt-test", "input": "after reset", "stream": false }))
.await
.json::<serde_json::Value>()
.await
.expect("json");
let secondary_after_reset = secondary
.post_responses(
json!({ "model": "gpt-test", "input": "after secondary reset", "stream": false }),
)
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(primary_before_reset["id"], "resp_000001");
assert_eq!(secondary_before_reset["id"], "resp_000001");
assert_eq!(primary_after_reset["id"], "resp_000001");
assert_eq!(secondary_after_reset["id"], "resp_000002");
assert_eq!(
primary_after_reset["output"][0]["content"][0]["text"],
"deterministic: after reset"
);
assert_eq!(
secondary_after_reset["output"][0]["content"][0]["text"],
"secondary still scripted"
);
let primary_logs = primary.request_logs().await;
let secondary_logs = secondary.request_logs().await;
assert_eq!(
primary_logs["requests"]
.as_array()
.expect("primary logs")
.len(),
1
);
assert_eq!(
secondary_logs["requests"]
.as_array()
.expect("secondary logs")
.len(),
2
);
assert_eq!(primary_logs["requests"][0]["input_text"], "after reset");
assert_eq!(
secondary_logs["requests"][1]["input_text"],
"after secondary reset"
);
}
#[tokio::test]
async fn admin_routes_accept_no_auth_but_reject_invalid_authorization_headers() {
let server = common::spawn_server().await.expect("server should start");
let unauthenticated = server
.client
.get(format!("{}/__admin/requests", server.base_url))
.send()
.await
.expect("admin logs should complete");
assert_eq!(unauthenticated.status(), 200);
for authorization in ["Bearer ", "Basic nope"] {
let response = server
.client
.get(format!("{}/__admin/requests", server.base_url))
.header(AUTHORIZATION, authorization)
.send()
.await
.expect("admin logs should complete");
assert_eq!(response.status(), 401);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["code"], "missing_bearer_token");
}
}
#[tokio::test]
async fn responses_supports_scripted_tool_call_and_continuation() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false },
"script": {
"kind": "success",
"tool_calls": [
{
"id": "call_weather",
"name": "lookup_weather",
"arguments": { "city": "Boston" }
}
],
"reasoning": ["tool reasoning"]
}
},
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": false, "input_contains": "72 and sunny" },
"script": { "kind": "success", "response_text": "The weather is 72 and sunny." }
}
]
}))
.await;
let first = server
.post_responses(json!({
"model": "gpt-test",
"input": "what is the weather?",
"stream": false,
"tools": [{ "type": "function", "name": "lookup_weather" }],
"tool_choice": {
"type": "function",
"name": "lookup_weather"
}
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(first["output"].as_array().expect("output").len(), 1);
assert_eq!(first["output"][0]["type"], "function_call");
assert_eq!(first["output"][0]["name"], "lookup_weather");
assert_eq!(first["output"][0]["id"], "fc_call_weather");
assert_eq!(first["output"][0]["call_id"], "call_weather");
assert_eq!(first["output"][0]["arguments"], "{\"city\":\"Boston\"}");
assert_eq!(first["reasoning"][0], "tool reasoning");
let continuation = server
.post_responses(json!({
"model": "gpt-test",
"stream": false,
"previous_response_id": first["id"],
"input": [
{
"type": "function_call_output",
"call_id": "call_weather",
"output": "72 and sunny"
}
]
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
continuation["output"][0]["content"][0]["text"],
"The weather is 72 and sunny."
);
}
#[tokio::test]
async fn responses_stream_supports_tool_only_turn_without_fabricated_text() {
let server = common::spawn_server().await.expect("server should start");
server
.enqueue_scenarios(json!({
"scenarios": [
{
"matcher": { "endpoint": "responses", "model": "gpt-test", "stream": true },
"script": {
"kind": "success",
"tool_calls": [
{
"id": "call_weather",
"name": "lookup_weather",
"arguments": { "city": "Boston" }
}
]
}
}
]
}))
.await;
let (status, chunks) = server
.post_responses_stream(json!({
"model": "gpt-test",
"input": "what is the weather?",
"stream": true,
"tools": [{ "type": "function", "name": "lookup_weather" }],
"tool_choice": {
"type": "function",
"name": "lookup_weather"
}
}))
.await;
let joined = chunks.join("");
let transcript = common::parse_sse_transcript(joined.as_bytes()).expect("valid sse");
let events = transcript
.events
.iter()
.filter_map(|event| event.event.as_deref())
.collect::<Vec<_>>();
assert_eq!(status, 200);
assert_eq!(events, vec![
"response.created",
"response.in_progress",
"response.output_item.added",
"response.output_item.done",
"response.output_item.added",
"response.function_call_arguments.delta",
"response.function_call_arguments.done",
"response.output_item.done",
"response.completed",
]);
assert!(joined.contains("\"id\":\"fc_call_weather\""));
assert!(joined.contains("\"call_id\":\"call_weather\""));
assert!(joined.contains("\"arguments\":\"{\\\"city\\\":\\\"Boston\\\"}\""));
assert!(!joined.contains("response.output_text.delta"));
}
#[tokio::test]
async fn responses_structured_output_support_is_explicit() {
let server = common::spawn_server().await.expect("server should start");
let json_object = server
.post_responses(json!({
"model": "gpt-test",
"input": "structured please",
"stream": false,
"text": {
"format": { "type": "json_object" }
}
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
json_object["output"][0]["content"][1]["type"],
"output_json"
);
assert_eq!(
json_object["output"][0]["content"][1]["json"]["message"],
"deterministic: structured please"
);
let json_schema = server
.post_responses(json!({
"model": "gpt-test",
"input": "schema please",
"stream": false,
"text": {
"format": {
"type": "json_schema",
"name": "response_schema",
"schema": {
"type": "object",
"properties": {
"message": { "type": "string" },
"ok": { "type": "boolean" }
}
},
"strict": true
}
}
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(json_schema["output"][0]["content"][1]["json"]["ok"], true);
let unsupported = server
.post_responses(json!({
"model": "gpt-test",
"input": "schema please",
"stream": false,
"text": {
"format": {
"type": "json_schema",
"name": "response_schema",
"schema": {
"type": "object",
"properties": {
"items": {
"type": "array",
"items": { "type": "string" }
}
}
}
}
}
}))
.await;
assert_eq!(unsupported.status(), 400);
let primitive_root = server
.post_responses(json!({
"model": "gpt-test",
"input": "schema please",
"stream": false,
"text": {
"format": {
"type": "json_schema",
"name": "response_schema",
"schema": {
"type": "string"
}
}
}
}))
.await;
assert_eq!(primitive_root.status(), 400);
}
#[tokio::test]
async fn responses_reasoning_and_continuation_fields_round_trip() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"stream": false,
"previous_response_id": "resp_before",
"reasoning": { "effort": "medium" },
"input": [
{ "role": "system", "content": "system prompt" },
{ "role": "assistant", "content": [{ "type": "text", "text": "prior assistant answer" }] },
{ "role": "user", "content": [{ "type": "input_text", "text": "continue carefully" }] },
{ "type": "function_call_output", "call_id": "call_continue", "output": "tool finished" }
]
}))
.await
.json::<serde_json::Value>()
.await
.expect("json");
assert_eq!(
response["reasoning"][0],
"reasoning: continue carefully tool finished"
);
assert_eq!(
response["output"][0]["content"][0]["text"],
"deterministic: continue carefully tool finished"
);
}
#[tokio::test]
async fn responses_reject_unsupported_text_format_type() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "bad format",
"text": {
"format": { "type": "xml" }
}
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn responses_reject_unsupported_tool_choice_shape() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "bad tools",
"tools": [{ "type": "function", "name": "lookup_weather" }],
"tool_choice": { "type": "required" }
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn responses_reject_object_stop_value() {
let server = common::spawn_server().await.expect("server should start");
let response = server
.post_responses(json!({
"model": "gpt-test",
"input": "bad stop",
"stop": { "type": "object" }
}))
.await;
assert_eq!(response.status(), 400);
let body = response.json::<serde_json::Value>().await.expect("json");
assert_eq!(body["error"]["type"], "invalid_request_error");
}